1d87f36a0SRajneesh Bhardwaj // SPDX-License-Identifier: GPL-2.0 OR MIT
2ca750681SFelix Kuehling /*
3d87f36a0SRajneesh Bhardwaj * Copyright 2016-2022 Advanced Micro Devices, Inc.
4ca750681SFelix Kuehling *
5ca750681SFelix Kuehling * Permission is hereby granted, free of charge, to any person obtaining a
6ca750681SFelix Kuehling * copy of this software and associated documentation files (the "Software"),
7ca750681SFelix Kuehling * to deal in the Software without restriction, including without limitation
8ca750681SFelix Kuehling * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9ca750681SFelix Kuehling * and/or sell copies of the Software, and to permit persons to whom the
10ca750681SFelix Kuehling * Software is furnished to do so, subject to the following conditions:
11ca750681SFelix Kuehling *
12ca750681SFelix Kuehling * The above copyright notice and this permission notice shall be included in
13ca750681SFelix Kuehling * all copies or substantial portions of the Software.
14ca750681SFelix Kuehling *
15ca750681SFelix Kuehling * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16ca750681SFelix Kuehling * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17ca750681SFelix Kuehling * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18ca750681SFelix Kuehling * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19ca750681SFelix Kuehling * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20ca750681SFelix Kuehling * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21ca750681SFelix Kuehling * OTHER DEALINGS IN THE SOFTWARE.
22ca750681SFelix Kuehling */
23ca750681SFelix Kuehling
24ca750681SFelix Kuehling #include "kfd_priv.h"
25ca750681SFelix Kuehling #include "kfd_events.h"
2612fb1ad7SJonathan Kim #include "kfd_debug.h"
27ca750681SFelix Kuehling #include "soc15_int.h"
28a53a11a8SYong Zhao #include "kfd_device_queue_manager.h"
29938a0650SAmber Lin #include "kfd_smi_events.h"
3020161e51SDennis Li
3112fb1ad7SJonathan Kim /*
3212fb1ad7SJonathan Kim * GFX9 SQ Interrupts
3312fb1ad7SJonathan Kim *
3412fb1ad7SJonathan Kim * There are 3 encoding types of interrupts sourced from SQ sent as a 44-bit
3512fb1ad7SJonathan Kim * packet to the Interrupt Handler:
3612fb1ad7SJonathan Kim * Auto - Generated by the SQG (various cmd overflows, timestamps etc)
3712fb1ad7SJonathan Kim * Wave - Generated by S_SENDMSG through a shader program
3812fb1ad7SJonathan Kim * Error - HW generated errors (Illegal instructions, Memviols, EDC etc)
3912fb1ad7SJonathan Kim *
4012fb1ad7SJonathan Kim * The 44-bit packet is mapped as {context_id1[7:0],context_id0[31:0]} plus
4112fb1ad7SJonathan Kim * 4-bits for VMID (SOC15_VMID_FROM_IH_ENTRY) as such:
4212fb1ad7SJonathan Kim *
4312fb1ad7SJonathan Kim * - context_id0[27:26]
4412fb1ad7SJonathan Kim * Encoding type (0 = Auto, 1 = Wave, 2 = Error)
4512fb1ad7SJonathan Kim *
4612fb1ad7SJonathan Kim * - context_id0[13]
4712fb1ad7SJonathan Kim * PRIV bit indicates that Wave S_SEND or error occurred within trap
4812fb1ad7SJonathan Kim *
4912fb1ad7SJonathan Kim * - {context_id1[7:0],context_id0[31:28],context_id0[11:0]}
5012fb1ad7SJonathan Kim * 24-bit data with the following layout per encoding type:
5112fb1ad7SJonathan Kim * Auto - only context_id0[8:0] is used, which reports various interrupts
5212fb1ad7SJonathan Kim * generated by SQG. The rest is 0.
5312fb1ad7SJonathan Kim * Wave - user data sent from m0 via S_SENDMSG
5412fb1ad7SJonathan Kim * Error - Error type (context_id1[7:4]), Error Details (rest of bits)
5512fb1ad7SJonathan Kim *
5612fb1ad7SJonathan Kim * The other context_id bits show coordinates (SE/SH/CU/SIMD/WAVE) for wave
5712fb1ad7SJonathan Kim * S_SENDMSG and Errors. These are 0 for Auto.
5812fb1ad7SJonathan Kim */
5912fb1ad7SJonathan Kim
6020161e51SDennis Li enum SQ_INTERRUPT_WORD_ENCODING {
6120161e51SDennis Li SQ_INTERRUPT_WORD_ENCODING_AUTO = 0x0,
6220161e51SDennis Li SQ_INTERRUPT_WORD_ENCODING_INST,
6320161e51SDennis Li SQ_INTERRUPT_WORD_ENCODING_ERROR,
6420161e51SDennis Li };
6520161e51SDennis Li
6620161e51SDennis Li enum SQ_INTERRUPT_ERROR_TYPE {
6720161e51SDennis Li SQ_INTERRUPT_ERROR_TYPE_EDC_FUE = 0x0,
6820161e51SDennis Li SQ_INTERRUPT_ERROR_TYPE_ILLEGAL_INST,
6920161e51SDennis Li SQ_INTERRUPT_ERROR_TYPE_MEMVIOL,
7020161e51SDennis Li SQ_INTERRUPT_ERROR_TYPE_EDC_FED,
7120161e51SDennis Li };
7220161e51SDennis Li
7320161e51SDennis Li /* SQ_INTERRUPT_WORD_AUTO_CTXID */
7420161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE__SHIFT 0
7520161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__WLT__SHIFT 1
7620161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_BUF_FULL__SHIFT 2
7720161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__REG_TIMESTAMP__SHIFT 3
7820161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__CMD_TIMESTAMP__SHIFT 4
7920161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_CMD_OVERFLOW__SHIFT 5
8020161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_REG_OVERFLOW__SHIFT 6
8120161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__IMMED_OVERFLOW__SHIFT 7
8220161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_UTC_ERROR__SHIFT 8
8320161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__SE_ID__SHIFT 24
8420161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__ENCODING__SHIFT 26
8520161e51SDennis Li
8620161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_MASK 0x00000001
8720161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__WLT_MASK 0x00000002
8820161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_BUF_FULL_MASK 0x00000004
8920161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__REG_TIMESTAMP_MASK 0x00000008
9020161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__CMD_TIMESTAMP_MASK 0x00000010
9120161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_CMD_OVERFLOW_MASK 0x00000020
9220161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_REG_OVERFLOW_MASK 0x00000040
9320161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__IMMED_OVERFLOW_MASK 0x00000080
9420161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_UTC_ERROR_MASK 0x00000100
9520161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__SE_ID_MASK 0x03000000
9620161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__ENCODING_MASK 0x0c000000
9720161e51SDennis Li
9820161e51SDennis Li /* SQ_INTERRUPT_WORD_WAVE_CTXID */
9920161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__DATA__SHIFT 0
10020161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__SH_ID__SHIFT 12
10120161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__PRIV__SHIFT 13
10220161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__WAVE_ID__SHIFT 14
10320161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__SIMD_ID__SHIFT 18
10420161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__CU_ID__SHIFT 20
10520161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__SE_ID__SHIFT 24
10620161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__ENCODING__SHIFT 26
10720161e51SDennis Li
10820161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__DATA_MASK 0x00000fff
10920161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__SH_ID_MASK 0x00001000
11020161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__PRIV_MASK 0x00002000
11120161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__WAVE_ID_MASK 0x0003c000
11220161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__SIMD_ID_MASK 0x000c0000
11320161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__CU_ID_MASK 0x00f00000
11420161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__SE_ID_MASK 0x03000000
11520161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__ENCODING_MASK 0x0c000000
11620161e51SDennis Li
11712fb1ad7SJonathan Kim /* GFX9 SQ interrupt 24-bit data from context_id<0,1> */
11820161e51SDennis Li #define KFD_CONTEXT_ID_GET_SQ_INT_DATA(ctx0, ctx1) \
11920161e51SDennis Li ((ctx0 & 0xfff) | ((ctx0 >> 16) & 0xf000) | ((ctx1 << 16) & 0xff0000))
12020161e51SDennis Li
12120161e51SDennis Li #define KFD_SQ_INT_DATA__ERR_TYPE_MASK 0xF00000
12220161e51SDennis Li #define KFD_SQ_INT_DATA__ERR_TYPE__SHIFT 20
123ca750681SFelix Kuehling
12412fb1ad7SJonathan Kim /*
12512fb1ad7SJonathan Kim * The debugger will send user data(m0) with PRIV=1 to indicate it requires
12612fb1ad7SJonathan Kim * notification from the KFD with the following queue id (DOORBELL_ID) and
12712fb1ad7SJonathan Kim * trap code (TRAP_CODE).
12812fb1ad7SJonathan Kim */
12912fb1ad7SJonathan Kim #define KFD_INT_DATA_DEBUG_DOORBELL_MASK 0x0003ff
13012fb1ad7SJonathan Kim #define KFD_INT_DATA_DEBUG_TRAP_CODE_SHIFT 10
13112fb1ad7SJonathan Kim #define KFD_INT_DATA_DEBUG_TRAP_CODE_MASK 0x07fc00
13212fb1ad7SJonathan Kim #define KFD_DEBUG_DOORBELL_ID(sq_int_data) ((sq_int_data) & \
13312fb1ad7SJonathan Kim KFD_INT_DATA_DEBUG_DOORBELL_MASK)
13412fb1ad7SJonathan Kim #define KFD_DEBUG_TRAP_CODE(sq_int_data) (((sq_int_data) & \
13512fb1ad7SJonathan Kim KFD_INT_DATA_DEBUG_TRAP_CODE_MASK) \
13612fb1ad7SJonathan Kim >> KFD_INT_DATA_DEBUG_TRAP_CODE_SHIFT)
13712fb1ad7SJonathan Kim #define KFD_DEBUG_CP_BAD_OP_ECODE_MASK 0x3fffc00
13812fb1ad7SJonathan Kim #define KFD_DEBUG_CP_BAD_OP_ECODE_SHIFT 10
13912fb1ad7SJonathan Kim #define KFD_DEBUG_CP_BAD_OP_ECODE(ctxid0) (((ctxid0) & \
14012fb1ad7SJonathan Kim KFD_DEBUG_CP_BAD_OP_ECODE_MASK) \
14112fb1ad7SJonathan Kim >> KFD_DEBUG_CP_BAD_OP_ECODE_SHIFT)
14212fb1ad7SJonathan Kim
event_interrupt_poison_consumption_v9(struct kfd_node * dev,uint16_t pasid,uint16_t client_id)1438dc1db31SMukul Joshi static void event_interrupt_poison_consumption_v9(struct kfd_node *dev,
1449d8a8d78STao Zhou uint16_t pasid, uint16_t client_id)
145b6485bedSTao Zhou {
146eed41975STao Zhou int old_poison, ret = -EINVAL;
147b6485bedSTao Zhou struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
148b6485bedSTao Zhou
149b6485bedSTao Zhou if (!p)
150b6485bedSTao Zhou return;
151b6485bedSTao Zhou
152b6485bedSTao Zhou /* all queues of a process will be unmapped in one time */
153eed41975STao Zhou old_poison = atomic_cmpxchg(&p->poison, 0, 1);
154b6485bedSTao Zhou kfd_unref_process(p);
155eed41975STao Zhou if (old_poison)
156b6485bedSTao Zhou return;
157b6485bedSTao Zhou
1589d8a8d78STao Zhou switch (client_id) {
1599d8a8d78STao Zhou case SOC15_IH_CLIENTID_SE0SH:
1609d8a8d78STao Zhou case SOC15_IH_CLIENTID_SE1SH:
1619d8a8d78STao Zhou case SOC15_IH_CLIENTID_SE2SH:
1629d8a8d78STao Zhou case SOC15_IH_CLIENTID_SE3SH:
1639d8a8d78STao Zhou case SOC15_IH_CLIENTID_UTCL2:
16429b440d2STao Zhou ret = kfd_dqm_evict_pasid(dev->dqm, pasid);
165b6485bedSTao Zhou break;
1669d8a8d78STao Zhou case SOC15_IH_CLIENTID_SDMA0:
1679d8a8d78STao Zhou case SOC15_IH_CLIENTID_SDMA1:
1689d8a8d78STao Zhou case SOC15_IH_CLIENTID_SDMA2:
1699d8a8d78STao Zhou case SOC15_IH_CLIENTID_SDMA3:
1709d8a8d78STao Zhou case SOC15_IH_CLIENTID_SDMA4:
1719d8a8d78STao Zhou break;
172b6485bedSTao Zhou default:
173b6485bedSTao Zhou break;
174b6485bedSTao Zhou }
175b6485bedSTao Zhou
176b6485bedSTao Zhou kfd_signal_poison_consumed_event(dev, pasid);
177b6485bedSTao Zhou
178b6485bedSTao Zhou /* resetting queue passes, do page retirement without gpu reset
1792243f493SRajneesh Bhardwaj * resetting queue fails, fallback to gpu reset solution
1802243f493SRajneesh Bhardwaj */
181ed94aca6STao Zhou if (!ret) {
182ed94aca6STao Zhou dev_warn(dev->adev->dev,
183ed94aca6STao Zhou "RAS poison consumption, unmap queue flow succeeded: client id %d\n",
184ed94aca6STao Zhou client_id);
185b6485bedSTao Zhou amdgpu_amdkfd_ras_poison_consumption_handler(dev->adev, false);
186ed94aca6STao Zhou } else {
187ed94aca6STao Zhou dev_warn(dev->adev->dev,
188ed94aca6STao Zhou "RAS poison consumption, fall back to gpu reset flow: client id %d\n",
189ed94aca6STao Zhou client_id);
190b6485bedSTao Zhou amdgpu_amdkfd_ras_poison_consumption_handler(dev->adev, true);
191b6485bedSTao Zhou }
192ed94aca6STao Zhou }
193b6485bedSTao Zhou
context_id_expected(struct kfd_dev * dev)194c3eb12dfSFelix Kuehling static bool context_id_expected(struct kfd_dev *dev)
195c3eb12dfSFelix Kuehling {
196c3eb12dfSFelix Kuehling switch (KFD_GC_VERSION(dev)) {
197c3eb12dfSFelix Kuehling case IP_VERSION(9, 0, 1):
198c3eb12dfSFelix Kuehling return dev->mec_fw_version >= 0x817a;
199c3eb12dfSFelix Kuehling case IP_VERSION(9, 1, 0):
200c3eb12dfSFelix Kuehling case IP_VERSION(9, 2, 1):
201c3eb12dfSFelix Kuehling case IP_VERSION(9, 2, 2):
202c3eb12dfSFelix Kuehling case IP_VERSION(9, 3, 0):
203c3eb12dfSFelix Kuehling case IP_VERSION(9, 4, 0):
204c3eb12dfSFelix Kuehling return dev->mec_fw_version >= 0x17a;
205c3eb12dfSFelix Kuehling default:
206c3eb12dfSFelix Kuehling /* Other GFXv9 and later GPUs always sent valid context IDs
207c3eb12dfSFelix Kuehling * on legitimate events
208c3eb12dfSFelix Kuehling */
209c3eb12dfSFelix Kuehling return KFD_GC_VERSION(dev) >= IP_VERSION(9, 4, 1);
210c3eb12dfSFelix Kuehling }
211c3eb12dfSFelix Kuehling }
212c3eb12dfSFelix Kuehling
event_interrupt_isr_v9(struct kfd_node * dev,const uint32_t * ih_ring_entry,uint32_t * patched_ihre,bool * patched_flag)2138dc1db31SMukul Joshi static bool event_interrupt_isr_v9(struct kfd_node *dev,
21458e69886SLan Xiao const uint32_t *ih_ring_entry,
21558e69886SLan Xiao uint32_t *patched_ihre,
21658e69886SLan Xiao bool *patched_flag)
217ca750681SFelix Kuehling {
218ca750681SFelix Kuehling uint16_t source_id, client_id, pasid, vmid;
219c129db12SFelix Kuehling const uint32_t *data = ih_ring_entry;
220c129db12SFelix Kuehling
221ca750681SFelix Kuehling source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry);
222ca750681SFelix Kuehling client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry);
22312fb1ad7SJonathan Kim
22412fb1ad7SJonathan Kim /* Only handle interrupts from KFD VMIDs */
22512fb1ad7SJonathan Kim vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry);
22612fb1ad7SJonathan Kim if (!KFD_IRQ_IS_FENCE(client_id, source_id) &&
22712fb1ad7SJonathan Kim (vmid < dev->vm_info.first_vmid_kfd ||
22812fb1ad7SJonathan Kim vmid > dev->vm_info.last_vmid_kfd))
22912fb1ad7SJonathan Kim return false;
23012fb1ad7SJonathan Kim
23100557f41SYong Zhao pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry);
232ca750681SFelix Kuehling
233ae279f69SAlex Deucher /* Only handle clients we care about */
234ae279f69SAlex Deucher if (client_id != SOC15_IH_CLIENTID_GRBM_CP &&
235ae279f69SAlex Deucher client_id != SOC15_IH_CLIENTID_SDMA0 &&
236ae279f69SAlex Deucher client_id != SOC15_IH_CLIENTID_SDMA1 &&
237ae279f69SAlex Deucher client_id != SOC15_IH_CLIENTID_SDMA2 &&
238ae279f69SAlex Deucher client_id != SOC15_IH_CLIENTID_SDMA3 &&
239ae279f69SAlex Deucher client_id != SOC15_IH_CLIENTID_SDMA4 &&
240ae279f69SAlex Deucher client_id != SOC15_IH_CLIENTID_SDMA5 &&
241ae279f69SAlex Deucher client_id != SOC15_IH_CLIENTID_SDMA6 &&
242ae279f69SAlex Deucher client_id != SOC15_IH_CLIENTID_SDMA7 &&
243ae279f69SAlex Deucher client_id != SOC15_IH_CLIENTID_VMC &&
244ae279f69SAlex Deucher client_id != SOC15_IH_CLIENTID_VMC1 &&
2457af103eaSTao Zhou client_id != SOC15_IH_CLIENTID_UTCL2 &&
2467af103eaSTao Zhou client_id != SOC15_IH_CLIENTID_SE0SH &&
2477af103eaSTao Zhou client_id != SOC15_IH_CLIENTID_SE1SH &&
2487af103eaSTao Zhou client_id != SOC15_IH_CLIENTID_SE2SH &&
24912fb1ad7SJonathan Kim client_id != SOC15_IH_CLIENTID_SE3SH &&
25012fb1ad7SJonathan Kim !KFD_IRQ_IS_FENCE(client_id, source_id))
251ae279f69SAlex Deucher return false;
252ae279f69SAlex Deucher
253a53a11a8SYong Zhao /* This is a known issue for gfx9. Under non HWS, pasid is not set
254a53a11a8SYong Zhao * in the interrupt payload, so we need to find out the pasid on our
255a53a11a8SYong Zhao * own.
256a53a11a8SYong Zhao */
257a53a11a8SYong Zhao if (!pasid && dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
258a53a11a8SYong Zhao const uint32_t pasid_mask = 0xffff;
259a53a11a8SYong Zhao
260a53a11a8SYong Zhao *patched_flag = true;
261a53a11a8SYong Zhao memcpy(patched_ihre, ih_ring_entry,
2628dc1db31SMukul Joshi dev->kfd->device_info.ih_ring_entry_size);
263a53a11a8SYong Zhao
2643fe023d4SYong Zhao pasid = dev->dqm->vmid_pasid[vmid];
265a53a11a8SYong Zhao
266a53a11a8SYong Zhao /* Patch the pasid field */
267a53a11a8SYong Zhao patched_ihre[3] = cpu_to_le32((le32_to_cpu(patched_ihre[3])
268a53a11a8SYong Zhao & ~pasid_mask) | pasid);
269a53a11a8SYong Zhao }
270a53a11a8SYong Zhao
27100557f41SYong Zhao pr_debug("client id 0x%x, source id %d, vmid %d, pasid 0x%x. raw data:\n",
27200557f41SYong Zhao client_id, source_id, vmid, pasid);
273ca750681SFelix Kuehling pr_debug("%8X, %8X, %8X, %8X, %8X, %8X, %8X, %8X.\n",
274ca750681SFelix Kuehling data[0], data[1], data[2], data[3],
275ca750681SFelix Kuehling data[4], data[5], data[6], data[7]);
276ca750681SFelix Kuehling
277a53a11a8SYong Zhao /* If there is no valid PASID, it's likely a bug */
278a53a11a8SYong Zhao if (WARN_ONCE(pasid == 0, "Bug: No PASID in KFD interrupt"))
2798c8e1f69SAishwarya Ramakrishnan return false;
28000557f41SYong Zhao
281c3eb12dfSFelix Kuehling /* Workaround CP firmware sending bogus signals with 0 context_id.
282c3eb12dfSFelix Kuehling * Those can be safely ignored on hardware and firmware versions that
283c3eb12dfSFelix Kuehling * include a valid context_id on legitimate signals. This avoids the
284c3eb12dfSFelix Kuehling * slow path in kfd_signal_event_interrupt that scans all event slots
285c3eb12dfSFelix Kuehling * for signaled events.
286c3eb12dfSFelix Kuehling */
287c3eb12dfSFelix Kuehling if (source_id == SOC15_INTSRC_CP_END_OF_PIPE) {
288c3eb12dfSFelix Kuehling uint32_t context_id =
289c3eb12dfSFelix Kuehling SOC15_CONTEXT_ID0_FROM_IH_ENTRY(ih_ring_entry);
290c3eb12dfSFelix Kuehling
2918dc1db31SMukul Joshi if (context_id == 0 && context_id_expected(dev->kfd))
292c3eb12dfSFelix Kuehling return false;
293c3eb12dfSFelix Kuehling }
294c3eb12dfSFelix Kuehling
295c129db12SFelix Kuehling /* Interrupt types we care about: various signals and faults.
296c129db12SFelix Kuehling * They will be forwarded to a work queue (see below).
297c129db12SFelix Kuehling */
298c129db12SFelix Kuehling return source_id == SOC15_INTSRC_CP_END_OF_PIPE ||
299ca750681SFelix Kuehling source_id == SOC15_INTSRC_SDMA_TRAP ||
3005b0ce2d4Syipechai source_id == SOC15_INTSRC_SDMA_ECC ||
301ca750681SFelix Kuehling source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG ||
3022640c3faSshaoyunl source_id == SOC15_INTSRC_CP_BAD_OPCODE ||
30312fb1ad7SJonathan Kim KFD_IRQ_IS_FENCE(client_id, source_id) ||
3046d909c5dSOak Zeng ((client_id == SOC15_IH_CLIENTID_VMC ||
3050ad8c5e2SYong Zhao client_id == SOC15_IH_CLIENTID_VMC1 ||
3066d909c5dSOak Zeng client_id == SOC15_IH_CLIENTID_UTCL2) &&
3076d909c5dSOak Zeng !amdgpu_no_queue_eviction_on_vm_fault);
308ca750681SFelix Kuehling }
309ca750681SFelix Kuehling
event_interrupt_wq_v9(struct kfd_node * dev,const uint32_t * ih_ring_entry)3108dc1db31SMukul Joshi static void event_interrupt_wq_v9(struct kfd_node *dev,
311ca750681SFelix Kuehling const uint32_t *ih_ring_entry)
312ca750681SFelix Kuehling {
313ca750681SFelix Kuehling uint16_t source_id, client_id, pasid, vmid;
31420161e51SDennis Li uint32_t context_id0, context_id1;
31520161e51SDennis Li uint32_t sq_intr_err, sq_int_data, encoding;
316ca750681SFelix Kuehling
317ca750681SFelix Kuehling source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry);
318ca750681SFelix Kuehling client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry);
319ca750681SFelix Kuehling pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry);
320ca750681SFelix Kuehling vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry);
32120161e51SDennis Li context_id0 = SOC15_CONTEXT_ID0_FROM_IH_ENTRY(ih_ring_entry);
32220161e51SDennis Li context_id1 = SOC15_CONTEXT_ID1_FROM_IH_ENTRY(ih_ring_entry);
323ca750681SFelix Kuehling
3247af103eaSTao Zhou if (client_id == SOC15_IH_CLIENTID_GRBM_CP ||
3257af103eaSTao Zhou client_id == SOC15_IH_CLIENTID_SE0SH ||
3267af103eaSTao Zhou client_id == SOC15_IH_CLIENTID_SE1SH ||
3277af103eaSTao Zhou client_id == SOC15_IH_CLIENTID_SE2SH ||
3287af103eaSTao Zhou client_id == SOC15_IH_CLIENTID_SE3SH) {
329ca750681SFelix Kuehling if (source_id == SOC15_INTSRC_CP_END_OF_PIPE)
33020161e51SDennis Li kfd_signal_event_interrupt(pasid, context_id0, 32);
33120161e51SDennis Li else if (source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG) {
33220161e51SDennis Li sq_int_data = KFD_CONTEXT_ID_GET_SQ_INT_DATA(context_id0, context_id1);
33320161e51SDennis Li encoding = REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, ENCODING);
33420161e51SDennis Li switch (encoding) {
33520161e51SDennis Li case SQ_INTERRUPT_WORD_ENCODING_AUTO:
336de1ce108SHarish Kasiviswanathan pr_debug_ratelimited(
33720161e51SDennis Li "sq_intr: auto, se %d, ttrace %d, wlt %d, ttrac_buf_full %d, reg_tms %d, cmd_tms %d, host_cmd_ovf %d, host_reg_ovf %d, immed_ovf %d, ttrace_utc_err %d\n",
33820161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, SE_ID),
33920161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, THREAD_TRACE),
34020161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, WLT),
34120161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, THREAD_TRACE_BUF_FULL),
34220161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, REG_TIMESTAMP),
34320161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, CMD_TIMESTAMP),
34420161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, HOST_CMD_OVERFLOW),
34520161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, HOST_REG_OVERFLOW),
34620161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, IMMED_OVERFLOW),
34720161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, THREAD_TRACE_UTC_ERROR));
34820161e51SDennis Li break;
34920161e51SDennis Li case SQ_INTERRUPT_WORD_ENCODING_INST:
350de1ce108SHarish Kasiviswanathan pr_debug_ratelimited("sq_intr: inst, se %d, data 0x%x, sh %d, priv %d, wave_id %d, simd_id %d, cu_id %d, intr_data 0x%x\n",
35120161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SE_ID),
35220161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, DATA),
35320161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SH_ID),
35420161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, PRIV),
35520161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, WAVE_ID),
35620161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SIMD_ID),
35720161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, CU_ID),
35820161e51SDennis Li sq_int_data);
35912fb1ad7SJonathan Kim if (context_id0 & SQ_INTERRUPT_WORD_WAVE_CTXID__PRIV_MASK) {
36012fb1ad7SJonathan Kim if (kfd_set_dbg_ev_from_interrupt(dev, pasid,
36112fb1ad7SJonathan Kim KFD_DEBUG_DOORBELL_ID(sq_int_data),
36212fb1ad7SJonathan Kim KFD_DEBUG_TRAP_CODE(sq_int_data),
36312fb1ad7SJonathan Kim NULL, 0))
36412fb1ad7SJonathan Kim return;
36512fb1ad7SJonathan Kim }
36620161e51SDennis Li break;
36720161e51SDennis Li case SQ_INTERRUPT_WORD_ENCODING_ERROR:
36820161e51SDennis Li sq_intr_err = REG_GET_FIELD(sq_int_data, KFD_SQ_INT_DATA, ERR_TYPE);
369de1ce108SHarish Kasiviswanathan pr_warn_ratelimited("sq_intr: error, se %d, data 0x%x, sh %d, priv %d, wave_id %d, simd_id %d, cu_id %d, err_type %d\n",
37020161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SE_ID),
37120161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, DATA),
37220161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SH_ID),
37320161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, PRIV),
37420161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, WAVE_ID),
37520161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SIMD_ID),
37620161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, CU_ID),
37720161e51SDennis Li sq_intr_err);
37820161e51SDennis Li if (sq_intr_err != SQ_INTERRUPT_ERROR_TYPE_ILLEGAL_INST &&
37920161e51SDennis Li sq_intr_err != SQ_INTERRUPT_ERROR_TYPE_MEMVIOL) {
380cc009e61SMukul Joshi event_interrupt_poison_consumption_v9(dev, pasid, client_id);
38120161e51SDennis Li return;
38220161e51SDennis Li }
38320161e51SDennis Li break;
38420161e51SDennis Li default:
38520161e51SDennis Li break;
38620161e51SDennis Li }
387a1fe9e9fSAlex Sierra kfd_signal_event_interrupt(pasid, sq_int_data, 24);
38841dc6791SJonathan Kim } else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE &&
38941dc6791SJonathan Kim KFD_DBG_EC_TYPE_IS_PACKET(KFD_DEBUG_CP_BAD_OP_ECODE(context_id0))) {
39012fb1ad7SJonathan Kim kfd_set_dbg_ev_from_interrupt(dev, pasid,
39112fb1ad7SJonathan Kim KFD_DEBUG_DOORBELL_ID(context_id0),
39212fb1ad7SJonathan Kim KFD_EC_MASK(KFD_DEBUG_CP_BAD_OP_ECODE(context_id0)),
39312fb1ad7SJonathan Kim NULL, 0);
39412fb1ad7SJonathan Kim }
395ae279f69SAlex Deucher } else if (client_id == SOC15_IH_CLIENTID_SDMA0 ||
396ae279f69SAlex Deucher client_id == SOC15_IH_CLIENTID_SDMA1 ||
397ae279f69SAlex Deucher client_id == SOC15_IH_CLIENTID_SDMA2 ||
398ae279f69SAlex Deucher client_id == SOC15_IH_CLIENTID_SDMA3 ||
399ae279f69SAlex Deucher client_id == SOC15_IH_CLIENTID_SDMA4 ||
400ae279f69SAlex Deucher client_id == SOC15_IH_CLIENTID_SDMA5 ||
401ae279f69SAlex Deucher client_id == SOC15_IH_CLIENTID_SDMA6 ||
402ae279f69SAlex Deucher client_id == SOC15_IH_CLIENTID_SDMA7) {
4034a1d4b6dSHawking Zhang if (source_id == SOC15_INTSRC_SDMA_TRAP) {
40420161e51SDennis Li kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28);
4054a1d4b6dSHawking Zhang } else if (source_id == SOC15_INTSRC_SDMA_ECC) {
406cc009e61SMukul Joshi event_interrupt_poison_consumption_v9(dev, pasid, client_id);
4074a1d4b6dSHawking Zhang return;
4084a1d4b6dSHawking Zhang }
409ae279f69SAlex Deucher } else if (client_id == SOC15_IH_CLIENTID_VMC ||
4100ad8c5e2SYong Zhao client_id == SOC15_IH_CLIENTID_VMC1 ||
411ca750681SFelix Kuehling client_id == SOC15_IH_CLIENTID_UTCL2) {
4122640c3faSshaoyunl struct kfd_vm_fault_info info = {0};
4132640c3faSshaoyunl uint16_t ring_id = SOC15_RING_ID_FROM_IH_ENTRY(ih_ring_entry);
41412fb1ad7SJonathan Kim struct kfd_hsa_memory_exception_data exception_data;
4152640c3faSshaoyunl
4161990e29bSTao Zhou if (client_id == SOC15_IH_CLIENTID_UTCL2 &&
4171990e29bSTao Zhou amdgpu_amdkfd_ras_query_utcl2_poison_status(dev->adev)) {
418cc009e61SMukul Joshi event_interrupt_poison_consumption_v9(dev, pasid, client_id);
4191990e29bSTao Zhou return;
4201990e29bSTao Zhou }
4211990e29bSTao Zhou
4222640c3faSshaoyunl info.vmid = vmid;
4232640c3faSshaoyunl info.mc_id = client_id;
4242640c3faSshaoyunl info.page_addr = ih_ring_entry[4] |
4252640c3faSshaoyunl (uint64_t)(ih_ring_entry[5] & 0xf) << 32;
4262640c3faSshaoyunl info.prot_valid = ring_id & 0x08;
4272640c3faSshaoyunl info.prot_read = ring_id & 0x10;
4282640c3faSshaoyunl info.prot_write = ring_id & 0x20;
4292640c3faSshaoyunl
43012fb1ad7SJonathan Kim memset(&exception_data, 0, sizeof(exception_data));
43112fb1ad7SJonathan Kim exception_data.gpu_id = dev->id;
43212fb1ad7SJonathan Kim exception_data.va = (info.page_addr) << PAGE_SHIFT;
43312fb1ad7SJonathan Kim exception_data.failure.NotPresent = info.prot_valid ? 1 : 0;
43412fb1ad7SJonathan Kim exception_data.failure.NoExecute = info.prot_exec ? 1 : 0;
43512fb1ad7SJonathan Kim exception_data.failure.ReadOnly = info.prot_write ? 1 : 0;
43612fb1ad7SJonathan Kim exception_data.failure.imprecise = 0;
43712fb1ad7SJonathan Kim
43812fb1ad7SJonathan Kim kfd_set_dbg_ev_from_interrupt(dev,
43912fb1ad7SJonathan Kim pasid,
44012fb1ad7SJonathan Kim -1,
44112fb1ad7SJonathan Kim KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION),
44212fb1ad7SJonathan Kim &exception_data,
44312fb1ad7SJonathan Kim sizeof(exception_data));
444938a0650SAmber Lin kfd_smi_event_update_vmfault(dev, pasid);
44512fb1ad7SJonathan Kim } else if (KFD_IRQ_IS_FENCE(client_id, source_id)) {
44612fb1ad7SJonathan Kim kfd_process_close_interrupt_drain(pasid);
447ca750681SFelix Kuehling }
448ca750681SFelix Kuehling }
449ca750681SFelix Kuehling
event_interrupt_isr_v9_4_3(struct kfd_node * node,const uint32_t * ih_ring_entry,uint32_t * patched_ihre,bool * patched_flag)450d4300362SMukul Joshi static bool event_interrupt_isr_v9_4_3(struct kfd_node *node,
451d4300362SMukul Joshi const uint32_t *ih_ring_entry,
452d4300362SMukul Joshi uint32_t *patched_ihre,
453d4300362SMukul Joshi bool *patched_flag)
454d4300362SMukul Joshi {
455d4300362SMukul Joshi uint16_t node_id, vmid;
456d4300362SMukul Joshi
457d4300362SMukul Joshi /*
458d4300362SMukul Joshi * For GFX 9.4.3, process the interrupt if:
459d4300362SMukul Joshi * - NodeID field in IH entry matches the corresponding bit
460d4300362SMukul Joshi * set in interrupt_bitmap Bits 0-15.
461d4300362SMukul Joshi * OR
462d4300362SMukul Joshi * - If partition mode is CPX and interrupt came from
463d4300362SMukul Joshi * Node_id 0,4,8,12, then check if the Bit (16 + client id)
464d4300362SMukul Joshi * is set in interrupt bitmap Bits 16-31.
465d4300362SMukul Joshi */
466d4300362SMukul Joshi node_id = SOC15_NODEID_FROM_IH_ENTRY(ih_ring_entry);
467d4300362SMukul Joshi vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry);
468d4300362SMukul Joshi if (kfd_irq_is_from_node(node, node_id, vmid))
469d4300362SMukul Joshi return event_interrupt_isr_v9(node, ih_ring_entry,
470d4300362SMukul Joshi patched_ihre, patched_flag);
471d4300362SMukul Joshi return false;
472d4300362SMukul Joshi }
473d4300362SMukul Joshi
474ca750681SFelix Kuehling const struct kfd_event_interrupt_class event_interrupt_class_v9 = {
475ca750681SFelix Kuehling .interrupt_isr = event_interrupt_isr_v9,
476ca750681SFelix Kuehling .interrupt_wq = event_interrupt_wq_v9,
477ca750681SFelix Kuehling };
478d4300362SMukul Joshi
479d4300362SMukul Joshi const struct kfd_event_interrupt_class event_interrupt_class_v9_4_3 = {
480d4300362SMukul Joshi .interrupt_isr = event_interrupt_isr_v9_4_3,
481d4300362SMukul Joshi .interrupt_wq = event_interrupt_wq_v9,
482d4300362SMukul Joshi };
483