1d87f36a0SRajneesh Bhardwaj // SPDX-License-Identifier: GPL-2.0 OR MIT 2ca750681SFelix Kuehling /* 3d87f36a0SRajneesh Bhardwaj * Copyright 2016-2022 Advanced Micro Devices, Inc. 4ca750681SFelix Kuehling * 5ca750681SFelix Kuehling * Permission is hereby granted, free of charge, to any person obtaining a 6ca750681SFelix Kuehling * copy of this software and associated documentation files (the "Software"), 7ca750681SFelix Kuehling * to deal in the Software without restriction, including without limitation 8ca750681SFelix Kuehling * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9ca750681SFelix Kuehling * and/or sell copies of the Software, and to permit persons to whom the 10ca750681SFelix Kuehling * Software is furnished to do so, subject to the following conditions: 11ca750681SFelix Kuehling * 12ca750681SFelix Kuehling * The above copyright notice and this permission notice shall be included in 13ca750681SFelix Kuehling * all copies or substantial portions of the Software. 14ca750681SFelix Kuehling * 15ca750681SFelix Kuehling * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16ca750681SFelix Kuehling * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17ca750681SFelix Kuehling * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18ca750681SFelix Kuehling * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19ca750681SFelix Kuehling * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20ca750681SFelix Kuehling * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21ca750681SFelix Kuehling * OTHER DEALINGS IN THE SOFTWARE. 22ca750681SFelix Kuehling */ 23ca750681SFelix Kuehling 24ca750681SFelix Kuehling #include "kfd_priv.h" 25ca750681SFelix Kuehling #include "kfd_events.h" 26ca750681SFelix Kuehling #include "soc15_int.h" 27a53a11a8SYong Zhao #include "kfd_device_queue_manager.h" 28938a0650SAmber Lin #include "kfd_smi_events.h" 2920161e51SDennis Li 3020161e51SDennis Li enum SQ_INTERRUPT_WORD_ENCODING { 3120161e51SDennis Li SQ_INTERRUPT_WORD_ENCODING_AUTO = 0x0, 3220161e51SDennis Li SQ_INTERRUPT_WORD_ENCODING_INST, 3320161e51SDennis Li SQ_INTERRUPT_WORD_ENCODING_ERROR, 3420161e51SDennis Li }; 3520161e51SDennis Li 3620161e51SDennis Li enum SQ_INTERRUPT_ERROR_TYPE { 3720161e51SDennis Li SQ_INTERRUPT_ERROR_TYPE_EDC_FUE = 0x0, 3820161e51SDennis Li SQ_INTERRUPT_ERROR_TYPE_ILLEGAL_INST, 3920161e51SDennis Li SQ_INTERRUPT_ERROR_TYPE_MEMVIOL, 4020161e51SDennis Li SQ_INTERRUPT_ERROR_TYPE_EDC_FED, 4120161e51SDennis Li }; 4220161e51SDennis Li 4320161e51SDennis Li /* SQ_INTERRUPT_WORD_AUTO_CTXID */ 4420161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE__SHIFT 0 4520161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__WLT__SHIFT 1 4620161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_BUF_FULL__SHIFT 2 4720161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__REG_TIMESTAMP__SHIFT 3 4820161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__CMD_TIMESTAMP__SHIFT 4 4920161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_CMD_OVERFLOW__SHIFT 5 5020161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_REG_OVERFLOW__SHIFT 6 5120161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__IMMED_OVERFLOW__SHIFT 7 5220161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_UTC_ERROR__SHIFT 8 5320161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__SE_ID__SHIFT 24 5420161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__ENCODING__SHIFT 26 5520161e51SDennis Li 5620161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_MASK 0x00000001 5720161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__WLT_MASK 0x00000002 5820161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_BUF_FULL_MASK 0x00000004 5920161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__REG_TIMESTAMP_MASK 0x00000008 6020161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__CMD_TIMESTAMP_MASK 0x00000010 6120161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_CMD_OVERFLOW_MASK 0x00000020 6220161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_REG_OVERFLOW_MASK 0x00000040 6320161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__IMMED_OVERFLOW_MASK 0x00000080 6420161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_UTC_ERROR_MASK 0x00000100 6520161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__SE_ID_MASK 0x03000000 6620161e51SDennis Li #define SQ_INTERRUPT_WORD_AUTO_CTXID__ENCODING_MASK 0x0c000000 6720161e51SDennis Li 6820161e51SDennis Li /* SQ_INTERRUPT_WORD_WAVE_CTXID */ 6920161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__DATA__SHIFT 0 7020161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__SH_ID__SHIFT 12 7120161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__PRIV__SHIFT 13 7220161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__WAVE_ID__SHIFT 14 7320161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__SIMD_ID__SHIFT 18 7420161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__CU_ID__SHIFT 20 7520161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__SE_ID__SHIFT 24 7620161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__ENCODING__SHIFT 26 7720161e51SDennis Li 7820161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__DATA_MASK 0x00000fff 7920161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__SH_ID_MASK 0x00001000 8020161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__PRIV_MASK 0x00002000 8120161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__WAVE_ID_MASK 0x0003c000 8220161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__SIMD_ID_MASK 0x000c0000 8320161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__CU_ID_MASK 0x00f00000 8420161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__SE_ID_MASK 0x03000000 8520161e51SDennis Li #define SQ_INTERRUPT_WORD_WAVE_CTXID__ENCODING_MASK 0x0c000000 8620161e51SDennis Li 8720161e51SDennis Li #define KFD_CONTEXT_ID_GET_SQ_INT_DATA(ctx0, ctx1) \ 8820161e51SDennis Li ((ctx0 & 0xfff) | ((ctx0 >> 16) & 0xf000) | ((ctx1 << 16) & 0xff0000)) 8920161e51SDennis Li 9020161e51SDennis Li #define KFD_SQ_INT_DATA__ERR_TYPE_MASK 0xF00000 9120161e51SDennis Li #define KFD_SQ_INT_DATA__ERR_TYPE__SHIFT 20 92ca750681SFelix Kuehling 93b6485bedSTao Zhou static void event_interrupt_poison_consumption(struct kfd_dev *dev, 94*9d8a8d78STao Zhou uint16_t pasid, uint16_t client_id) 95b6485bedSTao Zhou { 96eed41975STao Zhou int old_poison, ret = -EINVAL; 97b6485bedSTao Zhou struct kfd_process *p = kfd_lookup_process_by_pasid(pasid); 98b6485bedSTao Zhou 99b6485bedSTao Zhou if (!p) 100b6485bedSTao Zhou return; 101b6485bedSTao Zhou 102b6485bedSTao Zhou /* all queues of a process will be unmapped in one time */ 103eed41975STao Zhou old_poison = atomic_cmpxchg(&p->poison, 0, 1); 104b6485bedSTao Zhou kfd_unref_process(p); 105eed41975STao Zhou if (old_poison) 106b6485bedSTao Zhou return; 107b6485bedSTao Zhou 108*9d8a8d78STao Zhou pr_warn("RAS poison consumption handling: client id %d\n", client_id); 109b6485bedSTao Zhou 110*9d8a8d78STao Zhou switch (client_id) { 111*9d8a8d78STao Zhou case SOC15_IH_CLIENTID_SE0SH: 112*9d8a8d78STao Zhou case SOC15_IH_CLIENTID_SE1SH: 113*9d8a8d78STao Zhou case SOC15_IH_CLIENTID_SE2SH: 114*9d8a8d78STao Zhou case SOC15_IH_CLIENTID_SE3SH: 115*9d8a8d78STao Zhou case SOC15_IH_CLIENTID_UTCL2: 11629b440d2STao Zhou ret = kfd_dqm_evict_pasid(dev->dqm, pasid); 117b6485bedSTao Zhou break; 118*9d8a8d78STao Zhou case SOC15_IH_CLIENTID_SDMA0: 119*9d8a8d78STao Zhou case SOC15_IH_CLIENTID_SDMA1: 120*9d8a8d78STao Zhou case SOC15_IH_CLIENTID_SDMA2: 121*9d8a8d78STao Zhou case SOC15_IH_CLIENTID_SDMA3: 122*9d8a8d78STao Zhou case SOC15_IH_CLIENTID_SDMA4: 123*9d8a8d78STao Zhou break; 124b6485bedSTao Zhou default: 125b6485bedSTao Zhou break; 126b6485bedSTao Zhou } 127b6485bedSTao Zhou 128b6485bedSTao Zhou kfd_signal_poison_consumed_event(dev, pasid); 129b6485bedSTao Zhou 130b6485bedSTao Zhou /* resetting queue passes, do page retirement without gpu reset 1312243f493SRajneesh Bhardwaj * resetting queue fails, fallback to gpu reset solution 1322243f493SRajneesh Bhardwaj */ 133b6485bedSTao Zhou if (!ret) 134b6485bedSTao Zhou amdgpu_amdkfd_ras_poison_consumption_handler(dev->adev, false); 135b6485bedSTao Zhou else 136b6485bedSTao Zhou amdgpu_amdkfd_ras_poison_consumption_handler(dev->adev, true); 137b6485bedSTao Zhou } 138b6485bedSTao Zhou 139ca750681SFelix Kuehling static bool event_interrupt_isr_v9(struct kfd_dev *dev, 14058e69886SLan Xiao const uint32_t *ih_ring_entry, 14158e69886SLan Xiao uint32_t *patched_ihre, 14258e69886SLan Xiao bool *patched_flag) 143ca750681SFelix Kuehling { 144ca750681SFelix Kuehling uint16_t source_id, client_id, pasid, vmid; 145c129db12SFelix Kuehling const uint32_t *data = ih_ring_entry; 146c129db12SFelix Kuehling 147c129db12SFelix Kuehling /* Only handle interrupts from KFD VMIDs */ 148c129db12SFelix Kuehling vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry); 149c129db12SFelix Kuehling if (vmid < dev->vm_info.first_vmid_kfd || 150c129db12SFelix Kuehling vmid > dev->vm_info.last_vmid_kfd) 1518c8e1f69SAishwarya Ramakrishnan return false; 152c129db12SFelix Kuehling 153ca750681SFelix Kuehling source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry); 154ca750681SFelix Kuehling client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry); 15500557f41SYong Zhao pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry); 156ca750681SFelix Kuehling 157ae279f69SAlex Deucher /* Only handle clients we care about */ 158ae279f69SAlex Deucher if (client_id != SOC15_IH_CLIENTID_GRBM_CP && 159ae279f69SAlex Deucher client_id != SOC15_IH_CLIENTID_SDMA0 && 160ae279f69SAlex Deucher client_id != SOC15_IH_CLIENTID_SDMA1 && 161ae279f69SAlex Deucher client_id != SOC15_IH_CLIENTID_SDMA2 && 162ae279f69SAlex Deucher client_id != SOC15_IH_CLIENTID_SDMA3 && 163ae279f69SAlex Deucher client_id != SOC15_IH_CLIENTID_SDMA4 && 164ae279f69SAlex Deucher client_id != SOC15_IH_CLIENTID_SDMA5 && 165ae279f69SAlex Deucher client_id != SOC15_IH_CLIENTID_SDMA6 && 166ae279f69SAlex Deucher client_id != SOC15_IH_CLIENTID_SDMA7 && 167ae279f69SAlex Deucher client_id != SOC15_IH_CLIENTID_VMC && 168ae279f69SAlex Deucher client_id != SOC15_IH_CLIENTID_VMC1 && 1697af103eaSTao Zhou client_id != SOC15_IH_CLIENTID_UTCL2 && 1707af103eaSTao Zhou client_id != SOC15_IH_CLIENTID_SE0SH && 1717af103eaSTao Zhou client_id != SOC15_IH_CLIENTID_SE1SH && 1727af103eaSTao Zhou client_id != SOC15_IH_CLIENTID_SE2SH && 1737af103eaSTao Zhou client_id != SOC15_IH_CLIENTID_SE3SH) 174ae279f69SAlex Deucher return false; 175ae279f69SAlex Deucher 176a53a11a8SYong Zhao /* This is a known issue for gfx9. Under non HWS, pasid is not set 177a53a11a8SYong Zhao * in the interrupt payload, so we need to find out the pasid on our 178a53a11a8SYong Zhao * own. 179a53a11a8SYong Zhao */ 180a53a11a8SYong Zhao if (!pasid && dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 181a53a11a8SYong Zhao const uint32_t pasid_mask = 0xffff; 182a53a11a8SYong Zhao 183a53a11a8SYong Zhao *patched_flag = true; 184a53a11a8SYong Zhao memcpy(patched_ihre, ih_ring_entry, 185f0dc99a6SGraham Sider dev->device_info.ih_ring_entry_size); 186a53a11a8SYong Zhao 1873fe023d4SYong Zhao pasid = dev->dqm->vmid_pasid[vmid]; 188a53a11a8SYong Zhao 189a53a11a8SYong Zhao /* Patch the pasid field */ 190a53a11a8SYong Zhao patched_ihre[3] = cpu_to_le32((le32_to_cpu(patched_ihre[3]) 191a53a11a8SYong Zhao & ~pasid_mask) | pasid); 192a53a11a8SYong Zhao } 193a53a11a8SYong Zhao 19400557f41SYong Zhao pr_debug("client id 0x%x, source id %d, vmid %d, pasid 0x%x. raw data:\n", 19500557f41SYong Zhao client_id, source_id, vmid, pasid); 196ca750681SFelix Kuehling pr_debug("%8X, %8X, %8X, %8X, %8X, %8X, %8X, %8X.\n", 197ca750681SFelix Kuehling data[0], data[1], data[2], data[3], 198ca750681SFelix Kuehling data[4], data[5], data[6], data[7]); 199ca750681SFelix Kuehling 200a53a11a8SYong Zhao /* If there is no valid PASID, it's likely a bug */ 201a53a11a8SYong Zhao if (WARN_ONCE(pasid == 0, "Bug: No PASID in KFD interrupt")) 2028c8e1f69SAishwarya Ramakrishnan return false; 20300557f41SYong Zhao 204c129db12SFelix Kuehling /* Interrupt types we care about: various signals and faults. 205c129db12SFelix Kuehling * They will be forwarded to a work queue (see below). 206c129db12SFelix Kuehling */ 207c129db12SFelix Kuehling return source_id == SOC15_INTSRC_CP_END_OF_PIPE || 208ca750681SFelix Kuehling source_id == SOC15_INTSRC_SDMA_TRAP || 2095b0ce2d4Syipechai source_id == SOC15_INTSRC_SDMA_ECC || 210ca750681SFelix Kuehling source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG || 2112640c3faSshaoyunl source_id == SOC15_INTSRC_CP_BAD_OPCODE || 2126d909c5dSOak Zeng ((client_id == SOC15_IH_CLIENTID_VMC || 2130ad8c5e2SYong Zhao client_id == SOC15_IH_CLIENTID_VMC1 || 2146d909c5dSOak Zeng client_id == SOC15_IH_CLIENTID_UTCL2) && 2156d909c5dSOak Zeng !amdgpu_no_queue_eviction_on_vm_fault); 216ca750681SFelix Kuehling } 217ca750681SFelix Kuehling 218ca750681SFelix Kuehling static void event_interrupt_wq_v9(struct kfd_dev *dev, 219ca750681SFelix Kuehling const uint32_t *ih_ring_entry) 220ca750681SFelix Kuehling { 221ca750681SFelix Kuehling uint16_t source_id, client_id, pasid, vmid; 22220161e51SDennis Li uint32_t context_id0, context_id1; 22320161e51SDennis Li uint32_t sq_intr_err, sq_int_data, encoding; 224ca750681SFelix Kuehling 225ca750681SFelix Kuehling source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry); 226ca750681SFelix Kuehling client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry); 227ca750681SFelix Kuehling pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry); 228ca750681SFelix Kuehling vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry); 22920161e51SDennis Li context_id0 = SOC15_CONTEXT_ID0_FROM_IH_ENTRY(ih_ring_entry); 23020161e51SDennis Li context_id1 = SOC15_CONTEXT_ID1_FROM_IH_ENTRY(ih_ring_entry); 231ca750681SFelix Kuehling 2327af103eaSTao Zhou if (client_id == SOC15_IH_CLIENTID_GRBM_CP || 2337af103eaSTao Zhou client_id == SOC15_IH_CLIENTID_SE0SH || 2347af103eaSTao Zhou client_id == SOC15_IH_CLIENTID_SE1SH || 2357af103eaSTao Zhou client_id == SOC15_IH_CLIENTID_SE2SH || 2367af103eaSTao Zhou client_id == SOC15_IH_CLIENTID_SE3SH) { 237ca750681SFelix Kuehling if (source_id == SOC15_INTSRC_CP_END_OF_PIPE) 23820161e51SDennis Li kfd_signal_event_interrupt(pasid, context_id0, 32); 23920161e51SDennis Li else if (source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG) { 24020161e51SDennis Li sq_int_data = KFD_CONTEXT_ID_GET_SQ_INT_DATA(context_id0, context_id1); 24120161e51SDennis Li encoding = REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, ENCODING); 24220161e51SDennis Li switch (encoding) { 24320161e51SDennis Li case SQ_INTERRUPT_WORD_ENCODING_AUTO: 24420161e51SDennis Li pr_debug( 24520161e51SDennis Li "sq_intr: auto, se %d, ttrace %d, wlt %d, ttrac_buf_full %d, reg_tms %d, cmd_tms %d, host_cmd_ovf %d, host_reg_ovf %d, immed_ovf %d, ttrace_utc_err %d\n", 24620161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, SE_ID), 24720161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, THREAD_TRACE), 24820161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, WLT), 24920161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, THREAD_TRACE_BUF_FULL), 25020161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, REG_TIMESTAMP), 25120161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, CMD_TIMESTAMP), 25220161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, HOST_CMD_OVERFLOW), 25320161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, HOST_REG_OVERFLOW), 25420161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, IMMED_OVERFLOW), 25520161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, THREAD_TRACE_UTC_ERROR)); 25620161e51SDennis Li break; 25720161e51SDennis Li case SQ_INTERRUPT_WORD_ENCODING_INST: 25820161e51SDennis Li pr_debug("sq_intr: inst, se %d, data 0x%x, sh %d, priv %d, wave_id %d, simd_id %d, cu_id %d, intr_data 0x%x\n", 25920161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SE_ID), 26020161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, DATA), 26120161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SH_ID), 26220161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, PRIV), 26320161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, WAVE_ID), 26420161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SIMD_ID), 26520161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, CU_ID), 26620161e51SDennis Li sq_int_data); 26720161e51SDennis Li break; 26820161e51SDennis Li case SQ_INTERRUPT_WORD_ENCODING_ERROR: 26920161e51SDennis Li sq_intr_err = REG_GET_FIELD(sq_int_data, KFD_SQ_INT_DATA, ERR_TYPE); 27020161e51SDennis Li pr_warn("sq_intr: error, se %d, data 0x%x, sh %d, priv %d, wave_id %d, simd_id %d, cu_id %d, err_type %d\n", 27120161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SE_ID), 27220161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, DATA), 27320161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SH_ID), 27420161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, PRIV), 27520161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, WAVE_ID), 27620161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SIMD_ID), 27720161e51SDennis Li REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, CU_ID), 27820161e51SDennis Li sq_intr_err); 27920161e51SDennis Li if (sq_intr_err != SQ_INTERRUPT_ERROR_TYPE_ILLEGAL_INST && 28020161e51SDennis Li sq_intr_err != SQ_INTERRUPT_ERROR_TYPE_MEMVIOL) { 281*9d8a8d78STao Zhou event_interrupt_poison_consumption(dev, pasid, client_id); 28220161e51SDennis Li return; 28320161e51SDennis Li } 28420161e51SDennis Li break; 28520161e51SDennis Li default: 28620161e51SDennis Li break; 28720161e51SDennis Li } 28820161e51SDennis Li kfd_signal_event_interrupt(pasid, context_id0 & 0xffffff, 24); 28920161e51SDennis Li } else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE) 290ca750681SFelix Kuehling kfd_signal_hw_exception_event(pasid); 291ae279f69SAlex Deucher } else if (client_id == SOC15_IH_CLIENTID_SDMA0 || 292ae279f69SAlex Deucher client_id == SOC15_IH_CLIENTID_SDMA1 || 293ae279f69SAlex Deucher client_id == SOC15_IH_CLIENTID_SDMA2 || 294ae279f69SAlex Deucher client_id == SOC15_IH_CLIENTID_SDMA3 || 295ae279f69SAlex Deucher client_id == SOC15_IH_CLIENTID_SDMA4 || 296ae279f69SAlex Deucher client_id == SOC15_IH_CLIENTID_SDMA5 || 297ae279f69SAlex Deucher client_id == SOC15_IH_CLIENTID_SDMA6 || 298ae279f69SAlex Deucher client_id == SOC15_IH_CLIENTID_SDMA7) { 2994a1d4b6dSHawking Zhang if (source_id == SOC15_INTSRC_SDMA_TRAP) { 30020161e51SDennis Li kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28); 3014a1d4b6dSHawking Zhang } else if (source_id == SOC15_INTSRC_SDMA_ECC) { 302*9d8a8d78STao Zhou event_interrupt_poison_consumption(dev, pasid, client_id); 3034a1d4b6dSHawking Zhang return; 3044a1d4b6dSHawking Zhang } 305ae279f69SAlex Deucher } else if (client_id == SOC15_IH_CLIENTID_VMC || 3060ad8c5e2SYong Zhao client_id == SOC15_IH_CLIENTID_VMC1 || 307ca750681SFelix Kuehling client_id == SOC15_IH_CLIENTID_UTCL2) { 3082640c3faSshaoyunl struct kfd_vm_fault_info info = {0}; 3092640c3faSshaoyunl uint16_t ring_id = SOC15_RING_ID_FROM_IH_ENTRY(ih_ring_entry); 3102640c3faSshaoyunl 3112640c3faSshaoyunl info.vmid = vmid; 3122640c3faSshaoyunl info.mc_id = client_id; 3132640c3faSshaoyunl info.page_addr = ih_ring_entry[4] | 3142640c3faSshaoyunl (uint64_t)(ih_ring_entry[5] & 0xf) << 32; 3152640c3faSshaoyunl info.prot_valid = ring_id & 0x08; 3162640c3faSshaoyunl info.prot_read = ring_id & 0x10; 3172640c3faSshaoyunl info.prot_write = ring_id & 0x20; 3182640c3faSshaoyunl 319938a0650SAmber Lin kfd_smi_event_update_vmfault(dev, pasid); 32003e5b167STao Zhou kfd_dqm_evict_pasid(dev->dqm, pasid); 3212640c3faSshaoyunl kfd_signal_vm_fault_event(dev, pasid, &info); 322ca750681SFelix Kuehling } 323ca750681SFelix Kuehling } 324ca750681SFelix Kuehling 325ca750681SFelix Kuehling const struct kfd_event_interrupt_class event_interrupt_class_v9 = { 326ca750681SFelix Kuehling .interrupt_isr = event_interrupt_isr_v9, 327ca750681SFelix Kuehling .interrupt_wq = event_interrupt_wq_v9, 328ca750681SFelix Kuehling }; 329