1 /*
2  * Copyright 2016-2018 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  */
22 
23 #include "kfd_priv.h"
24 #include "kfd_events.h"
25 #include "soc15_int.h"
26 #include "kfd_device_queue_manager.h"
27 #include "kfd_smi_events.h"
28 
29 enum SQ_INTERRUPT_WORD_ENCODING {
30 	SQ_INTERRUPT_WORD_ENCODING_AUTO = 0x0,
31 	SQ_INTERRUPT_WORD_ENCODING_INST,
32 	SQ_INTERRUPT_WORD_ENCODING_ERROR,
33 };
34 
35 enum SQ_INTERRUPT_ERROR_TYPE {
36 	SQ_INTERRUPT_ERROR_TYPE_EDC_FUE = 0x0,
37 	SQ_INTERRUPT_ERROR_TYPE_ILLEGAL_INST,
38 	SQ_INTERRUPT_ERROR_TYPE_MEMVIOL,
39 	SQ_INTERRUPT_ERROR_TYPE_EDC_FED,
40 };
41 
42 /* SQ_INTERRUPT_WORD_AUTO_CTXID */
43 #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE__SHIFT 0
44 #define SQ_INTERRUPT_WORD_AUTO_CTXID__WLT__SHIFT 1
45 #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_BUF_FULL__SHIFT 2
46 #define SQ_INTERRUPT_WORD_AUTO_CTXID__REG_TIMESTAMP__SHIFT 3
47 #define SQ_INTERRUPT_WORD_AUTO_CTXID__CMD_TIMESTAMP__SHIFT 4
48 #define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_CMD_OVERFLOW__SHIFT 5
49 #define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_REG_OVERFLOW__SHIFT 6
50 #define SQ_INTERRUPT_WORD_AUTO_CTXID__IMMED_OVERFLOW__SHIFT 7
51 #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_UTC_ERROR__SHIFT 8
52 #define SQ_INTERRUPT_WORD_AUTO_CTXID__SE_ID__SHIFT 24
53 #define SQ_INTERRUPT_WORD_AUTO_CTXID__ENCODING__SHIFT 26
54 
55 #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_MASK 0x00000001
56 #define SQ_INTERRUPT_WORD_AUTO_CTXID__WLT_MASK 0x00000002
57 #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_BUF_FULL_MASK 0x00000004
58 #define SQ_INTERRUPT_WORD_AUTO_CTXID__REG_TIMESTAMP_MASK 0x00000008
59 #define SQ_INTERRUPT_WORD_AUTO_CTXID__CMD_TIMESTAMP_MASK 0x00000010
60 #define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_CMD_OVERFLOW_MASK 0x00000020
61 #define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_REG_OVERFLOW_MASK 0x00000040
62 #define SQ_INTERRUPT_WORD_AUTO_CTXID__IMMED_OVERFLOW_MASK 0x00000080
63 #define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_UTC_ERROR_MASK 0x00000100
64 #define SQ_INTERRUPT_WORD_AUTO_CTXID__SE_ID_MASK 0x03000000
65 #define SQ_INTERRUPT_WORD_AUTO_CTXID__ENCODING_MASK 0x0c000000
66 
67 /* SQ_INTERRUPT_WORD_WAVE_CTXID */
68 #define SQ_INTERRUPT_WORD_WAVE_CTXID__DATA__SHIFT 0
69 #define SQ_INTERRUPT_WORD_WAVE_CTXID__SH_ID__SHIFT 12
70 #define SQ_INTERRUPT_WORD_WAVE_CTXID__PRIV__SHIFT 13
71 #define SQ_INTERRUPT_WORD_WAVE_CTXID__WAVE_ID__SHIFT 14
72 #define SQ_INTERRUPT_WORD_WAVE_CTXID__SIMD_ID__SHIFT 18
73 #define SQ_INTERRUPT_WORD_WAVE_CTXID__CU_ID__SHIFT 20
74 #define SQ_INTERRUPT_WORD_WAVE_CTXID__SE_ID__SHIFT 24
75 #define SQ_INTERRUPT_WORD_WAVE_CTXID__ENCODING__SHIFT 26
76 
77 #define SQ_INTERRUPT_WORD_WAVE_CTXID__DATA_MASK 0x00000fff
78 #define SQ_INTERRUPT_WORD_WAVE_CTXID__SH_ID_MASK 0x00001000
79 #define SQ_INTERRUPT_WORD_WAVE_CTXID__PRIV_MASK 0x00002000
80 #define SQ_INTERRUPT_WORD_WAVE_CTXID__WAVE_ID_MASK 0x0003c000
81 #define SQ_INTERRUPT_WORD_WAVE_CTXID__SIMD_ID_MASK 0x000c0000
82 #define SQ_INTERRUPT_WORD_WAVE_CTXID__CU_ID_MASK 0x00f00000
83 #define SQ_INTERRUPT_WORD_WAVE_CTXID__SE_ID_MASK 0x03000000
84 #define SQ_INTERRUPT_WORD_WAVE_CTXID__ENCODING_MASK 0x0c000000
85 
86 #define KFD_CONTEXT_ID_GET_SQ_INT_DATA(ctx0, ctx1)                             \
87 	((ctx0 & 0xfff) | ((ctx0 >> 16) & 0xf000) | ((ctx1 << 16) & 0xff0000))
88 
89 #define KFD_SQ_INT_DATA__ERR_TYPE_MASK 0xF00000
90 #define KFD_SQ_INT_DATA__ERR_TYPE__SHIFT 20
91 
92 static bool event_interrupt_isr_v9(struct kfd_dev *dev,
93 					const uint32_t *ih_ring_entry,
94 					uint32_t *patched_ihre,
95 					bool *patched_flag)
96 {
97 	uint16_t source_id, client_id, pasid, vmid;
98 	const uint32_t *data = ih_ring_entry;
99 
100 	/* Only handle interrupts from KFD VMIDs */
101 	vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry);
102 	if (vmid < dev->vm_info.first_vmid_kfd ||
103 	    vmid > dev->vm_info.last_vmid_kfd)
104 		return false;
105 
106 	source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry);
107 	client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry);
108 	pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry);
109 
110 	/* Only handle clients we care about */
111 	if (client_id != SOC15_IH_CLIENTID_GRBM_CP &&
112 	    client_id != SOC15_IH_CLIENTID_SDMA0 &&
113 	    client_id != SOC15_IH_CLIENTID_SDMA1 &&
114 	    client_id != SOC15_IH_CLIENTID_SDMA2 &&
115 	    client_id != SOC15_IH_CLIENTID_SDMA3 &&
116 	    client_id != SOC15_IH_CLIENTID_SDMA4 &&
117 	    client_id != SOC15_IH_CLIENTID_SDMA5 &&
118 	    client_id != SOC15_IH_CLIENTID_SDMA6 &&
119 	    client_id != SOC15_IH_CLIENTID_SDMA7 &&
120 	    client_id != SOC15_IH_CLIENTID_VMC &&
121 	    client_id != SOC15_IH_CLIENTID_VMC1 &&
122 	    client_id != SOC15_IH_CLIENTID_UTCL2 &&
123 	    client_id != SOC15_IH_CLIENTID_SE0SH &&
124 	    client_id != SOC15_IH_CLIENTID_SE1SH &&
125 	    client_id != SOC15_IH_CLIENTID_SE2SH &&
126 	    client_id != SOC15_IH_CLIENTID_SE3SH)
127 		return false;
128 
129 	/* This is a known issue for gfx9. Under non HWS, pasid is not set
130 	 * in the interrupt payload, so we need to find out the pasid on our
131 	 * own.
132 	 */
133 	if (!pasid && dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
134 		const uint32_t pasid_mask = 0xffff;
135 
136 		*patched_flag = true;
137 		memcpy(patched_ihre, ih_ring_entry,
138 				dev->device_info->ih_ring_entry_size);
139 
140 		pasid = dev->dqm->vmid_pasid[vmid];
141 
142 		/* Patch the pasid field */
143 		patched_ihre[3] = cpu_to_le32((le32_to_cpu(patched_ihre[3])
144 					& ~pasid_mask) | pasid);
145 	}
146 
147 	pr_debug("client id 0x%x, source id %d, vmid %d, pasid 0x%x. raw data:\n",
148 		 client_id, source_id, vmid, pasid);
149 	pr_debug("%8X, %8X, %8X, %8X, %8X, %8X, %8X, %8X.\n",
150 		 data[0], data[1], data[2], data[3],
151 		 data[4], data[5], data[6], data[7]);
152 
153 	/* If there is no valid PASID, it's likely a bug */
154 	if (WARN_ONCE(pasid == 0, "Bug: No PASID in KFD interrupt"))
155 		return false;
156 
157 	/* Interrupt types we care about: various signals and faults.
158 	 * They will be forwarded to a work queue (see below).
159 	 */
160 	return source_id == SOC15_INTSRC_CP_END_OF_PIPE ||
161 		source_id == SOC15_INTSRC_SDMA_TRAP ||
162 		source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG ||
163 		source_id == SOC15_INTSRC_CP_BAD_OPCODE ||
164 		((client_id == SOC15_IH_CLIENTID_VMC ||
165 		client_id == SOC15_IH_CLIENTID_VMC1 ||
166 		client_id == SOC15_IH_CLIENTID_UTCL2) &&
167 		!amdgpu_no_queue_eviction_on_vm_fault);
168 }
169 
170 static void event_interrupt_wq_v9(struct kfd_dev *dev,
171 					const uint32_t *ih_ring_entry)
172 {
173 	uint16_t source_id, client_id, pasid, vmid;
174 	uint32_t context_id0, context_id1;
175 	uint32_t sq_intr_err, sq_int_data, encoding;
176 
177 	source_id = SOC15_SOURCE_ID_FROM_IH_ENTRY(ih_ring_entry);
178 	client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry);
179 	pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry);
180 	vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry);
181 	context_id0 = SOC15_CONTEXT_ID0_FROM_IH_ENTRY(ih_ring_entry);
182 	context_id1 = SOC15_CONTEXT_ID1_FROM_IH_ENTRY(ih_ring_entry);
183 
184 	if (client_id == SOC15_IH_CLIENTID_GRBM_CP ||
185 	    client_id == SOC15_IH_CLIENTID_SE0SH ||
186 	    client_id == SOC15_IH_CLIENTID_SE1SH ||
187 	    client_id == SOC15_IH_CLIENTID_SE2SH ||
188 	    client_id == SOC15_IH_CLIENTID_SE3SH) {
189 		if (source_id == SOC15_INTSRC_CP_END_OF_PIPE)
190 			kfd_signal_event_interrupt(pasid, context_id0, 32);
191 		else if (source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG) {
192 			sq_int_data = KFD_CONTEXT_ID_GET_SQ_INT_DATA(context_id0, context_id1);
193 			encoding = REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, ENCODING);
194 			switch (encoding) {
195 			case SQ_INTERRUPT_WORD_ENCODING_AUTO:
196 				pr_debug(
197 					"sq_intr: auto, se %d, ttrace %d, wlt %d, ttrac_buf_full %d, reg_tms %d, cmd_tms %d, host_cmd_ovf %d, host_reg_ovf %d, immed_ovf %d, ttrace_utc_err %d\n",
198 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, SE_ID),
199 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, THREAD_TRACE),
200 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, WLT),
201 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, THREAD_TRACE_BUF_FULL),
202 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, REG_TIMESTAMP),
203 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, CMD_TIMESTAMP),
204 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, HOST_CMD_OVERFLOW),
205 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, HOST_REG_OVERFLOW),
206 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, IMMED_OVERFLOW),
207 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_AUTO_CTXID, THREAD_TRACE_UTC_ERROR));
208 				break;
209 			case SQ_INTERRUPT_WORD_ENCODING_INST:
210 				pr_debug("sq_intr: inst, se %d, data 0x%x, sh %d, priv %d, wave_id %d, simd_id %d, cu_id %d, intr_data 0x%x\n",
211 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SE_ID),
212 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, DATA),
213 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SH_ID),
214 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, PRIV),
215 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, WAVE_ID),
216 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SIMD_ID),
217 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, CU_ID),
218 					sq_int_data);
219 				break;
220 			case SQ_INTERRUPT_WORD_ENCODING_ERROR:
221 				sq_intr_err = REG_GET_FIELD(sq_int_data, KFD_SQ_INT_DATA, ERR_TYPE);
222 				pr_warn("sq_intr: error, se %d, data 0x%x, sh %d, priv %d, wave_id %d, simd_id %d, cu_id %d, err_type %d\n",
223 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SE_ID),
224 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, DATA),
225 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SH_ID),
226 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, PRIV),
227 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, WAVE_ID),
228 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, SIMD_ID),
229 					REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID, CU_ID),
230 					sq_intr_err);
231 				if (sq_intr_err != SQ_INTERRUPT_ERROR_TYPE_ILLEGAL_INST &&
232 					sq_intr_err != SQ_INTERRUPT_ERROR_TYPE_MEMVIOL) {
233 					kfd_signal_poison_consumed_event(dev, pasid);
234 					amdgpu_amdkfd_ras_poison_consumption_handler(dev->kgd);
235 					return;
236 				}
237 				break;
238 			default:
239 				break;
240 			}
241 			kfd_signal_event_interrupt(pasid, context_id0 & 0xffffff, 24);
242 		} else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE)
243 			kfd_signal_hw_exception_event(pasid);
244 	} else if (client_id == SOC15_IH_CLIENTID_SDMA0 ||
245 		   client_id == SOC15_IH_CLIENTID_SDMA1 ||
246 		   client_id == SOC15_IH_CLIENTID_SDMA2 ||
247 		   client_id == SOC15_IH_CLIENTID_SDMA3 ||
248 		   client_id == SOC15_IH_CLIENTID_SDMA4 ||
249 		   client_id == SOC15_IH_CLIENTID_SDMA5 ||
250 		   client_id == SOC15_IH_CLIENTID_SDMA6 ||
251 		   client_id == SOC15_IH_CLIENTID_SDMA7) {
252 		if (source_id == SOC15_INTSRC_SDMA_TRAP) {
253 			kfd_signal_event_interrupt(pasid, context_id0 & 0xfffffff, 28);
254 		} else if (source_id == SOC15_INTSRC_SDMA_ECC) {
255 			kfd_signal_poison_consumed_event(dev, pasid);
256 			amdgpu_amdkfd_ras_poison_consumption_handler(dev->kgd);
257 			return;
258 		}
259 	} else if (client_id == SOC15_IH_CLIENTID_VMC ||
260 		   client_id == SOC15_IH_CLIENTID_VMC1 ||
261 		   client_id == SOC15_IH_CLIENTID_UTCL2) {
262 		struct kfd_vm_fault_info info = {0};
263 		uint16_t ring_id = SOC15_RING_ID_FROM_IH_ENTRY(ih_ring_entry);
264 
265 		info.vmid = vmid;
266 		info.mc_id = client_id;
267 		info.page_addr = ih_ring_entry[4] |
268 			(uint64_t)(ih_ring_entry[5] & 0xf) << 32;
269 		info.prot_valid = ring_id & 0x08;
270 		info.prot_read  = ring_id & 0x10;
271 		info.prot_write = ring_id & 0x20;
272 
273 		kfd_smi_event_update_vmfault(dev, pasid);
274 		kfd_process_vm_fault(dev->dqm, pasid);
275 		kfd_signal_vm_fault_event(dev, pasid, &info);
276 	}
277 }
278 
279 const struct kfd_event_interrupt_class event_interrupt_class_v9 = {
280 	.interrupt_isr = event_interrupt_isr_v9,
281 	.interrupt_wq = event_interrupt_wq_v9,
282 };
283