1 /*
2  * Copyright 2016 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 #include <drm/drmP.h>
24 #include "amdgpu.h"
25 #include "amdgpu_ih.h"
26 #include "soc15.h"
27 
28 #include "oss/osssys_4_0_offset.h"
29 #include "oss/osssys_4_0_sh_mask.h"
30 
31 #include "soc15_common.h"
32 #include "vega10_ih.h"
33 
34 
35 
36 static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev);
37 
38 /**
39  * vega10_ih_enable_interrupts - Enable the interrupt ring buffer
40  *
41  * @adev: amdgpu_device pointer
42  *
43  * Enable the interrupt ring buffer (VEGA10).
44  */
45 static void vega10_ih_enable_interrupts(struct amdgpu_device *adev)
46 {
47 	u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
48 
49 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
50 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1);
51 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
52 	adev->irq.ih.enabled = true;
53 }
54 
55 /**
56  * vega10_ih_disable_interrupts - Disable the interrupt ring buffer
57  *
58  * @adev: amdgpu_device pointer
59  *
60  * Disable the interrupt ring buffer (VEGA10).
61  */
62 static void vega10_ih_disable_interrupts(struct amdgpu_device *adev)
63 {
64 	u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
65 
66 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
67 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
68 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
69 	/* set rptr, wptr to 0 */
70 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
71 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
72 	adev->irq.ih.enabled = false;
73 	adev->irq.ih.rptr = 0;
74 }
75 
76 /**
77  * vega10_ih_irq_init - init and enable the interrupt ring
78  *
79  * @adev: amdgpu_device pointer
80  *
81  * Allocate a ring buffer for the interrupt controller,
82  * enable the RLC, disable interrupts, enable the IH
83  * ring buffer and enable it (VI).
84  * Called at device load and reume.
85  * Returns 0 for success, errors for failure.
86  */
87 static int vega10_ih_irq_init(struct amdgpu_device *adev)
88 {
89 	int ret = 0;
90 	int rb_bufsz;
91 	u32 ih_rb_cntl, ih_doorbell_rtpr;
92 	u32 tmp;
93 	u64 wptr_off;
94 
95 	/* disable irqs */
96 	vega10_ih_disable_interrupts(adev);
97 
98 	adev->nbio_funcs->ih_control(adev);
99 
100 	ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
101 	/* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
102 	if (adev->irq.ih.use_bus_addr) {
103 		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, adev->irq.ih.rb_dma_addr >> 8);
104 		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, ((u64)adev->irq.ih.rb_dma_addr >> 40) & 0xff);
105 		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 1);
106 	} else {
107 		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8);
108 		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (adev->irq.ih.gpu_addr >> 40) & 0xff);
109 		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 4);
110 	}
111 	rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);
112 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
113 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 1);
114 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
115 	/* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register value is written to memory */
116 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_WRITEBACK_ENABLE, 1);
117 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1);
118 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0);
119 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0);
120 
121 	if (adev->irq.msi_enabled)
122 		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, 1);
123 
124 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
125 
126 	/* set the writeback address whether it's enabled or not */
127 	if (adev->irq.ih.use_bus_addr)
128 		wptr_off = adev->irq.ih.rb_dma_addr + (adev->irq.ih.wptr_offs * 4);
129 	else
130 		wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4);
131 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO, lower_32_bits(wptr_off));
132 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI, upper_32_bits(wptr_off) & 0xFF);
133 
134 	/* set rptr, wptr to 0 */
135 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
136 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
137 
138 	ih_doorbell_rtpr = RREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR);
139 	if (adev->irq.ih.use_doorbell) {
140 		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
141 						 OFFSET, adev->irq.ih.doorbell_index);
142 		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
143 						 ENABLE, 1);
144 	} else {
145 		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
146 						 ENABLE, 0);
147 	}
148 	WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR, ih_doorbell_rtpr);
149 	adev->nbio_funcs->ih_doorbell_range(adev, adev->irq.ih.use_doorbell,
150 					    adev->irq.ih.doorbell_index);
151 
152 	tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL);
153 	tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL,
154 			    CLIENT18_IS_STORM_CLIENT, 1);
155 	WREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL, tmp);
156 
157 	tmp = RREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL);
158 	tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1);
159 	WREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL, tmp);
160 
161 	pci_set_master(adev->pdev);
162 
163 	/* enable interrupts */
164 	vega10_ih_enable_interrupts(adev);
165 
166 	return ret;
167 }
168 
169 /**
170  * vega10_ih_irq_disable - disable interrupts
171  *
172  * @adev: amdgpu_device pointer
173  *
174  * Disable interrupts on the hw (VEGA10).
175  */
176 static void vega10_ih_irq_disable(struct amdgpu_device *adev)
177 {
178 	vega10_ih_disable_interrupts(adev);
179 
180 	/* Wait and acknowledge irq */
181 	mdelay(1);
182 }
183 
184 /**
185  * vega10_ih_get_wptr - get the IH ring buffer wptr
186  *
187  * @adev: amdgpu_device pointer
188  *
189  * Get the IH ring buffer wptr from either the register
190  * or the writeback memory buffer (VEGA10).  Also check for
191  * ring buffer overflow and deal with it.
192  * Returns the value of the wptr.
193  */
194 static u32 vega10_ih_get_wptr(struct amdgpu_device *adev)
195 {
196 	u32 wptr, tmp;
197 
198 	if (adev->irq.ih.use_bus_addr)
199 		wptr = le32_to_cpu(adev->irq.ih.ring[adev->irq.ih.wptr_offs]);
200 	else
201 		wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]);
202 
203 	if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) {
204 		wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);
205 
206 		/* When a ring buffer overflow happen start parsing interrupt
207 		 * from the last not overwritten vector (wptr + 32). Hopefully
208 		 * this should allow us to catchup.
209 		 */
210 		tmp = (wptr + 32) & adev->irq.ih.ptr_mask;
211 		dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
212 			wptr, adev->irq.ih.rptr, tmp);
213 		adev->irq.ih.rptr = tmp;
214 
215 		tmp = RREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
216 		tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
217 		WREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), tmp);
218 	}
219 	return (wptr & adev->irq.ih.ptr_mask);
220 }
221 
222 /**
223  * vega10_ih_prescreen_iv - prescreen an interrupt vector
224  *
225  * @adev: amdgpu_device pointer
226  *
227  * Returns true if the interrupt vector should be further processed.
228  */
229 static bool vega10_ih_prescreen_iv(struct amdgpu_device *adev)
230 {
231 	u32 ring_index = adev->irq.ih.rptr >> 2;
232 	u32 dw0, dw3, dw4, dw5;
233 	u16 pasid;
234 	u64 addr, key;
235 	struct amdgpu_vm *vm;
236 	int r;
237 
238 	dw0 = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
239 	dw3 = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
240 	dw4 = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);
241 	dw5 = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);
242 
243 	/* Filter retry page faults, let only the first one pass. If
244 	 * there are too many outstanding faults, ignore them until
245 	 * some faults get cleared.
246 	 */
247 	switch (dw0 & 0xff) {
248 	case AMDGPU_IH_CLIENTID_VMC:
249 	case AMDGPU_IH_CLIENTID_UTCL2:
250 		break;
251 	default:
252 		/* Not a VM fault */
253 		return true;
254 	}
255 
256 	pasid = dw3 & 0xffff;
257 	/* No PASID, can't identify faulting process */
258 	if (!pasid)
259 		return true;
260 
261 	/* Not a retry fault, check fault credit */
262 	if (!(dw5 & 0x80)) {
263 		if (!amdgpu_vm_pasid_fault_credit(adev, pasid))
264 			goto ignore_iv;
265 		return true;
266 	}
267 
268 	addr = ((u64)(dw5 & 0xf) << 44) | ((u64)dw4 << 12);
269 	key = AMDGPU_VM_FAULT(pasid, addr);
270 	r = amdgpu_ih_add_fault(adev, key);
271 
272 	/* Hash table is full or the fault is already being processed,
273 	 * ignore further page faults
274 	 */
275 	if (r != 0)
276 		goto ignore_iv;
277 
278 	/* Track retry faults in per-VM fault FIFO. */
279 	spin_lock(&adev->vm_manager.pasid_lock);
280 	vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
281 	if (!vm) {
282 		/* VM not found, process it normally */
283 		spin_unlock(&adev->vm_manager.pasid_lock);
284 		amdgpu_ih_clear_fault(adev, key);
285 		return true;
286 	}
287 	/* No locking required with single writer and single reader */
288 	r = kfifo_put(&vm->faults, key);
289 	if (!r) {
290 		/* FIFO is full. Ignore it until there is space */
291 		spin_unlock(&adev->vm_manager.pasid_lock);
292 		amdgpu_ih_clear_fault(adev, key);
293 		goto ignore_iv;
294 	}
295 	spin_unlock(&adev->vm_manager.pasid_lock);
296 
297 	/* It's the first fault for this address, process it normally */
298 	return true;
299 
300 ignore_iv:
301 	adev->irq.ih.rptr += 32;
302 	return false;
303 }
304 
305 /**
306  * vega10_ih_decode_iv - decode an interrupt vector
307  *
308  * @adev: amdgpu_device pointer
309  *
310  * Decodes the interrupt vector at the current rptr
311  * position and also advance the position.
312  */
313 static void vega10_ih_decode_iv(struct amdgpu_device *adev,
314 				 struct amdgpu_iv_entry *entry)
315 {
316 	/* wptr/rptr are in bytes! */
317 	u32 ring_index = adev->irq.ih.rptr >> 2;
318 	uint32_t dw[8];
319 
320 	dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
321 	dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]);
322 	dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]);
323 	dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
324 	dw[4] = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);
325 	dw[5] = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);
326 	dw[6] = le32_to_cpu(adev->irq.ih.ring[ring_index + 6]);
327 	dw[7] = le32_to_cpu(adev->irq.ih.ring[ring_index + 7]);
328 
329 	entry->client_id = dw[0] & 0xff;
330 	entry->src_id = (dw[0] >> 8) & 0xff;
331 	entry->ring_id = (dw[0] >> 16) & 0xff;
332 	entry->vmid = (dw[0] >> 24) & 0xf;
333 	entry->vmid_src = (dw[0] >> 31);
334 	entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32);
335 	entry->timestamp_src = dw[2] >> 31;
336 	entry->pas_id = dw[3] & 0xffff;
337 	entry->pasid_src = dw[3] >> 31;
338 	entry->src_data[0] = dw[4];
339 	entry->src_data[1] = dw[5];
340 	entry->src_data[2] = dw[6];
341 	entry->src_data[3] = dw[7];
342 
343 
344 	/* wptr/rptr are in bytes! */
345 	adev->irq.ih.rptr += 32;
346 }
347 
348 /**
349  * vega10_ih_set_rptr - set the IH ring buffer rptr
350  *
351  * @adev: amdgpu_device pointer
352  *
353  * Set the IH ring buffer rptr.
354  */
355 static void vega10_ih_set_rptr(struct amdgpu_device *adev)
356 {
357 	if (adev->irq.ih.use_doorbell) {
358 		/* XXX check if swapping is necessary on BE */
359 		if (adev->irq.ih.use_bus_addr)
360 			adev->irq.ih.ring[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
361 		else
362 			adev->wb.wb[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
363 		WDOORBELL32(adev->irq.ih.doorbell_index, adev->irq.ih.rptr);
364 	} else {
365 		WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, adev->irq.ih.rptr);
366 	}
367 }
368 
369 static int vega10_ih_early_init(void *handle)
370 {
371 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
372 
373 	vega10_ih_set_interrupt_funcs(adev);
374 	return 0;
375 }
376 
377 static int vega10_ih_sw_init(void *handle)
378 {
379 	int r;
380 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
381 
382 	r = amdgpu_ih_ring_init(adev, 256 * 1024, true);
383 	if (r)
384 		return r;
385 
386 	adev->irq.ih.use_doorbell = true;
387 	adev->irq.ih.doorbell_index = AMDGPU_DOORBELL64_IH << 1;
388 
389 	adev->irq.ih.faults = kmalloc(sizeof(*adev->irq.ih.faults), GFP_KERNEL);
390 	if (!adev->irq.ih.faults)
391 		return -ENOMEM;
392 	INIT_CHASH_TABLE(adev->irq.ih.faults->hash,
393 			 AMDGPU_PAGEFAULT_HASH_BITS, 8, 0);
394 	spin_lock_init(&adev->irq.ih.faults->lock);
395 	adev->irq.ih.faults->count = 0;
396 
397 	r = amdgpu_irq_init(adev);
398 
399 	return r;
400 }
401 
402 static int vega10_ih_sw_fini(void *handle)
403 {
404 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
405 
406 	amdgpu_irq_fini(adev);
407 	amdgpu_ih_ring_fini(adev);
408 
409 	kfree(adev->irq.ih.faults);
410 	adev->irq.ih.faults = NULL;
411 
412 	return 0;
413 }
414 
415 static int vega10_ih_hw_init(void *handle)
416 {
417 	int r;
418 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
419 
420 	r = vega10_ih_irq_init(adev);
421 	if (r)
422 		return r;
423 
424 	return 0;
425 }
426 
427 static int vega10_ih_hw_fini(void *handle)
428 {
429 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
430 
431 	vega10_ih_irq_disable(adev);
432 
433 	return 0;
434 }
435 
436 static int vega10_ih_suspend(void *handle)
437 {
438 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
439 
440 	return vega10_ih_hw_fini(adev);
441 }
442 
443 static int vega10_ih_resume(void *handle)
444 {
445 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
446 
447 	return vega10_ih_hw_init(adev);
448 }
449 
450 static bool vega10_ih_is_idle(void *handle)
451 {
452 	/* todo */
453 	return true;
454 }
455 
456 static int vega10_ih_wait_for_idle(void *handle)
457 {
458 	/* todo */
459 	return -ETIMEDOUT;
460 }
461 
462 static int vega10_ih_soft_reset(void *handle)
463 {
464 	/* todo */
465 
466 	return 0;
467 }
468 
469 static int vega10_ih_set_clockgating_state(void *handle,
470 					  enum amd_clockgating_state state)
471 {
472 	return 0;
473 }
474 
475 static int vega10_ih_set_powergating_state(void *handle,
476 					  enum amd_powergating_state state)
477 {
478 	return 0;
479 }
480 
481 const struct amd_ip_funcs vega10_ih_ip_funcs = {
482 	.name = "vega10_ih",
483 	.early_init = vega10_ih_early_init,
484 	.late_init = NULL,
485 	.sw_init = vega10_ih_sw_init,
486 	.sw_fini = vega10_ih_sw_fini,
487 	.hw_init = vega10_ih_hw_init,
488 	.hw_fini = vega10_ih_hw_fini,
489 	.suspend = vega10_ih_suspend,
490 	.resume = vega10_ih_resume,
491 	.is_idle = vega10_ih_is_idle,
492 	.wait_for_idle = vega10_ih_wait_for_idle,
493 	.soft_reset = vega10_ih_soft_reset,
494 	.set_clockgating_state = vega10_ih_set_clockgating_state,
495 	.set_powergating_state = vega10_ih_set_powergating_state,
496 };
497 
498 static const struct amdgpu_ih_funcs vega10_ih_funcs = {
499 	.get_wptr = vega10_ih_get_wptr,
500 	.prescreen_iv = vega10_ih_prescreen_iv,
501 	.decode_iv = vega10_ih_decode_iv,
502 	.set_rptr = vega10_ih_set_rptr
503 };
504 
505 static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev)
506 {
507 	if (adev->irq.ih_funcs == NULL)
508 		adev->irq.ih_funcs = &vega10_ih_funcs;
509 }
510 
511 const struct amdgpu_ip_block_version vega10_ih_ip_block =
512 {
513 	.type = AMD_IP_BLOCK_TYPE_IH,
514 	.major = 4,
515 	.minor = 0,
516 	.rev = 0,
517 	.funcs = &vega10_ih_ip_funcs,
518 };
519