1 /*
2  * Copyright 2019 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/pci.h>
25 
26 #include "amdgpu.h"
27 #include "amdgpu_ih.h"
28 
29 #include "oss/osssys_5_0_0_offset.h"
30 #include "oss/osssys_5_0_0_sh_mask.h"
31 
32 #include "soc15_common.h"
33 #include "navi10_ih.h"
34 
35 
36 static void navi10_ih_set_interrupt_funcs(struct amdgpu_device *adev);
37 
38 /**
39  * navi10_ih_enable_interrupts - Enable the interrupt ring buffer
40  *
41  * @adev: amdgpu_device pointer
42  *
43  * Enable the interrupt ring buffer (NAVI10).
44  */
45 static void navi10_ih_enable_interrupts(struct amdgpu_device *adev)
46 {
47 	u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
48 
49 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
50 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1);
51 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
52 	adev->irq.ih.enabled = true;
53 }
54 
55 /**
56  * navi10_ih_disable_interrupts - Disable the interrupt ring buffer
57  *
58  * @adev: amdgpu_device pointer
59  *
60  * Disable the interrupt ring buffer (NAVI10).
61  */
62 static void navi10_ih_disable_interrupts(struct amdgpu_device *adev)
63 {
64 	u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
65 
66 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
67 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
68 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
69 	/* set rptr, wptr to 0 */
70 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
71 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
72 	adev->irq.ih.enabled = false;
73 	adev->irq.ih.rptr = 0;
74 }
75 
76 static uint32_t navi10_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl)
77 {
78 	int rb_bufsz = order_base_2(ih->ring_size / 4);
79 
80 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
81 				   MC_SPACE, ih->use_bus_addr ? 1 : 4);
82 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
83 				   WPTR_OVERFLOW_CLEAR, 1);
84 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
85 				   WPTR_OVERFLOW_ENABLE, 1);
86 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
87 	/* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register
88 	 * value is written to memory
89 	 */
90 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
91 				   WPTR_WRITEBACK_ENABLE, 1);
92 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1);
93 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0);
94 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0);
95 
96 	return ih_rb_cntl;
97 }
98 
99 /**
100  * navi10_ih_irq_init - init and enable the interrupt ring
101  *
102  * @adev: amdgpu_device pointer
103  *
104  * Allocate a ring buffer for the interrupt controller,
105  * enable the RLC, disable interrupts, enable the IH
106  * ring buffer and enable it (NAVI).
107  * Called at device load and reume.
108  * Returns 0 for success, errors for failure.
109  */
110 static int navi10_ih_irq_init(struct amdgpu_device *adev)
111 {
112 	struct amdgpu_ih_ring *ih = &adev->irq.ih;
113 	u32 ih_rb_cntl, ih_doorbell_rtpr, ih_chicken;
114 	u32 tmp;
115 
116 	/* disable irqs */
117 	navi10_ih_disable_interrupts(adev);
118 
119 	adev->nbio.funcs->ih_control(adev);
120 
121 	/* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
122 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, ih->gpu_addr >> 8);
123 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (ih->gpu_addr >> 40) & 0xff);
124 
125 	ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
126 	ih_rb_cntl = navi10_ih_rb_cntl(ih, ih_rb_cntl);
127 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM,
128 				   !!adev->irq.msi_enabled);
129 
130 	if (unlikely(adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT)) {
131 		if (ih->use_bus_addr) {
132 			ih_chicken = RREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN);
133 			ih_chicken = REG_SET_FIELD(ih_chicken,
134 					IH_CHICKEN, MC_SPACE_GPA_ENABLE, 1);
135 			WREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN, ih_chicken);
136 		}
137 	}
138 
139 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
140 
141 	/* set the writeback address whether it's enabled or not */
142 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO,
143 		     lower_32_bits(ih->wptr_addr));
144 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI,
145 		     upper_32_bits(ih->wptr_addr) & 0xFFFF);
146 
147 	/* set rptr, wptr to 0 */
148 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
149 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
150 
151 	ih_doorbell_rtpr = RREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR);
152 	if (ih->use_doorbell) {
153 		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
154 						 IH_DOORBELL_RPTR, OFFSET,
155 						 ih->doorbell_index);
156 		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
157 						 IH_DOORBELL_RPTR, ENABLE, 1);
158 	} else {
159 		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
160 						 IH_DOORBELL_RPTR, ENABLE, 0);
161 	}
162 	WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR, ih_doorbell_rtpr);
163 
164 	adev->nbio.funcs->ih_doorbell_range(adev, ih->use_doorbell,
165 					    ih->doorbell_index);
166 
167 	tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL);
168 	tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL,
169 			    CLIENT18_IS_STORM_CLIENT, 1);
170 	WREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL, tmp);
171 
172 	tmp = RREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL);
173 	tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1);
174 	WREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL, tmp);
175 
176 	pci_set_master(adev->pdev);
177 
178 	/* enable interrupts */
179 	navi10_ih_enable_interrupts(adev);
180 
181 	return 0;
182 }
183 
184 /**
185  * navi10_ih_irq_disable - disable interrupts
186  *
187  * @adev: amdgpu_device pointer
188  *
189  * Disable interrupts on the hw (NAVI10).
190  */
191 static void navi10_ih_irq_disable(struct amdgpu_device *adev)
192 {
193 	navi10_ih_disable_interrupts(adev);
194 
195 	/* Wait and acknowledge irq */
196 	mdelay(1);
197 }
198 
199 /**
200  * navi10_ih_get_wptr - get the IH ring buffer wptr
201  *
202  * @adev: amdgpu_device pointer
203  *
204  * Get the IH ring buffer wptr from either the register
205  * or the writeback memory buffer (NAVI10).  Also check for
206  * ring buffer overflow and deal with it.
207  * Returns the value of the wptr.
208  */
209 static u32 navi10_ih_get_wptr(struct amdgpu_device *adev,
210 			      struct amdgpu_ih_ring *ih)
211 {
212 	u32 wptr, reg, tmp;
213 
214 	wptr = le32_to_cpu(*ih->wptr_cpu);
215 
216 	if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
217 		goto out;
218 
219 	reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR);
220 	wptr = RREG32_NO_KIQ(reg);
221 	if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
222 		goto out;
223 	wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);
224 
225 	/* When a ring buffer overflow happen start parsing interrupt
226 	 * from the last not overwritten vector (wptr + 32). Hopefully
227 	 * this should allow us to catch up.
228 	 */
229 	tmp = (wptr + 32) & ih->ptr_mask;
230 	dev_warn(adev->dev, "IH ring buffer overflow "
231 		 "(0x%08X, 0x%08X, 0x%08X)\n",
232 		 wptr, ih->rptr, tmp);
233 	ih->rptr = tmp;
234 
235 	reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL);
236 	tmp = RREG32_NO_KIQ(reg);
237 	tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
238 	WREG32_NO_KIQ(reg, tmp);
239 out:
240 	return (wptr & ih->ptr_mask);
241 }
242 
243 /**
244  * navi10_ih_decode_iv - decode an interrupt vector
245  *
246  * @adev: amdgpu_device pointer
247  *
248  * Decodes the interrupt vector at the current rptr
249  * position and also advance the position.
250  */
251 static void navi10_ih_decode_iv(struct amdgpu_device *adev,
252 				struct amdgpu_ih_ring *ih,
253 				struct amdgpu_iv_entry *entry)
254 {
255 	/* wptr/rptr are in bytes! */
256 	u32 ring_index = ih->rptr >> 2;
257 	uint32_t dw[8];
258 
259 	dw[0] = le32_to_cpu(ih->ring[ring_index + 0]);
260 	dw[1] = le32_to_cpu(ih->ring[ring_index + 1]);
261 	dw[2] = le32_to_cpu(ih->ring[ring_index + 2]);
262 	dw[3] = le32_to_cpu(ih->ring[ring_index + 3]);
263 	dw[4] = le32_to_cpu(ih->ring[ring_index + 4]);
264 	dw[5] = le32_to_cpu(ih->ring[ring_index + 5]);
265 	dw[6] = le32_to_cpu(ih->ring[ring_index + 6]);
266 	dw[7] = le32_to_cpu(ih->ring[ring_index + 7]);
267 
268 	entry->client_id = dw[0] & 0xff;
269 	entry->src_id = (dw[0] >> 8) & 0xff;
270 	entry->ring_id = (dw[0] >> 16) & 0xff;
271 	entry->vmid = (dw[0] >> 24) & 0xf;
272 	entry->vmid_src = (dw[0] >> 31);
273 	entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32);
274 	entry->timestamp_src = dw[2] >> 31;
275 	entry->pasid = dw[3] & 0xffff;
276 	entry->pasid_src = dw[3] >> 31;
277 	entry->src_data[0] = dw[4];
278 	entry->src_data[1] = dw[5];
279 	entry->src_data[2] = dw[6];
280 	entry->src_data[3] = dw[7];
281 
282 	/* wptr/rptr are in bytes! */
283 	ih->rptr += 32;
284 }
285 
286 /**
287  * navi10_ih_set_rptr - set the IH ring buffer rptr
288  *
289  * @adev: amdgpu_device pointer
290  *
291  * Set the IH ring buffer rptr.
292  */
293 static void navi10_ih_set_rptr(struct amdgpu_device *adev,
294 			       struct amdgpu_ih_ring *ih)
295 {
296 	if (ih->use_doorbell) {
297 		/* XXX check if swapping is necessary on BE */
298 		*ih->rptr_cpu = ih->rptr;
299 		WDOORBELL32(ih->doorbell_index, ih->rptr);
300 	} else
301 		WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, ih->rptr);
302 }
303 
304 static int navi10_ih_early_init(void *handle)
305 {
306 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
307 
308 	navi10_ih_set_interrupt_funcs(adev);
309 	return 0;
310 }
311 
312 static int navi10_ih_sw_init(void *handle)
313 {
314 	int r;
315 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
316 	bool use_bus_addr;
317 
318 	/* use gpu virtual address for ih ring
319 	 * until ih_checken is programmed to allow
320 	 * use bus address for ih ring by psp bl */
321 	use_bus_addr =
322 		(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) ? false : true;
323 	r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 256 * 1024, use_bus_addr);
324 	if (r)
325 		return r;
326 
327 	adev->irq.ih.use_doorbell = true;
328 	adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1;
329 
330 	r = amdgpu_irq_init(adev);
331 
332 	return r;
333 }
334 
335 static int navi10_ih_sw_fini(void *handle)
336 {
337 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
338 
339 	amdgpu_irq_fini(adev);
340 	amdgpu_ih_ring_fini(adev, &adev->irq.ih);
341 
342 	return 0;
343 }
344 
345 static int navi10_ih_hw_init(void *handle)
346 {
347 	int r;
348 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
349 
350 	r = navi10_ih_irq_init(adev);
351 	if (r)
352 		return r;
353 
354 	return 0;
355 }
356 
357 static int navi10_ih_hw_fini(void *handle)
358 {
359 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
360 
361 	navi10_ih_irq_disable(adev);
362 
363 	return 0;
364 }
365 
366 static int navi10_ih_suspend(void *handle)
367 {
368 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
369 
370 	return navi10_ih_hw_fini(adev);
371 }
372 
373 static int navi10_ih_resume(void *handle)
374 {
375 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
376 
377 	return navi10_ih_hw_init(adev);
378 }
379 
380 static bool navi10_ih_is_idle(void *handle)
381 {
382 	/* todo */
383 	return true;
384 }
385 
386 static int navi10_ih_wait_for_idle(void *handle)
387 {
388 	/* todo */
389 	return -ETIMEDOUT;
390 }
391 
392 static int navi10_ih_soft_reset(void *handle)
393 {
394 	/* todo */
395 	return 0;
396 }
397 
398 static void navi10_ih_update_clockgating_state(struct amdgpu_device *adev,
399 					       bool enable)
400 {
401 	uint32_t data, def, field_val;
402 
403 	if (adev->cg_flags & AMD_CG_SUPPORT_IH_CG) {
404 		def = data = RREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL);
405 		field_val = enable ? 0 : 1;
406 		data = REG_SET_FIELD(data, IH_CLK_CTRL,
407 				     DBUS_MUX_CLK_SOFT_OVERRIDE, field_val);
408 		data = REG_SET_FIELD(data, IH_CLK_CTRL,
409 				     OSSSYS_SHARE_CLK_SOFT_OVERRIDE, field_val);
410 		data = REG_SET_FIELD(data, IH_CLK_CTRL,
411 				     LIMIT_SMN_CLK_SOFT_OVERRIDE, field_val);
412 		data = REG_SET_FIELD(data, IH_CLK_CTRL,
413 				     DYN_CLK_SOFT_OVERRIDE, field_val);
414 		data = REG_SET_FIELD(data, IH_CLK_CTRL,
415 				     REG_CLK_SOFT_OVERRIDE, field_val);
416 		if (def != data)
417 			WREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL, data);
418 	}
419 
420 	return;
421 }
422 
423 static int navi10_ih_set_clockgating_state(void *handle,
424 					   enum amd_clockgating_state state)
425 {
426 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
427 
428 	navi10_ih_update_clockgating_state(adev,
429 				state == AMD_CG_STATE_GATE);
430 	return 0;
431 }
432 
433 static int navi10_ih_set_powergating_state(void *handle,
434 					   enum amd_powergating_state state)
435 {
436 	return 0;
437 }
438 
439 static void navi10_ih_get_clockgating_state(void *handle, u32 *flags)
440 {
441 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
442 
443 	if (!RREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL))
444 		*flags |= AMD_CG_SUPPORT_IH_CG;
445 
446 	return;
447 }
448 
449 static const struct amd_ip_funcs navi10_ih_ip_funcs = {
450 	.name = "navi10_ih",
451 	.early_init = navi10_ih_early_init,
452 	.late_init = NULL,
453 	.sw_init = navi10_ih_sw_init,
454 	.sw_fini = navi10_ih_sw_fini,
455 	.hw_init = navi10_ih_hw_init,
456 	.hw_fini = navi10_ih_hw_fini,
457 	.suspend = navi10_ih_suspend,
458 	.resume = navi10_ih_resume,
459 	.is_idle = navi10_ih_is_idle,
460 	.wait_for_idle = navi10_ih_wait_for_idle,
461 	.soft_reset = navi10_ih_soft_reset,
462 	.set_clockgating_state = navi10_ih_set_clockgating_state,
463 	.set_powergating_state = navi10_ih_set_powergating_state,
464 	.get_clockgating_state = navi10_ih_get_clockgating_state,
465 };
466 
467 static const struct amdgpu_ih_funcs navi10_ih_funcs = {
468 	.get_wptr = navi10_ih_get_wptr,
469 	.decode_iv = navi10_ih_decode_iv,
470 	.set_rptr = navi10_ih_set_rptr
471 };
472 
473 static void navi10_ih_set_interrupt_funcs(struct amdgpu_device *adev)
474 {
475 	if (adev->irq.ih_funcs == NULL)
476 		adev->irq.ih_funcs = &navi10_ih_funcs;
477 }
478 
479 const struct amdgpu_ip_block_version navi10_ih_ip_block =
480 {
481 	.type = AMD_IP_BLOCK_TYPE_IH,
482 	.major = 5,
483 	.minor = 0,
484 	.rev = 0,
485 	.funcs = &navi10_ih_ip_funcs,
486 };
487