1 /*
2  * Copyright 2019 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include "amdgpu.h"
25 #include "gfxhub_v2_0.h"
26 
27 #include "gc/gc_10_1_0_offset.h"
28 #include "gc/gc_10_1_0_sh_mask.h"
29 #include "gc/gc_10_1_0_default.h"
30 #include "navi10_enum.h"
31 
32 #include "soc15_common.h"
33 
34 u64 gfxhub_v2_0_get_fb_location(struct amdgpu_device *adev)
35 {
36 	u64 base = RREG32_SOC15(GC, 0, mmGCMC_VM_FB_LOCATION_BASE);
37 
38 	base &= GCMC_VM_FB_LOCATION_BASE__FB_BASE_MASK;
39 	base <<= 24;
40 
41 	return base;
42 }
43 
44 u64 gfxhub_v2_0_get_mc_fb_offset(struct amdgpu_device *adev)
45 {
46 	return (u64)RREG32_SOC15(GC, 0, mmGCMC_VM_FB_OFFSET) << 24;
47 }
48 
49 static void gfxhub_v2_0_init_gart_pt_regs(struct amdgpu_device *adev)
50 {
51 	uint64_t value = amdgpu_gmc_pd_addr(adev->gart.bo);
52 
53 
54 	WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
55 		     lower_32_bits(value));
56 
57 	WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
58 		     upper_32_bits(value));
59 }
60 
61 static void gfxhub_v2_0_init_gart_aperture_regs(struct amdgpu_device *adev)
62 {
63 	gfxhub_v2_0_init_gart_pt_regs(adev);
64 
65 	WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
66 		     (u32)(adev->gmc.gart_start >> 12));
67 	WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
68 		     (u32)(adev->gmc.gart_start >> 44));
69 
70 	WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
71 		     (u32)(adev->gmc.gart_end >> 12));
72 	WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
73 		     (u32)(adev->gmc.gart_end >> 44));
74 }
75 
76 static void gfxhub_v2_0_init_system_aperture_regs(struct amdgpu_device *adev)
77 {
78 	uint64_t value;
79 
80 	/* Disable AGP. */
81 	WREG32_SOC15(GC, 0, mmGCMC_VM_AGP_BASE, 0);
82 	WREG32_SOC15(GC, 0, mmGCMC_VM_AGP_TOP, 0);
83 	WREG32_SOC15(GC, 0, mmGCMC_VM_AGP_BOT, 0x00FFFFFF);
84 
85 	/* Program the system aperture low logical page number. */
86 	WREG32_SOC15(GC, 0, mmGCMC_VM_SYSTEM_APERTURE_LOW_ADDR,
87 		     adev->gmc.vram_start >> 18);
88 	WREG32_SOC15(GC, 0, mmGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
89 		     adev->gmc.vram_end >> 18);
90 
91 	/* Set default page address. */
92 	value = adev->vram_scratch.gpu_addr - adev->gmc.vram_start
93 		+ adev->vm_manager.vram_base_offset;
94 	WREG32_SOC15(GC, 0, mmGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
95 		     (u32)(value >> 12));
96 	WREG32_SOC15(GC, 0, mmGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
97 		     (u32)(value >> 44));
98 
99 	/* Program "protection fault". */
100 	WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
101 		     (u32)(adev->dummy_page_addr >> 12));
102 	WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
103 		     (u32)((u64)adev->dummy_page_addr >> 44));
104 
105 	WREG32_FIELD15(GC, 0, GCVM_L2_PROTECTION_FAULT_CNTL2,
106 		       ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
107 }
108 
109 
110 static void gfxhub_v2_0_init_tlb_regs(struct amdgpu_device *adev)
111 {
112 	uint32_t tmp;
113 
114 	/* Setup TLB control */
115 	tmp = RREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL);
116 
117 	tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
118 	tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
119 	tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
120 			    ENABLE_ADVANCED_DRIVER_MODEL, 1);
121 	tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
122 			    SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
123 	tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
124 	tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
125 			    MTYPE, MTYPE_UC); /* UC, uncached */
126 
127 	WREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL, tmp);
128 }
129 
130 static void gfxhub_v2_0_init_cache_regs(struct amdgpu_device *adev)
131 {
132 	uint32_t tmp;
133 
134 	/* Setup L2 cache */
135 	tmp = RREG32_SOC15(GC, 0, mmGCVM_L2_CNTL);
136 	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, ENABLE_L2_CACHE, 1);
137 	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 0);
138 	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL,
139 			    ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1);
140 	/* XXX for emulation, Refer to closed source code.*/
141 	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL,
142 			    L2_PDE0_CACHE_TAG_GENERATION_MODE, 0);
143 	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 0);
144 	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
145 	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0);
146 	WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL, tmp);
147 
148 	tmp = RREG32_SOC15(GC, 0, mmGCVM_L2_CNTL2);
149 	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
150 	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
151 	WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL2, tmp);
152 
153 	tmp = mmGCVM_L2_CNTL3_DEFAULT;
154 	WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL3, tmp);
155 
156 	tmp = mmGCVM_L2_CNTL4_DEFAULT;
157 	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
158 	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
159 	WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL4, tmp);
160 }
161 
162 static void gfxhub_v2_0_enable_system_domain(struct amdgpu_device *adev)
163 {
164 	uint32_t tmp;
165 
166 	tmp = RREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_CNTL);
167 	tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
168 	tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
169 	WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_CNTL, tmp);
170 }
171 
172 static void gfxhub_v2_0_disable_identity_aperture(struct amdgpu_device *adev)
173 {
174 	WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
175 		     0xFFFFFFFF);
176 	WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
177 		     0x0000000F);
178 
179 	WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32,
180 		     0);
181 	WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32,
182 		     0);
183 
184 	WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 0);
185 	WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 0);
186 
187 }
188 
189 static void gfxhub_v2_0_setup_vmid_config(struct amdgpu_device *adev)
190 {
191 	int i;
192 	uint32_t tmp;
193 
194 	for (i = 0; i <= 14; i++) {
195 		tmp = RREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_CNTL, i);
196 		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
197 		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH,
198 				    adev->vm_manager.num_level);
199 		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
200 				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
201 		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
202 				DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
203 		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
204 				PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
205 		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
206 				VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
207 		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
208 				READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
209 		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
210 				WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
211 		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
212 				EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
213 		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
214 				PAGE_TABLE_BLOCK_SIZE,
215 				adev->vm_manager.block_size - 9);
216 		/* Send no-retry XNACK on fault to suppress VM fault storm. */
217 		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
218 				    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT,
219 				    !amdgpu_noretry);
220 		WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_CNTL, i, tmp);
221 		WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, i*2, 0);
222 		WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, i*2, 0);
223 		WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32,  i*2,
224 			lower_32_bits(adev->vm_manager.max_pfn - 1));
225 		WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, i*2,
226 			upper_32_bits(adev->vm_manager.max_pfn - 1));
227 	}
228 }
229 
230 static void gfxhub_v2_0_program_invalidation(struct amdgpu_device *adev)
231 {
232 	unsigned i;
233 
234 	for (i = 0 ; i < 18; ++i) {
235 		WREG32_SOC15_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
236 				    2 * i, 0xffffffff);
237 		WREG32_SOC15_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
238 				    2 * i, 0x1f);
239 	}
240 }
241 
242 int gfxhub_v2_0_gart_enable(struct amdgpu_device *adev)
243 {
244 	if (amdgpu_sriov_vf(adev)) {
245 		/*
246 		 * GCMC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase they are
247 		 * VF copy registers so vbios post doesn't program them, for
248 		 * SRIOV driver need to program them
249 		 */
250 		WREG32_SOC15(GC, 0, mmGCMC_VM_FB_LOCATION_BASE,
251 			     adev->gmc.vram_start >> 24);
252 		WREG32_SOC15(GC, 0, mmGCMC_VM_FB_LOCATION_TOP,
253 			     adev->gmc.vram_end >> 24);
254 	}
255 
256 	/* GART Enable. */
257 	gfxhub_v2_0_init_gart_aperture_regs(adev);
258 	gfxhub_v2_0_init_system_aperture_regs(adev);
259 	gfxhub_v2_0_init_tlb_regs(adev);
260 	gfxhub_v2_0_init_cache_regs(adev);
261 
262 	gfxhub_v2_0_enable_system_domain(adev);
263 	gfxhub_v2_0_disable_identity_aperture(adev);
264 	gfxhub_v2_0_setup_vmid_config(adev);
265 	gfxhub_v2_0_program_invalidation(adev);
266 
267 	return 0;
268 }
269 
270 void gfxhub_v2_0_gart_disable(struct amdgpu_device *adev)
271 {
272 	u32 tmp;
273 	u32 i;
274 
275 	/* Disable all tables */
276 	for (i = 0; i < 16; i++)
277 		WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_CNTL, i, 0);
278 
279 	/* Setup TLB control */
280 	tmp = RREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL);
281 	tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
282 	tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
283 			    ENABLE_ADVANCED_DRIVER_MODEL, 0);
284 	WREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL, tmp);
285 
286 	/* Setup L2 cache */
287 	WREG32_FIELD15(GC, 0, GCVM_L2_CNTL, ENABLE_L2_CACHE, 0);
288 	WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL3, 0);
289 }
290 
291 /**
292  * gfxhub_v2_0_set_fault_enable_default - update GART/VM fault handling
293  *
294  * @adev: amdgpu_device pointer
295  * @value: true redirects VM faults to the default page
296  */
297 void gfxhub_v2_0_set_fault_enable_default(struct amdgpu_device *adev,
298 					  bool value)
299 {
300 	u32 tmp;
301 	tmp = RREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL);
302 	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
303 			    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
304 	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
305 			    PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
306 	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
307 			    PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value);
308 	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
309 			    PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value);
310 	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
311 			    TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
312 			    value);
313 	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
314 			    NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value);
315 	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
316 			    DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
317 	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
318 			    VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
319 	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
320 			    READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
321 	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
322 			    WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
323 	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
324 			    EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
325 	if (!value) {
326 		tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
327 				CRASH_ON_NO_RETRY_FAULT, 1);
328 		tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
329 				CRASH_ON_RETRY_FAULT, 1);
330 	}
331 	WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL, tmp);
332 }
333 
334 void gfxhub_v2_0_init(struct amdgpu_device *adev)
335 {
336 	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0];
337 
338 	hub->ctx0_ptb_addr_lo32 =
339 		SOC15_REG_OFFSET(GC, 0,
340 				 mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32);
341 	hub->ctx0_ptb_addr_hi32 =
342 		SOC15_REG_OFFSET(GC, 0,
343 				 mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32);
344 	hub->vm_inv_eng0_req =
345 		SOC15_REG_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_REQ);
346 	hub->vm_inv_eng0_ack =
347 		SOC15_REG_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_ACK);
348 	hub->vm_context0_cntl =
349 		SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_CNTL);
350 	hub->vm_l2_pro_fault_status =
351 		SOC15_REG_OFFSET(GC, 0, mmGCVM_L2_PROTECTION_FAULT_STATUS);
352 	hub->vm_l2_pro_fault_cntl =
353 		SOC15_REG_OFFSET(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL);
354 }
355