1 /*
2  * Copyright 2016 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 #include "amdgpu.h"
24 #include "gfxhub_v1_0.h"
25 
26 #include "vega10/soc15ip.h"
27 #include "vega10/GC/gc_9_0_offset.h"
28 #include "vega10/GC/gc_9_0_sh_mask.h"
29 #include "vega10/GC/gc_9_0_default.h"
30 #include "vega10/vega10_enum.h"
31 
32 #include "soc15_common.h"
33 
34 u64 gfxhub_v1_0_get_mc_fb_offset(struct amdgpu_device *adev)
35 {
36 	return (u64)RREG32_SOC15(GC, 0, mmMC_VM_FB_OFFSET) << 24;
37 }
38 
39 static void gfxhub_v1_0_init_gart_pt_regs(struct amdgpu_device *adev)
40 {
41 	uint64_t value;
42 
43 	BUG_ON(adev->gart.table_addr & (~0x0000FFFFFFFFF000ULL));
44 	value = adev->gart.table_addr - adev->mc.vram_start
45 		+ adev->vm_manager.vram_base_offset;
46 	value &= 0x0000FFFFFFFFF000ULL;
47 	value |= 0x1; /*valid bit*/
48 
49 	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
50 		     lower_32_bits(value));
51 
52 	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
53 		     upper_32_bits(value));
54 }
55 
56 static void gfxhub_v1_0_init_gart_aperture_regs(struct amdgpu_device *adev)
57 {
58 	gfxhub_v1_0_init_gart_pt_regs(adev);
59 
60 	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
61 		     (u32)(adev->mc.gart_start >> 12));
62 	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
63 		     (u32)(adev->mc.gart_start >> 44));
64 
65 	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
66 		     (u32)(adev->mc.gart_end >> 12));
67 	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
68 		     (u32)(adev->mc.gart_end >> 44));
69 }
70 
71 static void gfxhub_v1_0_init_system_aperture_regs(struct amdgpu_device *adev)
72 {
73 	uint64_t value;
74 
75 	/* Disable AGP. */
76 	WREG32_SOC15(GC, 0, mmMC_VM_AGP_BASE, 0);
77 	WREG32_SOC15(GC, 0, mmMC_VM_AGP_TOP, 0);
78 	WREG32_SOC15(GC, 0, mmMC_VM_AGP_BOT, 0xFFFFFFFF);
79 
80 	/* Program the system aperture low logical page number. */
81 	WREG32_SOC15(GC, 0, mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
82 		     adev->mc.vram_start >> 18);
83 	WREG32_SOC15(GC, 0, mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
84 		     adev->mc.vram_end >> 18);
85 
86 	/* Set default page address. */
87 	value = adev->vram_scratch.gpu_addr - adev->mc.vram_start
88 		+ adev->vm_manager.vram_base_offset;
89 	WREG32_SOC15(GC, 0, mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
90 		     (u32)(value >> 12));
91 	WREG32_SOC15(GC, 0, mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
92 		     (u32)(value >> 44));
93 
94 	/* Program "protection fault". */
95 	WREG32_SOC15(GC, 0, mmVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
96 		     (u32)(adev->dummy_page.addr >> 12));
97 	WREG32_SOC15(GC, 0, mmVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
98 		     (u32)((u64)adev->dummy_page.addr >> 44));
99 
100 	WREG32_FIELD15(GC, 0, VM_L2_PROTECTION_FAULT_CNTL2,
101 		       ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
102 }
103 
104 static void gfxhub_v1_0_init_tlb_regs(struct amdgpu_device *adev)
105 {
106 	uint32_t tmp;
107 
108 	/* Setup TLB control */
109 	tmp = RREG32_SOC15(GC, 0, mmMC_VM_MX_L1_TLB_CNTL);
110 
111 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
112 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
113 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
114 			    ENABLE_ADVANCED_DRIVER_MODEL, 1);
115 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
116 			    SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
117 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
118 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
119 			    MTYPE, MTYPE_UC);/* XXX for emulation. */
120 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1);
121 
122 	WREG32_SOC15(GC, 0, mmMC_VM_MX_L1_TLB_CNTL, tmp);
123 }
124 
125 static void gfxhub_v1_0_init_cache_regs(struct amdgpu_device *adev)
126 {
127 	uint32_t tmp;
128 
129 	/* Setup L2 cache */
130 	tmp = RREG32_SOC15(GC, 0, mmVM_L2_CNTL);
131 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);
132 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1);
133 	/* XXX for emulation, Refer to closed source code.*/
134 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, L2_PDE0_CACHE_TAG_GENERATION_MODE,
135 			    0);
136 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 1);
137 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
138 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0);
139 	WREG32_SOC15(GC, 0, mmVM_L2_CNTL, tmp);
140 
141 	tmp = RREG32_SOC15(GC, 0, mmVM_L2_CNTL2);
142 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
143 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
144 	WREG32_SOC15(GC, 0, mmVM_L2_CNTL2, tmp);
145 
146 	tmp = mmVM_L2_CNTL3_DEFAULT;
147 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9);
148 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
149 	WREG32_SOC15(GC, 0, mmVM_L2_CNTL3, tmp);
150 
151 	tmp = mmVM_L2_CNTL4_DEFAULT;
152 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
153 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
154 	WREG32_SOC15(GC, 0, mmVM_L2_CNTL4, tmp);
155 }
156 
157 static void gfxhub_v1_0_enable_system_domain(struct amdgpu_device *adev)
158 {
159 	uint32_t tmp;
160 
161 	tmp = RREG32_SOC15(GC, 0, mmVM_CONTEXT0_CNTL);
162 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
163 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
164 	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_CNTL, tmp);
165 }
166 
167 static void gfxhub_v1_0_disable_identity_aperture(struct amdgpu_device *adev)
168 {
169 	WREG32_SOC15(GC, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
170 		     0XFFFFFFFF);
171 	WREG32_SOC15(GC, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
172 		     0x0000000F);
173 
174 	WREG32_SOC15(GC, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32,
175 		     0);
176 	WREG32_SOC15(GC, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32,
177 		     0);
178 
179 	WREG32_SOC15(GC, 0, mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 0);
180 	WREG32_SOC15(GC, 0, mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 0);
181 
182 }
183 
184 static void gfxhub_v1_0_setup_vmid_config(struct amdgpu_device *adev)
185 {
186 	int i;
187 	uint32_t tmp;
188 
189 	for (i = 0; i <= 14; i++) {
190 		tmp = RREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_CNTL, i);
191 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
192 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH,
193 				    adev->vm_manager.num_level);
194 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
195 				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
196 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
197 				DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
198 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
199 				PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
200 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
201 				VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
202 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
203 				READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
204 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
205 				WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
206 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
207 				EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
208 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
209 				PAGE_TABLE_BLOCK_SIZE,
210 				adev->vm_manager.block_size - 9);
211 		/* Send no-retry XNACK on fault to suppress VM fault storm. */
212 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
213 				    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
214 		WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_CNTL, i, tmp);
215 		WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, i*2, 0);
216 		WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, i*2, 0);
217 		WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32,  i*2,
218 			lower_32_bits(adev->vm_manager.max_pfn - 1));
219 		WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, i*2,
220 			upper_32_bits(adev->vm_manager.max_pfn - 1));
221 	}
222 }
223 
224 static void gfxhub_v1_0_program_invalidation(struct amdgpu_device *adev)
225 {
226 	unsigned i;
227 
228 	for (i = 0 ; i < 18; ++i) {
229 		WREG32_SOC15_OFFSET(GC, 0, mmVM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
230 				    2 * i, 0xffffffff);
231 		WREG32_SOC15_OFFSET(GC, 0, mmVM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
232 				    2 * i, 0x1f);
233 	}
234 }
235 
236 int gfxhub_v1_0_gart_enable(struct amdgpu_device *adev)
237 {
238 	if (amdgpu_sriov_vf(adev)) {
239 		/*
240 		 * MC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase they are
241 		 * VF copy registers so vbios post doesn't program them, for
242 		 * SRIOV driver need to program them
243 		 */
244 		WREG32_SOC15(GC, 0, mmMC_VM_FB_LOCATION_BASE,
245 			     adev->mc.vram_start >> 24);
246 		WREG32_SOC15(GC, 0, mmMC_VM_FB_LOCATION_TOP,
247 			     adev->mc.vram_end >> 24);
248 	}
249 
250 	/* GART Enable. */
251 	gfxhub_v1_0_init_gart_aperture_regs(adev);
252 	gfxhub_v1_0_init_system_aperture_regs(adev);
253 	gfxhub_v1_0_init_tlb_regs(adev);
254 	gfxhub_v1_0_init_cache_regs(adev);
255 
256 	gfxhub_v1_0_enable_system_domain(adev);
257 	gfxhub_v1_0_disable_identity_aperture(adev);
258 	gfxhub_v1_0_setup_vmid_config(adev);
259 	gfxhub_v1_0_program_invalidation(adev);
260 
261 	return 0;
262 }
263 
264 void gfxhub_v1_0_gart_disable(struct amdgpu_device *adev)
265 {
266 	u32 tmp;
267 	u32 i;
268 
269 	/* Disable all tables */
270 	for (i = 0; i < 16; i++)
271 		WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT0_CNTL, i, 0);
272 
273 	/* Setup TLB control */
274 	tmp = RREG32_SOC15(GC, 0, mmMC_VM_MX_L1_TLB_CNTL);
275 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
276 	tmp = REG_SET_FIELD(tmp,
277 				MC_VM_MX_L1_TLB_CNTL,
278 				ENABLE_ADVANCED_DRIVER_MODEL,
279 				0);
280 	WREG32_SOC15(GC, 0, mmMC_VM_MX_L1_TLB_CNTL, tmp);
281 
282 	/* Setup L2 cache */
283 	WREG32_FIELD15(GC, 0, VM_L2_CNTL, ENABLE_L2_CACHE, 0);
284 	WREG32_SOC15(GC, 0, mmVM_L2_CNTL3, 0);
285 }
286 
287 /**
288  * gfxhub_v1_0_set_fault_enable_default - update GART/VM fault handling
289  *
290  * @adev: amdgpu_device pointer
291  * @value: true redirects VM faults to the default page
292  */
293 void gfxhub_v1_0_set_fault_enable_default(struct amdgpu_device *adev,
294 					  bool value)
295 {
296 	u32 tmp;
297 	tmp = RREG32_SOC15(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL);
298 	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
299 			RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
300 	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
301 			PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
302 	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
303 			PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value);
304 	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
305 			PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value);
306 	tmp = REG_SET_FIELD(tmp,
307 			VM_L2_PROTECTION_FAULT_CNTL,
308 			TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
309 			value);
310 	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
311 			NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value);
312 	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
313 			DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
314 	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
315 			VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
316 	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
317 			READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
318 	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
319 			WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
320 	tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
321 			EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
322 	WREG32_SOC15(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL, tmp);
323 }
324 
325 void gfxhub_v1_0_init(struct amdgpu_device *adev)
326 {
327 	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB];
328 
329 	hub->ctx0_ptb_addr_lo32 =
330 		SOC15_REG_OFFSET(GC, 0,
331 				 mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32);
332 	hub->ctx0_ptb_addr_hi32 =
333 		SOC15_REG_OFFSET(GC, 0,
334 				 mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32);
335 	hub->vm_inv_eng0_req =
336 		SOC15_REG_OFFSET(GC, 0, mmVM_INVALIDATE_ENG0_REQ);
337 	hub->vm_inv_eng0_ack =
338 		SOC15_REG_OFFSET(GC, 0, mmVM_INVALIDATE_ENG0_ACK);
339 	hub->vm_context0_cntl =
340 		SOC15_REG_OFFSET(GC, 0, mmVM_CONTEXT0_CNTL);
341 	hub->vm_l2_pro_fault_status =
342 		SOC15_REG_OFFSET(GC, 0, mmVM_L2_PROTECTION_FAULT_STATUS);
343 	hub->vm_l2_pro_fault_cntl =
344 		SOC15_REG_OFFSET(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL);
345 }
346