1 /*
2  * Copyright 2018 Advanced Micro Devices, Inc.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sub license, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
20  *
21  * The above copyright notice and this permission notice (including the
22  * next paragraph) shall be included in all copies or substantial portions
23  * of the Software.
24  *
25  */
26 #ifndef __AMDGPU_GMC_H__
27 #define __AMDGPU_GMC_H__
28 
29 #include <linux/types.h>
30 
31 #include "amdgpu_irq.h"
32 #include "amdgpu_ras.h"
33 
34 /* VA hole for 48bit addresses on Vega10 */
35 #define AMDGPU_GMC_HOLE_START	0x0000800000000000ULL
36 #define AMDGPU_GMC_HOLE_END	0xffff800000000000ULL
37 
38 /*
39  * Hardware is programmed as if the hole doesn't exists with start and end
40  * address values.
41  *
42  * This mask is used to remove the upper 16bits of the VA and so come up with
43  * the linear addr value.
44  */
45 #define AMDGPU_GMC_HOLE_MASK	0x0000ffffffffffffULL
46 
47 /*
48  * Ring size as power of two for the log of recent faults.
49  */
50 #define AMDGPU_GMC_FAULT_RING_ORDER	8
51 #define AMDGPU_GMC_FAULT_RING_SIZE	(1 << AMDGPU_GMC_FAULT_RING_ORDER)
52 
53 /*
54  * Hash size as power of two for the log of recent faults
55  */
56 #define AMDGPU_GMC_FAULT_HASH_ORDER	8
57 #define AMDGPU_GMC_FAULT_HASH_SIZE	(1 << AMDGPU_GMC_FAULT_HASH_ORDER)
58 
59 /*
60  * Number of IH timestamp ticks until a fault is considered handled
61  */
62 #define AMDGPU_GMC_FAULT_TIMEOUT	5000ULL
63 
64 struct firmware;
65 
66 /*
67  * GMC page fault information
68  */
69 struct amdgpu_gmc_fault {
70 	uint64_t	timestamp:48;
71 	uint64_t	next:AMDGPU_GMC_FAULT_RING_ORDER;
72 	atomic64_t	key;
73 };
74 
75 /*
76  * VMHUB structures, functions & helpers
77  */
78 struct amdgpu_vmhub_funcs {
79 	void (*print_l2_protection_fault_status)(struct amdgpu_device *adev,
80 						 uint32_t status);
81 	uint32_t (*get_invalidate_req)(unsigned int vmid, uint32_t flush_type);
82 };
83 
84 struct amdgpu_vmhub {
85 	uint32_t	ctx0_ptb_addr_lo32;
86 	uint32_t	ctx0_ptb_addr_hi32;
87 	uint32_t	vm_inv_eng0_sem;
88 	uint32_t	vm_inv_eng0_req;
89 	uint32_t	vm_inv_eng0_ack;
90 	uint32_t	vm_context0_cntl;
91 	uint32_t	vm_l2_pro_fault_status;
92 	uint32_t	vm_l2_pro_fault_cntl;
93 
94 	/*
95 	 * store the register distances between two continuous context domain
96 	 * and invalidation engine.
97 	 */
98 	uint32_t	ctx_distance;
99 	uint32_t	ctx_addr_distance; /* include LO32/HI32 */
100 	uint32_t	eng_distance;
101 	uint32_t	eng_addr_distance; /* include LO32/HI32 */
102 
103 	uint32_t        vm_cntx_cntl;
104 	uint32_t	vm_cntx_cntl_vm_fault;
105 	uint32_t	vm_l2_bank_select_reserved_cid2;
106 
107 	const struct amdgpu_vmhub_funcs *vmhub_funcs;
108 };
109 
110 /*
111  * GPU MC structures, functions & helpers
112  */
113 struct amdgpu_gmc_funcs {
114 	/* flush the vm tlb via mmio */
115 	void (*flush_gpu_tlb)(struct amdgpu_device *adev, uint32_t vmid,
116 				uint32_t vmhub, uint32_t flush_type);
117 	/* flush the vm tlb via pasid */
118 	int (*flush_gpu_tlb_pasid)(struct amdgpu_device *adev, uint16_t pasid,
119 					uint32_t flush_type, bool all_hub);
120 	/* flush the vm tlb via ring */
121 	uint64_t (*emit_flush_gpu_tlb)(struct amdgpu_ring *ring, unsigned vmid,
122 				       uint64_t pd_addr);
123 	/* Change the VMID -> PASID mapping */
124 	void (*emit_pasid_mapping)(struct amdgpu_ring *ring, unsigned vmid,
125 				   unsigned pasid);
126 	/* enable/disable PRT support */
127 	void (*set_prt)(struct amdgpu_device *adev, bool enable);
128 	/* map mtype to hardware flags */
129 	uint64_t (*map_mtype)(struct amdgpu_device *adev, uint32_t flags);
130 	/* get the pde for a given mc addr */
131 	void (*get_vm_pde)(struct amdgpu_device *adev, int level,
132 			   u64 *dst, u64 *flags);
133 	/* get the pte flags to use for a BO VA mapping */
134 	void (*get_vm_pte)(struct amdgpu_device *adev,
135 			   struct amdgpu_bo_va_mapping *mapping,
136 			   uint64_t *flags);
137 	/* get the amount of memory used by the vbios for pre-OS console */
138 	unsigned int (*get_vbios_fb_size)(struct amdgpu_device *adev);
139 };
140 
141 struct amdgpu_xgmi_ras {
142 	struct amdgpu_ras_block_object ras_block;
143 };
144 
145 struct amdgpu_xgmi {
146 	/* from psp */
147 	u64 node_id;
148 	u64 hive_id;
149 	/* fixed per family */
150 	u64 node_segment_size;
151 	/* physical node (0-3) */
152 	unsigned physical_node_id;
153 	/* number of nodes (0-4) */
154 	unsigned num_physical_nodes;
155 	/* gpu list in the same hive */
156 	struct list_head head;
157 	bool supported;
158 	struct ras_common_if *ras_if;
159 	bool connected_to_cpu;
160 	bool pending_reset;
161 	struct amdgpu_xgmi_ras *ras;
162 };
163 
164 struct amdgpu_gmc {
165 	/* FB's physical address in MMIO space (for CPU to
166 	 * map FB). This is different compared to the agp/
167 	 * gart/vram_start/end field as the later is from
168 	 * GPU's view and aper_base is from CPU's view.
169 	 */
170 	resource_size_t		aper_size;
171 	resource_size_t		aper_base;
172 	/* for some chips with <= 32MB we need to lie
173 	 * about vram size near mc fb location */
174 	u64			mc_vram_size;
175 	u64			visible_vram_size;
176 	/* AGP aperture start and end in MC address space
177 	 * Driver find a hole in the MC address space
178 	 * to place AGP by setting MC_VM_AGP_BOT/TOP registers
179 	 * Under VMID0, logical address == MC address. AGP
180 	 * aperture maps to physical bus or IOVA addressed.
181 	 * AGP aperture is used to simulate FB in ZFB case.
182 	 * AGP aperture is also used for page table in system
183 	 * memory (mainly for APU).
184 	 *
185 	 */
186 	u64			agp_size;
187 	u64			agp_start;
188 	u64			agp_end;
189 	/* GART aperture start and end in MC address space
190 	 * Driver find a hole in the MC address space
191 	 * to place GART by setting VM_CONTEXT0_PAGE_TABLE_START/END_ADDR
192 	 * registers
193 	 * Under VMID0, logical address inside GART aperture will
194 	 * be translated through gpuvm gart page table to access
195 	 * paged system memory
196 	 */
197 	u64			gart_size;
198 	u64			gart_start;
199 	u64			gart_end;
200 	/* Frame buffer aperture of this GPU device. Different from
201 	 * fb_start (see below), this only covers the local GPU device.
202 	 * If driver uses FB aperture to access FB, driver get fb_start from
203 	 * MC_VM_FB_LOCATION_BASE (set by vbios) and calculate vram_start
204 	 * of this local device by adding an offset inside the XGMI hive.
205 	 * If driver uses GART table for VMID0 FB access, driver finds a hole in
206 	 * VMID0's virtual address space to place the SYSVM aperture inside
207 	 * which the first part is vram and the second part is gart (covering
208 	 * system ram).
209 	 */
210 	u64			vram_start;
211 	u64			vram_end;
212 	/* FB region , it's same as local vram region in single GPU, in XGMI
213 	 * configuration, this region covers all GPUs in the same hive ,
214 	 * each GPU in the hive has the same view of this FB region .
215 	 * GPU0's vram starts at offset (0 * segment size) ,
216 	 * GPU1 starts at offset (1 * segment size), etc.
217 	 */
218 	u64			fb_start;
219 	u64			fb_end;
220 	unsigned		vram_width;
221 	u64			real_vram_size;
222 	int			vram_mtrr;
223 	u64                     mc_mask;
224 	const struct firmware   *fw;	/* MC firmware */
225 	uint32_t                fw_version;
226 	struct amdgpu_irq_src	vm_fault;
227 	uint32_t		vram_type;
228 	uint8_t			vram_vendor;
229 	uint32_t                srbm_soft_reset;
230 	bool			prt_warning;
231 	uint32_t		sdpif_register;
232 	/* apertures */
233 	u64			shared_aperture_start;
234 	u64			shared_aperture_end;
235 	u64			private_aperture_start;
236 	u64			private_aperture_end;
237 	/* protects concurrent invalidation */
238 	spinlock_t		invalidate_lock;
239 	bool			translate_further;
240 	struct kfd_vm_fault_info *vm_fault_info;
241 	atomic_t		vm_fault_info_updated;
242 
243 	struct amdgpu_gmc_fault	fault_ring[AMDGPU_GMC_FAULT_RING_SIZE];
244 	struct {
245 		uint64_t	idx:AMDGPU_GMC_FAULT_RING_ORDER;
246 	} fault_hash[AMDGPU_GMC_FAULT_HASH_SIZE];
247 	uint64_t		last_fault:AMDGPU_GMC_FAULT_RING_ORDER;
248 
249 	bool tmz_enabled;
250 
251 	const struct amdgpu_gmc_funcs	*gmc_funcs;
252 
253 	struct amdgpu_xgmi xgmi;
254 	struct amdgpu_irq_src	ecc_irq;
255 	int noretry;
256 
257 	uint32_t	vmid0_page_table_block_size;
258 	uint32_t	vmid0_page_table_depth;
259 	struct amdgpu_bo		*pdb0_bo;
260 	/* CPU kmapped address of pdb0*/
261 	void				*ptr_pdb0;
262 
263 	/* MALL size */
264 	u64 mall_size;
265 	/* number of UMC instances */
266 	int num_umc;
267 	/* mode2 save restore */
268 	u64 VM_L2_CNTL;
269 	u64 VM_L2_CNTL2;
270 	u64 VM_DUMMY_PAGE_FAULT_CNTL;
271 	u64 VM_DUMMY_PAGE_FAULT_ADDR_LO32;
272 	u64 VM_DUMMY_PAGE_FAULT_ADDR_HI32;
273 	u64 VM_L2_PROTECTION_FAULT_CNTL;
274 	u64 VM_L2_PROTECTION_FAULT_CNTL2;
275 	u64 VM_L2_PROTECTION_FAULT_MM_CNTL3;
276 	u64 VM_L2_PROTECTION_FAULT_MM_CNTL4;
277 	u64 VM_L2_PROTECTION_FAULT_ADDR_LO32;
278 	u64 VM_L2_PROTECTION_FAULT_ADDR_HI32;
279 	u64 VM_DEBUG;
280 	u64 VM_L2_MM_GROUP_RT_CLASSES;
281 	u64 VM_L2_BANK_SELECT_RESERVED_CID;
282 	u64 VM_L2_BANK_SELECT_RESERVED_CID2;
283 	u64 VM_L2_CACHE_PARITY_CNTL;
284 	u64 VM_L2_IH_LOG_CNTL;
285 	u64 VM_CONTEXT_CNTL[16];
286 	u64 VM_CONTEXT_PAGE_TABLE_BASE_ADDR_LO32[16];
287 	u64 VM_CONTEXT_PAGE_TABLE_BASE_ADDR_HI32[16];
288 	u64 VM_CONTEXT_PAGE_TABLE_START_ADDR_LO32[16];
289 	u64 VM_CONTEXT_PAGE_TABLE_START_ADDR_HI32[16];
290 	u64 VM_CONTEXT_PAGE_TABLE_END_ADDR_LO32[16];
291 	u64 VM_CONTEXT_PAGE_TABLE_END_ADDR_HI32[16];
292 	u64 MC_VM_MX_L1_TLB_CNTL;
293 };
294 
295 #define amdgpu_gmc_flush_gpu_tlb(adev, vmid, vmhub, type) ((adev)->gmc.gmc_funcs->flush_gpu_tlb((adev), (vmid), (vmhub), (type)))
296 #define amdgpu_gmc_flush_gpu_tlb_pasid(adev, pasid, type, allhub) \
297 	((adev)->gmc.gmc_funcs->flush_gpu_tlb_pasid \
298 	((adev), (pasid), (type), (allhub)))
299 #define amdgpu_gmc_emit_flush_gpu_tlb(r, vmid, addr) (r)->adev->gmc.gmc_funcs->emit_flush_gpu_tlb((r), (vmid), (addr))
300 #define amdgpu_gmc_emit_pasid_mapping(r, vmid, pasid) (r)->adev->gmc.gmc_funcs->emit_pasid_mapping((r), (vmid), (pasid))
301 #define amdgpu_gmc_map_mtype(adev, flags) (adev)->gmc.gmc_funcs->map_mtype((adev),(flags))
302 #define amdgpu_gmc_get_vm_pde(adev, level, dst, flags) (adev)->gmc.gmc_funcs->get_vm_pde((adev), (level), (dst), (flags))
303 #define amdgpu_gmc_get_vm_pte(adev, mapping, flags) (adev)->gmc.gmc_funcs->get_vm_pte((adev), (mapping), (flags))
304 #define amdgpu_gmc_get_vbios_fb_size(adev) (adev)->gmc.gmc_funcs->get_vbios_fb_size((adev))
305 
306 /**
307  * amdgpu_gmc_vram_full_visible - Check if full VRAM is visible through the BAR
308  *
309  * @adev: amdgpu_device pointer
310  *
311  * Returns:
312  * True if full VRAM is visible through the BAR
313  */
314 static inline bool amdgpu_gmc_vram_full_visible(struct amdgpu_gmc *gmc)
315 {
316 	WARN_ON(gmc->real_vram_size < gmc->visible_vram_size);
317 
318 	return (gmc->real_vram_size == gmc->visible_vram_size);
319 }
320 
321 /**
322  * amdgpu_gmc_sign_extend - sign extend the given gmc address
323  *
324  * @addr: address to extend
325  */
326 static inline uint64_t amdgpu_gmc_sign_extend(uint64_t addr)
327 {
328 	if (addr >= AMDGPU_GMC_HOLE_START)
329 		addr |= AMDGPU_GMC_HOLE_END;
330 
331 	return addr;
332 }
333 
334 int amdgpu_gmc_pdb0_alloc(struct amdgpu_device *adev);
335 void amdgpu_gmc_get_pde_for_bo(struct amdgpu_bo *bo, int level,
336 			       uint64_t *addr, uint64_t *flags);
337 int amdgpu_gmc_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr,
338 				uint32_t gpu_page_idx, uint64_t addr,
339 				uint64_t flags);
340 uint64_t amdgpu_gmc_pd_addr(struct amdgpu_bo *bo);
341 uint64_t amdgpu_gmc_agp_addr(struct ttm_buffer_object *bo);
342 void amdgpu_gmc_sysvm_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc);
343 void amdgpu_gmc_vram_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc,
344 			      u64 base);
345 void amdgpu_gmc_gart_location(struct amdgpu_device *adev,
346 			      struct amdgpu_gmc *mc);
347 void amdgpu_gmc_agp_location(struct amdgpu_device *adev,
348 			     struct amdgpu_gmc *mc);
349 bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev,
350 			      struct amdgpu_ih_ring *ih, uint64_t addr,
351 			      uint16_t pasid, uint64_t timestamp);
352 void amdgpu_gmc_filter_faults_remove(struct amdgpu_device *adev, uint64_t addr,
353 				     uint16_t pasid);
354 int amdgpu_gmc_ras_early_init(struct amdgpu_device *adev);
355 int amdgpu_gmc_ras_late_init(struct amdgpu_device *adev);
356 void amdgpu_gmc_ras_fini(struct amdgpu_device *adev);
357 int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev);
358 
359 extern void amdgpu_gmc_tmz_set(struct amdgpu_device *adev);
360 extern void amdgpu_gmc_noretry_set(struct amdgpu_device *adev);
361 
362 extern void
363 amdgpu_gmc_set_vm_fault_masks(struct amdgpu_device *adev, int hub_type,
364 			      bool enable);
365 
366 void amdgpu_gmc_get_vbios_allocations(struct amdgpu_device *adev);
367 
368 void amdgpu_gmc_init_pdb0(struct amdgpu_device *adev);
369 uint64_t amdgpu_gmc_vram_mc2pa(struct amdgpu_device *adev, uint64_t mc_addr);
370 uint64_t amdgpu_gmc_vram_pa(struct amdgpu_device *adev, struct amdgpu_bo *bo);
371 uint64_t amdgpu_gmc_vram_cpu_pa(struct amdgpu_device *adev, struct amdgpu_bo *bo);
372 int amdgpu_gmc_vram_checking(struct amdgpu_device *adev);
373 #endif
374