1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
2 /*
3  * Copyright 2014-2022 Advanced Micro Devices, Inc.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21  * OTHER DEALINGS IN THE SOFTWARE.
22  *
23  */
24 
25 #ifndef KFD_DEVICE_QUEUE_MANAGER_H_
26 #define KFD_DEVICE_QUEUE_MANAGER_H_
27 
28 #include <linux/rwsem.h>
29 #include <linux/list.h>
30 #include <linux/mutex.h>
31 #include <linux/sched/mm.h>
32 #include "kfd_priv.h"
33 #include "kfd_mqd_manager.h"
34 
35 
36 #define VMID_NUM 16
37 
38 #define KFD_MES_PROCESS_QUANTUM		100000
39 #define KFD_MES_GANG_QUANTUM		10000
40 #define USE_DEFAULT_GRACE_PERIOD 0xffffffff
41 
42 struct device_process_node {
43 	struct qcm_process_device *qpd;
44 	struct list_head list;
45 };
46 
47 union SQ_CMD_BITS {
48 	struct {
49 		uint32_t cmd:3;
50 		uint32_t:1;
51 		uint32_t mode:3;
52 		uint32_t check_vmid:1;
53 		uint32_t trap_id:3;
54 		uint32_t:5;
55 		uint32_t wave_id:4;
56 		uint32_t simd_id:2;
57 		uint32_t:2;
58 		uint32_t queue_id:3;
59 		uint32_t:1;
60 		uint32_t vm_id:4;
61 	} bitfields, bits;
62 	uint32_t u32All;
63 	signed int i32All;
64 	float f32All;
65 };
66 
67 union GRBM_GFX_INDEX_BITS {
68 	struct {
69 		uint32_t instance_index:8;
70 		uint32_t sh_index:8;
71 		uint32_t se_index:8;
72 		uint32_t:5;
73 		uint32_t sh_broadcast_writes:1;
74 		uint32_t instance_broadcast_writes:1;
75 		uint32_t se_broadcast_writes:1;
76 	} bitfields, bits;
77 	uint32_t u32All;
78 	signed int i32All;
79 	float f32All;
80 };
81 
82 /**
83  * struct device_queue_manager_ops
84  *
85  * @create_queue: Queue creation routine.
86  *
87  * @destroy_queue: Queue destruction routine.
88  *
89  * @update_queue: Queue update routine.
90  *
91  * @exeute_queues: Dispatches the queues list to the H/W.
92  *
93  * @register_process: This routine associates a specific process with device.
94  *
95  * @unregister_process: destroys the associations between process to device.
96  *
97  * @initialize: Initializes the pipelines and memory module for that device.
98  *
99  * @start: Initializes the resources/modules the device needs for queues
100  * execution. This function is called on device initialization and after the
101  * system woke up after suspension.
102  *
103  * @stop: This routine stops execution of all the active queue running on the
104  * H/W and basically this function called on system suspend.
105  *
106  * @uninitialize: Destroys all the device queue manager resources allocated in
107  * initialize routine.
108  *
109  * @create_kernel_queue: Creates kernel queue. Used for debug queue.
110  *
111  * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue.
112  *
113  * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the
114  * memory apertures.
115  *
116  * @process_termination: Clears all process queues belongs to that device.
117  *
118  * @evict_process_queues: Evict all active queues of a process
119  *
120  * @restore_process_queues: Restore all evicted queues of a process
121  *
122  * @get_wave_state: Retrieves context save state and optionally copies the
123  * control stack, if kept in the MQD, to the given userspace address.
124  *
125  * @reset_queues: reset queues which consume RAS poison
126  * @get_queue_checkpoint_info: Retrieves queue size information for CRIU checkpoint.
127  *
128  * @checkpoint_mqd: checkpoint queue MQD contents for CRIU.
129  */
130 
131 struct device_queue_manager_ops {
132 	int	(*create_queue)(struct device_queue_manager *dqm,
133 				struct queue *q,
134 				struct qcm_process_device *qpd,
135 				const struct kfd_criu_queue_priv_data *qd,
136 				const void *restore_mqd,
137 				const void *restore_ctl_stack);
138 
139 	int	(*destroy_queue)(struct device_queue_manager *dqm,
140 				struct qcm_process_device *qpd,
141 				struct queue *q);
142 
143 	int	(*update_queue)(struct device_queue_manager *dqm,
144 				struct queue *q, struct mqd_update_info *minfo);
145 
146 	int	(*register_process)(struct device_queue_manager *dqm,
147 					struct qcm_process_device *qpd);
148 
149 	int	(*unregister_process)(struct device_queue_manager *dqm,
150 					struct qcm_process_device *qpd);
151 
152 	int	(*initialize)(struct device_queue_manager *dqm);
153 	int	(*start)(struct device_queue_manager *dqm);
154 	int	(*stop)(struct device_queue_manager *dqm);
155 	void	(*pre_reset)(struct device_queue_manager *dqm);
156 	void	(*uninitialize)(struct device_queue_manager *dqm);
157 	int	(*create_kernel_queue)(struct device_queue_manager *dqm,
158 					struct kernel_queue *kq,
159 					struct qcm_process_device *qpd);
160 
161 	void	(*destroy_kernel_queue)(struct device_queue_manager *dqm,
162 					struct kernel_queue *kq,
163 					struct qcm_process_device *qpd);
164 
165 	bool	(*set_cache_memory_policy)(struct device_queue_manager *dqm,
166 					   struct qcm_process_device *qpd,
167 					   enum cache_policy default_policy,
168 					   enum cache_policy alternate_policy,
169 					   void __user *alternate_aperture_base,
170 					   uint64_t alternate_aperture_size);
171 
172 	int (*process_termination)(struct device_queue_manager *dqm,
173 			struct qcm_process_device *qpd);
174 
175 	int (*evict_process_queues)(struct device_queue_manager *dqm,
176 				    struct qcm_process_device *qpd);
177 	int (*restore_process_queues)(struct device_queue_manager *dqm,
178 				      struct qcm_process_device *qpd);
179 
180 	int	(*get_wave_state)(struct device_queue_manager *dqm,
181 				  struct queue *q,
182 				  void __user *ctl_stack,
183 				  u32 *ctl_stack_used_size,
184 				  u32 *save_area_used_size);
185 
186 	int (*reset_queues)(struct device_queue_manager *dqm,
187 					uint16_t pasid);
188 	void	(*get_queue_checkpoint_info)(struct device_queue_manager *dqm,
189 				  const struct queue *q, u32 *mqd_size,
190 				  u32 *ctl_stack_size);
191 
192 	int	(*checkpoint_mqd)(struct device_queue_manager *dqm,
193 				  const struct queue *q,
194 				  void *mqd,
195 				  void *ctl_stack);
196 };
197 
198 struct device_queue_manager_asic_ops {
199 	int	(*update_qpd)(struct device_queue_manager *dqm,
200 					struct qcm_process_device *qpd);
201 	bool	(*set_cache_memory_policy)(struct device_queue_manager *dqm,
202 					   struct qcm_process_device *qpd,
203 					   enum cache_policy default_policy,
204 					   enum cache_policy alternate_policy,
205 					   void __user *alternate_aperture_base,
206 					   uint64_t alternate_aperture_size);
207 	void	(*init_sdma_vm)(struct device_queue_manager *dqm,
208 				struct queue *q,
209 				struct qcm_process_device *qpd);
210 	struct mqd_manager *	(*mqd_manager_init)(enum KFD_MQD_TYPE type,
211 				 struct kfd_node *dev);
212 };
213 
214 /**
215  * struct device_queue_manager
216  *
217  * This struct is a base class for the kfd queues scheduler in the
218  * device level. The device base class should expose the basic operations
219  * for queue creation and queue destruction. This base class hides the
220  * scheduling mode of the driver and the specific implementation of the
221  * concrete device. This class is the only class in the queues scheduler
222  * that configures the H/W.
223  *
224  */
225 
226 struct device_queue_manager {
227 	struct device_queue_manager_ops ops;
228 	struct device_queue_manager_asic_ops asic_ops;
229 
230 	struct mqd_manager	*mqd_mgrs[KFD_MQD_TYPE_MAX];
231 	struct packet_manager	packet_mgr;
232 	struct kfd_node		*dev;
233 	struct mutex		lock_hidden; /* use dqm_lock/unlock(dqm) */
234 	struct list_head	queues;
235 	unsigned int		saved_flags;
236 	unsigned int		processes_count;
237 	unsigned int		active_queue_count;
238 	unsigned int		active_cp_queue_count;
239 	unsigned int		gws_queue_count;
240 	unsigned int		total_queue_count;
241 	unsigned int		next_pipe_to_allocate;
242 	unsigned int		*allocated_queues;
243 	DECLARE_BITMAP(sdma_bitmap, KFD_MAX_SDMA_QUEUES);
244 	DECLARE_BITMAP(xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES);
245 	/* the pasid mapping for each kfd vmid */
246 	uint16_t		vmid_pasid[VMID_NUM];
247 	uint64_t		pipelines_addr;
248 	uint64_t		fence_gpu_addr;
249 	uint64_t		*fence_addr;
250 	struct kfd_mem_obj	*fence_mem;
251 	bool			active_runlist;
252 	int			sched_policy;
253 	uint32_t		trap_debug_vmid;
254 
255 	/* hw exception  */
256 	bool			is_hws_hang;
257 	bool			is_resetting;
258 	struct work_struct	hw_exception_work;
259 	struct kfd_mem_obj	hiq_sdma_mqd;
260 	bool			sched_running;
261 
262 	/* used for GFX 9.4.3 only */
263 	uint32_t		current_logical_xcc_start;
264 
265 	uint32_t		wait_times;
266 
267 	wait_queue_head_t	destroy_wait;
268 };
269 
270 void device_queue_manager_init_cik(
271 		struct device_queue_manager_asic_ops *asic_ops);
272 void device_queue_manager_init_vi(
273 		struct device_queue_manager_asic_ops *asic_ops);
274 void device_queue_manager_init_v9(
275 		struct device_queue_manager_asic_ops *asic_ops);
276 void device_queue_manager_init_v10(
277 		struct device_queue_manager_asic_ops *asic_ops);
278 void device_queue_manager_init_v11(
279 		struct device_queue_manager_asic_ops *asic_ops);
280 void program_sh_mem_settings(struct device_queue_manager *dqm,
281 					struct qcm_process_device *qpd);
282 unsigned int get_cp_queues_num(struct device_queue_manager *dqm);
283 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm);
284 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm);
285 unsigned int get_num_sdma_queues(struct device_queue_manager *dqm);
286 unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm);
287 int reserve_debug_trap_vmid(struct device_queue_manager *dqm,
288 			struct qcm_process_device *qpd);
289 int release_debug_trap_vmid(struct device_queue_manager *dqm,
290 			struct qcm_process_device *qpd);
291 int suspend_queues(struct kfd_process *p,
292 			uint32_t num_queues,
293 			uint32_t grace_period,
294 			uint64_t exception_clear_mask,
295 			uint32_t *usr_queue_id_array);
296 int resume_queues(struct kfd_process *p,
297 		uint32_t num_queues,
298 		uint32_t *usr_queue_id_array);
299 void set_queue_snapshot_entry(struct queue *q,
300 			      uint64_t exception_clear_mask,
301 			      struct kfd_queue_snapshot_entry *qss_entry);
302 int debug_lock_and_unmap(struct device_queue_manager *dqm);
303 int debug_map_and_unlock(struct device_queue_manager *dqm);
304 int debug_refresh_runlist(struct device_queue_manager *dqm);
305 
306 static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
307 {
308 	return (pdd->lds_base >> 16) & 0xFF;
309 }
310 
311 static inline unsigned int
312 get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd)
313 {
314 	return (pdd->lds_base >> 60) & 0x0E;
315 }
316 
317 /* The DQM lock can be taken in MMU notifiers. Make sure no reclaim-FS
318  * happens while holding this lock anywhere to prevent deadlocks when
319  * an MMU notifier runs in reclaim-FS context.
320  */
321 static inline void dqm_lock(struct device_queue_manager *dqm)
322 {
323 	mutex_lock(&dqm->lock_hidden);
324 	dqm->saved_flags = memalloc_noreclaim_save();
325 }
326 static inline void dqm_unlock(struct device_queue_manager *dqm)
327 {
328 	memalloc_noreclaim_restore(dqm->saved_flags);
329 	mutex_unlock(&dqm->lock_hidden);
330 }
331 
332 static inline int read_sdma_queue_counter(uint64_t __user *q_rptr, uint64_t *val)
333 {
334 	/* SDMA activity counter is stored at queue's RPTR + 0x8 location. */
335 	return get_user(*val, q_rptr + 1);
336 }
337 #endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */
338