1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */ 2 /* 3 * Copyright 2014-2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #ifndef KFD_DEVICE_QUEUE_MANAGER_H_ 26 #define KFD_DEVICE_QUEUE_MANAGER_H_ 27 28 #include <linux/rwsem.h> 29 #include <linux/list.h> 30 #include <linux/mutex.h> 31 #include <linux/sched/mm.h> 32 #include "kfd_priv.h" 33 #include "kfd_mqd_manager.h" 34 35 36 #define VMID_NUM 16 37 38 struct device_process_node { 39 struct qcm_process_device *qpd; 40 struct list_head list; 41 }; 42 43 union SQ_CMD_BITS { 44 struct { 45 uint32_t cmd:3; 46 uint32_t:1; 47 uint32_t mode:3; 48 uint32_t check_vmid:1; 49 uint32_t trap_id:3; 50 uint32_t:5; 51 uint32_t wave_id:4; 52 uint32_t simd_id:2; 53 uint32_t:2; 54 uint32_t queue_id:3; 55 uint32_t:1; 56 uint32_t vm_id:4; 57 } bitfields, bits; 58 uint32_t u32All; 59 signed int i32All; 60 float f32All; 61 }; 62 63 union GRBM_GFX_INDEX_BITS { 64 struct { 65 uint32_t instance_index:8; 66 uint32_t sh_index:8; 67 uint32_t se_index:8; 68 uint32_t:5; 69 uint32_t sh_broadcast_writes:1; 70 uint32_t instance_broadcast_writes:1; 71 uint32_t se_broadcast_writes:1; 72 } bitfields, bits; 73 uint32_t u32All; 74 signed int i32All; 75 float f32All; 76 }; 77 78 /** 79 * struct device_queue_manager_ops 80 * 81 * @create_queue: Queue creation routine. 82 * 83 * @destroy_queue: Queue destruction routine. 84 * 85 * @update_queue: Queue update routine. 86 * 87 * @exeute_queues: Dispatches the queues list to the H/W. 88 * 89 * @register_process: This routine associates a specific process with device. 90 * 91 * @unregister_process: destroys the associations between process to device. 92 * 93 * @initialize: Initializes the pipelines and memory module for that device. 94 * 95 * @start: Initializes the resources/modules the device needs for queues 96 * execution. This function is called on device initialization and after the 97 * system woke up after suspension. 98 * 99 * @stop: This routine stops execution of all the active queue running on the 100 * H/W and basically this function called on system suspend. 101 * 102 * @uninitialize: Destroys all the device queue manager resources allocated in 103 * initialize routine. 104 * 105 * @create_kernel_queue: Creates kernel queue. Used for debug queue. 106 * 107 * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue. 108 * 109 * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the 110 * memory apertures. 111 * 112 * @process_termination: Clears all process queues belongs to that device. 113 * 114 * @evict_process_queues: Evict all active queues of a process 115 * 116 * @restore_process_queues: Restore all evicted queues of a process 117 * 118 * @get_wave_state: Retrieves context save state and optionally copies the 119 * control stack, if kept in the MQD, to the given userspace address. 120 * 121 * @reset_queues: reset queues which consume RAS poison 122 * @get_queue_checkpoint_info: Retrieves queue size information for CRIU checkpoint. 123 * 124 * @checkpoint_mqd: checkpoint queue MQD contents for CRIU. 125 */ 126 127 struct device_queue_manager_ops { 128 int (*create_queue)(struct device_queue_manager *dqm, 129 struct queue *q, 130 struct qcm_process_device *qpd, 131 const struct kfd_criu_queue_priv_data *qd, 132 const void *restore_mqd, 133 const void *restore_ctl_stack); 134 135 int (*destroy_queue)(struct device_queue_manager *dqm, 136 struct qcm_process_device *qpd, 137 struct queue *q); 138 139 int (*update_queue)(struct device_queue_manager *dqm, 140 struct queue *q, struct mqd_update_info *minfo); 141 142 int (*register_process)(struct device_queue_manager *dqm, 143 struct qcm_process_device *qpd); 144 145 int (*unregister_process)(struct device_queue_manager *dqm, 146 struct qcm_process_device *qpd); 147 148 int (*initialize)(struct device_queue_manager *dqm); 149 int (*start)(struct device_queue_manager *dqm); 150 int (*stop)(struct device_queue_manager *dqm); 151 void (*pre_reset)(struct device_queue_manager *dqm); 152 void (*uninitialize)(struct device_queue_manager *dqm); 153 int (*create_kernel_queue)(struct device_queue_manager *dqm, 154 struct kernel_queue *kq, 155 struct qcm_process_device *qpd); 156 157 void (*destroy_kernel_queue)(struct device_queue_manager *dqm, 158 struct kernel_queue *kq, 159 struct qcm_process_device *qpd); 160 161 bool (*set_cache_memory_policy)(struct device_queue_manager *dqm, 162 struct qcm_process_device *qpd, 163 enum cache_policy default_policy, 164 enum cache_policy alternate_policy, 165 void __user *alternate_aperture_base, 166 uint64_t alternate_aperture_size); 167 168 int (*process_termination)(struct device_queue_manager *dqm, 169 struct qcm_process_device *qpd); 170 171 int (*evict_process_queues)(struct device_queue_manager *dqm, 172 struct qcm_process_device *qpd); 173 int (*restore_process_queues)(struct device_queue_manager *dqm, 174 struct qcm_process_device *qpd); 175 176 int (*get_wave_state)(struct device_queue_manager *dqm, 177 struct queue *q, 178 void __user *ctl_stack, 179 u32 *ctl_stack_used_size, 180 u32 *save_area_used_size); 181 182 int (*reset_queues)(struct device_queue_manager *dqm, 183 uint16_t pasid); 184 void (*get_queue_checkpoint_info)(struct device_queue_manager *dqm, 185 const struct queue *q, u32 *mqd_size, 186 u32 *ctl_stack_size); 187 188 int (*checkpoint_mqd)(struct device_queue_manager *dqm, 189 const struct queue *q, 190 void *mqd, 191 void *ctl_stack); 192 }; 193 194 struct device_queue_manager_asic_ops { 195 int (*update_qpd)(struct device_queue_manager *dqm, 196 struct qcm_process_device *qpd); 197 bool (*set_cache_memory_policy)(struct device_queue_manager *dqm, 198 struct qcm_process_device *qpd, 199 enum cache_policy default_policy, 200 enum cache_policy alternate_policy, 201 void __user *alternate_aperture_base, 202 uint64_t alternate_aperture_size); 203 void (*init_sdma_vm)(struct device_queue_manager *dqm, 204 struct queue *q, 205 struct qcm_process_device *qpd); 206 struct mqd_manager * (*mqd_manager_init)(enum KFD_MQD_TYPE type, 207 struct kfd_dev *dev); 208 }; 209 210 /** 211 * struct device_queue_manager 212 * 213 * This struct is a base class for the kfd queues scheduler in the 214 * device level. The device base class should expose the basic operations 215 * for queue creation and queue destruction. This base class hides the 216 * scheduling mode of the driver and the specific implementation of the 217 * concrete device. This class is the only class in the queues scheduler 218 * that configures the H/W. 219 * 220 */ 221 222 struct device_queue_manager { 223 struct device_queue_manager_ops ops; 224 struct device_queue_manager_asic_ops asic_ops; 225 226 struct mqd_manager *mqd_mgrs[KFD_MQD_TYPE_MAX]; 227 struct packet_manager packet_mgr; 228 struct kfd_dev *dev; 229 struct mutex lock_hidden; /* use dqm_lock/unlock(dqm) */ 230 struct list_head queues; 231 unsigned int saved_flags; 232 unsigned int processes_count; 233 unsigned int active_queue_count; 234 unsigned int active_cp_queue_count; 235 unsigned int gws_queue_count; 236 unsigned int total_queue_count; 237 unsigned int next_pipe_to_allocate; 238 unsigned int *allocated_queues; 239 uint64_t sdma_bitmap; 240 uint64_t xgmi_sdma_bitmap; 241 /* the pasid mapping for each kfd vmid */ 242 uint16_t vmid_pasid[VMID_NUM]; 243 uint64_t pipelines_addr; 244 uint64_t fence_gpu_addr; 245 uint64_t *fence_addr; 246 struct kfd_mem_obj *fence_mem; 247 bool active_runlist; 248 int sched_policy; 249 250 /* hw exception */ 251 bool is_hws_hang; 252 bool is_resetting; 253 struct work_struct hw_exception_work; 254 struct kfd_mem_obj hiq_sdma_mqd; 255 bool sched_running; 256 }; 257 258 void device_queue_manager_init_cik( 259 struct device_queue_manager_asic_ops *asic_ops); 260 void device_queue_manager_init_cik_hawaii( 261 struct device_queue_manager_asic_ops *asic_ops); 262 void device_queue_manager_init_vi( 263 struct device_queue_manager_asic_ops *asic_ops); 264 void device_queue_manager_init_vi_tonga( 265 struct device_queue_manager_asic_ops *asic_ops); 266 void device_queue_manager_init_v9( 267 struct device_queue_manager_asic_ops *asic_ops); 268 void device_queue_manager_init_v10_navi10( 269 struct device_queue_manager_asic_ops *asic_ops); 270 void program_sh_mem_settings(struct device_queue_manager *dqm, 271 struct qcm_process_device *qpd); 272 unsigned int get_cp_queues_num(struct device_queue_manager *dqm); 273 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm); 274 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm); 275 unsigned int get_num_sdma_queues(struct device_queue_manager *dqm); 276 unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm); 277 278 static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd) 279 { 280 return (pdd->lds_base >> 16) & 0xFF; 281 } 282 283 static inline unsigned int 284 get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd) 285 { 286 return (pdd->lds_base >> 60) & 0x0E; 287 } 288 289 /* The DQM lock can be taken in MMU notifiers. Make sure no reclaim-FS 290 * happens while holding this lock anywhere to prevent deadlocks when 291 * an MMU notifier runs in reclaim-FS context. 292 */ 293 static inline void dqm_lock(struct device_queue_manager *dqm) 294 { 295 mutex_lock(&dqm->lock_hidden); 296 dqm->saved_flags = memalloc_noreclaim_save(); 297 } 298 static inline void dqm_unlock(struct device_queue_manager *dqm) 299 { 300 memalloc_noreclaim_restore(dqm->saved_flags); 301 mutex_unlock(&dqm->lock_hidden); 302 } 303 304 static inline int read_sdma_queue_counter(uint64_t __user *q_rptr, uint64_t *val) 305 { 306 /* SDMA activity counter is stored at queue's RPTR + 0x8 location. */ 307 return get_user(*val, q_rptr + 1); 308 } 309 #endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */ 310