1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #ifndef KFD_DEVICE_QUEUE_MANAGER_H_ 25 #define KFD_DEVICE_QUEUE_MANAGER_H_ 26 27 #include <linux/rwsem.h> 28 #include <linux/list.h> 29 #include <linux/mutex.h> 30 #include <linux/sched/mm.h> 31 #include "kfd_priv.h" 32 #include "kfd_mqd_manager.h" 33 34 #define KFD_UNMAP_LATENCY_MS (4000) 35 #define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS (2 * KFD_UNMAP_LATENCY_MS + 1000) 36 #define KFD_SDMA_QUEUES_PER_ENGINE (2) 37 38 struct device_process_node { 39 struct qcm_process_device *qpd; 40 struct list_head list; 41 }; 42 43 /** 44 * struct device_queue_manager_ops 45 * 46 * @create_queue: Queue creation routine. 47 * 48 * @destroy_queue: Queue destruction routine. 49 * 50 * @update_queue: Queue update routine. 51 * 52 * @get_mqd_manager: Returns the mqd manager according to the mqd type. 53 * 54 * @exeute_queues: Dispatches the queues list to the H/W. 55 * 56 * @register_process: This routine associates a specific process with device. 57 * 58 * @unregister_process: destroys the associations between process to device. 59 * 60 * @initialize: Initializes the pipelines and memory module for that device. 61 * 62 * @start: Initializes the resources/modules the the device needs for queues 63 * execution. This function is called on device initialization and after the 64 * system woke up after suspension. 65 * 66 * @stop: This routine stops execution of all the active queue running on the 67 * H/W and basically this function called on system suspend. 68 * 69 * @uninitialize: Destroys all the device queue manager resources allocated in 70 * initialize routine. 71 * 72 * @create_kernel_queue: Creates kernel queue. Used for debug queue. 73 * 74 * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue. 75 * 76 * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the 77 * memory apertures. 78 * 79 * @process_termination: Clears all process queues belongs to that device. 80 * 81 * @evict_process_queues: Evict all active queues of a process 82 * 83 * @restore_process_queues: Restore all evicted queues queues of a process 84 * 85 */ 86 87 struct device_queue_manager_ops { 88 int (*create_queue)(struct device_queue_manager *dqm, 89 struct queue *q, 90 struct qcm_process_device *qpd); 91 92 int (*destroy_queue)(struct device_queue_manager *dqm, 93 struct qcm_process_device *qpd, 94 struct queue *q); 95 96 int (*update_queue)(struct device_queue_manager *dqm, 97 struct queue *q); 98 99 struct mqd_manager * (*get_mqd_manager) 100 (struct device_queue_manager *dqm, 101 enum KFD_MQD_TYPE type); 102 103 int (*register_process)(struct device_queue_manager *dqm, 104 struct qcm_process_device *qpd); 105 106 int (*unregister_process)(struct device_queue_manager *dqm, 107 struct qcm_process_device *qpd); 108 109 int (*initialize)(struct device_queue_manager *dqm); 110 int (*start)(struct device_queue_manager *dqm); 111 int (*stop)(struct device_queue_manager *dqm); 112 void (*uninitialize)(struct device_queue_manager *dqm); 113 int (*create_kernel_queue)(struct device_queue_manager *dqm, 114 struct kernel_queue *kq, 115 struct qcm_process_device *qpd); 116 117 void (*destroy_kernel_queue)(struct device_queue_manager *dqm, 118 struct kernel_queue *kq, 119 struct qcm_process_device *qpd); 120 121 bool (*set_cache_memory_policy)(struct device_queue_manager *dqm, 122 struct qcm_process_device *qpd, 123 enum cache_policy default_policy, 124 enum cache_policy alternate_policy, 125 void __user *alternate_aperture_base, 126 uint64_t alternate_aperture_size); 127 128 int (*set_trap_handler)(struct device_queue_manager *dqm, 129 struct qcm_process_device *qpd, 130 uint64_t tba_addr, 131 uint64_t tma_addr); 132 133 int (*process_termination)(struct device_queue_manager *dqm, 134 struct qcm_process_device *qpd); 135 136 int (*evict_process_queues)(struct device_queue_manager *dqm, 137 struct qcm_process_device *qpd); 138 int (*restore_process_queues)(struct device_queue_manager *dqm, 139 struct qcm_process_device *qpd); 140 }; 141 142 struct device_queue_manager_asic_ops { 143 int (*update_qpd)(struct device_queue_manager *dqm, 144 struct qcm_process_device *qpd); 145 bool (*set_cache_memory_policy)(struct device_queue_manager *dqm, 146 struct qcm_process_device *qpd, 147 enum cache_policy default_policy, 148 enum cache_policy alternate_policy, 149 void __user *alternate_aperture_base, 150 uint64_t alternate_aperture_size); 151 void (*init_sdma_vm)(struct device_queue_manager *dqm, 152 struct queue *q, 153 struct qcm_process_device *qpd); 154 }; 155 156 /** 157 * struct device_queue_manager 158 * 159 * This struct is a base class for the kfd queues scheduler in the 160 * device level. The device base class should expose the basic operations 161 * for queue creation and queue destruction. This base class hides the 162 * scheduling mode of the driver and the specific implementation of the 163 * concrete device. This class is the only class in the queues scheduler 164 * that configures the H/W. 165 * 166 */ 167 168 struct device_queue_manager { 169 struct device_queue_manager_ops ops; 170 struct device_queue_manager_asic_ops asic_ops; 171 172 struct mqd_manager *mqd_mgrs[KFD_MQD_TYPE_MAX]; 173 struct packet_manager packets; 174 struct kfd_dev *dev; 175 struct mutex lock_hidden; /* use dqm_lock/unlock(dqm) */ 176 struct list_head queues; 177 unsigned int saved_flags; 178 unsigned int processes_count; 179 unsigned int queue_count; 180 unsigned int sdma_queue_count; 181 unsigned int total_queue_count; 182 unsigned int next_pipe_to_allocate; 183 unsigned int *allocated_queues; 184 unsigned int sdma_bitmap; 185 unsigned int vmid_bitmap; 186 uint64_t pipelines_addr; 187 struct kfd_mem_obj *pipeline_mem; 188 uint64_t fence_gpu_addr; 189 unsigned int *fence_addr; 190 struct kfd_mem_obj *fence_mem; 191 bool active_runlist; 192 int sched_policy; 193 194 /* hw exception */ 195 bool is_hws_hang; 196 struct work_struct hw_exception_work; 197 }; 198 199 void device_queue_manager_init_cik( 200 struct device_queue_manager_asic_ops *asic_ops); 201 void device_queue_manager_init_cik_hawaii( 202 struct device_queue_manager_asic_ops *asic_ops); 203 void device_queue_manager_init_vi( 204 struct device_queue_manager_asic_ops *asic_ops); 205 void device_queue_manager_init_vi_tonga( 206 struct device_queue_manager_asic_ops *asic_ops); 207 void device_queue_manager_init_v9( 208 struct device_queue_manager_asic_ops *asic_ops); 209 void program_sh_mem_settings(struct device_queue_manager *dqm, 210 struct qcm_process_device *qpd); 211 unsigned int get_queues_num(struct device_queue_manager *dqm); 212 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm); 213 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm); 214 unsigned int get_num_sdma_queues(struct device_queue_manager *dqm); 215 216 static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd) 217 { 218 return (pdd->lds_base >> 16) & 0xFF; 219 } 220 221 static inline unsigned int 222 get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd) 223 { 224 return (pdd->lds_base >> 60) & 0x0E; 225 } 226 227 /* The DQM lock can be taken in MMU notifiers. Make sure no reclaim-FS 228 * happens while holding this lock anywhere to prevent deadlocks when 229 * an MMU notifier runs in reclaim-FS context. 230 */ 231 static inline void dqm_lock(struct device_queue_manager *dqm) 232 { 233 mutex_lock(&dqm->lock_hidden); 234 dqm->saved_flags = memalloc_nofs_save(); 235 } 236 static inline void dqm_unlock(struct device_queue_manager *dqm) 237 { 238 memalloc_nofs_restore(dqm->saved_flags); 239 mutex_unlock(&dqm->lock_hidden); 240 } 241 242 #endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */ 243