1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/printk.h> 25 #include <linux/slab.h> 26 #include "kfd_priv.h" 27 #include "kfd_mqd_manager.h" 28 #include "cik_regs.h" 29 #include "../../radeon/cik_reg.h" 30 31 inline void busy_wait(unsigned long ms) 32 { 33 while (time_before(jiffies, ms)) 34 cpu_relax(); 35 } 36 37 static inline struct cik_mqd *get_mqd(void *mqd) 38 { 39 return (struct cik_mqd *)mqd; 40 } 41 42 static int init_mqd(struct mqd_manager *mm, void **mqd, 43 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, 44 struct queue_properties *q) 45 { 46 uint64_t addr; 47 struct cik_mqd *m; 48 int retval; 49 50 BUG_ON(!mm || !q || !mqd); 51 52 pr_debug("kfd: In func %s\n", __func__); 53 54 retval = kfd2kgd->allocate_mem(mm->dev->kgd, 55 sizeof(struct cik_mqd), 56 256, 57 KFD_MEMPOOL_SYSTEM_WRITECOMBINE, 58 (struct kgd_mem **) mqd_mem_obj); 59 60 if (retval != 0) 61 return -ENOMEM; 62 63 m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; 64 addr = (*mqd_mem_obj)->gpu_addr; 65 66 memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); 67 68 m->header = 0xC0310800; 69 m->compute_pipelinestat_enable = 1; 70 m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; 71 m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; 72 m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; 73 m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; 74 75 /* 76 * Make sure to use the last queue state saved on mqd when the cp 77 * reassigns the queue, so when queue is switched on/off (e.g over 78 * subscription or quantum timeout) the context will be consistent 79 */ 80 m->cp_hqd_persistent_state = 81 DEFAULT_CP_HQD_PERSISTENT_STATE | PRELOAD_REQ; 82 83 m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; 84 m->cp_mqd_base_addr_lo = lower_32_bits(addr); 85 m->cp_mqd_base_addr_hi = upper_32_bits(addr); 86 87 m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE | IB_ATC_EN; 88 /* Although WinKFD writes this, I suspect it should not be necessary */ 89 m->cp_hqd_ib_control = IB_ATC_EN | DEFAULT_MIN_IB_AVAIL_SIZE; 90 91 m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | 92 QUANTUM_DURATION(10); 93 94 /* 95 * Pipe Priority 96 * Identifies the pipe relative priority when this queue is connected 97 * to the pipeline. The pipe priority is against the GFX pipe and HP3D. 98 * In KFD we are using a fixed pipe priority set to CS_MEDIUM. 99 * 0 = CS_LOW (typically below GFX) 100 * 1 = CS_MEDIUM (typically between HP3D and GFX 101 * 2 = CS_HIGH (typically above HP3D) 102 */ 103 m->cp_hqd_pipe_priority = 1; 104 m->cp_hqd_queue_priority = 15; 105 106 *mqd = m; 107 if (gart_addr != NULL) 108 *gart_addr = addr; 109 retval = mm->update_mqd(mm, m, q); 110 111 return retval; 112 } 113 114 static void uninit_mqd(struct mqd_manager *mm, void *mqd, 115 struct kfd_mem_obj *mqd_mem_obj) 116 { 117 BUG_ON(!mm || !mqd); 118 kfd2kgd->free_mem(mm->dev->kgd, (struct kgd_mem *) mqd_mem_obj); 119 } 120 121 static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id, 122 uint32_t queue_id, uint32_t __user *wptr) 123 { 124 return kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id, wptr); 125 126 } 127 128 static int update_mqd(struct mqd_manager *mm, void *mqd, 129 struct queue_properties *q) 130 { 131 struct cik_mqd *m; 132 133 BUG_ON(!mm || !q || !mqd); 134 135 pr_debug("kfd: In func %s\n", __func__); 136 137 m = get_mqd(mqd); 138 m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | 139 DEFAULT_MIN_AVAIL_SIZE | PQ_ATC_EN; 140 141 /* 142 * Calculating queue size which is log base 2 of actual queue size -1 143 * dwords and another -1 for ffs 144 */ 145 m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int)) 146 - 1 - 1; 147 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); 148 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); 149 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); 150 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); 151 m->cp_hqd_pq_doorbell_control = DOORBELL_EN | 152 DOORBELL_OFFSET(q->doorbell_off); 153 154 m->cp_hqd_vmid = q->vmid; 155 156 if (q->format == KFD_QUEUE_FORMAT_AQL) { 157 m->cp_hqd_iq_rptr = AQL_ENABLE; 158 m->cp_hqd_pq_control |= NO_UPDATE_RPTR; 159 } 160 161 m->cp_hqd_active = 0; 162 q->is_active = false; 163 if (q->queue_size > 0 && 164 q->queue_address != 0 && 165 q->queue_percent > 0) { 166 m->cp_hqd_active = 1; 167 q->is_active = true; 168 } 169 170 return 0; 171 } 172 173 static int destroy_mqd(struct mqd_manager *mm, void *mqd, 174 enum kfd_preempt_type type, 175 unsigned int timeout, uint32_t pipe_id, 176 uint32_t queue_id) 177 { 178 return kfd2kgd->hqd_destroy(mm->dev->kgd, type, timeout, 179 pipe_id, queue_id); 180 } 181 182 static bool is_occupied(struct mqd_manager *mm, void *mqd, 183 uint64_t queue_address, uint32_t pipe_id, 184 uint32_t queue_id) 185 { 186 187 return kfd2kgd->hqd_is_occupied(mm->dev->kgd, queue_address, 188 pipe_id, queue_id); 189 190 } 191 192 /* 193 * HIQ MQD Implementation, concrete implementation for HIQ MQD implementation. 194 * The HIQ queue in Kaveri is using the same MQD structure as all the user mode 195 * queues but with different initial values. 196 */ 197 198 static int init_mqd_hiq(struct mqd_manager *mm, void **mqd, 199 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, 200 struct queue_properties *q) 201 { 202 uint64_t addr; 203 struct cik_mqd *m; 204 int retval; 205 206 BUG_ON(!mm || !q || !mqd || !mqd_mem_obj); 207 208 pr_debug("kfd: In func %s\n", __func__); 209 210 retval = kfd2kgd->allocate_mem(mm->dev->kgd, 211 sizeof(struct cik_mqd), 212 256, 213 KFD_MEMPOOL_SYSTEM_WRITECOMBINE, 214 (struct kgd_mem **) mqd_mem_obj); 215 216 if (retval != 0) 217 return -ENOMEM; 218 219 m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; 220 addr = (*mqd_mem_obj)->gpu_addr; 221 222 memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); 223 224 m->header = 0xC0310800; 225 m->compute_pipelinestat_enable = 1; 226 m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; 227 m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; 228 m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; 229 m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; 230 231 m->cp_hqd_persistent_state = DEFAULT_CP_HQD_PERSISTENT_STATE | 232 PRELOAD_REQ; 233 m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | 234 QUANTUM_DURATION(10); 235 236 m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; 237 m->cp_mqd_base_addr_lo = lower_32_bits(addr); 238 m->cp_mqd_base_addr_hi = upper_32_bits(addr); 239 240 m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE; 241 242 /* 243 * Pipe Priority 244 * Identifies the pipe relative priority when this queue is connected 245 * to the pipeline. The pipe priority is against the GFX pipe and HP3D. 246 * In KFD we are using a fixed pipe priority set to CS_MEDIUM. 247 * 0 = CS_LOW (typically below GFX) 248 * 1 = CS_MEDIUM (typically between HP3D and GFX 249 * 2 = CS_HIGH (typically above HP3D) 250 */ 251 m->cp_hqd_pipe_priority = 1; 252 m->cp_hqd_queue_priority = 15; 253 254 *mqd = m; 255 if (gart_addr) 256 *gart_addr = addr; 257 retval = mm->update_mqd(mm, m, q); 258 259 return retval; 260 } 261 262 static int update_mqd_hiq(struct mqd_manager *mm, void *mqd, 263 struct queue_properties *q) 264 { 265 struct cik_mqd *m; 266 267 BUG_ON(!mm || !q || !mqd); 268 269 pr_debug("kfd: In func %s\n", __func__); 270 271 m = get_mqd(mqd); 272 m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | 273 DEFAULT_MIN_AVAIL_SIZE | 274 PRIV_STATE | 275 KMD_QUEUE; 276 277 /* 278 * Calculating queue size which is log base 2 of actual queue 279 * size -1 dwords 280 */ 281 m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int)) 282 - 1 - 1; 283 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); 284 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); 285 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); 286 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); 287 m->cp_hqd_pq_doorbell_control = DOORBELL_EN | 288 DOORBELL_OFFSET(q->doorbell_off); 289 290 m->cp_hqd_vmid = q->vmid; 291 292 m->cp_hqd_active = 0; 293 q->is_active = false; 294 if (q->queue_size > 0 && 295 q->queue_address != 0 && 296 q->queue_percent > 0) { 297 m->cp_hqd_active = 1; 298 q->is_active = true; 299 } 300 301 return 0; 302 } 303 304 struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type, 305 struct kfd_dev *dev) 306 { 307 struct mqd_manager *mqd; 308 309 BUG_ON(!dev); 310 BUG_ON(type >= KFD_MQD_TYPE_MAX); 311 312 pr_debug("kfd: In func %s\n", __func__); 313 314 mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL); 315 if (!mqd) 316 return NULL; 317 318 mqd->dev = dev; 319 320 switch (type) { 321 case KFD_MQD_TYPE_CIK_CP: 322 case KFD_MQD_TYPE_CIK_COMPUTE: 323 mqd->init_mqd = init_mqd; 324 mqd->uninit_mqd = uninit_mqd; 325 mqd->load_mqd = load_mqd; 326 mqd->update_mqd = update_mqd; 327 mqd->destroy_mqd = destroy_mqd; 328 mqd->is_occupied = is_occupied; 329 break; 330 case KFD_MQD_TYPE_CIK_HIQ: 331 mqd->init_mqd = init_mqd_hiq; 332 mqd->uninit_mqd = uninit_mqd; 333 mqd->load_mqd = load_mqd; 334 mqd->update_mqd = update_mqd_hiq; 335 mqd->destroy_mqd = destroy_mqd; 336 mqd->is_occupied = is_occupied; 337 break; 338 default: 339 kfree(mqd); 340 return NULL; 341 } 342 343 return mqd; 344 } 345 346 /* SDMA queues should be implemented here when the cp will supports them */ 347