1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/printk.h> 25 #include <linux/slab.h> 26 #include "kfd_priv.h" 27 #include "kfd_mqd_manager.h" 28 #include "cik_regs.h" 29 #include "../../radeon/cikd.h" 30 #include "../../radeon/cik_reg.h" 31 32 inline void busy_wait(unsigned long ms) 33 { 34 while (time_before(jiffies, ms)) 35 cpu_relax(); 36 } 37 38 static inline struct cik_mqd *get_mqd(void *mqd) 39 { 40 return (struct cik_mqd *)mqd; 41 } 42 43 static int init_mqd(struct mqd_manager *mm, void **mqd, 44 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, 45 struct queue_properties *q) 46 { 47 uint64_t addr; 48 struct cik_mqd *m; 49 int retval; 50 51 BUG_ON(!mm || !q || !mqd); 52 53 pr_debug("kfd: In func %s\n", __func__); 54 55 retval = kfd2kgd->allocate_mem(mm->dev->kgd, 56 sizeof(struct cik_mqd), 57 256, 58 KFD_MEMPOOL_SYSTEM_WRITECOMBINE, 59 (struct kgd_mem **) mqd_mem_obj); 60 61 if (retval != 0) 62 return -ENOMEM; 63 64 m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; 65 addr = (*mqd_mem_obj)->gpu_addr; 66 67 memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); 68 69 m->header = 0xC0310800; 70 m->compute_pipelinestat_enable = 1; 71 m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; 72 m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; 73 m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; 74 m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; 75 76 /* 77 * Make sure to use the last queue state saved on mqd when the cp 78 * reassigns the queue, so when queue is switched on/off (e.g over 79 * subscription or quantum timeout) the context will be consistent 80 */ 81 m->cp_hqd_persistent_state = 82 DEFAULT_CP_HQD_PERSISTENT_STATE | PRELOAD_REQ; 83 84 m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; 85 m->cp_mqd_base_addr_lo = lower_32_bits(addr); 86 m->cp_mqd_base_addr_hi = upper_32_bits(addr); 87 88 m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE | IB_ATC_EN; 89 /* Although WinKFD writes this, I suspect it should not be necessary */ 90 m->cp_hqd_ib_control = IB_ATC_EN | DEFAULT_MIN_IB_AVAIL_SIZE; 91 92 m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | 93 QUANTUM_DURATION(10); 94 95 /* 96 * Pipe Priority 97 * Identifies the pipe relative priority when this queue is connected 98 * to the pipeline. The pipe priority is against the GFX pipe and HP3D. 99 * In KFD we are using a fixed pipe priority set to CS_MEDIUM. 100 * 0 = CS_LOW (typically below GFX) 101 * 1 = CS_MEDIUM (typically between HP3D and GFX 102 * 2 = CS_HIGH (typically above HP3D) 103 */ 104 m->cp_hqd_pipe_priority = 1; 105 m->cp_hqd_queue_priority = 15; 106 107 *mqd = m; 108 if (gart_addr != NULL) 109 *gart_addr = addr; 110 retval = mm->update_mqd(mm, m, q); 111 112 return retval; 113 } 114 115 static int init_mqd_sdma(struct mqd_manager *mm, void **mqd, 116 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, 117 struct queue_properties *q) 118 { 119 int retval; 120 struct cik_sdma_rlc_registers *m; 121 122 BUG_ON(!mm || !mqd || !mqd_mem_obj); 123 124 retval = kfd2kgd->allocate_mem(mm->dev->kgd, 125 sizeof(struct cik_sdma_rlc_registers), 126 256, 127 KFD_MEMPOOL_SYSTEM_WRITECOMBINE, 128 (struct kgd_mem **) mqd_mem_obj); 129 130 if (retval != 0) 131 return -ENOMEM; 132 133 m = (struct cik_sdma_rlc_registers *) (*mqd_mem_obj)->cpu_ptr; 134 135 memset(m, 0, sizeof(struct cik_sdma_rlc_registers)); 136 137 *mqd = m; 138 if (gart_addr != NULL) 139 *gart_addr = (*mqd_mem_obj)->gpu_addr; 140 141 retval = mm->update_mqd(mm, m, q); 142 143 return retval; 144 } 145 146 static void uninit_mqd(struct mqd_manager *mm, void *mqd, 147 struct kfd_mem_obj *mqd_mem_obj) 148 { 149 BUG_ON(!mm || !mqd); 150 kfd2kgd->free_mem(mm->dev->kgd, (struct kgd_mem *) mqd_mem_obj); 151 } 152 153 static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd, 154 struct kfd_mem_obj *mqd_mem_obj) 155 { 156 BUG_ON(!mm || !mqd); 157 kfd2kgd->free_mem(mm->dev->kgd, (struct kgd_mem *) mqd_mem_obj); 158 } 159 160 static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id, 161 uint32_t queue_id, uint32_t __user *wptr) 162 { 163 return kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id, wptr); 164 } 165 166 static int load_mqd_sdma(struct mqd_manager *mm, void *mqd, 167 uint32_t pipe_id, uint32_t queue_id, 168 uint32_t __user *wptr) 169 { 170 return kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd); 171 } 172 173 static int update_mqd(struct mqd_manager *mm, void *mqd, 174 struct queue_properties *q) 175 { 176 struct cik_mqd *m; 177 178 BUG_ON(!mm || !q || !mqd); 179 180 pr_debug("kfd: In func %s\n", __func__); 181 182 m = get_mqd(mqd); 183 m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | 184 DEFAULT_MIN_AVAIL_SIZE | PQ_ATC_EN; 185 186 /* 187 * Calculating queue size which is log base 2 of actual queue size -1 188 * dwords and another -1 for ffs 189 */ 190 m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int)) 191 - 1 - 1; 192 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); 193 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); 194 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); 195 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); 196 m->cp_hqd_pq_doorbell_control = DOORBELL_EN | 197 DOORBELL_OFFSET(q->doorbell_off); 198 199 m->cp_hqd_vmid = q->vmid; 200 201 if (q->format == KFD_QUEUE_FORMAT_AQL) { 202 m->cp_hqd_iq_rptr = AQL_ENABLE; 203 m->cp_hqd_pq_control |= NO_UPDATE_RPTR; 204 } 205 206 m->cp_hqd_active = 0; 207 q->is_active = false; 208 if (q->queue_size > 0 && 209 q->queue_address != 0 && 210 q->queue_percent > 0) { 211 m->cp_hqd_active = 1; 212 q->is_active = true; 213 } 214 215 return 0; 216 } 217 218 static int update_mqd_sdma(struct mqd_manager *mm, void *mqd, 219 struct queue_properties *q) 220 { 221 struct cik_sdma_rlc_registers *m; 222 223 BUG_ON(!mm || !mqd || !q); 224 225 m = get_sdma_mqd(mqd); 226 m->sdma_rlc_rb_cntl = 227 SDMA_RB_SIZE((ffs(q->queue_size / sizeof(unsigned int)))) | 228 SDMA_RB_VMID(q->vmid) | 229 SDMA_RPTR_WRITEBACK_ENABLE | 230 SDMA_RPTR_WRITEBACK_TIMER(6); 231 232 m->sdma_rlc_rb_base = lower_32_bits(q->queue_address >> 8); 233 m->sdma_rlc_rb_base_hi = upper_32_bits(q->queue_address >> 8); 234 m->sdma_rlc_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr); 235 m->sdma_rlc_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr); 236 m->sdma_rlc_doorbell = SDMA_OFFSET(q->doorbell_off) | SDMA_DB_ENABLE; 237 m->sdma_rlc_virtual_addr = q->sdma_vm_addr; 238 239 m->sdma_engine_id = q->sdma_engine_id; 240 m->sdma_queue_id = q->sdma_queue_id; 241 242 q->is_active = false; 243 if (q->queue_size > 0 && 244 q->queue_address != 0 && 245 q->queue_percent > 0) { 246 m->sdma_rlc_rb_cntl |= SDMA_RB_ENABLE; 247 q->is_active = true; 248 } 249 250 return 0; 251 } 252 253 static int destroy_mqd(struct mqd_manager *mm, void *mqd, 254 enum kfd_preempt_type type, 255 unsigned int timeout, uint32_t pipe_id, 256 uint32_t queue_id) 257 { 258 return kfd2kgd->hqd_destroy(mm->dev->kgd, type, timeout, 259 pipe_id, queue_id); 260 } 261 262 /* 263 * preempt type here is ignored because there is only one way 264 * to preempt sdma queue 265 */ 266 static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd, 267 enum kfd_preempt_type type, 268 unsigned int timeout, uint32_t pipe_id, 269 uint32_t queue_id) 270 { 271 return kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout); 272 } 273 274 static bool is_occupied(struct mqd_manager *mm, void *mqd, 275 uint64_t queue_address, uint32_t pipe_id, 276 uint32_t queue_id) 277 { 278 279 return kfd2kgd->hqd_is_occupies(mm->dev->kgd, queue_address, 280 pipe_id, queue_id); 281 282 } 283 284 static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd, 285 uint64_t queue_address, uint32_t pipe_id, 286 uint32_t queue_id) 287 { 288 return kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd); 289 } 290 291 /* 292 * HIQ MQD Implementation, concrete implementation for HIQ MQD implementation. 293 * The HIQ queue in Kaveri is using the same MQD structure as all the user mode 294 * queues but with different initial values. 295 */ 296 297 static int init_mqd_hiq(struct mqd_manager *mm, void **mqd, 298 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, 299 struct queue_properties *q) 300 { 301 uint64_t addr; 302 struct cik_mqd *m; 303 int retval; 304 305 BUG_ON(!mm || !q || !mqd || !mqd_mem_obj); 306 307 pr_debug("kfd: In func %s\n", __func__); 308 309 retval = kfd2kgd->allocate_mem(mm->dev->kgd, 310 sizeof(struct cik_mqd), 311 256, 312 KFD_MEMPOOL_SYSTEM_WRITECOMBINE, 313 (struct kgd_mem **) mqd_mem_obj); 314 315 if (retval != 0) 316 return -ENOMEM; 317 318 m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; 319 addr = (*mqd_mem_obj)->gpu_addr; 320 321 memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); 322 323 m->header = 0xC0310800; 324 m->compute_pipelinestat_enable = 1; 325 m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; 326 m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; 327 m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; 328 m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; 329 330 m->cp_hqd_persistent_state = DEFAULT_CP_HQD_PERSISTENT_STATE | 331 PRELOAD_REQ; 332 m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | 333 QUANTUM_DURATION(10); 334 335 m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; 336 m->cp_mqd_base_addr_lo = lower_32_bits(addr); 337 m->cp_mqd_base_addr_hi = upper_32_bits(addr); 338 339 m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE; 340 341 /* 342 * Pipe Priority 343 * Identifies the pipe relative priority when this queue is connected 344 * to the pipeline. The pipe priority is against the GFX pipe and HP3D. 345 * In KFD we are using a fixed pipe priority set to CS_MEDIUM. 346 * 0 = CS_LOW (typically below GFX) 347 * 1 = CS_MEDIUM (typically between HP3D and GFX 348 * 2 = CS_HIGH (typically above HP3D) 349 */ 350 m->cp_hqd_pipe_priority = 1; 351 m->cp_hqd_queue_priority = 15; 352 353 *mqd = m; 354 if (gart_addr) 355 *gart_addr = addr; 356 retval = mm->update_mqd(mm, m, q); 357 358 return retval; 359 } 360 361 static int update_mqd_hiq(struct mqd_manager *mm, void *mqd, 362 struct queue_properties *q) 363 { 364 struct cik_mqd *m; 365 366 BUG_ON(!mm || !q || !mqd); 367 368 pr_debug("kfd: In func %s\n", __func__); 369 370 m = get_mqd(mqd); 371 m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | 372 DEFAULT_MIN_AVAIL_SIZE | 373 PRIV_STATE | 374 KMD_QUEUE; 375 376 /* 377 * Calculating queue size which is log base 2 of actual queue 378 * size -1 dwords 379 */ 380 m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int)) 381 - 1 - 1; 382 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); 383 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); 384 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); 385 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); 386 m->cp_hqd_pq_doorbell_control = DOORBELL_EN | 387 DOORBELL_OFFSET(q->doorbell_off); 388 389 m->cp_hqd_vmid = q->vmid; 390 391 m->cp_hqd_active = 0; 392 q->is_active = false; 393 if (q->queue_size > 0 && 394 q->queue_address != 0 && 395 q->queue_percent > 0) { 396 m->cp_hqd_active = 1; 397 q->is_active = true; 398 } 399 400 return 0; 401 } 402 403 /* 404 * SDMA MQD Implementation 405 */ 406 407 struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd) 408 { 409 struct cik_sdma_rlc_registers *m; 410 411 BUG_ON(!mqd); 412 413 m = (struct cik_sdma_rlc_registers *)mqd; 414 415 return m; 416 } 417 418 struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type, 419 struct kfd_dev *dev) 420 { 421 struct mqd_manager *mqd; 422 423 BUG_ON(!dev); 424 BUG_ON(type >= KFD_MQD_TYPE_MAX); 425 426 pr_debug("kfd: In func %s\n", __func__); 427 428 mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL); 429 if (!mqd) 430 return NULL; 431 432 mqd->dev = dev; 433 434 switch (type) { 435 case KFD_MQD_TYPE_CIK_CP: 436 case KFD_MQD_TYPE_CIK_COMPUTE: 437 mqd->init_mqd = init_mqd; 438 mqd->uninit_mqd = uninit_mqd; 439 mqd->load_mqd = load_mqd; 440 mqd->update_mqd = update_mqd; 441 mqd->destroy_mqd = destroy_mqd; 442 mqd->is_occupied = is_occupied; 443 break; 444 case KFD_MQD_TYPE_CIK_HIQ: 445 mqd->init_mqd = init_mqd_hiq; 446 mqd->uninit_mqd = uninit_mqd; 447 mqd->load_mqd = load_mqd; 448 mqd->update_mqd = update_mqd_hiq; 449 mqd->destroy_mqd = destroy_mqd; 450 mqd->is_occupied = is_occupied; 451 break; 452 case KFD_MQD_TYPE_CIK_SDMA: 453 mqd->init_mqd = init_mqd_sdma; 454 mqd->uninit_mqd = uninit_mqd_sdma; 455 mqd->load_mqd = load_mqd_sdma; 456 mqd->update_mqd = update_mqd_sdma; 457 mqd->destroy_mqd = destroy_mqd_sdma; 458 mqd->is_occupied = is_occupied_sdma; 459 break; 460 default: 461 kfree(mqd); 462 return NULL; 463 } 464 465 return mqd; 466 } 467 468 /* SDMA queues should be implemented here when the cp will supports them */ 469