1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/printk.h> 25 #include <linux/slab.h> 26 #include <linux/mm_types.h> 27 28 #include "kfd_priv.h" 29 #include "kfd_mqd_manager.h" 30 #include "cik_regs.h" 31 #include "cik_structs.h" 32 #include "oss/oss_2_4_sh_mask.h" 33 34 static inline struct cik_mqd *get_mqd(void *mqd) 35 { 36 return (struct cik_mqd *)mqd; 37 } 38 39 static int init_mqd(struct mqd_manager *mm, void **mqd, 40 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, 41 struct queue_properties *q) 42 { 43 uint64_t addr; 44 struct cik_mqd *m; 45 int retval; 46 47 BUG_ON(!mm || !q || !mqd); 48 49 pr_debug("kfd: In func %s\n", __func__); 50 51 retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd), 52 mqd_mem_obj); 53 54 if (retval != 0) 55 return -ENOMEM; 56 57 m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; 58 addr = (*mqd_mem_obj)->gpu_addr; 59 60 memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); 61 62 m->header = 0xC0310800; 63 m->compute_pipelinestat_enable = 1; 64 m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; 65 m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; 66 m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; 67 m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; 68 69 /* 70 * Make sure to use the last queue state saved on mqd when the cp 71 * reassigns the queue, so when queue is switched on/off (e.g over 72 * subscription or quantum timeout) the context will be consistent 73 */ 74 m->cp_hqd_persistent_state = 75 DEFAULT_CP_HQD_PERSISTENT_STATE | PRELOAD_REQ; 76 77 m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; 78 m->cp_mqd_base_addr_lo = lower_32_bits(addr); 79 m->cp_mqd_base_addr_hi = upper_32_bits(addr); 80 81 m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE | IB_ATC_EN; 82 /* Although WinKFD writes this, I suspect it should not be necessary */ 83 m->cp_hqd_ib_control = IB_ATC_EN | DEFAULT_MIN_IB_AVAIL_SIZE; 84 85 m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | 86 QUANTUM_DURATION(10); 87 88 /* 89 * Pipe Priority 90 * Identifies the pipe relative priority when this queue is connected 91 * to the pipeline. The pipe priority is against the GFX pipe and HP3D. 92 * In KFD we are using a fixed pipe priority set to CS_MEDIUM. 93 * 0 = CS_LOW (typically below GFX) 94 * 1 = CS_MEDIUM (typically between HP3D and GFX 95 * 2 = CS_HIGH (typically above HP3D) 96 */ 97 m->cp_hqd_pipe_priority = 1; 98 m->cp_hqd_queue_priority = 15; 99 100 if (q->format == KFD_QUEUE_FORMAT_AQL) 101 m->cp_hqd_iq_rptr = AQL_ENABLE; 102 103 *mqd = m; 104 if (gart_addr != NULL) 105 *gart_addr = addr; 106 retval = mm->update_mqd(mm, m, q); 107 108 return retval; 109 } 110 111 static int init_mqd_sdma(struct mqd_manager *mm, void **mqd, 112 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, 113 struct queue_properties *q) 114 { 115 int retval; 116 struct cik_sdma_rlc_registers *m; 117 118 BUG_ON(!mm || !mqd || !mqd_mem_obj); 119 120 retval = kfd_gtt_sa_allocate(mm->dev, 121 sizeof(struct cik_sdma_rlc_registers), 122 mqd_mem_obj); 123 124 if (retval != 0) 125 return -ENOMEM; 126 127 m = (struct cik_sdma_rlc_registers *) (*mqd_mem_obj)->cpu_ptr; 128 129 memset(m, 0, sizeof(struct cik_sdma_rlc_registers)); 130 131 *mqd = m; 132 if (gart_addr != NULL) 133 *gart_addr = (*mqd_mem_obj)->gpu_addr; 134 135 retval = mm->update_mqd(mm, m, q); 136 137 return retval; 138 } 139 140 static void uninit_mqd(struct mqd_manager *mm, void *mqd, 141 struct kfd_mem_obj *mqd_mem_obj) 142 { 143 BUG_ON(!mm || !mqd); 144 kfd_gtt_sa_free(mm->dev, mqd_mem_obj); 145 } 146 147 static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd, 148 struct kfd_mem_obj *mqd_mem_obj) 149 { 150 BUG_ON(!mm || !mqd); 151 kfd_gtt_sa_free(mm->dev, mqd_mem_obj); 152 } 153 154 static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id, 155 uint32_t queue_id, uint32_t __user *wptr) 156 { 157 return mm->dev->kfd2kgd->hqd_load 158 (mm->dev->kgd, mqd, pipe_id, queue_id, wptr); 159 } 160 161 static int load_mqd_sdma(struct mqd_manager *mm, void *mqd, 162 uint32_t pipe_id, uint32_t queue_id, 163 uint32_t __user *wptr) 164 { 165 return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd); 166 } 167 168 static int update_mqd(struct mqd_manager *mm, void *mqd, 169 struct queue_properties *q) 170 { 171 struct cik_mqd *m; 172 173 BUG_ON(!mm || !q || !mqd); 174 175 pr_debug("kfd: In func %s\n", __func__); 176 177 m = get_mqd(mqd); 178 m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | 179 DEFAULT_MIN_AVAIL_SIZE | PQ_ATC_EN; 180 181 /* 182 * Calculating queue size which is log base 2 of actual queue size -1 183 * dwords and another -1 for ffs 184 */ 185 m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int)) 186 - 1 - 1; 187 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); 188 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); 189 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); 190 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); 191 m->cp_hqd_pq_doorbell_control = DOORBELL_EN | 192 DOORBELL_OFFSET(q->doorbell_off); 193 194 m->cp_hqd_vmid = q->vmid; 195 196 if (q->format == KFD_QUEUE_FORMAT_AQL) { 197 m->cp_hqd_pq_control |= NO_UPDATE_RPTR; 198 } 199 200 m->cp_hqd_active = 0; 201 q->is_active = false; 202 if (q->queue_size > 0 && 203 q->queue_address != 0 && 204 q->queue_percent > 0) { 205 m->cp_hqd_active = 1; 206 q->is_active = true; 207 } 208 209 return 0; 210 } 211 212 static int update_mqd_sdma(struct mqd_manager *mm, void *mqd, 213 struct queue_properties *q) 214 { 215 struct cik_sdma_rlc_registers *m; 216 217 BUG_ON(!mm || !mqd || !q); 218 219 m = get_sdma_mqd(mqd); 220 m->sdma_rlc_rb_cntl = ffs(q->queue_size / sizeof(unsigned int)) << 221 SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT | 222 q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT | 223 1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT | 224 6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT; 225 226 m->sdma_rlc_rb_base = lower_32_bits(q->queue_address >> 8); 227 m->sdma_rlc_rb_base_hi = upper_32_bits(q->queue_address >> 8); 228 m->sdma_rlc_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr); 229 m->sdma_rlc_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr); 230 m->sdma_rlc_doorbell = q->doorbell_off << 231 SDMA0_RLC0_DOORBELL__OFFSET__SHIFT | 232 1 << SDMA0_RLC0_DOORBELL__ENABLE__SHIFT; 233 234 m->sdma_rlc_virtual_addr = q->sdma_vm_addr; 235 236 m->sdma_engine_id = q->sdma_engine_id; 237 m->sdma_queue_id = q->sdma_queue_id; 238 239 q->is_active = false; 240 if (q->queue_size > 0 && 241 q->queue_address != 0 && 242 q->queue_percent > 0) { 243 m->sdma_rlc_rb_cntl |= 244 1 << SDMA0_RLC0_RB_CNTL__RB_ENABLE__SHIFT; 245 246 q->is_active = true; 247 } 248 249 return 0; 250 } 251 252 static int destroy_mqd(struct mqd_manager *mm, void *mqd, 253 enum kfd_preempt_type type, 254 unsigned int timeout, uint32_t pipe_id, 255 uint32_t queue_id) 256 { 257 return mm->dev->kfd2kgd->hqd_destroy(mm->dev->kgd, type, timeout, 258 pipe_id, queue_id); 259 } 260 261 /* 262 * preempt type here is ignored because there is only one way 263 * to preempt sdma queue 264 */ 265 static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd, 266 enum kfd_preempt_type type, 267 unsigned int timeout, uint32_t pipe_id, 268 uint32_t queue_id) 269 { 270 return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout); 271 } 272 273 static bool is_occupied(struct mqd_manager *mm, void *mqd, 274 uint64_t queue_address, uint32_t pipe_id, 275 uint32_t queue_id) 276 { 277 278 return mm->dev->kfd2kgd->hqd_is_occupied(mm->dev->kgd, queue_address, 279 pipe_id, queue_id); 280 281 } 282 283 static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd, 284 uint64_t queue_address, uint32_t pipe_id, 285 uint32_t queue_id) 286 { 287 return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd); 288 } 289 290 /* 291 * HIQ MQD Implementation, concrete implementation for HIQ MQD implementation. 292 * The HIQ queue in Kaveri is using the same MQD structure as all the user mode 293 * queues but with different initial values. 294 */ 295 296 static int init_mqd_hiq(struct mqd_manager *mm, void **mqd, 297 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, 298 struct queue_properties *q) 299 { 300 uint64_t addr; 301 struct cik_mqd *m; 302 int retval; 303 304 BUG_ON(!mm || !q || !mqd || !mqd_mem_obj); 305 306 pr_debug("kfd: In func %s\n", __func__); 307 308 retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd), 309 mqd_mem_obj); 310 311 if (retval != 0) 312 return -ENOMEM; 313 314 m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; 315 addr = (*mqd_mem_obj)->gpu_addr; 316 317 memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); 318 319 m->header = 0xC0310800; 320 m->compute_pipelinestat_enable = 1; 321 m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; 322 m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; 323 m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; 324 m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; 325 326 m->cp_hqd_persistent_state = DEFAULT_CP_HQD_PERSISTENT_STATE | 327 PRELOAD_REQ; 328 m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | 329 QUANTUM_DURATION(10); 330 331 m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; 332 m->cp_mqd_base_addr_lo = lower_32_bits(addr); 333 m->cp_mqd_base_addr_hi = upper_32_bits(addr); 334 335 m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE; 336 337 /* 338 * Pipe Priority 339 * Identifies the pipe relative priority when this queue is connected 340 * to the pipeline. The pipe priority is against the GFX pipe and HP3D. 341 * In KFD we are using a fixed pipe priority set to CS_MEDIUM. 342 * 0 = CS_LOW (typically below GFX) 343 * 1 = CS_MEDIUM (typically between HP3D and GFX 344 * 2 = CS_HIGH (typically above HP3D) 345 */ 346 m->cp_hqd_pipe_priority = 1; 347 m->cp_hqd_queue_priority = 15; 348 349 *mqd = m; 350 if (gart_addr) 351 *gart_addr = addr; 352 retval = mm->update_mqd(mm, m, q); 353 354 return retval; 355 } 356 357 static int update_mqd_hiq(struct mqd_manager *mm, void *mqd, 358 struct queue_properties *q) 359 { 360 struct cik_mqd *m; 361 362 BUG_ON(!mm || !q || !mqd); 363 364 pr_debug("kfd: In func %s\n", __func__); 365 366 m = get_mqd(mqd); 367 m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | 368 DEFAULT_MIN_AVAIL_SIZE | 369 PRIV_STATE | 370 KMD_QUEUE; 371 372 /* 373 * Calculating queue size which is log base 2 of actual queue 374 * size -1 dwords 375 */ 376 m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int)) 377 - 1 - 1; 378 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); 379 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); 380 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); 381 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); 382 m->cp_hqd_pq_doorbell_control = DOORBELL_EN | 383 DOORBELL_OFFSET(q->doorbell_off); 384 385 m->cp_hqd_vmid = q->vmid; 386 387 m->cp_hqd_active = 0; 388 q->is_active = false; 389 if (q->queue_size > 0 && 390 q->queue_address != 0 && 391 q->queue_percent > 0) { 392 m->cp_hqd_active = 1; 393 q->is_active = true; 394 } 395 396 return 0; 397 } 398 399 struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd) 400 { 401 struct cik_sdma_rlc_registers *m; 402 403 BUG_ON(!mqd); 404 405 m = (struct cik_sdma_rlc_registers *)mqd; 406 407 return m; 408 } 409 410 struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type, 411 struct kfd_dev *dev) 412 { 413 struct mqd_manager *mqd; 414 415 BUG_ON(!dev); 416 BUG_ON(type >= KFD_MQD_TYPE_MAX); 417 418 pr_debug("kfd: In func %s\n", __func__); 419 420 mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL); 421 if (!mqd) 422 return NULL; 423 424 mqd->dev = dev; 425 426 switch (type) { 427 case KFD_MQD_TYPE_CP: 428 case KFD_MQD_TYPE_COMPUTE: 429 mqd->init_mqd = init_mqd; 430 mqd->uninit_mqd = uninit_mqd; 431 mqd->load_mqd = load_mqd; 432 mqd->update_mqd = update_mqd; 433 mqd->destroy_mqd = destroy_mqd; 434 mqd->is_occupied = is_occupied; 435 break; 436 case KFD_MQD_TYPE_HIQ: 437 mqd->init_mqd = init_mqd_hiq; 438 mqd->uninit_mqd = uninit_mqd; 439 mqd->load_mqd = load_mqd; 440 mqd->update_mqd = update_mqd_hiq; 441 mqd->destroy_mqd = destroy_mqd; 442 mqd->is_occupied = is_occupied; 443 break; 444 case KFD_MQD_TYPE_SDMA: 445 mqd->init_mqd = init_mqd_sdma; 446 mqd->uninit_mqd = uninit_mqd_sdma; 447 mqd->load_mqd = load_mqd_sdma; 448 mqd->update_mqd = update_mqd_sdma; 449 mqd->destroy_mqd = destroy_mqd_sdma; 450 mqd->is_occupied = is_occupied_sdma; 451 break; 452 default: 453 kfree(mqd); 454 return NULL; 455 } 456 457 return mqd; 458 } 459 460