1 /* 2 * Copyright 2017 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include "amdgpu_ids.h" 24 25 #include <linux/idr.h> 26 #include <linux/dma-fence-array.h> 27 28 29 #include "amdgpu.h" 30 #include "amdgpu_trace.h" 31 32 /* 33 * PASID manager 34 * 35 * PASIDs are global address space identifiers that can be shared 36 * between the GPU, an IOMMU and the driver. VMs on different devices 37 * may use the same PASID if they share the same address 38 * space. Therefore PASIDs are allocated using a global IDA. VMs are 39 * looked up from the PASID per amdgpu_device. 40 */ 41 static DEFINE_IDA(amdgpu_pasid_ida); 42 43 /* Helper to free pasid from a fence callback */ 44 struct amdgpu_pasid_cb { 45 struct dma_fence_cb cb; 46 u32 pasid; 47 }; 48 49 /** 50 * amdgpu_pasid_alloc - Allocate a PASID 51 * @bits: Maximum width of the PASID in bits, must be at least 1 52 * 53 * Allocates a PASID of the given width while keeping smaller PASIDs 54 * available if possible. 55 * 56 * Returns a positive integer on success. Returns %-EINVAL if bits==0. 57 * Returns %-ENOSPC if no PASID was available. Returns %-ENOMEM on 58 * memory allocation failure. 59 */ 60 int amdgpu_pasid_alloc(unsigned int bits) 61 { 62 int pasid = -EINVAL; 63 64 for (bits = min(bits, 31U); bits > 0; bits--) { 65 pasid = ida_simple_get(&amdgpu_pasid_ida, 66 1U << (bits - 1), 1U << bits, 67 GFP_KERNEL); 68 if (pasid != -ENOSPC) 69 break; 70 } 71 72 if (pasid >= 0) 73 trace_amdgpu_pasid_allocated(pasid); 74 75 return pasid; 76 } 77 78 /** 79 * amdgpu_pasid_free - Free a PASID 80 * @pasid: PASID to free 81 */ 82 void amdgpu_pasid_free(u32 pasid) 83 { 84 trace_amdgpu_pasid_freed(pasid); 85 ida_simple_remove(&amdgpu_pasid_ida, pasid); 86 } 87 88 static void amdgpu_pasid_free_cb(struct dma_fence *fence, 89 struct dma_fence_cb *_cb) 90 { 91 struct amdgpu_pasid_cb *cb = 92 container_of(_cb, struct amdgpu_pasid_cb, cb); 93 94 amdgpu_pasid_free(cb->pasid); 95 dma_fence_put(fence); 96 kfree(cb); 97 } 98 99 /** 100 * amdgpu_pasid_free_delayed - free pasid when fences signal 101 * 102 * @resv: reservation object with the fences to wait for 103 * @pasid: pasid to free 104 * 105 * Free the pasid only after all the fences in resv are signaled. 106 */ 107 void amdgpu_pasid_free_delayed(struct dma_resv *resv, 108 u32 pasid) 109 { 110 struct amdgpu_pasid_cb *cb; 111 struct dma_fence *fence; 112 int r; 113 114 r = dma_resv_get_singleton(resv, DMA_RESV_USAGE_BOOKKEEP, &fence); 115 if (r) 116 goto fallback; 117 118 if (!fence) { 119 amdgpu_pasid_free(pasid); 120 return; 121 } 122 123 cb = kmalloc(sizeof(*cb), GFP_KERNEL); 124 if (!cb) { 125 /* Last resort when we are OOM */ 126 dma_fence_wait(fence, false); 127 dma_fence_put(fence); 128 amdgpu_pasid_free(pasid); 129 } else { 130 cb->pasid = pasid; 131 if (dma_fence_add_callback(fence, &cb->cb, 132 amdgpu_pasid_free_cb)) 133 amdgpu_pasid_free_cb(fence, &cb->cb); 134 } 135 136 return; 137 138 fallback: 139 /* Not enough memory for the delayed delete, as last resort 140 * block for all the fences to complete. 141 */ 142 dma_resv_wait_timeout(resv, DMA_RESV_USAGE_BOOKKEEP, 143 false, MAX_SCHEDULE_TIMEOUT); 144 amdgpu_pasid_free(pasid); 145 } 146 147 /* 148 * VMID manager 149 * 150 * VMIDs are a per VMHUB identifier for page tables handling. 151 */ 152 153 /** 154 * amdgpu_vmid_had_gpu_reset - check if reset occured since last use 155 * 156 * @adev: amdgpu_device pointer 157 * @id: VMID structure 158 * 159 * Check if GPU reset occured since last use of the VMID. 160 */ 161 bool amdgpu_vmid_had_gpu_reset(struct amdgpu_device *adev, 162 struct amdgpu_vmid *id) 163 { 164 return id->current_gpu_reset_count != 165 atomic_read(&adev->gpu_reset_counter); 166 } 167 168 /** 169 * amdgpu_vmid_grab_idle - grab idle VMID 170 * 171 * @vm: vm to allocate id for 172 * @ring: ring we want to submit job to 173 * @idle: resulting idle VMID 174 * @fence: fence to wait for if no id could be grabbed 175 * 176 * Try to find an idle VMID, if none is idle add a fence to wait to the sync 177 * object. Returns -ENOMEM when we are out of memory. 178 */ 179 static int amdgpu_vmid_grab_idle(struct amdgpu_vm *vm, 180 struct amdgpu_ring *ring, 181 struct amdgpu_vmid **idle, 182 struct dma_fence **fence) 183 { 184 struct amdgpu_device *adev = ring->adev; 185 unsigned vmhub = ring->funcs->vmhub; 186 struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; 187 struct dma_fence **fences; 188 unsigned i; 189 190 if (!dma_fence_is_signaled(ring->vmid_wait)) { 191 *fence = dma_fence_get(ring->vmid_wait); 192 return 0; 193 } 194 195 fences = kmalloc_array(id_mgr->num_ids, sizeof(void *), GFP_KERNEL); 196 if (!fences) 197 return -ENOMEM; 198 199 /* Check if we have an idle VMID */ 200 i = 0; 201 list_for_each_entry((*idle), &id_mgr->ids_lru, list) { 202 /* Don't use per engine and per process VMID at the same time */ 203 struct amdgpu_ring *r = adev->vm_manager.concurrent_flush ? 204 NULL : ring; 205 206 fences[i] = amdgpu_sync_peek_fence(&(*idle)->active, r); 207 if (!fences[i]) 208 break; 209 ++i; 210 } 211 212 /* If we can't find a idle VMID to use, wait till one becomes available */ 213 if (&(*idle)->list == &id_mgr->ids_lru) { 214 u64 fence_context = adev->vm_manager.fence_context + ring->idx; 215 unsigned seqno = ++adev->vm_manager.seqno[ring->idx]; 216 struct dma_fence_array *array; 217 unsigned j; 218 219 *idle = NULL; 220 for (j = 0; j < i; ++j) 221 dma_fence_get(fences[j]); 222 223 array = dma_fence_array_create(i, fences, fence_context, 224 seqno, true); 225 if (!array) { 226 for (j = 0; j < i; ++j) 227 dma_fence_put(fences[j]); 228 kfree(fences); 229 return -ENOMEM; 230 } 231 232 *fence = dma_fence_get(&array->base); 233 dma_fence_put(ring->vmid_wait); 234 ring->vmid_wait = &array->base; 235 return 0; 236 } 237 kfree(fences); 238 239 return 0; 240 } 241 242 /** 243 * amdgpu_vmid_grab_reserved - try to assign reserved VMID 244 * 245 * @vm: vm to allocate id for 246 * @ring: ring we want to submit job to 247 * @job: job who wants to use the VMID 248 * @id: resulting VMID 249 * @fence: fence to wait for if no id could be grabbed 250 * 251 * Try to assign a reserved VMID. 252 */ 253 static int amdgpu_vmid_grab_reserved(struct amdgpu_vm *vm, 254 struct amdgpu_ring *ring, 255 struct amdgpu_job *job, 256 struct amdgpu_vmid **id, 257 struct dma_fence **fence) 258 { 259 struct amdgpu_device *adev = ring->adev; 260 unsigned vmhub = ring->funcs->vmhub; 261 uint64_t fence_context = adev->fence_context + ring->idx; 262 bool needs_flush = vm->use_cpu_for_update; 263 uint64_t updates = amdgpu_vm_tlb_seq(vm); 264 int r; 265 266 *id = vm->reserved_vmid[vmhub]; 267 if ((*id)->owner != vm->immediate.fence_context || 268 (*id)->pd_gpu_addr != job->vm_pd_addr || 269 (*id)->flushed_updates < updates || 270 !(*id)->last_flush || 271 ((*id)->last_flush->context != fence_context && 272 !dma_fence_is_signaled((*id)->last_flush))) { 273 struct dma_fence *tmp; 274 275 /* Don't use per engine and per process VMID at the same time */ 276 if (adev->vm_manager.concurrent_flush) 277 ring = NULL; 278 279 /* to prevent one context starved by another context */ 280 (*id)->pd_gpu_addr = 0; 281 tmp = amdgpu_sync_peek_fence(&(*id)->active, ring); 282 if (tmp) { 283 *id = NULL; 284 *fence = dma_fence_get(tmp); 285 return 0; 286 } 287 needs_flush = true; 288 } 289 290 /* Good we can use this VMID. Remember this submission as 291 * user of the VMID. 292 */ 293 r = amdgpu_sync_fence(&(*id)->active, &job->base.s_fence->finished); 294 if (r) 295 return r; 296 297 (*id)->flushed_updates = updates; 298 job->vm_needs_flush = needs_flush; 299 return 0; 300 } 301 302 /** 303 * amdgpu_vmid_grab_used - try to reuse a VMID 304 * 305 * @vm: vm to allocate id for 306 * @ring: ring we want to submit job to 307 * @job: job who wants to use the VMID 308 * @id: resulting VMID 309 * @fence: fence to wait for if no id could be grabbed 310 * 311 * Try to reuse a VMID for this submission. 312 */ 313 static int amdgpu_vmid_grab_used(struct amdgpu_vm *vm, 314 struct amdgpu_ring *ring, 315 struct amdgpu_job *job, 316 struct amdgpu_vmid **id, 317 struct dma_fence **fence) 318 { 319 struct amdgpu_device *adev = ring->adev; 320 unsigned vmhub = ring->funcs->vmhub; 321 struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; 322 uint64_t fence_context = adev->fence_context + ring->idx; 323 uint64_t updates = amdgpu_vm_tlb_seq(vm); 324 int r; 325 326 job->vm_needs_flush = vm->use_cpu_for_update; 327 328 /* Check if we can use a VMID already assigned to this VM */ 329 list_for_each_entry_reverse((*id), &id_mgr->ids_lru, list) { 330 bool needs_flush = vm->use_cpu_for_update; 331 332 /* Check all the prerequisites to using this VMID */ 333 if ((*id)->owner != vm->immediate.fence_context) 334 continue; 335 336 if ((*id)->pd_gpu_addr != job->vm_pd_addr) 337 continue; 338 339 if (!(*id)->last_flush || 340 ((*id)->last_flush->context != fence_context && 341 !dma_fence_is_signaled((*id)->last_flush))) 342 needs_flush = true; 343 344 if ((*id)->flushed_updates < updates) 345 needs_flush = true; 346 347 if (needs_flush && !adev->vm_manager.concurrent_flush) 348 continue; 349 350 /* Good, we can use this VMID. Remember this submission as 351 * user of the VMID. 352 */ 353 r = amdgpu_sync_fence(&(*id)->active, 354 &job->base.s_fence->finished); 355 if (r) 356 return r; 357 358 (*id)->flushed_updates = updates; 359 job->vm_needs_flush |= needs_flush; 360 return 0; 361 } 362 363 *id = NULL; 364 return 0; 365 } 366 367 /** 368 * amdgpu_vmid_grab - allocate the next free VMID 369 * 370 * @vm: vm to allocate id for 371 * @ring: ring we want to submit job to 372 * @job: job who wants to use the VMID 373 * @fence: fence to wait for if no id could be grabbed 374 * 375 * Allocate an id for the vm, adding fences to the sync obj as necessary. 376 */ 377 int amdgpu_vmid_grab(struct amdgpu_vm *vm, struct amdgpu_ring *ring, 378 struct amdgpu_job *job, struct dma_fence **fence) 379 { 380 struct amdgpu_device *adev = ring->adev; 381 unsigned vmhub = ring->funcs->vmhub; 382 struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; 383 struct amdgpu_vmid *idle = NULL; 384 struct amdgpu_vmid *id = NULL; 385 int r = 0; 386 387 mutex_lock(&id_mgr->lock); 388 r = amdgpu_vmid_grab_idle(vm, ring, &idle, fence); 389 if (r || !idle) 390 goto error; 391 392 if (vm->reserved_vmid[vmhub]) { 393 r = amdgpu_vmid_grab_reserved(vm, ring, job, &id, fence); 394 if (r || !id) 395 goto error; 396 } else { 397 r = amdgpu_vmid_grab_used(vm, ring, job, &id, fence); 398 if (r) 399 goto error; 400 401 if (!id) { 402 /* Still no ID to use? Then use the idle one found earlier */ 403 id = idle; 404 405 /* Remember this submission as user of the VMID */ 406 r = amdgpu_sync_fence(&id->active, 407 &job->base.s_fence->finished); 408 if (r) 409 goto error; 410 411 id->flushed_updates = amdgpu_vm_tlb_seq(vm); 412 job->vm_needs_flush = true; 413 } 414 415 list_move_tail(&id->list, &id_mgr->ids_lru); 416 } 417 418 id->pd_gpu_addr = job->vm_pd_addr; 419 id->owner = vm->immediate.fence_context; 420 421 if (job->vm_needs_flush) { 422 dma_fence_put(id->last_flush); 423 id->last_flush = NULL; 424 } 425 job->vmid = id - id_mgr->ids; 426 job->pasid = vm->pasid; 427 trace_amdgpu_vm_grab_id(vm, ring, job); 428 429 error: 430 mutex_unlock(&id_mgr->lock); 431 return r; 432 } 433 434 int amdgpu_vmid_alloc_reserved(struct amdgpu_device *adev, 435 struct amdgpu_vm *vm, 436 unsigned vmhub) 437 { 438 struct amdgpu_vmid_mgr *id_mgr; 439 struct amdgpu_vmid *idle; 440 int r = 0; 441 442 id_mgr = &adev->vm_manager.id_mgr[vmhub]; 443 mutex_lock(&id_mgr->lock); 444 if (vm->reserved_vmid[vmhub]) 445 goto unlock; 446 if (atomic_inc_return(&id_mgr->reserved_vmid_num) > 447 AMDGPU_VM_MAX_RESERVED_VMID) { 448 DRM_ERROR("Over limitation of reserved vmid\n"); 449 atomic_dec(&id_mgr->reserved_vmid_num); 450 r = -EINVAL; 451 goto unlock; 452 } 453 /* Select the first entry VMID */ 454 idle = list_first_entry(&id_mgr->ids_lru, struct amdgpu_vmid, list); 455 list_del_init(&idle->list); 456 vm->reserved_vmid[vmhub] = idle; 457 mutex_unlock(&id_mgr->lock); 458 459 return 0; 460 unlock: 461 mutex_unlock(&id_mgr->lock); 462 return r; 463 } 464 465 void amdgpu_vmid_free_reserved(struct amdgpu_device *adev, 466 struct amdgpu_vm *vm, 467 unsigned vmhub) 468 { 469 struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; 470 471 mutex_lock(&id_mgr->lock); 472 if (vm->reserved_vmid[vmhub]) { 473 list_add(&vm->reserved_vmid[vmhub]->list, 474 &id_mgr->ids_lru); 475 vm->reserved_vmid[vmhub] = NULL; 476 atomic_dec(&id_mgr->reserved_vmid_num); 477 } 478 mutex_unlock(&id_mgr->lock); 479 } 480 481 /** 482 * amdgpu_vmid_reset - reset VMID to zero 483 * 484 * @adev: amdgpu device structure 485 * @vmhub: vmhub type 486 * @vmid: vmid number to use 487 * 488 * Reset saved GDW, GWS and OA to force switch on next flush. 489 */ 490 void amdgpu_vmid_reset(struct amdgpu_device *adev, unsigned vmhub, 491 unsigned vmid) 492 { 493 struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; 494 struct amdgpu_vmid *id = &id_mgr->ids[vmid]; 495 496 mutex_lock(&id_mgr->lock); 497 id->owner = 0; 498 id->gds_base = 0; 499 id->gds_size = 0; 500 id->gws_base = 0; 501 id->gws_size = 0; 502 id->oa_base = 0; 503 id->oa_size = 0; 504 mutex_unlock(&id_mgr->lock); 505 } 506 507 /** 508 * amdgpu_vmid_reset_all - reset VMID to zero 509 * 510 * @adev: amdgpu device structure 511 * 512 * Reset VMID to force flush on next use 513 */ 514 void amdgpu_vmid_reset_all(struct amdgpu_device *adev) 515 { 516 unsigned i, j; 517 518 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { 519 struct amdgpu_vmid_mgr *id_mgr = 520 &adev->vm_manager.id_mgr[i]; 521 522 for (j = 1; j < id_mgr->num_ids; ++j) 523 amdgpu_vmid_reset(adev, i, j); 524 } 525 } 526 527 /** 528 * amdgpu_vmid_mgr_init - init the VMID manager 529 * 530 * @adev: amdgpu_device pointer 531 * 532 * Initialize the VM manager structures 533 */ 534 void amdgpu_vmid_mgr_init(struct amdgpu_device *adev) 535 { 536 unsigned i, j; 537 538 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { 539 struct amdgpu_vmid_mgr *id_mgr = 540 &adev->vm_manager.id_mgr[i]; 541 542 mutex_init(&id_mgr->lock); 543 INIT_LIST_HEAD(&id_mgr->ids_lru); 544 atomic_set(&id_mgr->reserved_vmid_num, 0); 545 546 /* manage only VMIDs not used by KFD */ 547 id_mgr->num_ids = adev->vm_manager.first_kfd_vmid; 548 549 /* skip over VMID 0, since it is the system VM */ 550 for (j = 1; j < id_mgr->num_ids; ++j) { 551 amdgpu_vmid_reset(adev, i, j); 552 amdgpu_sync_create(&id_mgr->ids[j].active); 553 list_add_tail(&id_mgr->ids[j].list, &id_mgr->ids_lru); 554 } 555 } 556 } 557 558 /** 559 * amdgpu_vmid_mgr_fini - cleanup VM manager 560 * 561 * @adev: amdgpu_device pointer 562 * 563 * Cleanup the VM manager and free resources. 564 */ 565 void amdgpu_vmid_mgr_fini(struct amdgpu_device *adev) 566 { 567 unsigned i, j; 568 569 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { 570 struct amdgpu_vmid_mgr *id_mgr = 571 &adev->vm_manager.id_mgr[i]; 572 573 mutex_destroy(&id_mgr->lock); 574 for (j = 0; j < AMDGPU_NUM_VMID; ++j) { 575 struct amdgpu_vmid *id = &id_mgr->ids[j]; 576 577 amdgpu_sync_free(&id->active); 578 dma_fence_put(id->last_flush); 579 dma_fence_put(id->pasid_mapping); 580 } 581 } 582 } 583