1 /* 2 * Copyright 2018 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 #include "nouveau_svm.h" 23 #include "nouveau_drv.h" 24 #include "nouveau_chan.h" 25 #include "nouveau_dmem.h" 26 27 #include <nvif/notify.h> 28 #include <nvif/object.h> 29 #include <nvif/vmm.h> 30 31 #include <nvif/class.h> 32 #include <nvif/clb069.h> 33 #include <nvif/ifc00d.h> 34 35 #include <linux/sched/mm.h> 36 #include <linux/sort.h> 37 #include <linux/hmm.h> 38 39 struct nouveau_svm { 40 struct nouveau_drm *drm; 41 struct mutex mutex; 42 struct list_head inst; 43 44 struct nouveau_svm_fault_buffer { 45 int id; 46 struct nvif_object object; 47 u32 entries; 48 u32 getaddr; 49 u32 putaddr; 50 u32 get; 51 u32 put; 52 struct nvif_notify notify; 53 54 struct nouveau_svm_fault { 55 u64 inst; 56 u64 addr; 57 u64 time; 58 u32 engine; 59 u8 gpc; 60 u8 hub; 61 u8 access; 62 u8 client; 63 u8 fault; 64 struct nouveau_svmm *svmm; 65 } **fault; 66 int fault_nr; 67 } buffer[1]; 68 }; 69 70 #define SVM_DBG(s,f,a...) NV_DEBUG((s)->drm, "svm: "f"\n", ##a) 71 #define SVM_ERR(s,f,a...) NV_WARN((s)->drm, "svm: "f"\n", ##a) 72 73 struct nouveau_ivmm { 74 struct nouveau_svmm *svmm; 75 u64 inst; 76 struct list_head head; 77 }; 78 79 static struct nouveau_ivmm * 80 nouveau_ivmm_find(struct nouveau_svm *svm, u64 inst) 81 { 82 struct nouveau_ivmm *ivmm; 83 list_for_each_entry(ivmm, &svm->inst, head) { 84 if (ivmm->inst == inst) 85 return ivmm; 86 } 87 return NULL; 88 } 89 90 struct nouveau_svmm { 91 struct nouveau_vmm *vmm; 92 struct { 93 unsigned long start; 94 unsigned long limit; 95 } unmanaged; 96 97 struct mutex mutex; 98 99 struct mm_struct *mm; 100 struct hmm_mirror mirror; 101 }; 102 103 #define SVMM_DBG(s,f,a...) \ 104 NV_DEBUG((s)->vmm->cli->drm, "svm-%p: "f"\n", (s), ##a) 105 #define SVMM_ERR(s,f,a...) \ 106 NV_WARN((s)->vmm->cli->drm, "svm-%p: "f"\n", (s), ##a) 107 108 int 109 nouveau_svmm_bind(struct drm_device *dev, void *data, 110 struct drm_file *file_priv) 111 { 112 struct nouveau_cli *cli = nouveau_cli(file_priv); 113 struct drm_nouveau_svm_bind *args = data; 114 unsigned target, cmd, priority; 115 unsigned long addr, end, size; 116 struct mm_struct *mm; 117 118 args->va_start &= PAGE_MASK; 119 args->va_end &= PAGE_MASK; 120 121 /* Sanity check arguments */ 122 if (args->reserved0 || args->reserved1) 123 return -EINVAL; 124 if (args->header & (~NOUVEAU_SVM_BIND_VALID_MASK)) 125 return -EINVAL; 126 if (args->va_start >= args->va_end) 127 return -EINVAL; 128 if (!args->npages) 129 return -EINVAL; 130 131 cmd = args->header >> NOUVEAU_SVM_BIND_COMMAND_SHIFT; 132 cmd &= NOUVEAU_SVM_BIND_COMMAND_MASK; 133 switch (cmd) { 134 case NOUVEAU_SVM_BIND_COMMAND__MIGRATE: 135 break; 136 default: 137 return -EINVAL; 138 } 139 140 priority = args->header >> NOUVEAU_SVM_BIND_PRIORITY_SHIFT; 141 priority &= NOUVEAU_SVM_BIND_PRIORITY_MASK; 142 143 /* FIXME support CPU target ie all target value < GPU_VRAM */ 144 target = args->header >> NOUVEAU_SVM_BIND_TARGET_SHIFT; 145 target &= NOUVEAU_SVM_BIND_TARGET_MASK; 146 switch (target) { 147 case NOUVEAU_SVM_BIND_TARGET__GPU_VRAM: 148 break; 149 default: 150 return -EINVAL; 151 } 152 153 /* 154 * FIXME: For now refuse non 0 stride, we need to change the migrate 155 * kernel function to handle stride to avoid to create a mess within 156 * each device driver. 157 */ 158 if (args->stride) 159 return -EINVAL; 160 161 size = ((unsigned long)args->npages) << PAGE_SHIFT; 162 if ((args->va_start + size) <= args->va_start) 163 return -EINVAL; 164 if ((args->va_start + size) > args->va_end) 165 return -EINVAL; 166 167 /* 168 * Ok we are ask to do something sane, for now we only support migrate 169 * commands but we will add things like memory policy (what to do on 170 * page fault) and maybe some other commands. 171 */ 172 173 mm = get_task_mm(current); 174 down_read(&mm->mmap_sem); 175 176 for (addr = args->va_start, end = args->va_start + size; addr < end;) { 177 struct vm_area_struct *vma; 178 unsigned long next; 179 180 vma = find_vma_intersection(mm, addr, end); 181 if (!vma) 182 break; 183 184 next = min(vma->vm_end, end); 185 /* This is a best effort so we ignore errors */ 186 nouveau_dmem_migrate_vma(cli->drm, vma, addr, next); 187 addr = next; 188 } 189 190 /* 191 * FIXME Return the number of page we have migrated, again we need to 192 * update the migrate API to return that information so that we can 193 * report it to user space. 194 */ 195 args->result = 0; 196 197 up_read(&mm->mmap_sem); 198 mmput(mm); 199 200 return 0; 201 } 202 203 /* Unlink channel instance from SVMM. */ 204 void 205 nouveau_svmm_part(struct nouveau_svmm *svmm, u64 inst) 206 { 207 struct nouveau_ivmm *ivmm; 208 if (svmm) { 209 mutex_lock(&svmm->vmm->cli->drm->svm->mutex); 210 ivmm = nouveau_ivmm_find(svmm->vmm->cli->drm->svm, inst); 211 if (ivmm) { 212 list_del(&ivmm->head); 213 kfree(ivmm); 214 } 215 mutex_unlock(&svmm->vmm->cli->drm->svm->mutex); 216 } 217 } 218 219 /* Link channel instance to SVMM. */ 220 int 221 nouveau_svmm_join(struct nouveau_svmm *svmm, u64 inst) 222 { 223 struct nouveau_ivmm *ivmm; 224 if (svmm) { 225 if (!(ivmm = kmalloc(sizeof(*ivmm), GFP_KERNEL))) 226 return -ENOMEM; 227 ivmm->svmm = svmm; 228 ivmm->inst = inst; 229 230 mutex_lock(&svmm->vmm->cli->drm->svm->mutex); 231 list_add(&ivmm->head, &svmm->vmm->cli->drm->svm->inst); 232 mutex_unlock(&svmm->vmm->cli->drm->svm->mutex); 233 } 234 return 0; 235 } 236 237 /* Invalidate SVMM address-range on GPU. */ 238 static void 239 nouveau_svmm_invalidate(struct nouveau_svmm *svmm, u64 start, u64 limit) 240 { 241 if (limit > start) { 242 bool super = svmm->vmm->vmm.object.client->super; 243 svmm->vmm->vmm.object.client->super = true; 244 nvif_object_mthd(&svmm->vmm->vmm.object, NVIF_VMM_V0_PFNCLR, 245 &(struct nvif_vmm_pfnclr_v0) { 246 .addr = start, 247 .size = limit - start, 248 }, sizeof(struct nvif_vmm_pfnclr_v0)); 249 svmm->vmm->vmm.object.client->super = super; 250 } 251 } 252 253 static int 254 nouveau_svmm_sync_cpu_device_pagetables(struct hmm_mirror *mirror, 255 const struct mmu_notifier_range *update) 256 { 257 struct nouveau_svmm *svmm = container_of(mirror, typeof(*svmm), mirror); 258 unsigned long start = update->start; 259 unsigned long limit = update->end; 260 261 if (!mmu_notifier_range_blockable(update)) 262 return -EAGAIN; 263 264 SVMM_DBG(svmm, "invalidate %016lx-%016lx", start, limit); 265 266 mutex_lock(&svmm->mutex); 267 if (limit > svmm->unmanaged.start && start < svmm->unmanaged.limit) { 268 if (start < svmm->unmanaged.start) { 269 nouveau_svmm_invalidate(svmm, start, 270 svmm->unmanaged.limit); 271 } 272 start = svmm->unmanaged.limit; 273 } 274 275 nouveau_svmm_invalidate(svmm, start, limit); 276 mutex_unlock(&svmm->mutex); 277 return 0; 278 } 279 280 static void 281 nouveau_svmm_release(struct hmm_mirror *mirror) 282 { 283 } 284 285 static const struct hmm_mirror_ops 286 nouveau_svmm = { 287 .sync_cpu_device_pagetables = nouveau_svmm_sync_cpu_device_pagetables, 288 .release = nouveau_svmm_release, 289 }; 290 291 void 292 nouveau_svmm_fini(struct nouveau_svmm **psvmm) 293 { 294 struct nouveau_svmm *svmm = *psvmm; 295 if (svmm) { 296 hmm_mirror_unregister(&svmm->mirror); 297 kfree(*psvmm); 298 *psvmm = NULL; 299 } 300 } 301 302 int 303 nouveau_svmm_init(struct drm_device *dev, void *data, 304 struct drm_file *file_priv) 305 { 306 struct nouveau_cli *cli = nouveau_cli(file_priv); 307 struct nouveau_svmm *svmm; 308 struct drm_nouveau_svm_init *args = data; 309 int ret; 310 311 /* Allocate tracking for SVM-enabled VMM. */ 312 if (!(svmm = kzalloc(sizeof(*svmm), GFP_KERNEL))) 313 return -ENOMEM; 314 svmm->vmm = &cli->svm; 315 svmm->unmanaged.start = args->unmanaged_addr; 316 svmm->unmanaged.limit = args->unmanaged_addr + args->unmanaged_size; 317 mutex_init(&svmm->mutex); 318 319 /* Check that SVM isn't already enabled for the client. */ 320 mutex_lock(&cli->mutex); 321 if (cli->svm.cli) { 322 ret = -EBUSY; 323 goto done; 324 } 325 326 /* Allocate a new GPU VMM that can support SVM (managed by the 327 * client, with replayable faults enabled). 328 * 329 * All future channel/memory allocations will make use of this 330 * VMM instead of the standard one. 331 */ 332 ret = nvif_vmm_init(&cli->mmu, cli->vmm.vmm.object.oclass, true, 333 args->unmanaged_addr, args->unmanaged_size, 334 &(struct gp100_vmm_v0) { 335 .fault_replay = true, 336 }, sizeof(struct gp100_vmm_v0), &cli->svm.vmm); 337 if (ret) 338 goto done; 339 340 /* Enable HMM mirroring of CPU address-space to VMM. */ 341 svmm->mm = get_task_mm(current); 342 down_write(&svmm->mm->mmap_sem); 343 svmm->mirror.ops = &nouveau_svmm; 344 ret = hmm_mirror_register(&svmm->mirror, svmm->mm); 345 if (ret == 0) { 346 cli->svm.svmm = svmm; 347 cli->svm.cli = cli; 348 } 349 up_write(&svmm->mm->mmap_sem); 350 mmput(svmm->mm); 351 352 done: 353 if (ret) 354 nouveau_svmm_fini(&svmm); 355 mutex_unlock(&cli->mutex); 356 return ret; 357 } 358 359 static const u64 360 nouveau_svm_pfn_flags[HMM_PFN_FLAG_MAX] = { 361 [HMM_PFN_VALID ] = NVIF_VMM_PFNMAP_V0_V, 362 [HMM_PFN_WRITE ] = NVIF_VMM_PFNMAP_V0_W, 363 [HMM_PFN_DEVICE_PRIVATE] = NVIF_VMM_PFNMAP_V0_VRAM, 364 }; 365 366 static const u64 367 nouveau_svm_pfn_values[HMM_PFN_VALUE_MAX] = { 368 [HMM_PFN_ERROR ] = ~NVIF_VMM_PFNMAP_V0_V, 369 [HMM_PFN_NONE ] = NVIF_VMM_PFNMAP_V0_NONE, 370 [HMM_PFN_SPECIAL] = ~NVIF_VMM_PFNMAP_V0_V, 371 }; 372 373 /* Issue fault replay for GPU to retry accesses that faulted previously. */ 374 static void 375 nouveau_svm_fault_replay(struct nouveau_svm *svm) 376 { 377 SVM_DBG(svm, "replay"); 378 WARN_ON(nvif_object_mthd(&svm->drm->client.vmm.vmm.object, 379 GP100_VMM_VN_FAULT_REPLAY, 380 &(struct gp100_vmm_fault_replay_vn) {}, 381 sizeof(struct gp100_vmm_fault_replay_vn))); 382 } 383 384 /* Cancel a replayable fault that could not be handled. 385 * 386 * Cancelling the fault will trigger recovery to reset the engine 387 * and kill the offending channel (ie. GPU SIGSEGV). 388 */ 389 static void 390 nouveau_svm_fault_cancel(struct nouveau_svm *svm, 391 u64 inst, u8 hub, u8 gpc, u8 client) 392 { 393 SVM_DBG(svm, "cancel %016llx %d %02x %02x", inst, hub, gpc, client); 394 WARN_ON(nvif_object_mthd(&svm->drm->client.vmm.vmm.object, 395 GP100_VMM_VN_FAULT_CANCEL, 396 &(struct gp100_vmm_fault_cancel_v0) { 397 .hub = hub, 398 .gpc = gpc, 399 .client = client, 400 .inst = inst, 401 }, sizeof(struct gp100_vmm_fault_cancel_v0))); 402 } 403 404 static void 405 nouveau_svm_fault_cancel_fault(struct nouveau_svm *svm, 406 struct nouveau_svm_fault *fault) 407 { 408 nouveau_svm_fault_cancel(svm, fault->inst, 409 fault->hub, 410 fault->gpc, 411 fault->client); 412 } 413 414 static int 415 nouveau_svm_fault_cmp(const void *a, const void *b) 416 { 417 const struct nouveau_svm_fault *fa = *(struct nouveau_svm_fault **)a; 418 const struct nouveau_svm_fault *fb = *(struct nouveau_svm_fault **)b; 419 int ret; 420 if ((ret = (s64)fa->inst - fb->inst)) 421 return ret; 422 if ((ret = (s64)fa->addr - fb->addr)) 423 return ret; 424 /*XXX: atomic? */ 425 return (fa->access == 0 || fa->access == 3) - 426 (fb->access == 0 || fb->access == 3); 427 } 428 429 static void 430 nouveau_svm_fault_cache(struct nouveau_svm *svm, 431 struct nouveau_svm_fault_buffer *buffer, u32 offset) 432 { 433 struct nvif_object *memory = &buffer->object; 434 const u32 instlo = nvif_rd32(memory, offset + 0x00); 435 const u32 insthi = nvif_rd32(memory, offset + 0x04); 436 const u32 addrlo = nvif_rd32(memory, offset + 0x08); 437 const u32 addrhi = nvif_rd32(memory, offset + 0x0c); 438 const u32 timelo = nvif_rd32(memory, offset + 0x10); 439 const u32 timehi = nvif_rd32(memory, offset + 0x14); 440 const u32 engine = nvif_rd32(memory, offset + 0x18); 441 const u32 info = nvif_rd32(memory, offset + 0x1c); 442 const u64 inst = (u64)insthi << 32 | instlo; 443 const u8 gpc = (info & 0x1f000000) >> 24; 444 const u8 hub = (info & 0x00100000) >> 20; 445 const u8 client = (info & 0x00007f00) >> 8; 446 struct nouveau_svm_fault *fault; 447 448 //XXX: i think we're supposed to spin waiting */ 449 if (WARN_ON(!(info & 0x80000000))) 450 return; 451 452 nvif_mask(memory, offset + 0x1c, 0x80000000, 0x00000000); 453 454 if (!buffer->fault[buffer->fault_nr]) { 455 fault = kmalloc(sizeof(*fault), GFP_KERNEL); 456 if (WARN_ON(!fault)) { 457 nouveau_svm_fault_cancel(svm, inst, hub, gpc, client); 458 return; 459 } 460 buffer->fault[buffer->fault_nr] = fault; 461 } 462 463 fault = buffer->fault[buffer->fault_nr++]; 464 fault->inst = inst; 465 fault->addr = (u64)addrhi << 32 | addrlo; 466 fault->time = (u64)timehi << 32 | timelo; 467 fault->engine = engine; 468 fault->gpc = gpc; 469 fault->hub = hub; 470 fault->access = (info & 0x000f0000) >> 16; 471 fault->client = client; 472 fault->fault = (info & 0x0000001f); 473 474 SVM_DBG(svm, "fault %016llx %016llx %02x", 475 fault->inst, fault->addr, fault->access); 476 } 477 478 static inline bool 479 nouveau_range_done(struct hmm_range *range) 480 { 481 bool ret = hmm_range_valid(range); 482 483 hmm_range_unregister(range); 484 return ret; 485 } 486 487 static int 488 nouveau_range_fault(struct nouveau_svmm *svmm, struct hmm_range *range) 489 { 490 long ret; 491 492 range->default_flags = 0; 493 range->pfn_flags_mask = -1UL; 494 495 ret = hmm_range_register(range, &svmm->mirror); 496 if (ret) { 497 up_read(&svmm->mm->mmap_sem); 498 return (int)ret; 499 } 500 501 if (!hmm_range_wait_until_valid(range, HMM_RANGE_DEFAULT_TIMEOUT)) { 502 up_read(&svmm->mm->mmap_sem); 503 return -EBUSY; 504 } 505 506 ret = hmm_range_fault(range, 0); 507 if (ret <= 0) { 508 if (ret == 0) 509 ret = -EBUSY; 510 up_read(&svmm->mm->mmap_sem); 511 hmm_range_unregister(range); 512 return ret; 513 } 514 return 0; 515 } 516 517 static int 518 nouveau_svm_fault(struct nvif_notify *notify) 519 { 520 struct nouveau_svm_fault_buffer *buffer = 521 container_of(notify, typeof(*buffer), notify); 522 struct nouveau_svm *svm = 523 container_of(buffer, typeof(*svm), buffer[buffer->id]); 524 struct nvif_object *device = &svm->drm->client.device.object; 525 struct nouveau_svmm *svmm; 526 struct { 527 struct { 528 struct nvif_ioctl_v0 i; 529 struct nvif_ioctl_mthd_v0 m; 530 struct nvif_vmm_pfnmap_v0 p; 531 } i; 532 u64 phys[16]; 533 } args; 534 struct hmm_range range; 535 struct vm_area_struct *vma; 536 u64 inst, start, limit; 537 int fi, fn, pi, fill; 538 int replay = 0, ret; 539 540 /* Parse available fault buffer entries into a cache, and update 541 * the GET pointer so HW can reuse the entries. 542 */ 543 SVM_DBG(svm, "fault handler"); 544 if (buffer->get == buffer->put) { 545 buffer->put = nvif_rd32(device, buffer->putaddr); 546 buffer->get = nvif_rd32(device, buffer->getaddr); 547 if (buffer->get == buffer->put) 548 return NVIF_NOTIFY_KEEP; 549 } 550 buffer->fault_nr = 0; 551 552 SVM_DBG(svm, "get %08x put %08x", buffer->get, buffer->put); 553 while (buffer->get != buffer->put) { 554 nouveau_svm_fault_cache(svm, buffer, buffer->get * 0x20); 555 if (++buffer->get == buffer->entries) 556 buffer->get = 0; 557 } 558 nvif_wr32(device, buffer->getaddr, buffer->get); 559 SVM_DBG(svm, "%d fault(s) pending", buffer->fault_nr); 560 561 /* Sort parsed faults by instance pointer to prevent unnecessary 562 * instance to SVMM translations, followed by address and access 563 * type to reduce the amount of work when handling the faults. 564 */ 565 sort(buffer->fault, buffer->fault_nr, sizeof(*buffer->fault), 566 nouveau_svm_fault_cmp, NULL); 567 568 /* Lookup SVMM structure for each unique instance pointer. */ 569 mutex_lock(&svm->mutex); 570 for (fi = 0, svmm = NULL; fi < buffer->fault_nr; fi++) { 571 if (!svmm || buffer->fault[fi]->inst != inst) { 572 struct nouveau_ivmm *ivmm = 573 nouveau_ivmm_find(svm, buffer->fault[fi]->inst); 574 svmm = ivmm ? ivmm->svmm : NULL; 575 inst = buffer->fault[fi]->inst; 576 SVM_DBG(svm, "inst %016llx -> svm-%p", inst, svmm); 577 } 578 buffer->fault[fi]->svmm = svmm; 579 } 580 mutex_unlock(&svm->mutex); 581 582 /* Process list of faults. */ 583 args.i.i.version = 0; 584 args.i.i.type = NVIF_IOCTL_V0_MTHD; 585 args.i.m.version = 0; 586 args.i.m.method = NVIF_VMM_V0_PFNMAP; 587 args.i.p.version = 0; 588 589 for (fi = 0; fn = fi + 1, fi < buffer->fault_nr; fi = fn) { 590 /* Cancel any faults from non-SVM channels. */ 591 if (!(svmm = buffer->fault[fi]->svmm)) { 592 nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]); 593 continue; 594 } 595 SVMM_DBG(svmm, "addr %016llx", buffer->fault[fi]->addr); 596 597 /* We try and group handling of faults within a small 598 * window into a single update. 599 */ 600 start = buffer->fault[fi]->addr; 601 limit = start + (ARRAY_SIZE(args.phys) << PAGE_SHIFT); 602 if (start < svmm->unmanaged.limit) 603 limit = min_t(u64, limit, svmm->unmanaged.start); 604 else 605 if (limit > svmm->unmanaged.start) 606 start = max_t(u64, start, svmm->unmanaged.limit); 607 SVMM_DBG(svmm, "wndw %016llx-%016llx", start, limit); 608 609 /* Intersect fault window with the CPU VMA, cancelling 610 * the fault if the address is invalid. 611 */ 612 down_read(&svmm->mm->mmap_sem); 613 vma = find_vma_intersection(svmm->mm, start, limit); 614 if (!vma) { 615 SVMM_ERR(svmm, "wndw %016llx-%016llx", start, limit); 616 up_read(&svmm->mm->mmap_sem); 617 nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]); 618 continue; 619 } 620 start = max_t(u64, start, vma->vm_start); 621 limit = min_t(u64, limit, vma->vm_end); 622 SVMM_DBG(svmm, "wndw %016llx-%016llx", start, limit); 623 624 if (buffer->fault[fi]->addr != start) { 625 SVMM_ERR(svmm, "addr %016llx", buffer->fault[fi]->addr); 626 up_read(&svmm->mm->mmap_sem); 627 nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]); 628 continue; 629 } 630 631 /* Prepare the GPU-side update of all pages within the 632 * fault window, determining required pages and access 633 * permissions based on pending faults. 634 */ 635 args.i.p.page = PAGE_SHIFT; 636 args.i.p.addr = start; 637 for (fn = fi, pi = 0;;) { 638 /* Determine required permissions based on GPU fault 639 * access flags. 640 *XXX: atomic? 641 */ 642 if (buffer->fault[fn]->access != 0 /* READ. */ && 643 buffer->fault[fn]->access != 3 /* PREFETCH. */) { 644 args.phys[pi++] = NVIF_VMM_PFNMAP_V0_V | 645 NVIF_VMM_PFNMAP_V0_W; 646 } else { 647 args.phys[pi++] = NVIF_VMM_PFNMAP_V0_V; 648 } 649 args.i.p.size = pi << PAGE_SHIFT; 650 651 /* It's okay to skip over duplicate addresses from the 652 * same SVMM as faults are ordered by access type such 653 * that only the first one needs to be handled. 654 * 655 * ie. WRITE faults appear first, thus any handling of 656 * pending READ faults will already be satisfied. 657 */ 658 while (++fn < buffer->fault_nr && 659 buffer->fault[fn]->svmm == svmm && 660 buffer->fault[fn ]->addr == 661 buffer->fault[fn - 1]->addr); 662 663 /* If the next fault is outside the window, or all GPU 664 * faults have been dealt with, we're done here. 665 */ 666 if (fn >= buffer->fault_nr || 667 buffer->fault[fn]->svmm != svmm || 668 buffer->fault[fn]->addr >= limit) 669 break; 670 671 /* Fill in the gap between this fault and the next. */ 672 fill = (buffer->fault[fn ]->addr - 673 buffer->fault[fn - 1]->addr) >> PAGE_SHIFT; 674 while (--fill) 675 args.phys[pi++] = NVIF_VMM_PFNMAP_V0_NONE; 676 } 677 678 SVMM_DBG(svmm, "wndw %016llx-%016llx covering %d fault(s)", 679 args.i.p.addr, 680 args.i.p.addr + args.i.p.size, fn - fi); 681 682 /* Have HMM fault pages within the fault window to the GPU. */ 683 range.start = args.i.p.addr; 684 range.end = args.i.p.addr + args.i.p.size; 685 range.pfns = args.phys; 686 range.flags = nouveau_svm_pfn_flags; 687 range.values = nouveau_svm_pfn_values; 688 range.pfn_shift = NVIF_VMM_PFNMAP_V0_ADDR_SHIFT; 689 again: 690 ret = nouveau_range_fault(svmm, &range); 691 if (ret == 0) { 692 mutex_lock(&svmm->mutex); 693 if (!nouveau_range_done(&range)) { 694 mutex_unlock(&svmm->mutex); 695 goto again; 696 } 697 698 nouveau_dmem_convert_pfn(svm->drm, &range); 699 700 svmm->vmm->vmm.object.client->super = true; 701 ret = nvif_object_ioctl(&svmm->vmm->vmm.object, 702 &args, sizeof(args.i) + 703 pi * sizeof(args.phys[0]), 704 NULL); 705 svmm->vmm->vmm.object.client->super = false; 706 mutex_unlock(&svmm->mutex); 707 up_read(&svmm->mm->mmap_sem); 708 } 709 710 /* Cancel any faults in the window whose pages didn't manage 711 * to keep their valid bit, or stay writeable when required. 712 * 713 * If handling failed completely, cancel all faults. 714 */ 715 while (fi < fn) { 716 struct nouveau_svm_fault *fault = buffer->fault[fi++]; 717 pi = (fault->addr - range.start) >> PAGE_SHIFT; 718 if (ret || 719 !(range.pfns[pi] & NVIF_VMM_PFNMAP_V0_V) || 720 (!(range.pfns[pi] & NVIF_VMM_PFNMAP_V0_W) && 721 fault->access != 0 && fault->access != 3)) { 722 nouveau_svm_fault_cancel_fault(svm, fault); 723 continue; 724 } 725 replay++; 726 } 727 } 728 729 /* Issue fault replay to the GPU. */ 730 if (replay) 731 nouveau_svm_fault_replay(svm); 732 return NVIF_NOTIFY_KEEP; 733 } 734 735 static void 736 nouveau_svm_fault_buffer_fini(struct nouveau_svm *svm, int id) 737 { 738 struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id]; 739 nvif_notify_put(&buffer->notify); 740 } 741 742 static int 743 nouveau_svm_fault_buffer_init(struct nouveau_svm *svm, int id) 744 { 745 struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id]; 746 struct nvif_object *device = &svm->drm->client.device.object; 747 buffer->get = nvif_rd32(device, buffer->getaddr); 748 buffer->put = nvif_rd32(device, buffer->putaddr); 749 SVM_DBG(svm, "get %08x put %08x (init)", buffer->get, buffer->put); 750 return nvif_notify_get(&buffer->notify); 751 } 752 753 static void 754 nouveau_svm_fault_buffer_dtor(struct nouveau_svm *svm, int id) 755 { 756 struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id]; 757 int i; 758 759 if (buffer->fault) { 760 for (i = 0; buffer->fault[i] && i < buffer->entries; i++) 761 kfree(buffer->fault[i]); 762 kvfree(buffer->fault); 763 } 764 765 nouveau_svm_fault_buffer_fini(svm, id); 766 767 nvif_notify_fini(&buffer->notify); 768 nvif_object_fini(&buffer->object); 769 } 770 771 static int 772 nouveau_svm_fault_buffer_ctor(struct nouveau_svm *svm, s32 oclass, int id) 773 { 774 struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id]; 775 struct nouveau_drm *drm = svm->drm; 776 struct nvif_object *device = &drm->client.device.object; 777 struct nvif_clb069_v0 args = {}; 778 int ret; 779 780 buffer->id = id; 781 782 ret = nvif_object_init(device, 0, oclass, &args, sizeof(args), 783 &buffer->object); 784 if (ret < 0) { 785 SVM_ERR(svm, "Fault buffer allocation failed: %d", ret); 786 return ret; 787 } 788 789 nvif_object_map(&buffer->object, NULL, 0); 790 buffer->entries = args.entries; 791 buffer->getaddr = args.get; 792 buffer->putaddr = args.put; 793 794 ret = nvif_notify_init(&buffer->object, nouveau_svm_fault, true, 795 NVB069_V0_NTFY_FAULT, NULL, 0, 0, 796 &buffer->notify); 797 if (ret) 798 return ret; 799 800 buffer->fault = kvzalloc(sizeof(*buffer->fault) * buffer->entries, GFP_KERNEL); 801 if (!buffer->fault) 802 return -ENOMEM; 803 804 return nouveau_svm_fault_buffer_init(svm, id); 805 } 806 807 void 808 nouveau_svm_resume(struct nouveau_drm *drm) 809 { 810 struct nouveau_svm *svm = drm->svm; 811 if (svm) 812 nouveau_svm_fault_buffer_init(svm, 0); 813 } 814 815 void 816 nouveau_svm_suspend(struct nouveau_drm *drm) 817 { 818 struct nouveau_svm *svm = drm->svm; 819 if (svm) 820 nouveau_svm_fault_buffer_fini(svm, 0); 821 } 822 823 void 824 nouveau_svm_fini(struct nouveau_drm *drm) 825 { 826 struct nouveau_svm *svm = drm->svm; 827 if (svm) { 828 nouveau_svm_fault_buffer_dtor(svm, 0); 829 kfree(drm->svm); 830 drm->svm = NULL; 831 } 832 } 833 834 void 835 nouveau_svm_init(struct nouveau_drm *drm) 836 { 837 static const struct nvif_mclass buffers[] = { 838 { VOLTA_FAULT_BUFFER_A, 0 }, 839 { MAXWELL_FAULT_BUFFER_A, 0 }, 840 {} 841 }; 842 struct nouveau_svm *svm; 843 int ret; 844 845 /* Disable on Volta and newer until channel recovery is fixed, 846 * otherwise clients will have a trivial way to trash the GPU 847 * for everyone. 848 */ 849 if (drm->client.device.info.family > NV_DEVICE_INFO_V0_PASCAL) 850 return; 851 852 if (!(drm->svm = svm = kzalloc(sizeof(*drm->svm), GFP_KERNEL))) 853 return; 854 855 drm->svm->drm = drm; 856 mutex_init(&drm->svm->mutex); 857 INIT_LIST_HEAD(&drm->svm->inst); 858 859 ret = nvif_mclass(&drm->client.device.object, buffers); 860 if (ret < 0) { 861 SVM_DBG(svm, "No supported fault buffer class"); 862 nouveau_svm_fini(drm); 863 return; 864 } 865 866 ret = nouveau_svm_fault_buffer_ctor(svm, buffers[ret].oclass, 0); 867 if (ret) { 868 nouveau_svm_fini(drm); 869 return; 870 } 871 872 SVM_DBG(svm, "Initialised"); 873 } 874