1 /* 2 * Copyright 2018 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 #include "nouveau_dmem.h" 23 #include "nouveau_drv.h" 24 #include "nouveau_chan.h" 25 #include "nouveau_dma.h" 26 #include "nouveau_mem.h" 27 #include "nouveau_bo.h" 28 29 #include <nvif/class.h> 30 #include <nvif/object.h> 31 #include <nvif/if500b.h> 32 #include <nvif/if900b.h> 33 34 #include <linux/sched/mm.h> 35 #include <linux/hmm.h> 36 37 /* 38 * FIXME: this is ugly right now we are using TTM to allocate vram and we pin 39 * it in vram while in use. We likely want to overhaul memory management for 40 * nouveau to be more page like (not necessarily with system page size but a 41 * bigger page size) at lowest level and have some shim layer on top that would 42 * provide the same functionality as TTM. 43 */ 44 #define DMEM_CHUNK_SIZE (2UL << 20) 45 #define DMEM_CHUNK_NPAGES (DMEM_CHUNK_SIZE >> PAGE_SHIFT) 46 47 struct nouveau_migrate; 48 49 enum nouveau_aper { 50 NOUVEAU_APER_VIRT, 51 NOUVEAU_APER_VRAM, 52 NOUVEAU_APER_HOST, 53 }; 54 55 typedef int (*nouveau_migrate_copy_t)(struct nouveau_drm *drm, u64 npages, 56 enum nouveau_aper, u64 dst_addr, 57 enum nouveau_aper, u64 src_addr); 58 59 struct nouveau_dmem_chunk { 60 struct list_head list; 61 struct nouveau_bo *bo; 62 struct nouveau_drm *drm; 63 unsigned long pfn_first; 64 unsigned long callocated; 65 unsigned long bitmap[BITS_TO_LONGS(DMEM_CHUNK_NPAGES)]; 66 spinlock_t lock; 67 }; 68 69 struct nouveau_dmem_migrate { 70 nouveau_migrate_copy_t copy_func; 71 struct nouveau_channel *chan; 72 }; 73 74 struct nouveau_dmem { 75 struct hmm_devmem *devmem; 76 struct nouveau_dmem_migrate migrate; 77 struct list_head chunk_free; 78 struct list_head chunk_full; 79 struct list_head chunk_empty; 80 struct mutex mutex; 81 }; 82 83 struct nouveau_dmem_fault { 84 struct nouveau_drm *drm; 85 struct nouveau_fence *fence; 86 dma_addr_t *dma; 87 unsigned long npages; 88 }; 89 90 struct nouveau_migrate { 91 struct vm_area_struct *vma; 92 struct nouveau_drm *drm; 93 struct nouveau_fence *fence; 94 unsigned long npages; 95 dma_addr_t *dma; 96 unsigned long dma_nr; 97 }; 98 99 static void 100 nouveau_dmem_free(struct hmm_devmem *devmem, struct page *page) 101 { 102 struct nouveau_dmem_chunk *chunk; 103 struct nouveau_drm *drm; 104 unsigned long idx; 105 106 chunk = (void *)hmm_devmem_page_get_drvdata(page); 107 idx = page_to_pfn(page) - chunk->pfn_first; 108 drm = chunk->drm; 109 110 /* 111 * FIXME: 112 * 113 * This is really a bad example, we need to overhaul nouveau memory 114 * management to be more page focus and allow lighter locking scheme 115 * to be use in the process. 116 */ 117 spin_lock(&chunk->lock); 118 clear_bit(idx, chunk->bitmap); 119 WARN_ON(!chunk->callocated); 120 chunk->callocated--; 121 /* 122 * FIXME when chunk->callocated reach 0 we should add the chunk to 123 * a reclaim list so that it can be freed in case of memory pressure. 124 */ 125 spin_unlock(&chunk->lock); 126 } 127 128 static void 129 nouveau_dmem_fault_alloc_and_copy(struct vm_area_struct *vma, 130 const unsigned long *src_pfns, 131 unsigned long *dst_pfns, 132 unsigned long start, 133 unsigned long end, 134 void *private) 135 { 136 struct nouveau_dmem_fault *fault = private; 137 struct nouveau_drm *drm = fault->drm; 138 struct device *dev = drm->dev->dev; 139 unsigned long addr, i, npages = 0; 140 nouveau_migrate_copy_t copy; 141 int ret; 142 143 144 /* First allocate new memory */ 145 for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) { 146 struct page *dpage, *spage; 147 148 dst_pfns[i] = 0; 149 spage = migrate_pfn_to_page(src_pfns[i]); 150 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) 151 continue; 152 153 dpage = hmm_vma_alloc_locked_page(vma, addr); 154 if (!dpage) { 155 dst_pfns[i] = MIGRATE_PFN_ERROR; 156 continue; 157 } 158 159 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) | 160 MIGRATE_PFN_LOCKED; 161 npages++; 162 } 163 164 /* Allocate storage for DMA addresses, so we can unmap later. */ 165 fault->dma = kmalloc(sizeof(*fault->dma) * npages, GFP_KERNEL); 166 if (!fault->dma) 167 goto error; 168 169 /* Copy things over */ 170 copy = drm->dmem->migrate.copy_func; 171 for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) { 172 struct nouveau_dmem_chunk *chunk; 173 struct page *spage, *dpage; 174 u64 src_addr, dst_addr; 175 176 dpage = migrate_pfn_to_page(dst_pfns[i]); 177 if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR) 178 continue; 179 180 spage = migrate_pfn_to_page(src_pfns[i]); 181 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) { 182 dst_pfns[i] = MIGRATE_PFN_ERROR; 183 __free_page(dpage); 184 continue; 185 } 186 187 fault->dma[fault->npages] = 188 dma_map_page_attrs(dev, dpage, 0, PAGE_SIZE, 189 PCI_DMA_BIDIRECTIONAL, 190 DMA_ATTR_SKIP_CPU_SYNC); 191 if (dma_mapping_error(dev, fault->dma[fault->npages])) { 192 dst_pfns[i] = MIGRATE_PFN_ERROR; 193 __free_page(dpage); 194 continue; 195 } 196 197 dst_addr = fault->dma[fault->npages++]; 198 199 chunk = (void *)hmm_devmem_page_get_drvdata(spage); 200 src_addr = page_to_pfn(spage) - chunk->pfn_first; 201 src_addr = (src_addr << PAGE_SHIFT) + chunk->bo->bo.offset; 202 203 ret = copy(drm, 1, NOUVEAU_APER_HOST, dst_addr, 204 NOUVEAU_APER_VRAM, src_addr); 205 if (ret) { 206 dst_pfns[i] = MIGRATE_PFN_ERROR; 207 __free_page(dpage); 208 continue; 209 } 210 } 211 212 nouveau_fence_new(drm->dmem->migrate.chan, false, &fault->fence); 213 214 return; 215 216 error: 217 for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) { 218 struct page *page; 219 220 if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR) 221 continue; 222 223 page = migrate_pfn_to_page(dst_pfns[i]); 224 dst_pfns[i] = MIGRATE_PFN_ERROR; 225 if (page == NULL) 226 continue; 227 228 __free_page(page); 229 } 230 } 231 232 void nouveau_dmem_fault_finalize_and_map(struct vm_area_struct *vma, 233 const unsigned long *src_pfns, 234 const unsigned long *dst_pfns, 235 unsigned long start, 236 unsigned long end, 237 void *private) 238 { 239 struct nouveau_dmem_fault *fault = private; 240 struct nouveau_drm *drm = fault->drm; 241 242 if (fault->fence) { 243 nouveau_fence_wait(fault->fence, true, false); 244 nouveau_fence_unref(&fault->fence); 245 } else { 246 /* 247 * FIXME wait for channel to be IDLE before calling finalizing 248 * the hmem object below (nouveau_migrate_hmem_fini()). 249 */ 250 } 251 252 while (fault->npages--) { 253 dma_unmap_page(drm->dev->dev, fault->dma[fault->npages], 254 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 255 } 256 kfree(fault->dma); 257 } 258 259 static const struct migrate_vma_ops nouveau_dmem_fault_migrate_ops = { 260 .alloc_and_copy = nouveau_dmem_fault_alloc_and_copy, 261 .finalize_and_map = nouveau_dmem_fault_finalize_and_map, 262 }; 263 264 static vm_fault_t 265 nouveau_dmem_fault(struct hmm_devmem *devmem, 266 struct vm_area_struct *vma, 267 unsigned long addr, 268 const struct page *page, 269 unsigned int flags, 270 pmd_t *pmdp) 271 { 272 struct drm_device *drm_dev = dev_get_drvdata(devmem->device); 273 unsigned long src[1] = {0}, dst[1] = {0}; 274 struct nouveau_dmem_fault fault = {0}; 275 int ret; 276 277 278 279 /* 280 * FIXME what we really want is to find some heuristic to migrate more 281 * than just one page on CPU fault. When such fault happens it is very 282 * likely that more surrounding page will CPU fault too. 283 */ 284 fault.drm = nouveau_drm(drm_dev); 285 ret = migrate_vma(&nouveau_dmem_fault_migrate_ops, vma, addr, 286 addr + PAGE_SIZE, src, dst, &fault); 287 if (ret) 288 return VM_FAULT_SIGBUS; 289 290 if (dst[0] == MIGRATE_PFN_ERROR) 291 return VM_FAULT_SIGBUS; 292 293 return 0; 294 } 295 296 static const struct hmm_devmem_ops 297 nouveau_dmem_devmem_ops = { 298 .free = nouveau_dmem_free, 299 .fault = nouveau_dmem_fault, 300 }; 301 302 static int 303 nouveau_dmem_chunk_alloc(struct nouveau_drm *drm) 304 { 305 struct nouveau_dmem_chunk *chunk; 306 int ret; 307 308 if (drm->dmem == NULL) 309 return -EINVAL; 310 311 mutex_lock(&drm->dmem->mutex); 312 chunk = list_first_entry_or_null(&drm->dmem->chunk_empty, 313 struct nouveau_dmem_chunk, 314 list); 315 if (chunk == NULL) { 316 mutex_unlock(&drm->dmem->mutex); 317 return -ENOMEM; 318 } 319 320 list_del(&chunk->list); 321 mutex_unlock(&drm->dmem->mutex); 322 323 ret = nouveau_bo_new(&drm->client, DMEM_CHUNK_SIZE, 0, 324 TTM_PL_FLAG_VRAM, 0, 0, NULL, NULL, 325 &chunk->bo); 326 if (ret) 327 goto out; 328 329 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false); 330 if (ret) { 331 nouveau_bo_ref(NULL, &chunk->bo); 332 goto out; 333 } 334 335 bitmap_zero(chunk->bitmap, DMEM_CHUNK_NPAGES); 336 spin_lock_init(&chunk->lock); 337 338 out: 339 mutex_lock(&drm->dmem->mutex); 340 if (chunk->bo) 341 list_add(&chunk->list, &drm->dmem->chunk_empty); 342 else 343 list_add_tail(&chunk->list, &drm->dmem->chunk_empty); 344 mutex_unlock(&drm->dmem->mutex); 345 346 return ret; 347 } 348 349 static struct nouveau_dmem_chunk * 350 nouveau_dmem_chunk_first_free_locked(struct nouveau_drm *drm) 351 { 352 struct nouveau_dmem_chunk *chunk; 353 354 chunk = list_first_entry_or_null(&drm->dmem->chunk_free, 355 struct nouveau_dmem_chunk, 356 list); 357 if (chunk) 358 return chunk; 359 360 chunk = list_first_entry_or_null(&drm->dmem->chunk_empty, 361 struct nouveau_dmem_chunk, 362 list); 363 if (chunk->bo) 364 return chunk; 365 366 return NULL; 367 } 368 369 static int 370 nouveau_dmem_pages_alloc(struct nouveau_drm *drm, 371 unsigned long npages, 372 unsigned long *pages) 373 { 374 struct nouveau_dmem_chunk *chunk; 375 unsigned long c; 376 int ret; 377 378 memset(pages, 0xff, npages * sizeof(*pages)); 379 380 mutex_lock(&drm->dmem->mutex); 381 for (c = 0; c < npages;) { 382 unsigned long i; 383 384 chunk = nouveau_dmem_chunk_first_free_locked(drm); 385 if (chunk == NULL) { 386 mutex_unlock(&drm->dmem->mutex); 387 ret = nouveau_dmem_chunk_alloc(drm); 388 if (ret) { 389 if (c) 390 break; 391 return ret; 392 } 393 continue; 394 } 395 396 spin_lock(&chunk->lock); 397 i = find_first_zero_bit(chunk->bitmap, DMEM_CHUNK_NPAGES); 398 while (i < DMEM_CHUNK_NPAGES && c < npages) { 399 pages[c] = chunk->pfn_first + i; 400 set_bit(i, chunk->bitmap); 401 chunk->callocated++; 402 c++; 403 404 i = find_next_zero_bit(chunk->bitmap, 405 DMEM_CHUNK_NPAGES, i); 406 } 407 spin_unlock(&chunk->lock); 408 } 409 mutex_unlock(&drm->dmem->mutex); 410 411 return 0; 412 } 413 414 static struct page * 415 nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm) 416 { 417 unsigned long pfns[1]; 418 struct page *page; 419 int ret; 420 421 /* FIXME stop all the miss-match API ... */ 422 ret = nouveau_dmem_pages_alloc(drm, 1, pfns); 423 if (ret) 424 return NULL; 425 426 page = pfn_to_page(pfns[0]); 427 get_page(page); 428 lock_page(page); 429 return page; 430 } 431 432 static void 433 nouveau_dmem_page_free_locked(struct nouveau_drm *drm, struct page *page) 434 { 435 unlock_page(page); 436 put_page(page); 437 } 438 439 void 440 nouveau_dmem_resume(struct nouveau_drm *drm) 441 { 442 struct nouveau_dmem_chunk *chunk; 443 int ret; 444 445 if (drm->dmem == NULL) 446 return; 447 448 mutex_lock(&drm->dmem->mutex); 449 list_for_each_entry (chunk, &drm->dmem->chunk_free, list) { 450 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false); 451 /* FIXME handle pin failure */ 452 WARN_ON(ret); 453 } 454 list_for_each_entry (chunk, &drm->dmem->chunk_full, list) { 455 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false); 456 /* FIXME handle pin failure */ 457 WARN_ON(ret); 458 } 459 list_for_each_entry (chunk, &drm->dmem->chunk_empty, list) { 460 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false); 461 /* FIXME handle pin failure */ 462 WARN_ON(ret); 463 } 464 mutex_unlock(&drm->dmem->mutex); 465 } 466 467 void 468 nouveau_dmem_suspend(struct nouveau_drm *drm) 469 { 470 struct nouveau_dmem_chunk *chunk; 471 472 if (drm->dmem == NULL) 473 return; 474 475 mutex_lock(&drm->dmem->mutex); 476 list_for_each_entry (chunk, &drm->dmem->chunk_free, list) { 477 nouveau_bo_unpin(chunk->bo); 478 } 479 list_for_each_entry (chunk, &drm->dmem->chunk_full, list) { 480 nouveau_bo_unpin(chunk->bo); 481 } 482 list_for_each_entry (chunk, &drm->dmem->chunk_empty, list) { 483 nouveau_bo_unpin(chunk->bo); 484 } 485 mutex_unlock(&drm->dmem->mutex); 486 } 487 488 void 489 nouveau_dmem_fini(struct nouveau_drm *drm) 490 { 491 struct nouveau_dmem_chunk *chunk, *tmp; 492 493 if (drm->dmem == NULL) 494 return; 495 496 mutex_lock(&drm->dmem->mutex); 497 498 WARN_ON(!list_empty(&drm->dmem->chunk_free)); 499 WARN_ON(!list_empty(&drm->dmem->chunk_full)); 500 501 list_for_each_entry_safe (chunk, tmp, &drm->dmem->chunk_empty, list) { 502 if (chunk->bo) { 503 nouveau_bo_unpin(chunk->bo); 504 nouveau_bo_ref(NULL, &chunk->bo); 505 } 506 list_del(&chunk->list); 507 kfree(chunk); 508 } 509 510 mutex_unlock(&drm->dmem->mutex); 511 } 512 513 static int 514 nvc0b5_migrate_copy(struct nouveau_drm *drm, u64 npages, 515 enum nouveau_aper dst_aper, u64 dst_addr, 516 enum nouveau_aper src_aper, u64 src_addr) 517 { 518 struct nouveau_channel *chan = drm->dmem->migrate.chan; 519 u32 launch_dma = (1 << 9) /* MULTI_LINE_ENABLE. */ | 520 (1 << 8) /* DST_MEMORY_LAYOUT_PITCH. */ | 521 (1 << 7) /* SRC_MEMORY_LAYOUT_PITCH. */ | 522 (1 << 2) /* FLUSH_ENABLE_TRUE. */ | 523 (2 << 0) /* DATA_TRANSFER_TYPE_NON_PIPELINED. */; 524 int ret; 525 526 ret = RING_SPACE(chan, 13); 527 if (ret) 528 return ret; 529 530 if (src_aper != NOUVEAU_APER_VIRT) { 531 switch (src_aper) { 532 case NOUVEAU_APER_VRAM: 533 BEGIN_IMC0(chan, NvSubCopy, 0x0260, 0); 534 break; 535 case NOUVEAU_APER_HOST: 536 BEGIN_IMC0(chan, NvSubCopy, 0x0260, 1); 537 break; 538 default: 539 return -EINVAL; 540 } 541 launch_dma |= 0x00001000; /* SRC_TYPE_PHYSICAL. */ 542 } 543 544 if (dst_aper != NOUVEAU_APER_VIRT) { 545 switch (dst_aper) { 546 case NOUVEAU_APER_VRAM: 547 BEGIN_IMC0(chan, NvSubCopy, 0x0264, 0); 548 break; 549 case NOUVEAU_APER_HOST: 550 BEGIN_IMC0(chan, NvSubCopy, 0x0264, 1); 551 break; 552 default: 553 return -EINVAL; 554 } 555 launch_dma |= 0x00002000; /* DST_TYPE_PHYSICAL. */ 556 } 557 558 BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8); 559 OUT_RING (chan, upper_32_bits(src_addr)); 560 OUT_RING (chan, lower_32_bits(src_addr)); 561 OUT_RING (chan, upper_32_bits(dst_addr)); 562 OUT_RING (chan, lower_32_bits(dst_addr)); 563 OUT_RING (chan, PAGE_SIZE); 564 OUT_RING (chan, PAGE_SIZE); 565 OUT_RING (chan, PAGE_SIZE); 566 OUT_RING (chan, npages); 567 BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1); 568 OUT_RING (chan, launch_dma); 569 return 0; 570 } 571 572 static int 573 nouveau_dmem_migrate_init(struct nouveau_drm *drm) 574 { 575 switch (drm->ttm.copy.oclass) { 576 case PASCAL_DMA_COPY_A: 577 case PASCAL_DMA_COPY_B: 578 case VOLTA_DMA_COPY_A: 579 case TURING_DMA_COPY_A: 580 drm->dmem->migrate.copy_func = nvc0b5_migrate_copy; 581 drm->dmem->migrate.chan = drm->ttm.chan; 582 return 0; 583 default: 584 break; 585 } 586 return -ENODEV; 587 } 588 589 void 590 nouveau_dmem_init(struct nouveau_drm *drm) 591 { 592 struct device *device = drm->dev->dev; 593 unsigned long i, size; 594 int ret; 595 596 /* This only make sense on PASCAL or newer */ 597 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_PASCAL) 598 return; 599 600 if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL))) 601 return; 602 603 mutex_init(&drm->dmem->mutex); 604 INIT_LIST_HEAD(&drm->dmem->chunk_free); 605 INIT_LIST_HEAD(&drm->dmem->chunk_full); 606 INIT_LIST_HEAD(&drm->dmem->chunk_empty); 607 608 size = ALIGN(drm->client.device.info.ram_user, DMEM_CHUNK_SIZE); 609 610 /* Initialize migration dma helpers before registering memory */ 611 ret = nouveau_dmem_migrate_init(drm); 612 if (ret) { 613 kfree(drm->dmem); 614 drm->dmem = NULL; 615 return; 616 } 617 618 /* 619 * FIXME we need some kind of policy to decide how much VRAM we 620 * want to register with HMM. For now just register everything 621 * and latter if we want to do thing like over commit then we 622 * could revisit this. 623 */ 624 drm->dmem->devmem = hmm_devmem_add(&nouveau_dmem_devmem_ops, 625 device, size); 626 if (drm->dmem->devmem == NULL) { 627 kfree(drm->dmem); 628 drm->dmem = NULL; 629 return; 630 } 631 632 for (i = 0; i < (size / DMEM_CHUNK_SIZE); ++i) { 633 struct nouveau_dmem_chunk *chunk; 634 struct page *page; 635 unsigned long j; 636 637 chunk = kzalloc(sizeof(*chunk), GFP_KERNEL); 638 if (chunk == NULL) { 639 nouveau_dmem_fini(drm); 640 return; 641 } 642 643 chunk->drm = drm; 644 chunk->pfn_first = drm->dmem->devmem->pfn_first; 645 chunk->pfn_first += (i * DMEM_CHUNK_NPAGES); 646 list_add_tail(&chunk->list, &drm->dmem->chunk_empty); 647 648 page = pfn_to_page(chunk->pfn_first); 649 for (j = 0; j < DMEM_CHUNK_NPAGES; ++j, ++page) { 650 hmm_devmem_page_set_drvdata(page, (long)chunk); 651 } 652 } 653 654 NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", size >> 20); 655 } 656 657 static void 658 nouveau_dmem_migrate_alloc_and_copy(struct vm_area_struct *vma, 659 const unsigned long *src_pfns, 660 unsigned long *dst_pfns, 661 unsigned long start, 662 unsigned long end, 663 void *private) 664 { 665 struct nouveau_migrate *migrate = private; 666 struct nouveau_drm *drm = migrate->drm; 667 struct device *dev = drm->dev->dev; 668 unsigned long addr, i, npages = 0; 669 nouveau_migrate_copy_t copy; 670 int ret; 671 672 /* First allocate new memory */ 673 for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) { 674 struct page *dpage, *spage; 675 676 dst_pfns[i] = 0; 677 spage = migrate_pfn_to_page(src_pfns[i]); 678 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) 679 continue; 680 681 dpage = nouveau_dmem_page_alloc_locked(drm); 682 if (!dpage) 683 continue; 684 685 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) | 686 MIGRATE_PFN_LOCKED | 687 MIGRATE_PFN_DEVICE; 688 npages++; 689 } 690 691 if (!npages) 692 return; 693 694 /* Allocate storage for DMA addresses, so we can unmap later. */ 695 migrate->dma = kmalloc(sizeof(*migrate->dma) * npages, GFP_KERNEL); 696 if (!migrate->dma) 697 goto error; 698 699 /* Copy things over */ 700 copy = drm->dmem->migrate.copy_func; 701 for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) { 702 struct nouveau_dmem_chunk *chunk; 703 struct page *spage, *dpage; 704 u64 src_addr, dst_addr; 705 706 dpage = migrate_pfn_to_page(dst_pfns[i]); 707 if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR) 708 continue; 709 710 chunk = (void *)hmm_devmem_page_get_drvdata(dpage); 711 dst_addr = page_to_pfn(dpage) - chunk->pfn_first; 712 dst_addr = (dst_addr << PAGE_SHIFT) + chunk->bo->bo.offset; 713 714 spage = migrate_pfn_to_page(src_pfns[i]); 715 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) { 716 nouveau_dmem_page_free_locked(drm, dpage); 717 dst_pfns[i] = 0; 718 continue; 719 } 720 721 migrate->dma[migrate->dma_nr] = 722 dma_map_page_attrs(dev, spage, 0, PAGE_SIZE, 723 PCI_DMA_BIDIRECTIONAL, 724 DMA_ATTR_SKIP_CPU_SYNC); 725 if (dma_mapping_error(dev, migrate->dma[migrate->dma_nr])) { 726 nouveau_dmem_page_free_locked(drm, dpage); 727 dst_pfns[i] = 0; 728 continue; 729 } 730 731 src_addr = migrate->dma[migrate->dma_nr++]; 732 733 ret = copy(drm, 1, NOUVEAU_APER_VRAM, dst_addr, 734 NOUVEAU_APER_HOST, src_addr); 735 if (ret) { 736 nouveau_dmem_page_free_locked(drm, dpage); 737 dst_pfns[i] = 0; 738 continue; 739 } 740 } 741 742 nouveau_fence_new(drm->dmem->migrate.chan, false, &migrate->fence); 743 744 return; 745 746 error: 747 for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) { 748 struct page *page; 749 750 if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR) 751 continue; 752 753 page = migrate_pfn_to_page(dst_pfns[i]); 754 dst_pfns[i] = MIGRATE_PFN_ERROR; 755 if (page == NULL) 756 continue; 757 758 __free_page(page); 759 } 760 } 761 762 void nouveau_dmem_migrate_finalize_and_map(struct vm_area_struct *vma, 763 const unsigned long *src_pfns, 764 const unsigned long *dst_pfns, 765 unsigned long start, 766 unsigned long end, 767 void *private) 768 { 769 struct nouveau_migrate *migrate = private; 770 struct nouveau_drm *drm = migrate->drm; 771 772 if (migrate->fence) { 773 nouveau_fence_wait(migrate->fence, true, false); 774 nouveau_fence_unref(&migrate->fence); 775 } else { 776 /* 777 * FIXME wait for channel to be IDLE before finalizing 778 * the hmem object below (nouveau_migrate_hmem_fini()) ? 779 */ 780 } 781 782 while (migrate->dma_nr--) { 783 dma_unmap_page(drm->dev->dev, migrate->dma[migrate->dma_nr], 784 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 785 } 786 kfree(migrate->dma); 787 788 /* 789 * FIXME optimization: update GPU page table to point to newly 790 * migrated memory. 791 */ 792 } 793 794 static const struct migrate_vma_ops nouveau_dmem_migrate_ops = { 795 .alloc_and_copy = nouveau_dmem_migrate_alloc_and_copy, 796 .finalize_and_map = nouveau_dmem_migrate_finalize_and_map, 797 }; 798 799 int 800 nouveau_dmem_migrate_vma(struct nouveau_drm *drm, 801 struct vm_area_struct *vma, 802 unsigned long start, 803 unsigned long end) 804 { 805 unsigned long *src_pfns, *dst_pfns, npages; 806 struct nouveau_migrate migrate = {0}; 807 unsigned long i, c, max; 808 int ret = 0; 809 810 npages = (end - start) >> PAGE_SHIFT; 811 max = min(SG_MAX_SINGLE_ALLOC, npages); 812 src_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL); 813 if (src_pfns == NULL) 814 return -ENOMEM; 815 dst_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL); 816 if (dst_pfns == NULL) { 817 kfree(src_pfns); 818 return -ENOMEM; 819 } 820 821 migrate.drm = drm; 822 migrate.vma = vma; 823 migrate.npages = npages; 824 for (i = 0; i < npages; i += c) { 825 unsigned long next; 826 827 c = min(SG_MAX_SINGLE_ALLOC, npages); 828 next = start + (c << PAGE_SHIFT); 829 ret = migrate_vma(&nouveau_dmem_migrate_ops, vma, start, 830 next, src_pfns, dst_pfns, &migrate); 831 if (ret) 832 goto out; 833 start = next; 834 } 835 836 out: 837 kfree(dst_pfns); 838 kfree(src_pfns); 839 return ret; 840 } 841 842 static inline bool 843 nouveau_dmem_page(struct nouveau_drm *drm, struct page *page) 844 { 845 if (!is_device_private_page(page)) 846 return false; 847 848 if (drm->dmem->devmem != page->pgmap->data) 849 return false; 850 851 return true; 852 } 853 854 void 855 nouveau_dmem_convert_pfn(struct nouveau_drm *drm, 856 struct hmm_range *range) 857 { 858 unsigned long i, npages; 859 860 npages = (range->end - range->start) >> PAGE_SHIFT; 861 for (i = 0; i < npages; ++i) { 862 struct nouveau_dmem_chunk *chunk; 863 struct page *page; 864 uint64_t addr; 865 866 page = hmm_pfn_to_page(range, range->pfns[i]); 867 if (page == NULL) 868 continue; 869 870 if (!(range->pfns[i] & range->flags[HMM_PFN_DEVICE_PRIVATE])) { 871 continue; 872 } 873 874 if (!nouveau_dmem_page(drm, page)) { 875 WARN(1, "Some unknown device memory !\n"); 876 range->pfns[i] = 0; 877 continue; 878 } 879 880 chunk = (void *)hmm_devmem_page_get_drvdata(page); 881 addr = page_to_pfn(page) - chunk->pfn_first; 882 addr = (addr + chunk->bo->bo.mem.start) << PAGE_SHIFT; 883 884 range->pfns[i] &= ((1UL << range->pfn_shift) - 1); 885 range->pfns[i] |= (addr >> PAGE_SHIFT) << range->pfn_shift; 886 } 887 } 888