1 /* 2 * Copyright 2018 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 #include "nouveau_dmem.h" 23 #include "nouveau_drv.h" 24 #include "nouveau_chan.h" 25 #include "nouveau_dma.h" 26 #include "nouveau_mem.h" 27 #include "nouveau_bo.h" 28 29 #include <nvif/class.h> 30 #include <nvif/object.h> 31 #include <nvif/if500b.h> 32 #include <nvif/if900b.h> 33 34 #include <linux/sched/mm.h> 35 #include <linux/hmm.h> 36 37 /* 38 * FIXME: this is ugly right now we are using TTM to allocate vram and we pin 39 * it in vram while in use. We likely want to overhaul memory management for 40 * nouveau to be more page like (not necessarily with system page size but a 41 * bigger page size) at lowest level and have some shim layer on top that would 42 * provide the same functionality as TTM. 43 */ 44 #define DMEM_CHUNK_SIZE (2UL << 20) 45 #define DMEM_CHUNK_NPAGES (DMEM_CHUNK_SIZE >> PAGE_SHIFT) 46 47 struct nouveau_migrate; 48 49 enum nouveau_aper { 50 NOUVEAU_APER_VIRT, 51 NOUVEAU_APER_VRAM, 52 NOUVEAU_APER_HOST, 53 }; 54 55 typedef int (*nouveau_migrate_copy_t)(struct nouveau_drm *drm, u64 npages, 56 enum nouveau_aper, u64 dst_addr, 57 enum nouveau_aper, u64 src_addr); 58 59 struct nouveau_dmem_chunk { 60 struct list_head list; 61 struct nouveau_bo *bo; 62 struct nouveau_drm *drm; 63 unsigned long pfn_first; 64 unsigned long callocated; 65 unsigned long bitmap[BITS_TO_LONGS(DMEM_CHUNK_NPAGES)]; 66 spinlock_t lock; 67 }; 68 69 struct nouveau_dmem_migrate { 70 nouveau_migrate_copy_t copy_func; 71 struct nouveau_channel *chan; 72 }; 73 74 struct nouveau_dmem { 75 struct nouveau_drm *drm; 76 struct dev_pagemap pagemap; 77 struct nouveau_dmem_migrate migrate; 78 struct list_head chunk_free; 79 struct list_head chunk_full; 80 struct list_head chunk_empty; 81 struct mutex mutex; 82 }; 83 84 static inline struct nouveau_dmem *page_to_dmem(struct page *page) 85 { 86 return container_of(page->pgmap, struct nouveau_dmem, pagemap); 87 } 88 89 struct nouveau_dmem_fault { 90 struct nouveau_drm *drm; 91 struct nouveau_fence *fence; 92 dma_addr_t *dma; 93 unsigned long npages; 94 }; 95 96 struct nouveau_migrate { 97 struct vm_area_struct *vma; 98 struct nouveau_drm *drm; 99 struct nouveau_fence *fence; 100 unsigned long npages; 101 dma_addr_t *dma; 102 unsigned long dma_nr; 103 }; 104 105 static void nouveau_dmem_page_free(struct page *page) 106 { 107 struct nouveau_dmem_chunk *chunk = page->zone_device_data; 108 unsigned long idx = page_to_pfn(page) - chunk->pfn_first; 109 110 /* 111 * FIXME: 112 * 113 * This is really a bad example, we need to overhaul nouveau memory 114 * management to be more page focus and allow lighter locking scheme 115 * to be use in the process. 116 */ 117 spin_lock(&chunk->lock); 118 clear_bit(idx, chunk->bitmap); 119 WARN_ON(!chunk->callocated); 120 chunk->callocated--; 121 /* 122 * FIXME when chunk->callocated reach 0 we should add the chunk to 123 * a reclaim list so that it can be freed in case of memory pressure. 124 */ 125 spin_unlock(&chunk->lock); 126 } 127 128 static void 129 nouveau_dmem_fault_alloc_and_copy(struct vm_area_struct *vma, 130 const unsigned long *src_pfns, 131 unsigned long *dst_pfns, 132 unsigned long start, 133 unsigned long end, 134 void *private) 135 { 136 struct nouveau_dmem_fault *fault = private; 137 struct nouveau_drm *drm = fault->drm; 138 struct device *dev = drm->dev->dev; 139 unsigned long addr, i, npages = 0; 140 nouveau_migrate_copy_t copy; 141 int ret; 142 143 144 /* First allocate new memory */ 145 for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) { 146 struct page *dpage, *spage; 147 148 dst_pfns[i] = 0; 149 spage = migrate_pfn_to_page(src_pfns[i]); 150 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) 151 continue; 152 153 dpage = alloc_page_vma(GFP_HIGHUSER, vma, addr); 154 if (!dpage) { 155 dst_pfns[i] = MIGRATE_PFN_ERROR; 156 continue; 157 } 158 lock_page(dpage); 159 160 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) | 161 MIGRATE_PFN_LOCKED; 162 npages++; 163 } 164 165 /* Allocate storage for DMA addresses, so we can unmap later. */ 166 fault->dma = kmalloc(sizeof(*fault->dma) * npages, GFP_KERNEL); 167 if (!fault->dma) 168 goto error; 169 170 /* Copy things over */ 171 copy = drm->dmem->migrate.copy_func; 172 for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) { 173 struct nouveau_dmem_chunk *chunk; 174 struct page *spage, *dpage; 175 u64 src_addr, dst_addr; 176 177 dpage = migrate_pfn_to_page(dst_pfns[i]); 178 if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR) 179 continue; 180 181 spage = migrate_pfn_to_page(src_pfns[i]); 182 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) { 183 dst_pfns[i] = MIGRATE_PFN_ERROR; 184 __free_page(dpage); 185 continue; 186 } 187 188 fault->dma[fault->npages] = 189 dma_map_page_attrs(dev, dpage, 0, PAGE_SIZE, 190 PCI_DMA_BIDIRECTIONAL, 191 DMA_ATTR_SKIP_CPU_SYNC); 192 if (dma_mapping_error(dev, fault->dma[fault->npages])) { 193 dst_pfns[i] = MIGRATE_PFN_ERROR; 194 __free_page(dpage); 195 continue; 196 } 197 198 dst_addr = fault->dma[fault->npages++]; 199 200 chunk = spage->zone_device_data; 201 src_addr = page_to_pfn(spage) - chunk->pfn_first; 202 src_addr = (src_addr << PAGE_SHIFT) + chunk->bo->bo.offset; 203 204 ret = copy(drm, 1, NOUVEAU_APER_HOST, dst_addr, 205 NOUVEAU_APER_VRAM, src_addr); 206 if (ret) { 207 dst_pfns[i] = MIGRATE_PFN_ERROR; 208 __free_page(dpage); 209 continue; 210 } 211 } 212 213 nouveau_fence_new(drm->dmem->migrate.chan, false, &fault->fence); 214 215 return; 216 217 error: 218 for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) { 219 struct page *page; 220 221 if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR) 222 continue; 223 224 page = migrate_pfn_to_page(dst_pfns[i]); 225 dst_pfns[i] = MIGRATE_PFN_ERROR; 226 if (page == NULL) 227 continue; 228 229 __free_page(page); 230 } 231 } 232 233 void nouveau_dmem_fault_finalize_and_map(struct vm_area_struct *vma, 234 const unsigned long *src_pfns, 235 const unsigned long *dst_pfns, 236 unsigned long start, 237 unsigned long end, 238 void *private) 239 { 240 struct nouveau_dmem_fault *fault = private; 241 struct nouveau_drm *drm = fault->drm; 242 243 if (fault->fence) { 244 nouveau_fence_wait(fault->fence, true, false); 245 nouveau_fence_unref(&fault->fence); 246 } else { 247 /* 248 * FIXME wait for channel to be IDLE before calling finalizing 249 * the hmem object below (nouveau_migrate_hmem_fini()). 250 */ 251 } 252 253 while (fault->npages--) { 254 dma_unmap_page(drm->dev->dev, fault->dma[fault->npages], 255 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 256 } 257 kfree(fault->dma); 258 } 259 260 static const struct migrate_vma_ops nouveau_dmem_fault_migrate_ops = { 261 .alloc_and_copy = nouveau_dmem_fault_alloc_and_copy, 262 .finalize_and_map = nouveau_dmem_fault_finalize_and_map, 263 }; 264 265 static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf) 266 { 267 struct nouveau_dmem *dmem = page_to_dmem(vmf->page); 268 unsigned long src[1] = {0}, dst[1] = {0}; 269 struct nouveau_dmem_fault fault = { .drm = dmem->drm }; 270 int ret; 271 272 /* 273 * FIXME what we really want is to find some heuristic to migrate more 274 * than just one page on CPU fault. When such fault happens it is very 275 * likely that more surrounding page will CPU fault too. 276 */ 277 ret = migrate_vma(&nouveau_dmem_fault_migrate_ops, vmf->vma, 278 vmf->address, vmf->address + PAGE_SIZE, 279 src, dst, &fault); 280 if (ret) 281 return VM_FAULT_SIGBUS; 282 283 if (dst[0] == MIGRATE_PFN_ERROR) 284 return VM_FAULT_SIGBUS; 285 286 return 0; 287 } 288 289 static const struct dev_pagemap_ops nouveau_dmem_pagemap_ops = { 290 .page_free = nouveau_dmem_page_free, 291 .migrate_to_ram = nouveau_dmem_migrate_to_ram, 292 }; 293 294 static int 295 nouveau_dmem_chunk_alloc(struct nouveau_drm *drm) 296 { 297 struct nouveau_dmem_chunk *chunk; 298 int ret; 299 300 if (drm->dmem == NULL) 301 return -EINVAL; 302 303 mutex_lock(&drm->dmem->mutex); 304 chunk = list_first_entry_or_null(&drm->dmem->chunk_empty, 305 struct nouveau_dmem_chunk, 306 list); 307 if (chunk == NULL) { 308 mutex_unlock(&drm->dmem->mutex); 309 return -ENOMEM; 310 } 311 312 list_del(&chunk->list); 313 mutex_unlock(&drm->dmem->mutex); 314 315 ret = nouveau_bo_new(&drm->client, DMEM_CHUNK_SIZE, 0, 316 TTM_PL_FLAG_VRAM, 0, 0, NULL, NULL, 317 &chunk->bo); 318 if (ret) 319 goto out; 320 321 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false); 322 if (ret) { 323 nouveau_bo_ref(NULL, &chunk->bo); 324 goto out; 325 } 326 327 bitmap_zero(chunk->bitmap, DMEM_CHUNK_NPAGES); 328 spin_lock_init(&chunk->lock); 329 330 out: 331 mutex_lock(&drm->dmem->mutex); 332 if (chunk->bo) 333 list_add(&chunk->list, &drm->dmem->chunk_empty); 334 else 335 list_add_tail(&chunk->list, &drm->dmem->chunk_empty); 336 mutex_unlock(&drm->dmem->mutex); 337 338 return ret; 339 } 340 341 static struct nouveau_dmem_chunk * 342 nouveau_dmem_chunk_first_free_locked(struct nouveau_drm *drm) 343 { 344 struct nouveau_dmem_chunk *chunk; 345 346 chunk = list_first_entry_or_null(&drm->dmem->chunk_free, 347 struct nouveau_dmem_chunk, 348 list); 349 if (chunk) 350 return chunk; 351 352 chunk = list_first_entry_or_null(&drm->dmem->chunk_empty, 353 struct nouveau_dmem_chunk, 354 list); 355 if (chunk->bo) 356 return chunk; 357 358 return NULL; 359 } 360 361 static int 362 nouveau_dmem_pages_alloc(struct nouveau_drm *drm, 363 unsigned long npages, 364 unsigned long *pages) 365 { 366 struct nouveau_dmem_chunk *chunk; 367 unsigned long c; 368 int ret; 369 370 memset(pages, 0xff, npages * sizeof(*pages)); 371 372 mutex_lock(&drm->dmem->mutex); 373 for (c = 0; c < npages;) { 374 unsigned long i; 375 376 chunk = nouveau_dmem_chunk_first_free_locked(drm); 377 if (chunk == NULL) { 378 mutex_unlock(&drm->dmem->mutex); 379 ret = nouveau_dmem_chunk_alloc(drm); 380 if (ret) { 381 if (c) 382 return 0; 383 return ret; 384 } 385 mutex_lock(&drm->dmem->mutex); 386 continue; 387 } 388 389 spin_lock(&chunk->lock); 390 i = find_first_zero_bit(chunk->bitmap, DMEM_CHUNK_NPAGES); 391 while (i < DMEM_CHUNK_NPAGES && c < npages) { 392 pages[c] = chunk->pfn_first + i; 393 set_bit(i, chunk->bitmap); 394 chunk->callocated++; 395 c++; 396 397 i = find_next_zero_bit(chunk->bitmap, 398 DMEM_CHUNK_NPAGES, i); 399 } 400 spin_unlock(&chunk->lock); 401 } 402 mutex_unlock(&drm->dmem->mutex); 403 404 return 0; 405 } 406 407 static struct page * 408 nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm) 409 { 410 unsigned long pfns[1]; 411 struct page *page; 412 int ret; 413 414 /* FIXME stop all the miss-match API ... */ 415 ret = nouveau_dmem_pages_alloc(drm, 1, pfns); 416 if (ret) 417 return NULL; 418 419 page = pfn_to_page(pfns[0]); 420 get_page(page); 421 lock_page(page); 422 return page; 423 } 424 425 static void 426 nouveau_dmem_page_free_locked(struct nouveau_drm *drm, struct page *page) 427 { 428 unlock_page(page); 429 put_page(page); 430 } 431 432 void 433 nouveau_dmem_resume(struct nouveau_drm *drm) 434 { 435 struct nouveau_dmem_chunk *chunk; 436 int ret; 437 438 if (drm->dmem == NULL) 439 return; 440 441 mutex_lock(&drm->dmem->mutex); 442 list_for_each_entry (chunk, &drm->dmem->chunk_free, list) { 443 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false); 444 /* FIXME handle pin failure */ 445 WARN_ON(ret); 446 } 447 list_for_each_entry (chunk, &drm->dmem->chunk_full, list) { 448 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false); 449 /* FIXME handle pin failure */ 450 WARN_ON(ret); 451 } 452 mutex_unlock(&drm->dmem->mutex); 453 } 454 455 void 456 nouveau_dmem_suspend(struct nouveau_drm *drm) 457 { 458 struct nouveau_dmem_chunk *chunk; 459 460 if (drm->dmem == NULL) 461 return; 462 463 mutex_lock(&drm->dmem->mutex); 464 list_for_each_entry (chunk, &drm->dmem->chunk_free, list) { 465 nouveau_bo_unpin(chunk->bo); 466 } 467 list_for_each_entry (chunk, &drm->dmem->chunk_full, list) { 468 nouveau_bo_unpin(chunk->bo); 469 } 470 mutex_unlock(&drm->dmem->mutex); 471 } 472 473 void 474 nouveau_dmem_fini(struct nouveau_drm *drm) 475 { 476 struct nouveau_dmem_chunk *chunk, *tmp; 477 478 if (drm->dmem == NULL) 479 return; 480 481 mutex_lock(&drm->dmem->mutex); 482 483 WARN_ON(!list_empty(&drm->dmem->chunk_free)); 484 WARN_ON(!list_empty(&drm->dmem->chunk_full)); 485 486 list_for_each_entry_safe (chunk, tmp, &drm->dmem->chunk_empty, list) { 487 if (chunk->bo) { 488 nouveau_bo_unpin(chunk->bo); 489 nouveau_bo_ref(NULL, &chunk->bo); 490 } 491 list_del(&chunk->list); 492 kfree(chunk); 493 } 494 495 mutex_unlock(&drm->dmem->mutex); 496 } 497 498 static int 499 nvc0b5_migrate_copy(struct nouveau_drm *drm, u64 npages, 500 enum nouveau_aper dst_aper, u64 dst_addr, 501 enum nouveau_aper src_aper, u64 src_addr) 502 { 503 struct nouveau_channel *chan = drm->dmem->migrate.chan; 504 u32 launch_dma = (1 << 9) /* MULTI_LINE_ENABLE. */ | 505 (1 << 8) /* DST_MEMORY_LAYOUT_PITCH. */ | 506 (1 << 7) /* SRC_MEMORY_LAYOUT_PITCH. */ | 507 (1 << 2) /* FLUSH_ENABLE_TRUE. */ | 508 (2 << 0) /* DATA_TRANSFER_TYPE_NON_PIPELINED. */; 509 int ret; 510 511 ret = RING_SPACE(chan, 13); 512 if (ret) 513 return ret; 514 515 if (src_aper != NOUVEAU_APER_VIRT) { 516 switch (src_aper) { 517 case NOUVEAU_APER_VRAM: 518 BEGIN_IMC0(chan, NvSubCopy, 0x0260, 0); 519 break; 520 case NOUVEAU_APER_HOST: 521 BEGIN_IMC0(chan, NvSubCopy, 0x0260, 1); 522 break; 523 default: 524 return -EINVAL; 525 } 526 launch_dma |= 0x00001000; /* SRC_TYPE_PHYSICAL. */ 527 } 528 529 if (dst_aper != NOUVEAU_APER_VIRT) { 530 switch (dst_aper) { 531 case NOUVEAU_APER_VRAM: 532 BEGIN_IMC0(chan, NvSubCopy, 0x0264, 0); 533 break; 534 case NOUVEAU_APER_HOST: 535 BEGIN_IMC0(chan, NvSubCopy, 0x0264, 1); 536 break; 537 default: 538 return -EINVAL; 539 } 540 launch_dma |= 0x00002000; /* DST_TYPE_PHYSICAL. */ 541 } 542 543 BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8); 544 OUT_RING (chan, upper_32_bits(src_addr)); 545 OUT_RING (chan, lower_32_bits(src_addr)); 546 OUT_RING (chan, upper_32_bits(dst_addr)); 547 OUT_RING (chan, lower_32_bits(dst_addr)); 548 OUT_RING (chan, PAGE_SIZE); 549 OUT_RING (chan, PAGE_SIZE); 550 OUT_RING (chan, PAGE_SIZE); 551 OUT_RING (chan, npages); 552 BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1); 553 OUT_RING (chan, launch_dma); 554 return 0; 555 } 556 557 static int 558 nouveau_dmem_migrate_init(struct nouveau_drm *drm) 559 { 560 switch (drm->ttm.copy.oclass) { 561 case PASCAL_DMA_COPY_A: 562 case PASCAL_DMA_COPY_B: 563 case VOLTA_DMA_COPY_A: 564 case TURING_DMA_COPY_A: 565 drm->dmem->migrate.copy_func = nvc0b5_migrate_copy; 566 drm->dmem->migrate.chan = drm->ttm.chan; 567 return 0; 568 default: 569 break; 570 } 571 return -ENODEV; 572 } 573 574 void 575 nouveau_dmem_init(struct nouveau_drm *drm) 576 { 577 struct device *device = drm->dev->dev; 578 struct resource *res; 579 unsigned long i, size, pfn_first; 580 int ret; 581 582 /* This only make sense on PASCAL or newer */ 583 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_PASCAL) 584 return; 585 586 if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL))) 587 return; 588 589 drm->dmem->drm = drm; 590 mutex_init(&drm->dmem->mutex); 591 INIT_LIST_HEAD(&drm->dmem->chunk_free); 592 INIT_LIST_HEAD(&drm->dmem->chunk_full); 593 INIT_LIST_HEAD(&drm->dmem->chunk_empty); 594 595 size = ALIGN(drm->client.device.info.ram_user, DMEM_CHUNK_SIZE); 596 597 /* Initialize migration dma helpers before registering memory */ 598 ret = nouveau_dmem_migrate_init(drm); 599 if (ret) 600 goto out_free; 601 602 /* 603 * FIXME we need some kind of policy to decide how much VRAM we 604 * want to register with HMM. For now just register everything 605 * and latter if we want to do thing like over commit then we 606 * could revisit this. 607 */ 608 res = devm_request_free_mem_region(device, &iomem_resource, size); 609 if (IS_ERR(res)) 610 goto out_free; 611 drm->dmem->pagemap.type = MEMORY_DEVICE_PRIVATE; 612 drm->dmem->pagemap.res = *res; 613 drm->dmem->pagemap.ops = &nouveau_dmem_pagemap_ops; 614 if (IS_ERR(devm_memremap_pages(device, &drm->dmem->pagemap))) 615 goto out_free; 616 617 pfn_first = res->start >> PAGE_SHIFT; 618 for (i = 0; i < (size / DMEM_CHUNK_SIZE); ++i) { 619 struct nouveau_dmem_chunk *chunk; 620 struct page *page; 621 unsigned long j; 622 623 chunk = kzalloc(sizeof(*chunk), GFP_KERNEL); 624 if (chunk == NULL) { 625 nouveau_dmem_fini(drm); 626 return; 627 } 628 629 chunk->drm = drm; 630 chunk->pfn_first = pfn_first + (i * DMEM_CHUNK_NPAGES); 631 list_add_tail(&chunk->list, &drm->dmem->chunk_empty); 632 633 page = pfn_to_page(chunk->pfn_first); 634 for (j = 0; j < DMEM_CHUNK_NPAGES; ++j, ++page) 635 page->zone_device_data = chunk; 636 } 637 638 NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", size >> 20); 639 return; 640 out_free: 641 kfree(drm->dmem); 642 drm->dmem = NULL; 643 } 644 645 static void 646 nouveau_dmem_migrate_alloc_and_copy(struct vm_area_struct *vma, 647 const unsigned long *src_pfns, 648 unsigned long *dst_pfns, 649 unsigned long start, 650 unsigned long end, 651 void *private) 652 { 653 struct nouveau_migrate *migrate = private; 654 struct nouveau_drm *drm = migrate->drm; 655 struct device *dev = drm->dev->dev; 656 unsigned long addr, i, npages = 0; 657 nouveau_migrate_copy_t copy; 658 int ret; 659 660 /* First allocate new memory */ 661 for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) { 662 struct page *dpage, *spage; 663 664 dst_pfns[i] = 0; 665 spage = migrate_pfn_to_page(src_pfns[i]); 666 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) 667 continue; 668 669 dpage = nouveau_dmem_page_alloc_locked(drm); 670 if (!dpage) 671 continue; 672 673 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) | 674 MIGRATE_PFN_LOCKED | 675 MIGRATE_PFN_DEVICE; 676 npages++; 677 } 678 679 if (!npages) 680 return; 681 682 /* Allocate storage for DMA addresses, so we can unmap later. */ 683 migrate->dma = kmalloc(sizeof(*migrate->dma) * npages, GFP_KERNEL); 684 if (!migrate->dma) 685 goto error; 686 687 /* Copy things over */ 688 copy = drm->dmem->migrate.copy_func; 689 for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) { 690 struct nouveau_dmem_chunk *chunk; 691 struct page *spage, *dpage; 692 u64 src_addr, dst_addr; 693 694 dpage = migrate_pfn_to_page(dst_pfns[i]); 695 if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR) 696 continue; 697 698 chunk = dpage->zone_device_data; 699 dst_addr = page_to_pfn(dpage) - chunk->pfn_first; 700 dst_addr = (dst_addr << PAGE_SHIFT) + chunk->bo->bo.offset; 701 702 spage = migrate_pfn_to_page(src_pfns[i]); 703 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) { 704 nouveau_dmem_page_free_locked(drm, dpage); 705 dst_pfns[i] = 0; 706 continue; 707 } 708 709 migrate->dma[migrate->dma_nr] = 710 dma_map_page_attrs(dev, spage, 0, PAGE_SIZE, 711 PCI_DMA_BIDIRECTIONAL, 712 DMA_ATTR_SKIP_CPU_SYNC); 713 if (dma_mapping_error(dev, migrate->dma[migrate->dma_nr])) { 714 nouveau_dmem_page_free_locked(drm, dpage); 715 dst_pfns[i] = 0; 716 continue; 717 } 718 719 src_addr = migrate->dma[migrate->dma_nr++]; 720 721 ret = copy(drm, 1, NOUVEAU_APER_VRAM, dst_addr, 722 NOUVEAU_APER_HOST, src_addr); 723 if (ret) { 724 nouveau_dmem_page_free_locked(drm, dpage); 725 dst_pfns[i] = 0; 726 continue; 727 } 728 } 729 730 nouveau_fence_new(drm->dmem->migrate.chan, false, &migrate->fence); 731 732 return; 733 734 error: 735 for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) { 736 struct page *page; 737 738 if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR) 739 continue; 740 741 page = migrate_pfn_to_page(dst_pfns[i]); 742 dst_pfns[i] = MIGRATE_PFN_ERROR; 743 if (page == NULL) 744 continue; 745 746 __free_page(page); 747 } 748 } 749 750 void nouveau_dmem_migrate_finalize_and_map(struct vm_area_struct *vma, 751 const unsigned long *src_pfns, 752 const unsigned long *dst_pfns, 753 unsigned long start, 754 unsigned long end, 755 void *private) 756 { 757 struct nouveau_migrate *migrate = private; 758 struct nouveau_drm *drm = migrate->drm; 759 760 if (migrate->fence) { 761 nouveau_fence_wait(migrate->fence, true, false); 762 nouveau_fence_unref(&migrate->fence); 763 } else { 764 /* 765 * FIXME wait for channel to be IDLE before finalizing 766 * the hmem object below (nouveau_migrate_hmem_fini()) ? 767 */ 768 } 769 770 while (migrate->dma_nr--) { 771 dma_unmap_page(drm->dev->dev, migrate->dma[migrate->dma_nr], 772 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 773 } 774 kfree(migrate->dma); 775 776 /* 777 * FIXME optimization: update GPU page table to point to newly 778 * migrated memory. 779 */ 780 } 781 782 static const struct migrate_vma_ops nouveau_dmem_migrate_ops = { 783 .alloc_and_copy = nouveau_dmem_migrate_alloc_and_copy, 784 .finalize_and_map = nouveau_dmem_migrate_finalize_and_map, 785 }; 786 787 int 788 nouveau_dmem_migrate_vma(struct nouveau_drm *drm, 789 struct vm_area_struct *vma, 790 unsigned long start, 791 unsigned long end) 792 { 793 unsigned long *src_pfns, *dst_pfns, npages; 794 struct nouveau_migrate migrate = {0}; 795 unsigned long i, c, max; 796 int ret = 0; 797 798 npages = (end - start) >> PAGE_SHIFT; 799 max = min(SG_MAX_SINGLE_ALLOC, npages); 800 src_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL); 801 if (src_pfns == NULL) 802 return -ENOMEM; 803 dst_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL); 804 if (dst_pfns == NULL) { 805 kfree(src_pfns); 806 return -ENOMEM; 807 } 808 809 migrate.drm = drm; 810 migrate.vma = vma; 811 migrate.npages = npages; 812 for (i = 0; i < npages; i += c) { 813 unsigned long next; 814 815 c = min(SG_MAX_SINGLE_ALLOC, npages); 816 next = start + (c << PAGE_SHIFT); 817 ret = migrate_vma(&nouveau_dmem_migrate_ops, vma, start, 818 next, src_pfns, dst_pfns, &migrate); 819 if (ret) 820 goto out; 821 start = next; 822 } 823 824 out: 825 kfree(dst_pfns); 826 kfree(src_pfns); 827 return ret; 828 } 829 830 static inline bool 831 nouveau_dmem_page(struct nouveau_drm *drm, struct page *page) 832 { 833 return is_device_private_page(page) && drm->dmem == page_to_dmem(page); 834 } 835 836 void 837 nouveau_dmem_convert_pfn(struct nouveau_drm *drm, 838 struct hmm_range *range) 839 { 840 unsigned long i, npages; 841 842 npages = (range->end - range->start) >> PAGE_SHIFT; 843 for (i = 0; i < npages; ++i) { 844 struct nouveau_dmem_chunk *chunk; 845 struct page *page; 846 uint64_t addr; 847 848 page = hmm_pfn_to_page(range, range->pfns[i]); 849 if (page == NULL) 850 continue; 851 852 if (!(range->pfns[i] & range->flags[HMM_PFN_DEVICE_PRIVATE])) { 853 continue; 854 } 855 856 if (!nouveau_dmem_page(drm, page)) { 857 WARN(1, "Some unknown device memory !\n"); 858 range->pfns[i] = 0; 859 continue; 860 } 861 862 chunk = page->zone_device_data; 863 addr = page_to_pfn(page) - chunk->pfn_first; 864 addr = (addr + chunk->bo->bo.mem.start) << PAGE_SHIFT; 865 866 range->pfns[i] &= ((1UL << range->pfn_shift) - 1); 867 range->pfns[i] |= (addr >> PAGE_SHIFT) << range->pfn_shift; 868 } 869 } 870