1 /* 2 * Copyright 2018 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 #include "nouveau_dmem.h" 23 #include "nouveau_drv.h" 24 #include "nouveau_chan.h" 25 #include "nouveau_dma.h" 26 #include "nouveau_mem.h" 27 #include "nouveau_bo.h" 28 29 #include <nvif/class.h> 30 #include <nvif/object.h> 31 #include <nvif/if000c.h> 32 #include <nvif/if500b.h> 33 #include <nvif/if900b.h> 34 35 #include <linux/sched/mm.h> 36 #include <linux/hmm.h> 37 38 /* 39 * FIXME: this is ugly right now we are using TTM to allocate vram and we pin 40 * it in vram while in use. We likely want to overhaul memory management for 41 * nouveau to be more page like (not necessarily with system page size but a 42 * bigger page size) at lowest level and have some shim layer on top that would 43 * provide the same functionality as TTM. 44 */ 45 #define DMEM_CHUNK_SIZE (2UL << 20) 46 #define DMEM_CHUNK_NPAGES (DMEM_CHUNK_SIZE >> PAGE_SHIFT) 47 48 enum nouveau_aper { 49 NOUVEAU_APER_VIRT, 50 NOUVEAU_APER_VRAM, 51 NOUVEAU_APER_HOST, 52 }; 53 54 typedef int (*nouveau_migrate_copy_t)(struct nouveau_drm *drm, u64 npages, 55 enum nouveau_aper, u64 dst_addr, 56 enum nouveau_aper, u64 src_addr); 57 58 struct nouveau_dmem_chunk { 59 struct list_head list; 60 struct nouveau_bo *bo; 61 struct nouveau_drm *drm; 62 unsigned long pfn_first; 63 unsigned long callocated; 64 unsigned long bitmap[BITS_TO_LONGS(DMEM_CHUNK_NPAGES)]; 65 spinlock_t lock; 66 }; 67 68 struct nouveau_dmem_migrate { 69 nouveau_migrate_copy_t copy_func; 70 struct nouveau_channel *chan; 71 }; 72 73 struct nouveau_dmem { 74 struct nouveau_drm *drm; 75 struct dev_pagemap pagemap; 76 struct nouveau_dmem_migrate migrate; 77 struct list_head chunk_free; 78 struct list_head chunk_full; 79 struct list_head chunk_empty; 80 struct mutex mutex; 81 }; 82 83 static inline struct nouveau_dmem *page_to_dmem(struct page *page) 84 { 85 return container_of(page->pgmap, struct nouveau_dmem, pagemap); 86 } 87 88 static unsigned long nouveau_dmem_page_addr(struct page *page) 89 { 90 struct nouveau_dmem_chunk *chunk = page->zone_device_data; 91 unsigned long idx = page_to_pfn(page) - chunk->pfn_first; 92 93 return (idx << PAGE_SHIFT) + chunk->bo->bo.offset; 94 } 95 96 static void nouveau_dmem_page_free(struct page *page) 97 { 98 struct nouveau_dmem_chunk *chunk = page->zone_device_data; 99 unsigned long idx = page_to_pfn(page) - chunk->pfn_first; 100 101 /* 102 * FIXME: 103 * 104 * This is really a bad example, we need to overhaul nouveau memory 105 * management to be more page focus and allow lighter locking scheme 106 * to be use in the process. 107 */ 108 spin_lock(&chunk->lock); 109 clear_bit(idx, chunk->bitmap); 110 WARN_ON(!chunk->callocated); 111 chunk->callocated--; 112 /* 113 * FIXME when chunk->callocated reach 0 we should add the chunk to 114 * a reclaim list so that it can be freed in case of memory pressure. 115 */ 116 spin_unlock(&chunk->lock); 117 } 118 119 static void nouveau_dmem_fence_done(struct nouveau_fence **fence) 120 { 121 if (fence) { 122 nouveau_fence_wait(*fence, true, false); 123 nouveau_fence_unref(fence); 124 } else { 125 /* 126 * FIXME wait for channel to be IDLE before calling finalizing 127 * the hmem object. 128 */ 129 } 130 } 131 132 static vm_fault_t nouveau_dmem_fault_copy_one(struct nouveau_drm *drm, 133 struct vm_fault *vmf, struct migrate_vma *args, 134 dma_addr_t *dma_addr) 135 { 136 struct device *dev = drm->dev->dev; 137 struct page *dpage, *spage; 138 139 spage = migrate_pfn_to_page(args->src[0]); 140 if (!spage || !(args->src[0] & MIGRATE_PFN_MIGRATE)) 141 return 0; 142 143 dpage = alloc_page_vma(GFP_HIGHUSER, vmf->vma, vmf->address); 144 if (!dpage) 145 return VM_FAULT_SIGBUS; 146 lock_page(dpage); 147 148 *dma_addr = dma_map_page(dev, dpage, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); 149 if (dma_mapping_error(dev, *dma_addr)) 150 goto error_free_page; 151 152 if (drm->dmem->migrate.copy_func(drm, 1, NOUVEAU_APER_HOST, *dma_addr, 153 NOUVEAU_APER_VRAM, nouveau_dmem_page_addr(spage))) 154 goto error_dma_unmap; 155 156 args->dst[0] = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED; 157 return 0; 158 159 error_dma_unmap: 160 dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); 161 error_free_page: 162 __free_page(dpage); 163 return VM_FAULT_SIGBUS; 164 } 165 166 static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf) 167 { 168 struct nouveau_dmem *dmem = page_to_dmem(vmf->page); 169 struct nouveau_drm *drm = dmem->drm; 170 struct nouveau_fence *fence; 171 unsigned long src = 0, dst = 0; 172 dma_addr_t dma_addr = 0; 173 vm_fault_t ret; 174 struct migrate_vma args = { 175 .vma = vmf->vma, 176 .start = vmf->address, 177 .end = vmf->address + PAGE_SIZE, 178 .src = &src, 179 .dst = &dst, 180 .src_owner = drm->dev, 181 }; 182 183 /* 184 * FIXME what we really want is to find some heuristic to migrate more 185 * than just one page on CPU fault. When such fault happens it is very 186 * likely that more surrounding page will CPU fault too. 187 */ 188 if (migrate_vma_setup(&args) < 0) 189 return VM_FAULT_SIGBUS; 190 if (!args.cpages) 191 return 0; 192 193 ret = nouveau_dmem_fault_copy_one(drm, vmf, &args, &dma_addr); 194 if (ret || dst == 0) 195 goto done; 196 197 nouveau_fence_new(dmem->migrate.chan, false, &fence); 198 migrate_vma_pages(&args); 199 nouveau_dmem_fence_done(&fence); 200 dma_unmap_page(drm->dev->dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); 201 done: 202 migrate_vma_finalize(&args); 203 return ret; 204 } 205 206 static const struct dev_pagemap_ops nouveau_dmem_pagemap_ops = { 207 .page_free = nouveau_dmem_page_free, 208 .migrate_to_ram = nouveau_dmem_migrate_to_ram, 209 }; 210 211 static int 212 nouveau_dmem_chunk_alloc(struct nouveau_drm *drm) 213 { 214 struct nouveau_dmem_chunk *chunk; 215 int ret; 216 217 if (drm->dmem == NULL) 218 return -EINVAL; 219 220 mutex_lock(&drm->dmem->mutex); 221 chunk = list_first_entry_or_null(&drm->dmem->chunk_empty, 222 struct nouveau_dmem_chunk, 223 list); 224 if (chunk == NULL) { 225 mutex_unlock(&drm->dmem->mutex); 226 return -ENOMEM; 227 } 228 229 list_del(&chunk->list); 230 mutex_unlock(&drm->dmem->mutex); 231 232 ret = nouveau_bo_new(&drm->client, DMEM_CHUNK_SIZE, 0, 233 TTM_PL_FLAG_VRAM, 0, 0, NULL, NULL, 234 &chunk->bo); 235 if (ret) 236 goto out; 237 238 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false); 239 if (ret) { 240 nouveau_bo_ref(NULL, &chunk->bo); 241 goto out; 242 } 243 244 bitmap_zero(chunk->bitmap, DMEM_CHUNK_NPAGES); 245 spin_lock_init(&chunk->lock); 246 247 out: 248 mutex_lock(&drm->dmem->mutex); 249 if (chunk->bo) 250 list_add(&chunk->list, &drm->dmem->chunk_empty); 251 else 252 list_add_tail(&chunk->list, &drm->dmem->chunk_empty); 253 mutex_unlock(&drm->dmem->mutex); 254 255 return ret; 256 } 257 258 static struct nouveau_dmem_chunk * 259 nouveau_dmem_chunk_first_free_locked(struct nouveau_drm *drm) 260 { 261 struct nouveau_dmem_chunk *chunk; 262 263 chunk = list_first_entry_or_null(&drm->dmem->chunk_free, 264 struct nouveau_dmem_chunk, 265 list); 266 if (chunk) 267 return chunk; 268 269 chunk = list_first_entry_or_null(&drm->dmem->chunk_empty, 270 struct nouveau_dmem_chunk, 271 list); 272 if (chunk->bo) 273 return chunk; 274 275 return NULL; 276 } 277 278 static int 279 nouveau_dmem_pages_alloc(struct nouveau_drm *drm, 280 unsigned long npages, 281 unsigned long *pages) 282 { 283 struct nouveau_dmem_chunk *chunk; 284 unsigned long c; 285 int ret; 286 287 memset(pages, 0xff, npages * sizeof(*pages)); 288 289 mutex_lock(&drm->dmem->mutex); 290 for (c = 0; c < npages;) { 291 unsigned long i; 292 293 chunk = nouveau_dmem_chunk_first_free_locked(drm); 294 if (chunk == NULL) { 295 mutex_unlock(&drm->dmem->mutex); 296 ret = nouveau_dmem_chunk_alloc(drm); 297 if (ret) { 298 if (c) 299 return 0; 300 return ret; 301 } 302 mutex_lock(&drm->dmem->mutex); 303 continue; 304 } 305 306 spin_lock(&chunk->lock); 307 i = find_first_zero_bit(chunk->bitmap, DMEM_CHUNK_NPAGES); 308 while (i < DMEM_CHUNK_NPAGES && c < npages) { 309 pages[c] = chunk->pfn_first + i; 310 set_bit(i, chunk->bitmap); 311 chunk->callocated++; 312 c++; 313 314 i = find_next_zero_bit(chunk->bitmap, 315 DMEM_CHUNK_NPAGES, i); 316 } 317 spin_unlock(&chunk->lock); 318 } 319 mutex_unlock(&drm->dmem->mutex); 320 321 return 0; 322 } 323 324 static struct page * 325 nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm) 326 { 327 unsigned long pfns[1]; 328 struct page *page; 329 int ret; 330 331 /* FIXME stop all the miss-match API ... */ 332 ret = nouveau_dmem_pages_alloc(drm, 1, pfns); 333 if (ret) 334 return NULL; 335 336 page = pfn_to_page(pfns[0]); 337 get_page(page); 338 lock_page(page); 339 return page; 340 } 341 342 static void 343 nouveau_dmem_page_free_locked(struct nouveau_drm *drm, struct page *page) 344 { 345 unlock_page(page); 346 put_page(page); 347 } 348 349 void 350 nouveau_dmem_resume(struct nouveau_drm *drm) 351 { 352 struct nouveau_dmem_chunk *chunk; 353 int ret; 354 355 if (drm->dmem == NULL) 356 return; 357 358 mutex_lock(&drm->dmem->mutex); 359 list_for_each_entry (chunk, &drm->dmem->chunk_free, list) { 360 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false); 361 /* FIXME handle pin failure */ 362 WARN_ON(ret); 363 } 364 list_for_each_entry (chunk, &drm->dmem->chunk_full, list) { 365 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false); 366 /* FIXME handle pin failure */ 367 WARN_ON(ret); 368 } 369 mutex_unlock(&drm->dmem->mutex); 370 } 371 372 void 373 nouveau_dmem_suspend(struct nouveau_drm *drm) 374 { 375 struct nouveau_dmem_chunk *chunk; 376 377 if (drm->dmem == NULL) 378 return; 379 380 mutex_lock(&drm->dmem->mutex); 381 list_for_each_entry (chunk, &drm->dmem->chunk_free, list) { 382 nouveau_bo_unpin(chunk->bo); 383 } 384 list_for_each_entry (chunk, &drm->dmem->chunk_full, list) { 385 nouveau_bo_unpin(chunk->bo); 386 } 387 mutex_unlock(&drm->dmem->mutex); 388 } 389 390 void 391 nouveau_dmem_fini(struct nouveau_drm *drm) 392 { 393 struct nouveau_dmem_chunk *chunk, *tmp; 394 395 if (drm->dmem == NULL) 396 return; 397 398 mutex_lock(&drm->dmem->mutex); 399 400 WARN_ON(!list_empty(&drm->dmem->chunk_free)); 401 WARN_ON(!list_empty(&drm->dmem->chunk_full)); 402 403 list_for_each_entry_safe (chunk, tmp, &drm->dmem->chunk_empty, list) { 404 if (chunk->bo) { 405 nouveau_bo_unpin(chunk->bo); 406 nouveau_bo_ref(NULL, &chunk->bo); 407 } 408 list_del(&chunk->list); 409 kfree(chunk); 410 } 411 412 mutex_unlock(&drm->dmem->mutex); 413 } 414 415 static int 416 nvc0b5_migrate_copy(struct nouveau_drm *drm, u64 npages, 417 enum nouveau_aper dst_aper, u64 dst_addr, 418 enum nouveau_aper src_aper, u64 src_addr) 419 { 420 struct nouveau_channel *chan = drm->dmem->migrate.chan; 421 u32 launch_dma = (1 << 9) /* MULTI_LINE_ENABLE. */ | 422 (1 << 8) /* DST_MEMORY_LAYOUT_PITCH. */ | 423 (1 << 7) /* SRC_MEMORY_LAYOUT_PITCH. */ | 424 (1 << 2) /* FLUSH_ENABLE_TRUE. */ | 425 (2 << 0) /* DATA_TRANSFER_TYPE_NON_PIPELINED. */; 426 int ret; 427 428 ret = RING_SPACE(chan, 13); 429 if (ret) 430 return ret; 431 432 if (src_aper != NOUVEAU_APER_VIRT) { 433 switch (src_aper) { 434 case NOUVEAU_APER_VRAM: 435 BEGIN_IMC0(chan, NvSubCopy, 0x0260, 0); 436 break; 437 case NOUVEAU_APER_HOST: 438 BEGIN_IMC0(chan, NvSubCopy, 0x0260, 1); 439 break; 440 default: 441 return -EINVAL; 442 } 443 launch_dma |= 0x00001000; /* SRC_TYPE_PHYSICAL. */ 444 } 445 446 if (dst_aper != NOUVEAU_APER_VIRT) { 447 switch (dst_aper) { 448 case NOUVEAU_APER_VRAM: 449 BEGIN_IMC0(chan, NvSubCopy, 0x0264, 0); 450 break; 451 case NOUVEAU_APER_HOST: 452 BEGIN_IMC0(chan, NvSubCopy, 0x0264, 1); 453 break; 454 default: 455 return -EINVAL; 456 } 457 launch_dma |= 0x00002000; /* DST_TYPE_PHYSICAL. */ 458 } 459 460 BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8); 461 OUT_RING (chan, upper_32_bits(src_addr)); 462 OUT_RING (chan, lower_32_bits(src_addr)); 463 OUT_RING (chan, upper_32_bits(dst_addr)); 464 OUT_RING (chan, lower_32_bits(dst_addr)); 465 OUT_RING (chan, PAGE_SIZE); 466 OUT_RING (chan, PAGE_SIZE); 467 OUT_RING (chan, PAGE_SIZE); 468 OUT_RING (chan, npages); 469 BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1); 470 OUT_RING (chan, launch_dma); 471 return 0; 472 } 473 474 static int 475 nouveau_dmem_migrate_init(struct nouveau_drm *drm) 476 { 477 switch (drm->ttm.copy.oclass) { 478 case PASCAL_DMA_COPY_A: 479 case PASCAL_DMA_COPY_B: 480 case VOLTA_DMA_COPY_A: 481 case TURING_DMA_COPY_A: 482 drm->dmem->migrate.copy_func = nvc0b5_migrate_copy; 483 drm->dmem->migrate.chan = drm->ttm.chan; 484 return 0; 485 default: 486 break; 487 } 488 return -ENODEV; 489 } 490 491 void 492 nouveau_dmem_init(struct nouveau_drm *drm) 493 { 494 struct device *device = drm->dev->dev; 495 struct resource *res; 496 unsigned long i, size, pfn_first; 497 int ret; 498 499 /* This only make sense on PASCAL or newer */ 500 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_PASCAL) 501 return; 502 503 if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL))) 504 return; 505 506 drm->dmem->drm = drm; 507 mutex_init(&drm->dmem->mutex); 508 INIT_LIST_HEAD(&drm->dmem->chunk_free); 509 INIT_LIST_HEAD(&drm->dmem->chunk_full); 510 INIT_LIST_HEAD(&drm->dmem->chunk_empty); 511 512 size = ALIGN(drm->client.device.info.ram_user, DMEM_CHUNK_SIZE); 513 514 /* Initialize migration dma helpers before registering memory */ 515 ret = nouveau_dmem_migrate_init(drm); 516 if (ret) 517 goto out_free; 518 519 /* 520 * FIXME we need some kind of policy to decide how much VRAM we 521 * want to register with HMM. For now just register everything 522 * and latter if we want to do thing like over commit then we 523 * could revisit this. 524 */ 525 res = devm_request_free_mem_region(device, &iomem_resource, size); 526 if (IS_ERR(res)) 527 goto out_free; 528 drm->dmem->pagemap.type = MEMORY_DEVICE_PRIVATE; 529 drm->dmem->pagemap.res = *res; 530 drm->dmem->pagemap.ops = &nouveau_dmem_pagemap_ops; 531 drm->dmem->pagemap.owner = drm->dev; 532 if (IS_ERR(devm_memremap_pages(device, &drm->dmem->pagemap))) 533 goto out_free; 534 535 pfn_first = res->start >> PAGE_SHIFT; 536 for (i = 0; i < (size / DMEM_CHUNK_SIZE); ++i) { 537 struct nouveau_dmem_chunk *chunk; 538 struct page *page; 539 unsigned long j; 540 541 chunk = kzalloc(sizeof(*chunk), GFP_KERNEL); 542 if (chunk == NULL) { 543 nouveau_dmem_fini(drm); 544 return; 545 } 546 547 chunk->drm = drm; 548 chunk->pfn_first = pfn_first + (i * DMEM_CHUNK_NPAGES); 549 list_add_tail(&chunk->list, &drm->dmem->chunk_empty); 550 551 page = pfn_to_page(chunk->pfn_first); 552 for (j = 0; j < DMEM_CHUNK_NPAGES; ++j, ++page) 553 page->zone_device_data = chunk; 554 } 555 556 NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", size >> 20); 557 return; 558 out_free: 559 kfree(drm->dmem); 560 drm->dmem = NULL; 561 } 562 563 static unsigned long nouveau_dmem_migrate_copy_one(struct nouveau_drm *drm, 564 unsigned long src, dma_addr_t *dma_addr) 565 { 566 struct device *dev = drm->dev->dev; 567 struct page *dpage, *spage; 568 569 spage = migrate_pfn_to_page(src); 570 if (!spage || !(src & MIGRATE_PFN_MIGRATE)) 571 goto out; 572 573 dpage = nouveau_dmem_page_alloc_locked(drm); 574 if (!dpage) 575 return 0; 576 577 *dma_addr = dma_map_page(dev, spage, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); 578 if (dma_mapping_error(dev, *dma_addr)) 579 goto out_free_page; 580 581 if (drm->dmem->migrate.copy_func(drm, 1, NOUVEAU_APER_VRAM, 582 nouveau_dmem_page_addr(dpage), NOUVEAU_APER_HOST, 583 *dma_addr)) 584 goto out_dma_unmap; 585 586 return migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED; 587 588 out_dma_unmap: 589 dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); 590 out_free_page: 591 nouveau_dmem_page_free_locked(drm, dpage); 592 out: 593 return 0; 594 } 595 596 static void nouveau_dmem_migrate_chunk(struct nouveau_drm *drm, 597 struct migrate_vma *args, dma_addr_t *dma_addrs) 598 { 599 struct nouveau_fence *fence; 600 unsigned long addr = args->start, nr_dma = 0, i; 601 602 for (i = 0; addr < args->end; i++) { 603 args->dst[i] = nouveau_dmem_migrate_copy_one(drm, args->src[i], 604 dma_addrs + nr_dma); 605 if (args->dst[i]) 606 nr_dma++; 607 addr += PAGE_SIZE; 608 } 609 610 nouveau_fence_new(drm->dmem->migrate.chan, false, &fence); 611 migrate_vma_pages(args); 612 nouveau_dmem_fence_done(&fence); 613 614 while (nr_dma--) { 615 dma_unmap_page(drm->dev->dev, dma_addrs[nr_dma], PAGE_SIZE, 616 DMA_BIDIRECTIONAL); 617 } 618 /* 619 * FIXME optimization: update GPU page table to point to newly migrated 620 * memory. 621 */ 622 migrate_vma_finalize(args); 623 } 624 625 int 626 nouveau_dmem_migrate_vma(struct nouveau_drm *drm, 627 struct vm_area_struct *vma, 628 unsigned long start, 629 unsigned long end) 630 { 631 unsigned long npages = (end - start) >> PAGE_SHIFT; 632 unsigned long max = min(SG_MAX_SINGLE_ALLOC, npages); 633 dma_addr_t *dma_addrs; 634 struct migrate_vma args = { 635 .vma = vma, 636 .start = start, 637 }; 638 unsigned long c, i; 639 int ret = -ENOMEM; 640 641 args.src = kcalloc(max, sizeof(*args.src), GFP_KERNEL); 642 if (!args.src) 643 goto out; 644 args.dst = kcalloc(max, sizeof(*args.dst), GFP_KERNEL); 645 if (!args.dst) 646 goto out_free_src; 647 648 dma_addrs = kmalloc_array(max, sizeof(*dma_addrs), GFP_KERNEL); 649 if (!dma_addrs) 650 goto out_free_dst; 651 652 for (i = 0; i < npages; i += c) { 653 c = min(SG_MAX_SINGLE_ALLOC, npages); 654 args.end = start + (c << PAGE_SHIFT); 655 ret = migrate_vma_setup(&args); 656 if (ret) 657 goto out_free_dma; 658 659 if (args.cpages) 660 nouveau_dmem_migrate_chunk(drm, &args, dma_addrs); 661 args.start = args.end; 662 } 663 664 ret = 0; 665 out_free_dma: 666 kfree(dma_addrs); 667 out_free_dst: 668 kfree(args.dst); 669 out_free_src: 670 kfree(args.src); 671 out: 672 return ret; 673 } 674 675 void 676 nouveau_dmem_convert_pfn(struct nouveau_drm *drm, 677 struct hmm_range *range) 678 { 679 unsigned long i, npages; 680 681 npages = (range->end - range->start) >> PAGE_SHIFT; 682 for (i = 0; i < npages; ++i) { 683 struct page *page; 684 uint64_t addr; 685 686 page = hmm_device_entry_to_page(range, range->pfns[i]); 687 if (page == NULL) 688 continue; 689 690 if (!is_device_private_page(page)) 691 continue; 692 693 addr = nouveau_dmem_page_addr(page); 694 range->pfns[i] &= ((1UL << range->pfn_shift) - 1); 695 range->pfns[i] |= (addr >> PAGE_SHIFT) << range->pfn_shift; 696 range->pfns[i] |= NVIF_VMM_PFNMAP_V0_VRAM; 697 } 698 } 699