1 /* 2 * Copyright 2018 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 #include "nouveau_dmem.h" 23 #include "nouveau_drv.h" 24 #include "nouveau_chan.h" 25 #include "nouveau_dma.h" 26 #include "nouveau_mem.h" 27 #include "nouveau_bo.h" 28 #include "nouveau_svm.h" 29 30 #include <nvif/class.h> 31 #include <nvif/object.h> 32 #include <nvif/push906f.h> 33 #include <nvif/if000c.h> 34 #include <nvif/if500b.h> 35 #include <nvif/if900b.h> 36 #include <nvif/if000c.h> 37 38 #include <nvhw/class/cla0b5.h> 39 40 #include <linux/sched/mm.h> 41 #include <linux/hmm.h> 42 43 /* 44 * FIXME: this is ugly right now we are using TTM to allocate vram and we pin 45 * it in vram while in use. We likely want to overhaul memory management for 46 * nouveau to be more page like (not necessarily with system page size but a 47 * bigger page size) at lowest level and have some shim layer on top that would 48 * provide the same functionality as TTM. 49 */ 50 #define DMEM_CHUNK_SIZE (2UL << 20) 51 #define DMEM_CHUNK_NPAGES (DMEM_CHUNK_SIZE >> PAGE_SHIFT) 52 53 enum nouveau_aper { 54 NOUVEAU_APER_VIRT, 55 NOUVEAU_APER_VRAM, 56 NOUVEAU_APER_HOST, 57 }; 58 59 typedef int (*nouveau_migrate_copy_t)(struct nouveau_drm *drm, u64 npages, 60 enum nouveau_aper, u64 dst_addr, 61 enum nouveau_aper, u64 src_addr); 62 typedef int (*nouveau_clear_page_t)(struct nouveau_drm *drm, u32 length, 63 enum nouveau_aper, u64 dst_addr); 64 65 struct nouveau_dmem_chunk { 66 struct list_head list; 67 struct nouveau_bo *bo; 68 struct nouveau_drm *drm; 69 unsigned long callocated; 70 struct dev_pagemap pagemap; 71 }; 72 73 struct nouveau_dmem_migrate { 74 nouveau_migrate_copy_t copy_func; 75 nouveau_clear_page_t clear_func; 76 struct nouveau_channel *chan; 77 }; 78 79 struct nouveau_dmem { 80 struct nouveau_drm *drm; 81 struct nouveau_dmem_migrate migrate; 82 struct list_head chunks; 83 struct mutex mutex; 84 struct page *free_pages; 85 spinlock_t lock; 86 }; 87 88 static struct nouveau_dmem_chunk *nouveau_page_to_chunk(struct page *page) 89 { 90 return container_of(page->pgmap, struct nouveau_dmem_chunk, pagemap); 91 } 92 93 static struct nouveau_drm *page_to_drm(struct page *page) 94 { 95 struct nouveau_dmem_chunk *chunk = nouveau_page_to_chunk(page); 96 97 return chunk->drm; 98 } 99 100 unsigned long nouveau_dmem_page_addr(struct page *page) 101 { 102 struct nouveau_dmem_chunk *chunk = nouveau_page_to_chunk(page); 103 unsigned long off = (page_to_pfn(page) << PAGE_SHIFT) - 104 chunk->pagemap.res.start; 105 106 return chunk->bo->offset + off; 107 } 108 109 static void nouveau_dmem_page_free(struct page *page) 110 { 111 struct nouveau_dmem_chunk *chunk = nouveau_page_to_chunk(page); 112 struct nouveau_dmem *dmem = chunk->drm->dmem; 113 114 spin_lock(&dmem->lock); 115 page->zone_device_data = dmem->free_pages; 116 dmem->free_pages = page; 117 118 WARN_ON(!chunk->callocated); 119 chunk->callocated--; 120 /* 121 * FIXME when chunk->callocated reach 0 we should add the chunk to 122 * a reclaim list so that it can be freed in case of memory pressure. 123 */ 124 spin_unlock(&dmem->lock); 125 } 126 127 static void nouveau_dmem_fence_done(struct nouveau_fence **fence) 128 { 129 if (fence) { 130 nouveau_fence_wait(*fence, true, false); 131 nouveau_fence_unref(fence); 132 } else { 133 /* 134 * FIXME wait for channel to be IDLE before calling finalizing 135 * the hmem object. 136 */ 137 } 138 } 139 140 static vm_fault_t nouveau_dmem_fault_copy_one(struct nouveau_drm *drm, 141 struct vm_fault *vmf, struct migrate_vma *args, 142 dma_addr_t *dma_addr) 143 { 144 struct device *dev = drm->dev->dev; 145 struct page *dpage, *spage; 146 147 spage = migrate_pfn_to_page(args->src[0]); 148 if (!spage || !(args->src[0] & MIGRATE_PFN_MIGRATE)) 149 return 0; 150 151 dpage = alloc_page_vma(GFP_HIGHUSER, vmf->vma, vmf->address); 152 if (!dpage) 153 return VM_FAULT_SIGBUS; 154 lock_page(dpage); 155 156 *dma_addr = dma_map_page(dev, dpage, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); 157 if (dma_mapping_error(dev, *dma_addr)) 158 goto error_free_page; 159 160 if (drm->dmem->migrate.copy_func(drm, 1, NOUVEAU_APER_HOST, *dma_addr, 161 NOUVEAU_APER_VRAM, nouveau_dmem_page_addr(spage))) 162 goto error_dma_unmap; 163 164 args->dst[0] = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED; 165 return 0; 166 167 error_dma_unmap: 168 dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); 169 error_free_page: 170 __free_page(dpage); 171 return VM_FAULT_SIGBUS; 172 } 173 174 static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf) 175 { 176 struct nouveau_drm *drm = page_to_drm(vmf->page); 177 struct nouveau_dmem *dmem = drm->dmem; 178 struct nouveau_fence *fence; 179 unsigned long src = 0, dst = 0; 180 dma_addr_t dma_addr = 0; 181 vm_fault_t ret; 182 struct migrate_vma args = { 183 .vma = vmf->vma, 184 .start = vmf->address, 185 .end = vmf->address + PAGE_SIZE, 186 .src = &src, 187 .dst = &dst, 188 .src_owner = drm->dev, 189 }; 190 191 /* 192 * FIXME what we really want is to find some heuristic to migrate more 193 * than just one page on CPU fault. When such fault happens it is very 194 * likely that more surrounding page will CPU fault too. 195 */ 196 if (migrate_vma_setup(&args) < 0) 197 return VM_FAULT_SIGBUS; 198 if (!args.cpages) 199 return 0; 200 201 ret = nouveau_dmem_fault_copy_one(drm, vmf, &args, &dma_addr); 202 if (ret || dst == 0) 203 goto done; 204 205 nouveau_fence_new(dmem->migrate.chan, false, &fence); 206 migrate_vma_pages(&args); 207 nouveau_dmem_fence_done(&fence); 208 dma_unmap_page(drm->dev->dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); 209 done: 210 migrate_vma_finalize(&args); 211 return ret; 212 } 213 214 static const struct dev_pagemap_ops nouveau_dmem_pagemap_ops = { 215 .page_free = nouveau_dmem_page_free, 216 .migrate_to_ram = nouveau_dmem_migrate_to_ram, 217 }; 218 219 static int 220 nouveau_dmem_chunk_alloc(struct nouveau_drm *drm, struct page **ppage) 221 { 222 struct nouveau_dmem_chunk *chunk; 223 struct resource *res; 224 struct page *page; 225 void *ptr; 226 unsigned long i, pfn_first; 227 int ret; 228 229 chunk = kzalloc(sizeof(*chunk), GFP_KERNEL); 230 if (chunk == NULL) { 231 ret = -ENOMEM; 232 goto out; 233 } 234 235 /* Allocate unused physical address space for device private pages. */ 236 res = request_free_mem_region(&iomem_resource, DMEM_CHUNK_SIZE, 237 "nouveau_dmem"); 238 if (IS_ERR(res)) { 239 ret = PTR_ERR(res); 240 goto out_free; 241 } 242 243 chunk->drm = drm; 244 chunk->pagemap.type = MEMORY_DEVICE_PRIVATE; 245 chunk->pagemap.res = *res; 246 chunk->pagemap.ops = &nouveau_dmem_pagemap_ops; 247 chunk->pagemap.owner = drm->dev; 248 249 ret = nouveau_bo_new(&drm->client, DMEM_CHUNK_SIZE, 0, 250 TTM_PL_FLAG_VRAM, 0, 0, NULL, NULL, 251 &chunk->bo); 252 if (ret) 253 goto out_release; 254 255 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false); 256 if (ret) 257 goto out_bo_free; 258 259 ptr = memremap_pages(&chunk->pagemap, numa_node_id()); 260 if (IS_ERR(ptr)) { 261 ret = PTR_ERR(ptr); 262 goto out_bo_unpin; 263 } 264 265 mutex_lock(&drm->dmem->mutex); 266 list_add(&chunk->list, &drm->dmem->chunks); 267 mutex_unlock(&drm->dmem->mutex); 268 269 pfn_first = chunk->pagemap.res.start >> PAGE_SHIFT; 270 page = pfn_to_page(pfn_first); 271 spin_lock(&drm->dmem->lock); 272 for (i = 0; i < DMEM_CHUNK_NPAGES - 1; ++i, ++page) { 273 page->zone_device_data = drm->dmem->free_pages; 274 drm->dmem->free_pages = page; 275 } 276 *ppage = page; 277 chunk->callocated++; 278 spin_unlock(&drm->dmem->lock); 279 280 NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", 281 DMEM_CHUNK_SIZE >> 20); 282 283 return 0; 284 285 out_bo_unpin: 286 nouveau_bo_unpin(chunk->bo); 287 out_bo_free: 288 nouveau_bo_ref(NULL, &chunk->bo); 289 out_release: 290 release_mem_region(chunk->pagemap.res.start, 291 resource_size(&chunk->pagemap.res)); 292 out_free: 293 kfree(chunk); 294 out: 295 return ret; 296 } 297 298 static struct page * 299 nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm) 300 { 301 struct nouveau_dmem_chunk *chunk; 302 struct page *page = NULL; 303 int ret; 304 305 spin_lock(&drm->dmem->lock); 306 if (drm->dmem->free_pages) { 307 page = drm->dmem->free_pages; 308 drm->dmem->free_pages = page->zone_device_data; 309 chunk = nouveau_page_to_chunk(page); 310 chunk->callocated++; 311 spin_unlock(&drm->dmem->lock); 312 } else { 313 spin_unlock(&drm->dmem->lock); 314 ret = nouveau_dmem_chunk_alloc(drm, &page); 315 if (ret) 316 return NULL; 317 } 318 319 get_page(page); 320 lock_page(page); 321 return page; 322 } 323 324 static void 325 nouveau_dmem_page_free_locked(struct nouveau_drm *drm, struct page *page) 326 { 327 unlock_page(page); 328 put_page(page); 329 } 330 331 void 332 nouveau_dmem_resume(struct nouveau_drm *drm) 333 { 334 struct nouveau_dmem_chunk *chunk; 335 int ret; 336 337 if (drm->dmem == NULL) 338 return; 339 340 mutex_lock(&drm->dmem->mutex); 341 list_for_each_entry(chunk, &drm->dmem->chunks, list) { 342 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false); 343 /* FIXME handle pin failure */ 344 WARN_ON(ret); 345 } 346 mutex_unlock(&drm->dmem->mutex); 347 } 348 349 void 350 nouveau_dmem_suspend(struct nouveau_drm *drm) 351 { 352 struct nouveau_dmem_chunk *chunk; 353 354 if (drm->dmem == NULL) 355 return; 356 357 mutex_lock(&drm->dmem->mutex); 358 list_for_each_entry(chunk, &drm->dmem->chunks, list) 359 nouveau_bo_unpin(chunk->bo); 360 mutex_unlock(&drm->dmem->mutex); 361 } 362 363 void 364 nouveau_dmem_fini(struct nouveau_drm *drm) 365 { 366 struct nouveau_dmem_chunk *chunk, *tmp; 367 368 if (drm->dmem == NULL) 369 return; 370 371 mutex_lock(&drm->dmem->mutex); 372 373 list_for_each_entry_safe(chunk, tmp, &drm->dmem->chunks, list) { 374 nouveau_bo_unpin(chunk->bo); 375 nouveau_bo_ref(NULL, &chunk->bo); 376 list_del(&chunk->list); 377 memunmap_pages(&chunk->pagemap); 378 release_mem_region(chunk->pagemap.res.start, 379 resource_size(&chunk->pagemap.res)); 380 kfree(chunk); 381 } 382 383 mutex_unlock(&drm->dmem->mutex); 384 } 385 386 static int 387 nvc0b5_migrate_copy(struct nouveau_drm *drm, u64 npages, 388 enum nouveau_aper dst_aper, u64 dst_addr, 389 enum nouveau_aper src_aper, u64 src_addr) 390 { 391 struct nvif_push *push = drm->dmem->migrate.chan->chan.push; 392 u32 launch_dma = 0; 393 int ret; 394 395 ret = PUSH_WAIT(push, 13); 396 if (ret) 397 return ret; 398 399 if (src_aper != NOUVEAU_APER_VIRT) { 400 switch (src_aper) { 401 case NOUVEAU_APER_VRAM: 402 PUSH_IMMD(push, NVA0B5, SET_SRC_PHYS_MODE, 403 NVDEF(NVA0B5, SET_SRC_PHYS_MODE, TARGET, LOCAL_FB)); 404 break; 405 case NOUVEAU_APER_HOST: 406 PUSH_IMMD(push, NVA0B5, SET_SRC_PHYS_MODE, 407 NVDEF(NVA0B5, SET_SRC_PHYS_MODE, TARGET, COHERENT_SYSMEM)); 408 break; 409 default: 410 return -EINVAL; 411 } 412 413 launch_dma |= NVDEF(NVA0B5, LAUNCH_DMA, SRC_TYPE, PHYSICAL); 414 } 415 416 if (dst_aper != NOUVEAU_APER_VIRT) { 417 switch (dst_aper) { 418 case NOUVEAU_APER_VRAM: 419 PUSH_IMMD(push, NVA0B5, SET_DST_PHYS_MODE, 420 NVDEF(NVA0B5, SET_DST_PHYS_MODE, TARGET, LOCAL_FB)); 421 break; 422 case NOUVEAU_APER_HOST: 423 PUSH_IMMD(push, NVA0B5, SET_DST_PHYS_MODE, 424 NVDEF(NVA0B5, SET_DST_PHYS_MODE, TARGET, COHERENT_SYSMEM)); 425 break; 426 default: 427 return -EINVAL; 428 } 429 430 launch_dma |= NVDEF(NVA0B5, LAUNCH_DMA, DST_TYPE, PHYSICAL); 431 } 432 433 PUSH_MTHD(push, NVA0B5, OFFSET_IN_UPPER, 434 NVVAL(NVA0B5, OFFSET_IN_UPPER, UPPER, upper_32_bits(src_addr)), 435 436 OFFSET_IN_LOWER, lower_32_bits(src_addr), 437 438 OFFSET_OUT_UPPER, 439 NVVAL(NVA0B5, OFFSET_OUT_UPPER, UPPER, upper_32_bits(dst_addr)), 440 441 OFFSET_OUT_LOWER, lower_32_bits(dst_addr), 442 PITCH_IN, PAGE_SIZE, 443 PITCH_OUT, PAGE_SIZE, 444 LINE_LENGTH_IN, PAGE_SIZE, 445 LINE_COUNT, npages); 446 447 PUSH_MTHD(push, NVA0B5, LAUNCH_DMA, launch_dma | 448 NVDEF(NVA0B5, LAUNCH_DMA, DATA_TRANSFER_TYPE, NON_PIPELINED) | 449 NVDEF(NVA0B5, LAUNCH_DMA, FLUSH_ENABLE, TRUE) | 450 NVDEF(NVA0B5, LAUNCH_DMA, SEMAPHORE_TYPE, NONE) | 451 NVDEF(NVA0B5, LAUNCH_DMA, INTERRUPT_TYPE, NONE) | 452 NVDEF(NVA0B5, LAUNCH_DMA, SRC_MEMORY_LAYOUT, PITCH) | 453 NVDEF(NVA0B5, LAUNCH_DMA, DST_MEMORY_LAYOUT, PITCH) | 454 NVDEF(NVA0B5, LAUNCH_DMA, MULTI_LINE_ENABLE, TRUE) | 455 NVDEF(NVA0B5, LAUNCH_DMA, REMAP_ENABLE, FALSE) | 456 NVDEF(NVA0B5, LAUNCH_DMA, BYPASS_L2, USE_PTE_SETTING)); 457 return 0; 458 } 459 460 static int 461 nvc0b5_migrate_clear(struct nouveau_drm *drm, u32 length, 462 enum nouveau_aper dst_aper, u64 dst_addr) 463 { 464 struct nvif_push *push = drm->dmem->migrate.chan->chan.push; 465 u32 launch_dma = 0; 466 int ret; 467 468 ret = PUSH_WAIT(push, 12); 469 if (ret) 470 return ret; 471 472 switch (dst_aper) { 473 case NOUVEAU_APER_VRAM: 474 PUSH_IMMD(push, NVA0B5, SET_DST_PHYS_MODE, 475 NVDEF(NVA0B5, SET_DST_PHYS_MODE, TARGET, LOCAL_FB)); 476 break; 477 case NOUVEAU_APER_HOST: 478 PUSH_IMMD(push, NVA0B5, SET_DST_PHYS_MODE, 479 NVDEF(NVA0B5, SET_DST_PHYS_MODE, TARGET, COHERENT_SYSMEM)); 480 break; 481 default: 482 return -EINVAL; 483 } 484 485 launch_dma |= NVDEF(NVA0B5, LAUNCH_DMA, DST_TYPE, PHYSICAL); 486 487 PUSH_MTHD(push, NVA0B5, SET_REMAP_CONST_A, 0, 488 SET_REMAP_CONST_B, 0, 489 490 SET_REMAP_COMPONENTS, 491 NVDEF(NVA0B5, SET_REMAP_COMPONENTS, DST_X, CONST_A) | 492 NVDEF(NVA0B5, SET_REMAP_COMPONENTS, DST_Y, CONST_B) | 493 NVDEF(NVA0B5, SET_REMAP_COMPONENTS, COMPONENT_SIZE, FOUR) | 494 NVDEF(NVA0B5, SET_REMAP_COMPONENTS, NUM_DST_COMPONENTS, TWO)); 495 496 PUSH_MTHD(push, NVA0B5, OFFSET_OUT_UPPER, 497 NVVAL(NVA0B5, OFFSET_OUT_UPPER, UPPER, upper_32_bits(dst_addr)), 498 499 OFFSET_OUT_LOWER, lower_32_bits(dst_addr)); 500 501 PUSH_MTHD(push, NVA0B5, LINE_LENGTH_IN, length >> 3); 502 503 PUSH_MTHD(push, NVA0B5, LAUNCH_DMA, launch_dma | 504 NVDEF(NVA0B5, LAUNCH_DMA, DATA_TRANSFER_TYPE, NON_PIPELINED) | 505 NVDEF(NVA0B5, LAUNCH_DMA, FLUSH_ENABLE, TRUE) | 506 NVDEF(NVA0B5, LAUNCH_DMA, SEMAPHORE_TYPE, NONE) | 507 NVDEF(NVA0B5, LAUNCH_DMA, INTERRUPT_TYPE, NONE) | 508 NVDEF(NVA0B5, LAUNCH_DMA, SRC_MEMORY_LAYOUT, PITCH) | 509 NVDEF(NVA0B5, LAUNCH_DMA, DST_MEMORY_LAYOUT, PITCH) | 510 NVDEF(NVA0B5, LAUNCH_DMA, MULTI_LINE_ENABLE, FALSE) | 511 NVDEF(NVA0B5, LAUNCH_DMA, REMAP_ENABLE, TRUE) | 512 NVDEF(NVA0B5, LAUNCH_DMA, BYPASS_L2, USE_PTE_SETTING)); 513 return 0; 514 } 515 516 static int 517 nouveau_dmem_migrate_init(struct nouveau_drm *drm) 518 { 519 switch (drm->ttm.copy.oclass) { 520 case PASCAL_DMA_COPY_A: 521 case PASCAL_DMA_COPY_B: 522 case VOLTA_DMA_COPY_A: 523 case TURING_DMA_COPY_A: 524 drm->dmem->migrate.copy_func = nvc0b5_migrate_copy; 525 drm->dmem->migrate.clear_func = nvc0b5_migrate_clear; 526 drm->dmem->migrate.chan = drm->ttm.chan; 527 return 0; 528 default: 529 break; 530 } 531 return -ENODEV; 532 } 533 534 void 535 nouveau_dmem_init(struct nouveau_drm *drm) 536 { 537 int ret; 538 539 /* This only make sense on PASCAL or newer */ 540 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_PASCAL) 541 return; 542 543 if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL))) 544 return; 545 546 drm->dmem->drm = drm; 547 mutex_init(&drm->dmem->mutex); 548 INIT_LIST_HEAD(&drm->dmem->chunks); 549 mutex_init(&drm->dmem->mutex); 550 spin_lock_init(&drm->dmem->lock); 551 552 /* Initialize migration dma helpers before registering memory */ 553 ret = nouveau_dmem_migrate_init(drm); 554 if (ret) { 555 kfree(drm->dmem); 556 drm->dmem = NULL; 557 } 558 } 559 560 static unsigned long nouveau_dmem_migrate_copy_one(struct nouveau_drm *drm, 561 unsigned long src, dma_addr_t *dma_addr, u64 *pfn) 562 { 563 struct device *dev = drm->dev->dev; 564 struct page *dpage, *spage; 565 unsigned long paddr; 566 567 spage = migrate_pfn_to_page(src); 568 if (!(src & MIGRATE_PFN_MIGRATE)) 569 goto out; 570 571 dpage = nouveau_dmem_page_alloc_locked(drm); 572 if (!dpage) 573 goto out; 574 575 paddr = nouveau_dmem_page_addr(dpage); 576 if (spage) { 577 *dma_addr = dma_map_page(dev, spage, 0, page_size(spage), 578 DMA_BIDIRECTIONAL); 579 if (dma_mapping_error(dev, *dma_addr)) 580 goto out_free_page; 581 if (drm->dmem->migrate.copy_func(drm, 1, 582 NOUVEAU_APER_VRAM, paddr, NOUVEAU_APER_HOST, *dma_addr)) 583 goto out_dma_unmap; 584 } else { 585 *dma_addr = DMA_MAPPING_ERROR; 586 if (drm->dmem->migrate.clear_func(drm, page_size(dpage), 587 NOUVEAU_APER_VRAM, paddr)) 588 goto out_free_page; 589 } 590 591 *pfn = NVIF_VMM_PFNMAP_V0_V | NVIF_VMM_PFNMAP_V0_VRAM | 592 ((paddr >> PAGE_SHIFT) << NVIF_VMM_PFNMAP_V0_ADDR_SHIFT); 593 if (src & MIGRATE_PFN_WRITE) 594 *pfn |= NVIF_VMM_PFNMAP_V0_W; 595 return migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED; 596 597 out_dma_unmap: 598 dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); 599 out_free_page: 600 nouveau_dmem_page_free_locked(drm, dpage); 601 out: 602 *pfn = NVIF_VMM_PFNMAP_V0_NONE; 603 return 0; 604 } 605 606 static void nouveau_dmem_migrate_chunk(struct nouveau_drm *drm, 607 struct nouveau_svmm *svmm, struct migrate_vma *args, 608 dma_addr_t *dma_addrs, u64 *pfns) 609 { 610 struct nouveau_fence *fence; 611 unsigned long addr = args->start, nr_dma = 0, i; 612 613 for (i = 0; addr < args->end; i++) { 614 args->dst[i] = nouveau_dmem_migrate_copy_one(drm, args->src[i], 615 dma_addrs + nr_dma, pfns + i); 616 if (!dma_mapping_error(drm->dev->dev, dma_addrs[nr_dma])) 617 nr_dma++; 618 addr += PAGE_SIZE; 619 } 620 621 nouveau_fence_new(drm->dmem->migrate.chan, false, &fence); 622 migrate_vma_pages(args); 623 nouveau_dmem_fence_done(&fence); 624 nouveau_pfns_map(svmm, args->vma->vm_mm, args->start, pfns, i); 625 626 while (nr_dma--) { 627 dma_unmap_page(drm->dev->dev, dma_addrs[nr_dma], PAGE_SIZE, 628 DMA_BIDIRECTIONAL); 629 } 630 migrate_vma_finalize(args); 631 } 632 633 int 634 nouveau_dmem_migrate_vma(struct nouveau_drm *drm, 635 struct nouveau_svmm *svmm, 636 struct vm_area_struct *vma, 637 unsigned long start, 638 unsigned long end) 639 { 640 unsigned long npages = (end - start) >> PAGE_SHIFT; 641 unsigned long max = min(SG_MAX_SINGLE_ALLOC, npages); 642 dma_addr_t *dma_addrs; 643 struct migrate_vma args = { 644 .vma = vma, 645 .start = start, 646 }; 647 unsigned long i; 648 u64 *pfns; 649 int ret = -ENOMEM; 650 651 if (drm->dmem == NULL) 652 return -ENODEV; 653 654 args.src = kcalloc(max, sizeof(*args.src), GFP_KERNEL); 655 if (!args.src) 656 goto out; 657 args.dst = kcalloc(max, sizeof(*args.dst), GFP_KERNEL); 658 if (!args.dst) 659 goto out_free_src; 660 661 dma_addrs = kmalloc_array(max, sizeof(*dma_addrs), GFP_KERNEL); 662 if (!dma_addrs) 663 goto out_free_dst; 664 665 pfns = nouveau_pfns_alloc(max); 666 if (!pfns) 667 goto out_free_dma; 668 669 for (i = 0; i < npages; i += max) { 670 args.end = start + (max << PAGE_SHIFT); 671 ret = migrate_vma_setup(&args); 672 if (ret) 673 goto out_free_pfns; 674 675 if (args.cpages) 676 nouveau_dmem_migrate_chunk(drm, svmm, &args, dma_addrs, 677 pfns); 678 args.start = args.end; 679 } 680 681 ret = 0; 682 out_free_pfns: 683 nouveau_pfns_free(pfns); 684 out_free_dma: 685 kfree(dma_addrs); 686 out_free_dst: 687 kfree(args.dst); 688 out_free_src: 689 kfree(args.src); 690 out: 691 return ret; 692 } 693