1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/err.h> 5 #include <linux/spinlock.h> 6 7 #include <linux/mm.h> 8 #include <linux/memremap.h> 9 #include <linux/pagemap.h> 10 #include <linux/rmap.h> 11 #include <linux/swap.h> 12 #include <linux/swapops.h> 13 14 #include <linux/sched/signal.h> 15 #include <linux/rwsem.h> 16 #include <linux/hugetlb.h> 17 #include <linux/migrate.h> 18 #include <linux/mm_inline.h> 19 #include <linux/sched/mm.h> 20 21 #include <asm/mmu_context.h> 22 #include <asm/tlbflush.h> 23 24 #include "internal.h" 25 26 struct follow_page_context { 27 struct dev_pagemap *pgmap; 28 unsigned int page_mask; 29 }; 30 31 static void hpage_pincount_add(struct page *page, int refs) 32 { 33 VM_BUG_ON_PAGE(!hpage_pincount_available(page), page); 34 VM_BUG_ON_PAGE(page != compound_head(page), page); 35 36 atomic_add(refs, compound_pincount_ptr(page)); 37 } 38 39 static void hpage_pincount_sub(struct page *page, int refs) 40 { 41 VM_BUG_ON_PAGE(!hpage_pincount_available(page), page); 42 VM_BUG_ON_PAGE(page != compound_head(page), page); 43 44 atomic_sub(refs, compound_pincount_ptr(page)); 45 } 46 47 /* 48 * Return the compound head page with ref appropriately incremented, 49 * or NULL if that failed. 50 */ 51 static inline struct page *try_get_compound_head(struct page *page, int refs) 52 { 53 struct page *head = compound_head(page); 54 55 if (WARN_ON_ONCE(page_ref_count(head) < 0)) 56 return NULL; 57 if (unlikely(!page_cache_add_speculative(head, refs))) 58 return NULL; 59 return head; 60 } 61 62 /* 63 * try_grab_compound_head() - attempt to elevate a page's refcount, by a 64 * flags-dependent amount. 65 * 66 * "grab" names in this file mean, "look at flags to decide whether to use 67 * FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount. 68 * 69 * Either FOLL_PIN or FOLL_GET (or neither) must be set, but not both at the 70 * same time. (That's true throughout the get_user_pages*() and 71 * pin_user_pages*() APIs.) Cases: 72 * 73 * FOLL_GET: page's refcount will be incremented by 1. 74 * FOLL_PIN: page's refcount will be incremented by GUP_PIN_COUNTING_BIAS. 75 * 76 * Return: head page (with refcount appropriately incremented) for success, or 77 * NULL upon failure. If neither FOLL_GET nor FOLL_PIN was set, that's 78 * considered failure, and furthermore, a likely bug in the caller, so a warning 79 * is also emitted. 80 */ 81 __maybe_unused struct page *try_grab_compound_head(struct page *page, 82 int refs, unsigned int flags) 83 { 84 if (flags & FOLL_GET) 85 return try_get_compound_head(page, refs); 86 else if (flags & FOLL_PIN) { 87 int orig_refs = refs; 88 89 /* 90 * Can't do FOLL_LONGTERM + FOLL_PIN with CMA in the gup fast 91 * path, so fail and let the caller fall back to the slow path. 92 */ 93 if (unlikely(flags & FOLL_LONGTERM) && 94 is_migrate_cma_page(page)) 95 return NULL; 96 97 /* 98 * When pinning a compound page of order > 1 (which is what 99 * hpage_pincount_available() checks for), use an exact count to 100 * track it, via hpage_pincount_add/_sub(). 101 * 102 * However, be sure to *also* increment the normal page refcount 103 * field at least once, so that the page really is pinned. 104 */ 105 if (!hpage_pincount_available(page)) 106 refs *= GUP_PIN_COUNTING_BIAS; 107 108 page = try_get_compound_head(page, refs); 109 if (!page) 110 return NULL; 111 112 if (hpage_pincount_available(page)) 113 hpage_pincount_add(page, refs); 114 115 mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED, 116 orig_refs); 117 118 return page; 119 } 120 121 WARN_ON_ONCE(1); 122 return NULL; 123 } 124 125 static void put_compound_head(struct page *page, int refs, unsigned int flags) 126 { 127 if (flags & FOLL_PIN) { 128 mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_RELEASED, 129 refs); 130 131 if (hpage_pincount_available(page)) 132 hpage_pincount_sub(page, refs); 133 else 134 refs *= GUP_PIN_COUNTING_BIAS; 135 } 136 137 VM_BUG_ON_PAGE(page_ref_count(page) < refs, page); 138 /* 139 * Calling put_page() for each ref is unnecessarily slow. Only the last 140 * ref needs a put_page(). 141 */ 142 if (refs > 1) 143 page_ref_sub(page, refs - 1); 144 put_page(page); 145 } 146 147 /** 148 * try_grab_page() - elevate a page's refcount by a flag-dependent amount 149 * 150 * This might not do anything at all, depending on the flags argument. 151 * 152 * "grab" names in this file mean, "look at flags to decide whether to use 153 * FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount. 154 * 155 * @page: pointer to page to be grabbed 156 * @flags: gup flags: these are the FOLL_* flag values. 157 * 158 * Either FOLL_PIN or FOLL_GET (or neither) may be set, but not both at the same 159 * time. Cases: 160 * 161 * FOLL_GET: page's refcount will be incremented by 1. 162 * FOLL_PIN: page's refcount will be incremented by GUP_PIN_COUNTING_BIAS. 163 * 164 * Return: true for success, or if no action was required (if neither FOLL_PIN 165 * nor FOLL_GET was set, nothing is done). False for failure: FOLL_GET or 166 * FOLL_PIN was set, but the page could not be grabbed. 167 */ 168 bool __must_check try_grab_page(struct page *page, unsigned int flags) 169 { 170 WARN_ON_ONCE((flags & (FOLL_GET | FOLL_PIN)) == (FOLL_GET | FOLL_PIN)); 171 172 if (flags & FOLL_GET) 173 return try_get_page(page); 174 else if (flags & FOLL_PIN) { 175 int refs = 1; 176 177 page = compound_head(page); 178 179 if (WARN_ON_ONCE(page_ref_count(page) <= 0)) 180 return false; 181 182 if (hpage_pincount_available(page)) 183 hpage_pincount_add(page, 1); 184 else 185 refs = GUP_PIN_COUNTING_BIAS; 186 187 /* 188 * Similar to try_grab_compound_head(): even if using the 189 * hpage_pincount_add/_sub() routines, be sure to 190 * *also* increment the normal page refcount field at least 191 * once, so that the page really is pinned. 192 */ 193 page_ref_add(page, refs); 194 195 mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED, 1); 196 } 197 198 return true; 199 } 200 201 /** 202 * unpin_user_page() - release a dma-pinned page 203 * @page: pointer to page to be released 204 * 205 * Pages that were pinned via pin_user_pages*() must be released via either 206 * unpin_user_page(), or one of the unpin_user_pages*() routines. This is so 207 * that such pages can be separately tracked and uniquely handled. In 208 * particular, interactions with RDMA and filesystems need special handling. 209 */ 210 void unpin_user_page(struct page *page) 211 { 212 put_compound_head(compound_head(page), 1, FOLL_PIN); 213 } 214 EXPORT_SYMBOL(unpin_user_page); 215 216 /** 217 * unpin_user_pages_dirty_lock() - release and optionally dirty gup-pinned pages 218 * @pages: array of pages to be maybe marked dirty, and definitely released. 219 * @npages: number of pages in the @pages array. 220 * @make_dirty: whether to mark the pages dirty 221 * 222 * "gup-pinned page" refers to a page that has had one of the get_user_pages() 223 * variants called on that page. 224 * 225 * For each page in the @pages array, make that page (or its head page, if a 226 * compound page) dirty, if @make_dirty is true, and if the page was previously 227 * listed as clean. In any case, releases all pages using unpin_user_page(), 228 * possibly via unpin_user_pages(), for the non-dirty case. 229 * 230 * Please see the unpin_user_page() documentation for details. 231 * 232 * set_page_dirty_lock() is used internally. If instead, set_page_dirty() is 233 * required, then the caller should a) verify that this is really correct, 234 * because _lock() is usually required, and b) hand code it: 235 * set_page_dirty_lock(), unpin_user_page(). 236 * 237 */ 238 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages, 239 bool make_dirty) 240 { 241 unsigned long index; 242 243 /* 244 * TODO: this can be optimized for huge pages: if a series of pages is 245 * physically contiguous and part of the same compound page, then a 246 * single operation to the head page should suffice. 247 */ 248 249 if (!make_dirty) { 250 unpin_user_pages(pages, npages); 251 return; 252 } 253 254 for (index = 0; index < npages; index++) { 255 struct page *page = compound_head(pages[index]); 256 /* 257 * Checking PageDirty at this point may race with 258 * clear_page_dirty_for_io(), but that's OK. Two key 259 * cases: 260 * 261 * 1) This code sees the page as already dirty, so it 262 * skips the call to set_page_dirty(). That could happen 263 * because clear_page_dirty_for_io() called 264 * page_mkclean(), followed by set_page_dirty(). 265 * However, now the page is going to get written back, 266 * which meets the original intention of setting it 267 * dirty, so all is well: clear_page_dirty_for_io() goes 268 * on to call TestClearPageDirty(), and write the page 269 * back. 270 * 271 * 2) This code sees the page as clean, so it calls 272 * set_page_dirty(). The page stays dirty, despite being 273 * written back, so it gets written back again in the 274 * next writeback cycle. This is harmless. 275 */ 276 if (!PageDirty(page)) 277 set_page_dirty_lock(page); 278 unpin_user_page(page); 279 } 280 } 281 EXPORT_SYMBOL(unpin_user_pages_dirty_lock); 282 283 /** 284 * unpin_user_pages() - release an array of gup-pinned pages. 285 * @pages: array of pages to be marked dirty and released. 286 * @npages: number of pages in the @pages array. 287 * 288 * For each page in the @pages array, release the page using unpin_user_page(). 289 * 290 * Please see the unpin_user_page() documentation for details. 291 */ 292 void unpin_user_pages(struct page **pages, unsigned long npages) 293 { 294 unsigned long index; 295 296 /* 297 * If this WARN_ON() fires, then the system *might* be leaking pages (by 298 * leaving them pinned), but probably not. More likely, gup/pup returned 299 * a hard -ERRNO error to the caller, who erroneously passed it here. 300 */ 301 if (WARN_ON(IS_ERR_VALUE(npages))) 302 return; 303 /* 304 * TODO: this can be optimized for huge pages: if a series of pages is 305 * physically contiguous and part of the same compound page, then a 306 * single operation to the head page should suffice. 307 */ 308 for (index = 0; index < npages; index++) 309 unpin_user_page(pages[index]); 310 } 311 EXPORT_SYMBOL(unpin_user_pages); 312 313 #ifdef CONFIG_MMU 314 static struct page *no_page_table(struct vm_area_struct *vma, 315 unsigned int flags) 316 { 317 /* 318 * When core dumping an enormous anonymous area that nobody 319 * has touched so far, we don't want to allocate unnecessary pages or 320 * page tables. Return error instead of NULL to skip handle_mm_fault, 321 * then get_dump_page() will return NULL to leave a hole in the dump. 322 * But we can only make this optimization where a hole would surely 323 * be zero-filled if handle_mm_fault() actually did handle it. 324 */ 325 if ((flags & FOLL_DUMP) && 326 (vma_is_anonymous(vma) || !vma->vm_ops->fault)) 327 return ERR_PTR(-EFAULT); 328 return NULL; 329 } 330 331 static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address, 332 pte_t *pte, unsigned int flags) 333 { 334 /* No page to get reference */ 335 if (flags & FOLL_GET) 336 return -EFAULT; 337 338 if (flags & FOLL_TOUCH) { 339 pte_t entry = *pte; 340 341 if (flags & FOLL_WRITE) 342 entry = pte_mkdirty(entry); 343 entry = pte_mkyoung(entry); 344 345 if (!pte_same(*pte, entry)) { 346 set_pte_at(vma->vm_mm, address, pte, entry); 347 update_mmu_cache(vma, address, pte); 348 } 349 } 350 351 /* Proper page table entry exists, but no corresponding struct page */ 352 return -EEXIST; 353 } 354 355 /* 356 * FOLL_FORCE can write to even unwritable pte's, but only 357 * after we've gone through a COW cycle and they are dirty. 358 */ 359 static inline bool can_follow_write_pte(pte_t pte, unsigned int flags) 360 { 361 return pte_write(pte) || 362 ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pte_dirty(pte)); 363 } 364 365 static struct page *follow_page_pte(struct vm_area_struct *vma, 366 unsigned long address, pmd_t *pmd, unsigned int flags, 367 struct dev_pagemap **pgmap) 368 { 369 struct mm_struct *mm = vma->vm_mm; 370 struct page *page; 371 spinlock_t *ptl; 372 pte_t *ptep, pte; 373 int ret; 374 375 /* FOLL_GET and FOLL_PIN are mutually exclusive. */ 376 if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) == 377 (FOLL_PIN | FOLL_GET))) 378 return ERR_PTR(-EINVAL); 379 retry: 380 if (unlikely(pmd_bad(*pmd))) 381 return no_page_table(vma, flags); 382 383 ptep = pte_offset_map_lock(mm, pmd, address, &ptl); 384 pte = *ptep; 385 if (!pte_present(pte)) { 386 swp_entry_t entry; 387 /* 388 * KSM's break_ksm() relies upon recognizing a ksm page 389 * even while it is being migrated, so for that case we 390 * need migration_entry_wait(). 391 */ 392 if (likely(!(flags & FOLL_MIGRATION))) 393 goto no_page; 394 if (pte_none(pte)) 395 goto no_page; 396 entry = pte_to_swp_entry(pte); 397 if (!is_migration_entry(entry)) 398 goto no_page; 399 pte_unmap_unlock(ptep, ptl); 400 migration_entry_wait(mm, pmd, address); 401 goto retry; 402 } 403 if ((flags & FOLL_NUMA) && pte_protnone(pte)) 404 goto no_page; 405 if ((flags & FOLL_WRITE) && !can_follow_write_pte(pte, flags)) { 406 pte_unmap_unlock(ptep, ptl); 407 return NULL; 408 } 409 410 page = vm_normal_page(vma, address, pte); 411 if (!page && pte_devmap(pte) && (flags & (FOLL_GET | FOLL_PIN))) { 412 /* 413 * Only return device mapping pages in the FOLL_GET or FOLL_PIN 414 * case since they are only valid while holding the pgmap 415 * reference. 416 */ 417 *pgmap = get_dev_pagemap(pte_pfn(pte), *pgmap); 418 if (*pgmap) 419 page = pte_page(pte); 420 else 421 goto no_page; 422 } else if (unlikely(!page)) { 423 if (flags & FOLL_DUMP) { 424 /* Avoid special (like zero) pages in core dumps */ 425 page = ERR_PTR(-EFAULT); 426 goto out; 427 } 428 429 if (is_zero_pfn(pte_pfn(pte))) { 430 page = pte_page(pte); 431 } else { 432 ret = follow_pfn_pte(vma, address, ptep, flags); 433 page = ERR_PTR(ret); 434 goto out; 435 } 436 } 437 438 if (flags & FOLL_SPLIT && PageTransCompound(page)) { 439 get_page(page); 440 pte_unmap_unlock(ptep, ptl); 441 lock_page(page); 442 ret = split_huge_page(page); 443 unlock_page(page); 444 put_page(page); 445 if (ret) 446 return ERR_PTR(ret); 447 goto retry; 448 } 449 450 /* try_grab_page() does nothing unless FOLL_GET or FOLL_PIN is set. */ 451 if (unlikely(!try_grab_page(page, flags))) { 452 page = ERR_PTR(-ENOMEM); 453 goto out; 454 } 455 /* 456 * We need to make the page accessible if and only if we are going 457 * to access its content (the FOLL_PIN case). Please see 458 * Documentation/core-api/pin_user_pages.rst for details. 459 */ 460 if (flags & FOLL_PIN) { 461 ret = arch_make_page_accessible(page); 462 if (ret) { 463 unpin_user_page(page); 464 page = ERR_PTR(ret); 465 goto out; 466 } 467 } 468 if (flags & FOLL_TOUCH) { 469 if ((flags & FOLL_WRITE) && 470 !pte_dirty(pte) && !PageDirty(page)) 471 set_page_dirty(page); 472 /* 473 * pte_mkyoung() would be more correct here, but atomic care 474 * is needed to avoid losing the dirty bit: it is easier to use 475 * mark_page_accessed(). 476 */ 477 mark_page_accessed(page); 478 } 479 if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) { 480 /* Do not mlock pte-mapped THP */ 481 if (PageTransCompound(page)) 482 goto out; 483 484 /* 485 * The preliminary mapping check is mainly to avoid the 486 * pointless overhead of lock_page on the ZERO_PAGE 487 * which might bounce very badly if there is contention. 488 * 489 * If the page is already locked, we don't need to 490 * handle it now - vmscan will handle it later if and 491 * when it attempts to reclaim the page. 492 */ 493 if (page->mapping && trylock_page(page)) { 494 lru_add_drain(); /* push cached pages to LRU */ 495 /* 496 * Because we lock page here, and migration is 497 * blocked by the pte's page reference, and we 498 * know the page is still mapped, we don't even 499 * need to check for file-cache page truncation. 500 */ 501 mlock_vma_page(page); 502 unlock_page(page); 503 } 504 } 505 out: 506 pte_unmap_unlock(ptep, ptl); 507 return page; 508 no_page: 509 pte_unmap_unlock(ptep, ptl); 510 if (!pte_none(pte)) 511 return NULL; 512 return no_page_table(vma, flags); 513 } 514 515 static struct page *follow_pmd_mask(struct vm_area_struct *vma, 516 unsigned long address, pud_t *pudp, 517 unsigned int flags, 518 struct follow_page_context *ctx) 519 { 520 pmd_t *pmd, pmdval; 521 spinlock_t *ptl; 522 struct page *page; 523 struct mm_struct *mm = vma->vm_mm; 524 525 pmd = pmd_offset(pudp, address); 526 /* 527 * The READ_ONCE() will stabilize the pmdval in a register or 528 * on the stack so that it will stop changing under the code. 529 */ 530 pmdval = READ_ONCE(*pmd); 531 if (pmd_none(pmdval)) 532 return no_page_table(vma, flags); 533 if (pmd_huge(pmdval) && is_vm_hugetlb_page(vma)) { 534 page = follow_huge_pmd(mm, address, pmd, flags); 535 if (page) 536 return page; 537 return no_page_table(vma, flags); 538 } 539 if (is_hugepd(__hugepd(pmd_val(pmdval)))) { 540 page = follow_huge_pd(vma, address, 541 __hugepd(pmd_val(pmdval)), flags, 542 PMD_SHIFT); 543 if (page) 544 return page; 545 return no_page_table(vma, flags); 546 } 547 retry: 548 if (!pmd_present(pmdval)) { 549 if (likely(!(flags & FOLL_MIGRATION))) 550 return no_page_table(vma, flags); 551 VM_BUG_ON(thp_migration_supported() && 552 !is_pmd_migration_entry(pmdval)); 553 if (is_pmd_migration_entry(pmdval)) 554 pmd_migration_entry_wait(mm, pmd); 555 pmdval = READ_ONCE(*pmd); 556 /* 557 * MADV_DONTNEED may convert the pmd to null because 558 * mmap_lock is held in read mode 559 */ 560 if (pmd_none(pmdval)) 561 return no_page_table(vma, flags); 562 goto retry; 563 } 564 if (pmd_devmap(pmdval)) { 565 ptl = pmd_lock(mm, pmd); 566 page = follow_devmap_pmd(vma, address, pmd, flags, &ctx->pgmap); 567 spin_unlock(ptl); 568 if (page) 569 return page; 570 } 571 if (likely(!pmd_trans_huge(pmdval))) 572 return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap); 573 574 if ((flags & FOLL_NUMA) && pmd_protnone(pmdval)) 575 return no_page_table(vma, flags); 576 577 retry_locked: 578 ptl = pmd_lock(mm, pmd); 579 if (unlikely(pmd_none(*pmd))) { 580 spin_unlock(ptl); 581 return no_page_table(vma, flags); 582 } 583 if (unlikely(!pmd_present(*pmd))) { 584 spin_unlock(ptl); 585 if (likely(!(flags & FOLL_MIGRATION))) 586 return no_page_table(vma, flags); 587 pmd_migration_entry_wait(mm, pmd); 588 goto retry_locked; 589 } 590 if (unlikely(!pmd_trans_huge(*pmd))) { 591 spin_unlock(ptl); 592 return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap); 593 } 594 if (flags & (FOLL_SPLIT | FOLL_SPLIT_PMD)) { 595 int ret; 596 page = pmd_page(*pmd); 597 if (is_huge_zero_page(page)) { 598 spin_unlock(ptl); 599 ret = 0; 600 split_huge_pmd(vma, pmd, address); 601 if (pmd_trans_unstable(pmd)) 602 ret = -EBUSY; 603 } else if (flags & FOLL_SPLIT) { 604 if (unlikely(!try_get_page(page))) { 605 spin_unlock(ptl); 606 return ERR_PTR(-ENOMEM); 607 } 608 spin_unlock(ptl); 609 lock_page(page); 610 ret = split_huge_page(page); 611 unlock_page(page); 612 put_page(page); 613 if (pmd_none(*pmd)) 614 return no_page_table(vma, flags); 615 } else { /* flags & FOLL_SPLIT_PMD */ 616 spin_unlock(ptl); 617 split_huge_pmd(vma, pmd, address); 618 ret = pte_alloc(mm, pmd) ? -ENOMEM : 0; 619 } 620 621 return ret ? ERR_PTR(ret) : 622 follow_page_pte(vma, address, pmd, flags, &ctx->pgmap); 623 } 624 page = follow_trans_huge_pmd(vma, address, pmd, flags); 625 spin_unlock(ptl); 626 ctx->page_mask = HPAGE_PMD_NR - 1; 627 return page; 628 } 629 630 static struct page *follow_pud_mask(struct vm_area_struct *vma, 631 unsigned long address, p4d_t *p4dp, 632 unsigned int flags, 633 struct follow_page_context *ctx) 634 { 635 pud_t *pud; 636 spinlock_t *ptl; 637 struct page *page; 638 struct mm_struct *mm = vma->vm_mm; 639 640 pud = pud_offset(p4dp, address); 641 if (pud_none(*pud)) 642 return no_page_table(vma, flags); 643 if (pud_huge(*pud) && is_vm_hugetlb_page(vma)) { 644 page = follow_huge_pud(mm, address, pud, flags); 645 if (page) 646 return page; 647 return no_page_table(vma, flags); 648 } 649 if (is_hugepd(__hugepd(pud_val(*pud)))) { 650 page = follow_huge_pd(vma, address, 651 __hugepd(pud_val(*pud)), flags, 652 PUD_SHIFT); 653 if (page) 654 return page; 655 return no_page_table(vma, flags); 656 } 657 if (pud_devmap(*pud)) { 658 ptl = pud_lock(mm, pud); 659 page = follow_devmap_pud(vma, address, pud, flags, &ctx->pgmap); 660 spin_unlock(ptl); 661 if (page) 662 return page; 663 } 664 if (unlikely(pud_bad(*pud))) 665 return no_page_table(vma, flags); 666 667 return follow_pmd_mask(vma, address, pud, flags, ctx); 668 } 669 670 static struct page *follow_p4d_mask(struct vm_area_struct *vma, 671 unsigned long address, pgd_t *pgdp, 672 unsigned int flags, 673 struct follow_page_context *ctx) 674 { 675 p4d_t *p4d; 676 struct page *page; 677 678 p4d = p4d_offset(pgdp, address); 679 if (p4d_none(*p4d)) 680 return no_page_table(vma, flags); 681 BUILD_BUG_ON(p4d_huge(*p4d)); 682 if (unlikely(p4d_bad(*p4d))) 683 return no_page_table(vma, flags); 684 685 if (is_hugepd(__hugepd(p4d_val(*p4d)))) { 686 page = follow_huge_pd(vma, address, 687 __hugepd(p4d_val(*p4d)), flags, 688 P4D_SHIFT); 689 if (page) 690 return page; 691 return no_page_table(vma, flags); 692 } 693 return follow_pud_mask(vma, address, p4d, flags, ctx); 694 } 695 696 /** 697 * follow_page_mask - look up a page descriptor from a user-virtual address 698 * @vma: vm_area_struct mapping @address 699 * @address: virtual address to look up 700 * @flags: flags modifying lookup behaviour 701 * @ctx: contains dev_pagemap for %ZONE_DEVICE memory pinning and a 702 * pointer to output page_mask 703 * 704 * @flags can have FOLL_ flags set, defined in <linux/mm.h> 705 * 706 * When getting pages from ZONE_DEVICE memory, the @ctx->pgmap caches 707 * the device's dev_pagemap metadata to avoid repeating expensive lookups. 708 * 709 * On output, the @ctx->page_mask is set according to the size of the page. 710 * 711 * Return: the mapped (struct page *), %NULL if no mapping exists, or 712 * an error pointer if there is a mapping to something not represented 713 * by a page descriptor (see also vm_normal_page()). 714 */ 715 static struct page *follow_page_mask(struct vm_area_struct *vma, 716 unsigned long address, unsigned int flags, 717 struct follow_page_context *ctx) 718 { 719 pgd_t *pgd; 720 struct page *page; 721 struct mm_struct *mm = vma->vm_mm; 722 723 ctx->page_mask = 0; 724 725 /* make this handle hugepd */ 726 page = follow_huge_addr(mm, address, flags & FOLL_WRITE); 727 if (!IS_ERR(page)) { 728 WARN_ON_ONCE(flags & (FOLL_GET | FOLL_PIN)); 729 return page; 730 } 731 732 pgd = pgd_offset(mm, address); 733 734 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) 735 return no_page_table(vma, flags); 736 737 if (pgd_huge(*pgd)) { 738 page = follow_huge_pgd(mm, address, pgd, flags); 739 if (page) 740 return page; 741 return no_page_table(vma, flags); 742 } 743 if (is_hugepd(__hugepd(pgd_val(*pgd)))) { 744 page = follow_huge_pd(vma, address, 745 __hugepd(pgd_val(*pgd)), flags, 746 PGDIR_SHIFT); 747 if (page) 748 return page; 749 return no_page_table(vma, flags); 750 } 751 752 return follow_p4d_mask(vma, address, pgd, flags, ctx); 753 } 754 755 struct page *follow_page(struct vm_area_struct *vma, unsigned long address, 756 unsigned int foll_flags) 757 { 758 struct follow_page_context ctx = { NULL }; 759 struct page *page; 760 761 page = follow_page_mask(vma, address, foll_flags, &ctx); 762 if (ctx.pgmap) 763 put_dev_pagemap(ctx.pgmap); 764 return page; 765 } 766 767 static int get_gate_page(struct mm_struct *mm, unsigned long address, 768 unsigned int gup_flags, struct vm_area_struct **vma, 769 struct page **page) 770 { 771 pgd_t *pgd; 772 p4d_t *p4d; 773 pud_t *pud; 774 pmd_t *pmd; 775 pte_t *pte; 776 int ret = -EFAULT; 777 778 /* user gate pages are read-only */ 779 if (gup_flags & FOLL_WRITE) 780 return -EFAULT; 781 if (address > TASK_SIZE) 782 pgd = pgd_offset_k(address); 783 else 784 pgd = pgd_offset_gate(mm, address); 785 if (pgd_none(*pgd)) 786 return -EFAULT; 787 p4d = p4d_offset(pgd, address); 788 if (p4d_none(*p4d)) 789 return -EFAULT; 790 pud = pud_offset(p4d, address); 791 if (pud_none(*pud)) 792 return -EFAULT; 793 pmd = pmd_offset(pud, address); 794 if (!pmd_present(*pmd)) 795 return -EFAULT; 796 VM_BUG_ON(pmd_trans_huge(*pmd)); 797 pte = pte_offset_map(pmd, address); 798 if (pte_none(*pte)) 799 goto unmap; 800 *vma = get_gate_vma(mm); 801 if (!page) 802 goto out; 803 *page = vm_normal_page(*vma, address, *pte); 804 if (!*page) { 805 if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(*pte))) 806 goto unmap; 807 *page = pte_page(*pte); 808 } 809 if (unlikely(!try_grab_page(*page, gup_flags))) { 810 ret = -ENOMEM; 811 goto unmap; 812 } 813 out: 814 ret = 0; 815 unmap: 816 pte_unmap(pte); 817 return ret; 818 } 819 820 /* 821 * mmap_lock must be held on entry. If @locked != NULL and *@flags 822 * does not include FOLL_NOWAIT, the mmap_lock may be released. If it 823 * is, *@locked will be set to 0 and -EBUSY returned. 824 */ 825 static int faultin_page(struct vm_area_struct *vma, 826 unsigned long address, unsigned int *flags, int *locked) 827 { 828 unsigned int fault_flags = 0; 829 vm_fault_t ret; 830 831 /* mlock all present pages, but do not fault in new pages */ 832 if ((*flags & (FOLL_POPULATE | FOLL_MLOCK)) == FOLL_MLOCK) 833 return -ENOENT; 834 if (*flags & FOLL_WRITE) 835 fault_flags |= FAULT_FLAG_WRITE; 836 if (*flags & FOLL_REMOTE) 837 fault_flags |= FAULT_FLAG_REMOTE; 838 if (locked) 839 fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; 840 if (*flags & FOLL_NOWAIT) 841 fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT; 842 if (*flags & FOLL_TRIED) { 843 /* 844 * Note: FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_TRIED 845 * can co-exist 846 */ 847 fault_flags |= FAULT_FLAG_TRIED; 848 } 849 850 ret = handle_mm_fault(vma, address, fault_flags, NULL); 851 if (ret & VM_FAULT_ERROR) { 852 int err = vm_fault_to_errno(ret, *flags); 853 854 if (err) 855 return err; 856 BUG(); 857 } 858 859 if (ret & VM_FAULT_RETRY) { 860 if (locked && !(fault_flags & FAULT_FLAG_RETRY_NOWAIT)) 861 *locked = 0; 862 return -EBUSY; 863 } 864 865 /* 866 * The VM_FAULT_WRITE bit tells us that do_wp_page has broken COW when 867 * necessary, even if maybe_mkwrite decided not to set pte_write. We 868 * can thus safely do subsequent page lookups as if they were reads. 869 * But only do so when looping for pte_write is futile: in some cases 870 * userspace may also be wanting to write to the gotten user page, 871 * which a read fault here might prevent (a readonly page might get 872 * reCOWed by userspace write). 873 */ 874 if ((ret & VM_FAULT_WRITE) && !(vma->vm_flags & VM_WRITE)) 875 *flags |= FOLL_COW; 876 return 0; 877 } 878 879 static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) 880 { 881 vm_flags_t vm_flags = vma->vm_flags; 882 int write = (gup_flags & FOLL_WRITE); 883 int foreign = (gup_flags & FOLL_REMOTE); 884 885 if (vm_flags & (VM_IO | VM_PFNMAP)) 886 return -EFAULT; 887 888 if (gup_flags & FOLL_ANON && !vma_is_anonymous(vma)) 889 return -EFAULT; 890 891 if ((gup_flags & FOLL_LONGTERM) && vma_is_fsdax(vma)) 892 return -EOPNOTSUPP; 893 894 if (write) { 895 if (!(vm_flags & VM_WRITE)) { 896 if (!(gup_flags & FOLL_FORCE)) 897 return -EFAULT; 898 /* 899 * We used to let the write,force case do COW in a 900 * VM_MAYWRITE VM_SHARED !VM_WRITE vma, so ptrace could 901 * set a breakpoint in a read-only mapping of an 902 * executable, without corrupting the file (yet only 903 * when that file had been opened for writing!). 904 * Anon pages in shared mappings are surprising: now 905 * just reject it. 906 */ 907 if (!is_cow_mapping(vm_flags)) 908 return -EFAULT; 909 } 910 } else if (!(vm_flags & VM_READ)) { 911 if (!(gup_flags & FOLL_FORCE)) 912 return -EFAULT; 913 /* 914 * Is there actually any vma we can reach here which does not 915 * have VM_MAYREAD set? 916 */ 917 if (!(vm_flags & VM_MAYREAD)) 918 return -EFAULT; 919 } 920 /* 921 * gups are always data accesses, not instruction 922 * fetches, so execute=false here 923 */ 924 if (!arch_vma_access_permitted(vma, write, false, foreign)) 925 return -EFAULT; 926 return 0; 927 } 928 929 /** 930 * __get_user_pages() - pin user pages in memory 931 * @mm: mm_struct of target mm 932 * @start: starting user address 933 * @nr_pages: number of pages from start to pin 934 * @gup_flags: flags modifying pin behaviour 935 * @pages: array that receives pointers to the pages pinned. 936 * Should be at least nr_pages long. Or NULL, if caller 937 * only intends to ensure the pages are faulted in. 938 * @vmas: array of pointers to vmas corresponding to each page. 939 * Or NULL if the caller does not require them. 940 * @locked: whether we're still with the mmap_lock held 941 * 942 * Returns either number of pages pinned (which may be less than the 943 * number requested), or an error. Details about the return value: 944 * 945 * -- If nr_pages is 0, returns 0. 946 * -- If nr_pages is >0, but no pages were pinned, returns -errno. 947 * -- If nr_pages is >0, and some pages were pinned, returns the number of 948 * pages pinned. Again, this may be less than nr_pages. 949 * -- 0 return value is possible when the fault would need to be retried. 950 * 951 * The caller is responsible for releasing returned @pages, via put_page(). 952 * 953 * @vmas are valid only as long as mmap_lock is held. 954 * 955 * Must be called with mmap_lock held. It may be released. See below. 956 * 957 * __get_user_pages walks a process's page tables and takes a reference to 958 * each struct page that each user address corresponds to at a given 959 * instant. That is, it takes the page that would be accessed if a user 960 * thread accesses the given user virtual address at that instant. 961 * 962 * This does not guarantee that the page exists in the user mappings when 963 * __get_user_pages returns, and there may even be a completely different 964 * page there in some cases (eg. if mmapped pagecache has been invalidated 965 * and subsequently re faulted). However it does guarantee that the page 966 * won't be freed completely. And mostly callers simply care that the page 967 * contains data that was valid *at some point in time*. Typically, an IO 968 * or similar operation cannot guarantee anything stronger anyway because 969 * locks can't be held over the syscall boundary. 970 * 971 * If @gup_flags & FOLL_WRITE == 0, the page must not be written to. If 972 * the page is written to, set_page_dirty (or set_page_dirty_lock, as 973 * appropriate) must be called after the page is finished with, and 974 * before put_page is called. 975 * 976 * If @locked != NULL, *@locked will be set to 0 when mmap_lock is 977 * released by an up_read(). That can happen if @gup_flags does not 978 * have FOLL_NOWAIT. 979 * 980 * A caller using such a combination of @locked and @gup_flags 981 * must therefore hold the mmap_lock for reading only, and recognize 982 * when it's been released. Otherwise, it must be held for either 983 * reading or writing and will not be released. 984 * 985 * In most cases, get_user_pages or get_user_pages_fast should be used 986 * instead of __get_user_pages. __get_user_pages should be used only if 987 * you need some special @gup_flags. 988 */ 989 static long __get_user_pages(struct mm_struct *mm, 990 unsigned long start, unsigned long nr_pages, 991 unsigned int gup_flags, struct page **pages, 992 struct vm_area_struct **vmas, int *locked) 993 { 994 long ret = 0, i = 0; 995 struct vm_area_struct *vma = NULL; 996 struct follow_page_context ctx = { NULL }; 997 998 if (!nr_pages) 999 return 0; 1000 1001 start = untagged_addr(start); 1002 1003 VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN))); 1004 1005 /* 1006 * If FOLL_FORCE is set then do not force a full fault as the hinting 1007 * fault information is unrelated to the reference behaviour of a task 1008 * using the address space 1009 */ 1010 if (!(gup_flags & FOLL_FORCE)) 1011 gup_flags |= FOLL_NUMA; 1012 1013 do { 1014 struct page *page; 1015 unsigned int foll_flags = gup_flags; 1016 unsigned int page_increm; 1017 1018 /* first iteration or cross vma bound */ 1019 if (!vma || start >= vma->vm_end) { 1020 vma = find_extend_vma(mm, start); 1021 if (!vma && in_gate_area(mm, start)) { 1022 ret = get_gate_page(mm, start & PAGE_MASK, 1023 gup_flags, &vma, 1024 pages ? &pages[i] : NULL); 1025 if (ret) 1026 goto out; 1027 ctx.page_mask = 0; 1028 goto next_page; 1029 } 1030 1031 if (!vma) { 1032 ret = -EFAULT; 1033 goto out; 1034 } 1035 ret = check_vma_flags(vma, gup_flags); 1036 if (ret) 1037 goto out; 1038 1039 if (is_vm_hugetlb_page(vma)) { 1040 i = follow_hugetlb_page(mm, vma, pages, vmas, 1041 &start, &nr_pages, i, 1042 gup_flags, locked); 1043 if (locked && *locked == 0) { 1044 /* 1045 * We've got a VM_FAULT_RETRY 1046 * and we've lost mmap_lock. 1047 * We must stop here. 1048 */ 1049 BUG_ON(gup_flags & FOLL_NOWAIT); 1050 BUG_ON(ret != 0); 1051 goto out; 1052 } 1053 continue; 1054 } 1055 } 1056 retry: 1057 /* 1058 * If we have a pending SIGKILL, don't keep faulting pages and 1059 * potentially allocating memory. 1060 */ 1061 if (fatal_signal_pending(current)) { 1062 ret = -EINTR; 1063 goto out; 1064 } 1065 cond_resched(); 1066 1067 page = follow_page_mask(vma, start, foll_flags, &ctx); 1068 if (!page) { 1069 ret = faultin_page(vma, start, &foll_flags, locked); 1070 switch (ret) { 1071 case 0: 1072 goto retry; 1073 case -EBUSY: 1074 ret = 0; 1075 fallthrough; 1076 case -EFAULT: 1077 case -ENOMEM: 1078 case -EHWPOISON: 1079 goto out; 1080 case -ENOENT: 1081 goto next_page; 1082 } 1083 BUG(); 1084 } else if (PTR_ERR(page) == -EEXIST) { 1085 /* 1086 * Proper page table entry exists, but no corresponding 1087 * struct page. 1088 */ 1089 goto next_page; 1090 } else if (IS_ERR(page)) { 1091 ret = PTR_ERR(page); 1092 goto out; 1093 } 1094 if (pages) { 1095 pages[i] = page; 1096 flush_anon_page(vma, page, start); 1097 flush_dcache_page(page); 1098 ctx.page_mask = 0; 1099 } 1100 next_page: 1101 if (vmas) { 1102 vmas[i] = vma; 1103 ctx.page_mask = 0; 1104 } 1105 page_increm = 1 + (~(start >> PAGE_SHIFT) & ctx.page_mask); 1106 if (page_increm > nr_pages) 1107 page_increm = nr_pages; 1108 i += page_increm; 1109 start += page_increm * PAGE_SIZE; 1110 nr_pages -= page_increm; 1111 } while (nr_pages); 1112 out: 1113 if (ctx.pgmap) 1114 put_dev_pagemap(ctx.pgmap); 1115 return i ? i : ret; 1116 } 1117 1118 static bool vma_permits_fault(struct vm_area_struct *vma, 1119 unsigned int fault_flags) 1120 { 1121 bool write = !!(fault_flags & FAULT_FLAG_WRITE); 1122 bool foreign = !!(fault_flags & FAULT_FLAG_REMOTE); 1123 vm_flags_t vm_flags = write ? VM_WRITE : VM_READ; 1124 1125 if (!(vm_flags & vma->vm_flags)) 1126 return false; 1127 1128 /* 1129 * The architecture might have a hardware protection 1130 * mechanism other than read/write that can deny access. 1131 * 1132 * gup always represents data access, not instruction 1133 * fetches, so execute=false here: 1134 */ 1135 if (!arch_vma_access_permitted(vma, write, false, foreign)) 1136 return false; 1137 1138 return true; 1139 } 1140 1141 /** 1142 * fixup_user_fault() - manually resolve a user page fault 1143 * @mm: mm_struct of target mm 1144 * @address: user address 1145 * @fault_flags:flags to pass down to handle_mm_fault() 1146 * @unlocked: did we unlock the mmap_lock while retrying, maybe NULL if caller 1147 * does not allow retry. If NULL, the caller must guarantee 1148 * that fault_flags does not contain FAULT_FLAG_ALLOW_RETRY. 1149 * 1150 * This is meant to be called in the specific scenario where for locking reasons 1151 * we try to access user memory in atomic context (within a pagefault_disable() 1152 * section), this returns -EFAULT, and we want to resolve the user fault before 1153 * trying again. 1154 * 1155 * Typically this is meant to be used by the futex code. 1156 * 1157 * The main difference with get_user_pages() is that this function will 1158 * unconditionally call handle_mm_fault() which will in turn perform all the 1159 * necessary SW fixup of the dirty and young bits in the PTE, while 1160 * get_user_pages() only guarantees to update these in the struct page. 1161 * 1162 * This is important for some architectures where those bits also gate the 1163 * access permission to the page because they are maintained in software. On 1164 * such architectures, gup() will not be enough to make a subsequent access 1165 * succeed. 1166 * 1167 * This function will not return with an unlocked mmap_lock. So it has not the 1168 * same semantics wrt the @mm->mmap_lock as does filemap_fault(). 1169 */ 1170 int fixup_user_fault(struct mm_struct *mm, 1171 unsigned long address, unsigned int fault_flags, 1172 bool *unlocked) 1173 { 1174 struct vm_area_struct *vma; 1175 vm_fault_t ret, major = 0; 1176 1177 address = untagged_addr(address); 1178 1179 if (unlocked) 1180 fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; 1181 1182 retry: 1183 vma = find_extend_vma(mm, address); 1184 if (!vma || address < vma->vm_start) 1185 return -EFAULT; 1186 1187 if (!vma_permits_fault(vma, fault_flags)) 1188 return -EFAULT; 1189 1190 if ((fault_flags & FAULT_FLAG_KILLABLE) && 1191 fatal_signal_pending(current)) 1192 return -EINTR; 1193 1194 ret = handle_mm_fault(vma, address, fault_flags, NULL); 1195 major |= ret & VM_FAULT_MAJOR; 1196 if (ret & VM_FAULT_ERROR) { 1197 int err = vm_fault_to_errno(ret, 0); 1198 1199 if (err) 1200 return err; 1201 BUG(); 1202 } 1203 1204 if (ret & VM_FAULT_RETRY) { 1205 mmap_read_lock(mm); 1206 *unlocked = true; 1207 fault_flags |= FAULT_FLAG_TRIED; 1208 goto retry; 1209 } 1210 1211 return 0; 1212 } 1213 EXPORT_SYMBOL_GPL(fixup_user_fault); 1214 1215 /* 1216 * Please note that this function, unlike __get_user_pages will not 1217 * return 0 for nr_pages > 0 without FOLL_NOWAIT 1218 */ 1219 static __always_inline long __get_user_pages_locked(struct mm_struct *mm, 1220 unsigned long start, 1221 unsigned long nr_pages, 1222 struct page **pages, 1223 struct vm_area_struct **vmas, 1224 int *locked, 1225 unsigned int flags) 1226 { 1227 long ret, pages_done; 1228 bool lock_dropped; 1229 1230 if (locked) { 1231 /* if VM_FAULT_RETRY can be returned, vmas become invalid */ 1232 BUG_ON(vmas); 1233 /* check caller initialized locked */ 1234 BUG_ON(*locked != 1); 1235 } 1236 1237 if (flags & FOLL_PIN) 1238 atomic_set(&mm->has_pinned, 1); 1239 1240 /* 1241 * FOLL_PIN and FOLL_GET are mutually exclusive. Traditional behavior 1242 * is to set FOLL_GET if the caller wants pages[] filled in (but has 1243 * carelessly failed to specify FOLL_GET), so keep doing that, but only 1244 * for FOLL_GET, not for the newer FOLL_PIN. 1245 * 1246 * FOLL_PIN always expects pages to be non-null, but no need to assert 1247 * that here, as any failures will be obvious enough. 1248 */ 1249 if (pages && !(flags & FOLL_PIN)) 1250 flags |= FOLL_GET; 1251 1252 pages_done = 0; 1253 lock_dropped = false; 1254 for (;;) { 1255 ret = __get_user_pages(mm, start, nr_pages, flags, pages, 1256 vmas, locked); 1257 if (!locked) 1258 /* VM_FAULT_RETRY couldn't trigger, bypass */ 1259 return ret; 1260 1261 /* VM_FAULT_RETRY cannot return errors */ 1262 if (!*locked) { 1263 BUG_ON(ret < 0); 1264 BUG_ON(ret >= nr_pages); 1265 } 1266 1267 if (ret > 0) { 1268 nr_pages -= ret; 1269 pages_done += ret; 1270 if (!nr_pages) 1271 break; 1272 } 1273 if (*locked) { 1274 /* 1275 * VM_FAULT_RETRY didn't trigger or it was a 1276 * FOLL_NOWAIT. 1277 */ 1278 if (!pages_done) 1279 pages_done = ret; 1280 break; 1281 } 1282 /* 1283 * VM_FAULT_RETRY triggered, so seek to the faulting offset. 1284 * For the prefault case (!pages) we only update counts. 1285 */ 1286 if (likely(pages)) 1287 pages += ret; 1288 start += ret << PAGE_SHIFT; 1289 lock_dropped = true; 1290 1291 retry: 1292 /* 1293 * Repeat on the address that fired VM_FAULT_RETRY 1294 * with both FAULT_FLAG_ALLOW_RETRY and 1295 * FAULT_FLAG_TRIED. Note that GUP can be interrupted 1296 * by fatal signals, so we need to check it before we 1297 * start trying again otherwise it can loop forever. 1298 */ 1299 1300 if (fatal_signal_pending(current)) { 1301 if (!pages_done) 1302 pages_done = -EINTR; 1303 break; 1304 } 1305 1306 ret = mmap_read_lock_killable(mm); 1307 if (ret) { 1308 BUG_ON(ret > 0); 1309 if (!pages_done) 1310 pages_done = ret; 1311 break; 1312 } 1313 1314 *locked = 1; 1315 ret = __get_user_pages(mm, start, 1, flags | FOLL_TRIED, 1316 pages, NULL, locked); 1317 if (!*locked) { 1318 /* Continue to retry until we succeeded */ 1319 BUG_ON(ret != 0); 1320 goto retry; 1321 } 1322 if (ret != 1) { 1323 BUG_ON(ret > 1); 1324 if (!pages_done) 1325 pages_done = ret; 1326 break; 1327 } 1328 nr_pages--; 1329 pages_done++; 1330 if (!nr_pages) 1331 break; 1332 if (likely(pages)) 1333 pages++; 1334 start += PAGE_SIZE; 1335 } 1336 if (lock_dropped && *locked) { 1337 /* 1338 * We must let the caller know we temporarily dropped the lock 1339 * and so the critical section protected by it was lost. 1340 */ 1341 mmap_read_unlock(mm); 1342 *locked = 0; 1343 } 1344 return pages_done; 1345 } 1346 1347 /** 1348 * populate_vma_page_range() - populate a range of pages in the vma. 1349 * @vma: target vma 1350 * @start: start address 1351 * @end: end address 1352 * @locked: whether the mmap_lock is still held 1353 * 1354 * This takes care of mlocking the pages too if VM_LOCKED is set. 1355 * 1356 * Return either number of pages pinned in the vma, or a negative error 1357 * code on error. 1358 * 1359 * vma->vm_mm->mmap_lock must be held. 1360 * 1361 * If @locked is NULL, it may be held for read or write and will 1362 * be unperturbed. 1363 * 1364 * If @locked is non-NULL, it must held for read only and may be 1365 * released. If it's released, *@locked will be set to 0. 1366 */ 1367 long populate_vma_page_range(struct vm_area_struct *vma, 1368 unsigned long start, unsigned long end, int *locked) 1369 { 1370 struct mm_struct *mm = vma->vm_mm; 1371 unsigned long nr_pages = (end - start) / PAGE_SIZE; 1372 int gup_flags; 1373 1374 VM_BUG_ON(start & ~PAGE_MASK); 1375 VM_BUG_ON(end & ~PAGE_MASK); 1376 VM_BUG_ON_VMA(start < vma->vm_start, vma); 1377 VM_BUG_ON_VMA(end > vma->vm_end, vma); 1378 mmap_assert_locked(mm); 1379 1380 gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK; 1381 if (vma->vm_flags & VM_LOCKONFAULT) 1382 gup_flags &= ~FOLL_POPULATE; 1383 /* 1384 * We want to touch writable mappings with a write fault in order 1385 * to break COW, except for shared mappings because these don't COW 1386 * and we would not want to dirty them for nothing. 1387 */ 1388 if ((vma->vm_flags & (VM_WRITE | VM_SHARED)) == VM_WRITE) 1389 gup_flags |= FOLL_WRITE; 1390 1391 /* 1392 * We want mlock to succeed for regions that have any permissions 1393 * other than PROT_NONE. 1394 */ 1395 if (vma_is_accessible(vma)) 1396 gup_flags |= FOLL_FORCE; 1397 1398 /* 1399 * We made sure addr is within a VMA, so the following will 1400 * not result in a stack expansion that recurses back here. 1401 */ 1402 return __get_user_pages(mm, start, nr_pages, gup_flags, 1403 NULL, NULL, locked); 1404 } 1405 1406 /* 1407 * __mm_populate - populate and/or mlock pages within a range of address space. 1408 * 1409 * This is used to implement mlock() and the MAP_POPULATE / MAP_LOCKED mmap 1410 * flags. VMAs must be already marked with the desired vm_flags, and 1411 * mmap_lock must not be held. 1412 */ 1413 int __mm_populate(unsigned long start, unsigned long len, int ignore_errors) 1414 { 1415 struct mm_struct *mm = current->mm; 1416 unsigned long end, nstart, nend; 1417 struct vm_area_struct *vma = NULL; 1418 int locked = 0; 1419 long ret = 0; 1420 1421 end = start + len; 1422 1423 for (nstart = start; nstart < end; nstart = nend) { 1424 /* 1425 * We want to fault in pages for [nstart; end) address range. 1426 * Find first corresponding VMA. 1427 */ 1428 if (!locked) { 1429 locked = 1; 1430 mmap_read_lock(mm); 1431 vma = find_vma(mm, nstart); 1432 } else if (nstart >= vma->vm_end) 1433 vma = vma->vm_next; 1434 if (!vma || vma->vm_start >= end) 1435 break; 1436 /* 1437 * Set [nstart; nend) to intersection of desired address 1438 * range with the first VMA. Also, skip undesirable VMA types. 1439 */ 1440 nend = min(end, vma->vm_end); 1441 if (vma->vm_flags & (VM_IO | VM_PFNMAP)) 1442 continue; 1443 if (nstart < vma->vm_start) 1444 nstart = vma->vm_start; 1445 /* 1446 * Now fault in a range of pages. populate_vma_page_range() 1447 * double checks the vma flags, so that it won't mlock pages 1448 * if the vma was already munlocked. 1449 */ 1450 ret = populate_vma_page_range(vma, nstart, nend, &locked); 1451 if (ret < 0) { 1452 if (ignore_errors) { 1453 ret = 0; 1454 continue; /* continue at next VMA */ 1455 } 1456 break; 1457 } 1458 nend = nstart + ret * PAGE_SIZE; 1459 ret = 0; 1460 } 1461 if (locked) 1462 mmap_read_unlock(mm); 1463 return ret; /* 0 or negative error code */ 1464 } 1465 #else /* CONFIG_MMU */ 1466 static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start, 1467 unsigned long nr_pages, struct page **pages, 1468 struct vm_area_struct **vmas, int *locked, 1469 unsigned int foll_flags) 1470 { 1471 struct vm_area_struct *vma; 1472 unsigned long vm_flags; 1473 int i; 1474 1475 /* calculate required read or write permissions. 1476 * If FOLL_FORCE is set, we only require the "MAY" flags. 1477 */ 1478 vm_flags = (foll_flags & FOLL_WRITE) ? 1479 (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD); 1480 vm_flags &= (foll_flags & FOLL_FORCE) ? 1481 (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE); 1482 1483 for (i = 0; i < nr_pages; i++) { 1484 vma = find_vma(mm, start); 1485 if (!vma) 1486 goto finish_or_fault; 1487 1488 /* protect what we can, including chardevs */ 1489 if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) || 1490 !(vm_flags & vma->vm_flags)) 1491 goto finish_or_fault; 1492 1493 if (pages) { 1494 pages[i] = virt_to_page(start); 1495 if (pages[i]) 1496 get_page(pages[i]); 1497 } 1498 if (vmas) 1499 vmas[i] = vma; 1500 start = (start + PAGE_SIZE) & PAGE_MASK; 1501 } 1502 1503 return i; 1504 1505 finish_or_fault: 1506 return i ? : -EFAULT; 1507 } 1508 #endif /* !CONFIG_MMU */ 1509 1510 /** 1511 * get_dump_page() - pin user page in memory while writing it to core dump 1512 * @addr: user address 1513 * 1514 * Returns struct page pointer of user page pinned for dump, 1515 * to be freed afterwards by put_page(). 1516 * 1517 * Returns NULL on any kind of failure - a hole must then be inserted into 1518 * the corefile, to preserve alignment with its headers; and also returns 1519 * NULL wherever the ZERO_PAGE, or an anonymous pte_none, has been found - 1520 * allowing a hole to be left in the corefile to save diskspace. 1521 * 1522 * Called without mmap_lock (takes and releases the mmap_lock by itself). 1523 */ 1524 #ifdef CONFIG_ELF_CORE 1525 struct page *get_dump_page(unsigned long addr) 1526 { 1527 struct mm_struct *mm = current->mm; 1528 struct page *page; 1529 int locked = 1; 1530 int ret; 1531 1532 if (mmap_read_lock_killable(mm)) 1533 return NULL; 1534 ret = __get_user_pages_locked(mm, addr, 1, &page, NULL, &locked, 1535 FOLL_FORCE | FOLL_DUMP | FOLL_GET); 1536 if (locked) 1537 mmap_read_unlock(mm); 1538 1539 if (ret == 1 && is_page_poisoned(page)) 1540 return NULL; 1541 1542 return (ret == 1) ? page : NULL; 1543 } 1544 #endif /* CONFIG_ELF_CORE */ 1545 1546 #ifdef CONFIG_CMA 1547 static long check_and_migrate_cma_pages(struct mm_struct *mm, 1548 unsigned long start, 1549 unsigned long nr_pages, 1550 struct page **pages, 1551 struct vm_area_struct **vmas, 1552 unsigned int gup_flags) 1553 { 1554 unsigned long i; 1555 unsigned long step; 1556 bool drain_allow = true; 1557 bool migrate_allow = true; 1558 LIST_HEAD(cma_page_list); 1559 long ret = nr_pages; 1560 struct migration_target_control mtc = { 1561 .nid = NUMA_NO_NODE, 1562 .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_NOWARN, 1563 }; 1564 1565 check_again: 1566 for (i = 0; i < nr_pages;) { 1567 1568 struct page *head = compound_head(pages[i]); 1569 1570 /* 1571 * gup may start from a tail page. Advance step by the left 1572 * part. 1573 */ 1574 step = compound_nr(head) - (pages[i] - head); 1575 /* 1576 * If we get a page from the CMA zone, since we are going to 1577 * be pinning these entries, we might as well move them out 1578 * of the CMA zone if possible. 1579 */ 1580 if (is_migrate_cma_page(head)) { 1581 if (PageHuge(head)) 1582 isolate_huge_page(head, &cma_page_list); 1583 else { 1584 if (!PageLRU(head) && drain_allow) { 1585 lru_add_drain_all(); 1586 drain_allow = false; 1587 } 1588 1589 if (!isolate_lru_page(head)) { 1590 list_add_tail(&head->lru, &cma_page_list); 1591 mod_node_page_state(page_pgdat(head), 1592 NR_ISOLATED_ANON + 1593 page_is_file_lru(head), 1594 thp_nr_pages(head)); 1595 } 1596 } 1597 } 1598 1599 i += step; 1600 } 1601 1602 if (!list_empty(&cma_page_list)) { 1603 /* 1604 * drop the above get_user_pages reference. 1605 */ 1606 if (gup_flags & FOLL_PIN) 1607 unpin_user_pages(pages, nr_pages); 1608 else 1609 for (i = 0; i < nr_pages; i++) 1610 put_page(pages[i]); 1611 1612 if (migrate_pages(&cma_page_list, alloc_migration_target, NULL, 1613 (unsigned long)&mtc, MIGRATE_SYNC, MR_CONTIG_RANGE)) { 1614 /* 1615 * some of the pages failed migration. Do get_user_pages 1616 * without migration. 1617 */ 1618 migrate_allow = false; 1619 1620 if (!list_empty(&cma_page_list)) 1621 putback_movable_pages(&cma_page_list); 1622 } 1623 /* 1624 * We did migrate all the pages, Try to get the page references 1625 * again migrating any new CMA pages which we failed to isolate 1626 * earlier. 1627 */ 1628 ret = __get_user_pages_locked(mm, start, nr_pages, 1629 pages, vmas, NULL, 1630 gup_flags); 1631 1632 if ((ret > 0) && migrate_allow) { 1633 nr_pages = ret; 1634 drain_allow = true; 1635 goto check_again; 1636 } 1637 } 1638 1639 return ret; 1640 } 1641 #else 1642 static long check_and_migrate_cma_pages(struct mm_struct *mm, 1643 unsigned long start, 1644 unsigned long nr_pages, 1645 struct page **pages, 1646 struct vm_area_struct **vmas, 1647 unsigned int gup_flags) 1648 { 1649 return nr_pages; 1650 } 1651 #endif /* CONFIG_CMA */ 1652 1653 /* 1654 * __gup_longterm_locked() is a wrapper for __get_user_pages_locked which 1655 * allows us to process the FOLL_LONGTERM flag. 1656 */ 1657 static long __gup_longterm_locked(struct mm_struct *mm, 1658 unsigned long start, 1659 unsigned long nr_pages, 1660 struct page **pages, 1661 struct vm_area_struct **vmas, 1662 unsigned int gup_flags) 1663 { 1664 unsigned long flags = 0; 1665 long rc; 1666 1667 if (gup_flags & FOLL_LONGTERM) 1668 flags = memalloc_nocma_save(); 1669 1670 rc = __get_user_pages_locked(mm, start, nr_pages, pages, vmas, NULL, 1671 gup_flags); 1672 1673 if (gup_flags & FOLL_LONGTERM) { 1674 if (rc > 0) 1675 rc = check_and_migrate_cma_pages(mm, start, rc, pages, 1676 vmas, gup_flags); 1677 memalloc_nocma_restore(flags); 1678 } 1679 return rc; 1680 } 1681 1682 static bool is_valid_gup_flags(unsigned int gup_flags) 1683 { 1684 /* 1685 * FOLL_PIN must only be set internally by the pin_user_pages*() APIs, 1686 * never directly by the caller, so enforce that with an assertion: 1687 */ 1688 if (WARN_ON_ONCE(gup_flags & FOLL_PIN)) 1689 return false; 1690 /* 1691 * FOLL_PIN is a prerequisite to FOLL_LONGTERM. Another way of saying 1692 * that is, FOLL_LONGTERM is a specific case, more restrictive case of 1693 * FOLL_PIN. 1694 */ 1695 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM)) 1696 return false; 1697 1698 return true; 1699 } 1700 1701 #ifdef CONFIG_MMU 1702 static long __get_user_pages_remote(struct mm_struct *mm, 1703 unsigned long start, unsigned long nr_pages, 1704 unsigned int gup_flags, struct page **pages, 1705 struct vm_area_struct **vmas, int *locked) 1706 { 1707 /* 1708 * Parts of FOLL_LONGTERM behavior are incompatible with 1709 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on 1710 * vmas. However, this only comes up if locked is set, and there are 1711 * callers that do request FOLL_LONGTERM, but do not set locked. So, 1712 * allow what we can. 1713 */ 1714 if (gup_flags & FOLL_LONGTERM) { 1715 if (WARN_ON_ONCE(locked)) 1716 return -EINVAL; 1717 /* 1718 * This will check the vmas (even if our vmas arg is NULL) 1719 * and return -ENOTSUPP if DAX isn't allowed in this case: 1720 */ 1721 return __gup_longterm_locked(mm, start, nr_pages, pages, 1722 vmas, gup_flags | FOLL_TOUCH | 1723 FOLL_REMOTE); 1724 } 1725 1726 return __get_user_pages_locked(mm, start, nr_pages, pages, vmas, 1727 locked, 1728 gup_flags | FOLL_TOUCH | FOLL_REMOTE); 1729 } 1730 1731 /** 1732 * get_user_pages_remote() - pin user pages in memory 1733 * @mm: mm_struct of target mm 1734 * @start: starting user address 1735 * @nr_pages: number of pages from start to pin 1736 * @gup_flags: flags modifying lookup behaviour 1737 * @pages: array that receives pointers to the pages pinned. 1738 * Should be at least nr_pages long. Or NULL, if caller 1739 * only intends to ensure the pages are faulted in. 1740 * @vmas: array of pointers to vmas corresponding to each page. 1741 * Or NULL if the caller does not require them. 1742 * @locked: pointer to lock flag indicating whether lock is held and 1743 * subsequently whether VM_FAULT_RETRY functionality can be 1744 * utilised. Lock must initially be held. 1745 * 1746 * Returns either number of pages pinned (which may be less than the 1747 * number requested), or an error. Details about the return value: 1748 * 1749 * -- If nr_pages is 0, returns 0. 1750 * -- If nr_pages is >0, but no pages were pinned, returns -errno. 1751 * -- If nr_pages is >0, and some pages were pinned, returns the number of 1752 * pages pinned. Again, this may be less than nr_pages. 1753 * 1754 * The caller is responsible for releasing returned @pages, via put_page(). 1755 * 1756 * @vmas are valid only as long as mmap_lock is held. 1757 * 1758 * Must be called with mmap_lock held for read or write. 1759 * 1760 * get_user_pages_remote walks a process's page tables and takes a reference 1761 * to each struct page that each user address corresponds to at a given 1762 * instant. That is, it takes the page that would be accessed if a user 1763 * thread accesses the given user virtual address at that instant. 1764 * 1765 * This does not guarantee that the page exists in the user mappings when 1766 * get_user_pages_remote returns, and there may even be a completely different 1767 * page there in some cases (eg. if mmapped pagecache has been invalidated 1768 * and subsequently re faulted). However it does guarantee that the page 1769 * won't be freed completely. And mostly callers simply care that the page 1770 * contains data that was valid *at some point in time*. Typically, an IO 1771 * or similar operation cannot guarantee anything stronger anyway because 1772 * locks can't be held over the syscall boundary. 1773 * 1774 * If gup_flags & FOLL_WRITE == 0, the page must not be written to. If the page 1775 * is written to, set_page_dirty (or set_page_dirty_lock, as appropriate) must 1776 * be called after the page is finished with, and before put_page is called. 1777 * 1778 * get_user_pages_remote is typically used for fewer-copy IO operations, 1779 * to get a handle on the memory by some means other than accesses 1780 * via the user virtual addresses. The pages may be submitted for 1781 * DMA to devices or accessed via their kernel linear mapping (via the 1782 * kmap APIs). Care should be taken to use the correct cache flushing APIs. 1783 * 1784 * See also get_user_pages_fast, for performance critical applications. 1785 * 1786 * get_user_pages_remote should be phased out in favor of 1787 * get_user_pages_locked|unlocked or get_user_pages_fast. Nothing 1788 * should use get_user_pages_remote because it cannot pass 1789 * FAULT_FLAG_ALLOW_RETRY to handle_mm_fault. 1790 */ 1791 long get_user_pages_remote(struct mm_struct *mm, 1792 unsigned long start, unsigned long nr_pages, 1793 unsigned int gup_flags, struct page **pages, 1794 struct vm_area_struct **vmas, int *locked) 1795 { 1796 if (!is_valid_gup_flags(gup_flags)) 1797 return -EINVAL; 1798 1799 return __get_user_pages_remote(mm, start, nr_pages, gup_flags, 1800 pages, vmas, locked); 1801 } 1802 EXPORT_SYMBOL(get_user_pages_remote); 1803 1804 #else /* CONFIG_MMU */ 1805 long get_user_pages_remote(struct mm_struct *mm, 1806 unsigned long start, unsigned long nr_pages, 1807 unsigned int gup_flags, struct page **pages, 1808 struct vm_area_struct **vmas, int *locked) 1809 { 1810 return 0; 1811 } 1812 1813 static long __get_user_pages_remote(struct mm_struct *mm, 1814 unsigned long start, unsigned long nr_pages, 1815 unsigned int gup_flags, struct page **pages, 1816 struct vm_area_struct **vmas, int *locked) 1817 { 1818 return 0; 1819 } 1820 #endif /* !CONFIG_MMU */ 1821 1822 /** 1823 * get_user_pages() - pin user pages in memory 1824 * @start: starting user address 1825 * @nr_pages: number of pages from start to pin 1826 * @gup_flags: flags modifying lookup behaviour 1827 * @pages: array that receives pointers to the pages pinned. 1828 * Should be at least nr_pages long. Or NULL, if caller 1829 * only intends to ensure the pages are faulted in. 1830 * @vmas: array of pointers to vmas corresponding to each page. 1831 * Or NULL if the caller does not require them. 1832 * 1833 * This is the same as get_user_pages_remote(), just with a less-flexible 1834 * calling convention where we assume that the mm being operated on belongs to 1835 * the current task, and doesn't allow passing of a locked parameter. We also 1836 * obviously don't pass FOLL_REMOTE in here. 1837 */ 1838 long get_user_pages(unsigned long start, unsigned long nr_pages, 1839 unsigned int gup_flags, struct page **pages, 1840 struct vm_area_struct **vmas) 1841 { 1842 if (!is_valid_gup_flags(gup_flags)) 1843 return -EINVAL; 1844 1845 return __gup_longterm_locked(current->mm, start, nr_pages, 1846 pages, vmas, gup_flags | FOLL_TOUCH); 1847 } 1848 EXPORT_SYMBOL(get_user_pages); 1849 1850 /** 1851 * get_user_pages_locked() - variant of get_user_pages() 1852 * 1853 * @start: starting user address 1854 * @nr_pages: number of pages from start to pin 1855 * @gup_flags: flags modifying lookup behaviour 1856 * @pages: array that receives pointers to the pages pinned. 1857 * Should be at least nr_pages long. Or NULL, if caller 1858 * only intends to ensure the pages are faulted in. 1859 * @locked: pointer to lock flag indicating whether lock is held and 1860 * subsequently whether VM_FAULT_RETRY functionality can be 1861 * utilised. Lock must initially be held. 1862 * 1863 * It is suitable to replace the form: 1864 * 1865 * mmap_read_lock(mm); 1866 * do_something() 1867 * get_user_pages(mm, ..., pages, NULL); 1868 * mmap_read_unlock(mm); 1869 * 1870 * to: 1871 * 1872 * int locked = 1; 1873 * mmap_read_lock(mm); 1874 * do_something() 1875 * get_user_pages_locked(mm, ..., pages, &locked); 1876 * if (locked) 1877 * mmap_read_unlock(mm); 1878 * 1879 * We can leverage the VM_FAULT_RETRY functionality in the page fault 1880 * paths better by using either get_user_pages_locked() or 1881 * get_user_pages_unlocked(). 1882 * 1883 */ 1884 long get_user_pages_locked(unsigned long start, unsigned long nr_pages, 1885 unsigned int gup_flags, struct page **pages, 1886 int *locked) 1887 { 1888 /* 1889 * FIXME: Current FOLL_LONGTERM behavior is incompatible with 1890 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on 1891 * vmas. As there are no users of this flag in this call we simply 1892 * disallow this option for now. 1893 */ 1894 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM)) 1895 return -EINVAL; 1896 /* 1897 * FOLL_PIN must only be set internally by the pin_user_pages*() APIs, 1898 * never directly by the caller, so enforce that: 1899 */ 1900 if (WARN_ON_ONCE(gup_flags & FOLL_PIN)) 1901 return -EINVAL; 1902 1903 return __get_user_pages_locked(current->mm, start, nr_pages, 1904 pages, NULL, locked, 1905 gup_flags | FOLL_TOUCH); 1906 } 1907 EXPORT_SYMBOL(get_user_pages_locked); 1908 1909 /* 1910 * get_user_pages_unlocked() is suitable to replace the form: 1911 * 1912 * mmap_read_lock(mm); 1913 * get_user_pages(mm, ..., pages, NULL); 1914 * mmap_read_unlock(mm); 1915 * 1916 * with: 1917 * 1918 * get_user_pages_unlocked(mm, ..., pages); 1919 * 1920 * It is functionally equivalent to get_user_pages_fast so 1921 * get_user_pages_fast should be used instead if specific gup_flags 1922 * (e.g. FOLL_FORCE) are not required. 1923 */ 1924 long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, 1925 struct page **pages, unsigned int gup_flags) 1926 { 1927 struct mm_struct *mm = current->mm; 1928 int locked = 1; 1929 long ret; 1930 1931 /* 1932 * FIXME: Current FOLL_LONGTERM behavior is incompatible with 1933 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on 1934 * vmas. As there are no users of this flag in this call we simply 1935 * disallow this option for now. 1936 */ 1937 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM)) 1938 return -EINVAL; 1939 1940 mmap_read_lock(mm); 1941 ret = __get_user_pages_locked(mm, start, nr_pages, pages, NULL, 1942 &locked, gup_flags | FOLL_TOUCH); 1943 if (locked) 1944 mmap_read_unlock(mm); 1945 return ret; 1946 } 1947 EXPORT_SYMBOL(get_user_pages_unlocked); 1948 1949 /* 1950 * Fast GUP 1951 * 1952 * get_user_pages_fast attempts to pin user pages by walking the page 1953 * tables directly and avoids taking locks. Thus the walker needs to be 1954 * protected from page table pages being freed from under it, and should 1955 * block any THP splits. 1956 * 1957 * One way to achieve this is to have the walker disable interrupts, and 1958 * rely on IPIs from the TLB flushing code blocking before the page table 1959 * pages are freed. This is unsuitable for architectures that do not need 1960 * to broadcast an IPI when invalidating TLBs. 1961 * 1962 * Another way to achieve this is to batch up page table containing pages 1963 * belonging to more than one mm_user, then rcu_sched a callback to free those 1964 * pages. Disabling interrupts will allow the fast_gup walker to both block 1965 * the rcu_sched callback, and an IPI that we broadcast for splitting THPs 1966 * (which is a relatively rare event). The code below adopts this strategy. 1967 * 1968 * Before activating this code, please be aware that the following assumptions 1969 * are currently made: 1970 * 1971 * *) Either MMU_GATHER_RCU_TABLE_FREE is enabled, and tlb_remove_table() is used to 1972 * free pages containing page tables or TLB flushing requires IPI broadcast. 1973 * 1974 * *) ptes can be read atomically by the architecture. 1975 * 1976 * *) access_ok is sufficient to validate userspace address ranges. 1977 * 1978 * The last two assumptions can be relaxed by the addition of helper functions. 1979 * 1980 * This code is based heavily on the PowerPC implementation by Nick Piggin. 1981 */ 1982 #ifdef CONFIG_HAVE_FAST_GUP 1983 1984 static void __maybe_unused undo_dev_pagemap(int *nr, int nr_start, 1985 unsigned int flags, 1986 struct page **pages) 1987 { 1988 while ((*nr) - nr_start) { 1989 struct page *page = pages[--(*nr)]; 1990 1991 ClearPageReferenced(page); 1992 if (flags & FOLL_PIN) 1993 unpin_user_page(page); 1994 else 1995 put_page(page); 1996 } 1997 } 1998 1999 #ifdef CONFIG_ARCH_HAS_PTE_SPECIAL 2000 static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, 2001 unsigned int flags, struct page **pages, int *nr) 2002 { 2003 struct dev_pagemap *pgmap = NULL; 2004 int nr_start = *nr, ret = 0; 2005 pte_t *ptep, *ptem; 2006 2007 ptem = ptep = pte_offset_map(&pmd, addr); 2008 do { 2009 pte_t pte = ptep_get_lockless(ptep); 2010 struct page *head, *page; 2011 2012 /* 2013 * Similar to the PMD case below, NUMA hinting must take slow 2014 * path using the pte_protnone check. 2015 */ 2016 if (pte_protnone(pte)) 2017 goto pte_unmap; 2018 2019 if (!pte_access_permitted(pte, flags & FOLL_WRITE)) 2020 goto pte_unmap; 2021 2022 if (pte_devmap(pte)) { 2023 if (unlikely(flags & FOLL_LONGTERM)) 2024 goto pte_unmap; 2025 2026 pgmap = get_dev_pagemap(pte_pfn(pte), pgmap); 2027 if (unlikely(!pgmap)) { 2028 undo_dev_pagemap(nr, nr_start, flags, pages); 2029 goto pte_unmap; 2030 } 2031 } else if (pte_special(pte)) 2032 goto pte_unmap; 2033 2034 VM_BUG_ON(!pfn_valid(pte_pfn(pte))); 2035 page = pte_page(pte); 2036 2037 head = try_grab_compound_head(page, 1, flags); 2038 if (!head) 2039 goto pte_unmap; 2040 2041 if (unlikely(pte_val(pte) != pte_val(*ptep))) { 2042 put_compound_head(head, 1, flags); 2043 goto pte_unmap; 2044 } 2045 2046 VM_BUG_ON_PAGE(compound_head(page) != head, page); 2047 2048 /* 2049 * We need to make the page accessible if and only if we are 2050 * going to access its content (the FOLL_PIN case). Please 2051 * see Documentation/core-api/pin_user_pages.rst for 2052 * details. 2053 */ 2054 if (flags & FOLL_PIN) { 2055 ret = arch_make_page_accessible(page); 2056 if (ret) { 2057 unpin_user_page(page); 2058 goto pte_unmap; 2059 } 2060 } 2061 SetPageReferenced(page); 2062 pages[*nr] = page; 2063 (*nr)++; 2064 2065 } while (ptep++, addr += PAGE_SIZE, addr != end); 2066 2067 ret = 1; 2068 2069 pte_unmap: 2070 if (pgmap) 2071 put_dev_pagemap(pgmap); 2072 pte_unmap(ptem); 2073 return ret; 2074 } 2075 #else 2076 2077 /* 2078 * If we can't determine whether or not a pte is special, then fail immediately 2079 * for ptes. Note, we can still pin HugeTLB and THP as these are guaranteed not 2080 * to be special. 2081 * 2082 * For a futex to be placed on a THP tail page, get_futex_key requires a 2083 * get_user_pages_fast_only implementation that can pin pages. Thus it's still 2084 * useful to have gup_huge_pmd even if we can't operate on ptes. 2085 */ 2086 static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, 2087 unsigned int flags, struct page **pages, int *nr) 2088 { 2089 return 0; 2090 } 2091 #endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */ 2092 2093 #if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && defined(CONFIG_TRANSPARENT_HUGEPAGE) 2094 static int __gup_device_huge(unsigned long pfn, unsigned long addr, 2095 unsigned long end, unsigned int flags, 2096 struct page **pages, int *nr) 2097 { 2098 int nr_start = *nr; 2099 struct dev_pagemap *pgmap = NULL; 2100 2101 do { 2102 struct page *page = pfn_to_page(pfn); 2103 2104 pgmap = get_dev_pagemap(pfn, pgmap); 2105 if (unlikely(!pgmap)) { 2106 undo_dev_pagemap(nr, nr_start, flags, pages); 2107 return 0; 2108 } 2109 SetPageReferenced(page); 2110 pages[*nr] = page; 2111 if (unlikely(!try_grab_page(page, flags))) { 2112 undo_dev_pagemap(nr, nr_start, flags, pages); 2113 return 0; 2114 } 2115 (*nr)++; 2116 pfn++; 2117 } while (addr += PAGE_SIZE, addr != end); 2118 2119 if (pgmap) 2120 put_dev_pagemap(pgmap); 2121 return 1; 2122 } 2123 2124 static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, 2125 unsigned long end, unsigned int flags, 2126 struct page **pages, int *nr) 2127 { 2128 unsigned long fault_pfn; 2129 int nr_start = *nr; 2130 2131 fault_pfn = pmd_pfn(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); 2132 if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr)) 2133 return 0; 2134 2135 if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) { 2136 undo_dev_pagemap(nr, nr_start, flags, pages); 2137 return 0; 2138 } 2139 return 1; 2140 } 2141 2142 static int __gup_device_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, 2143 unsigned long end, unsigned int flags, 2144 struct page **pages, int *nr) 2145 { 2146 unsigned long fault_pfn; 2147 int nr_start = *nr; 2148 2149 fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); 2150 if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr)) 2151 return 0; 2152 2153 if (unlikely(pud_val(orig) != pud_val(*pudp))) { 2154 undo_dev_pagemap(nr, nr_start, flags, pages); 2155 return 0; 2156 } 2157 return 1; 2158 } 2159 #else 2160 static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, 2161 unsigned long end, unsigned int flags, 2162 struct page **pages, int *nr) 2163 { 2164 BUILD_BUG(); 2165 return 0; 2166 } 2167 2168 static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr, 2169 unsigned long end, unsigned int flags, 2170 struct page **pages, int *nr) 2171 { 2172 BUILD_BUG(); 2173 return 0; 2174 } 2175 #endif 2176 2177 static int record_subpages(struct page *page, unsigned long addr, 2178 unsigned long end, struct page **pages) 2179 { 2180 int nr; 2181 2182 for (nr = 0; addr != end; addr += PAGE_SIZE) 2183 pages[nr++] = page++; 2184 2185 return nr; 2186 } 2187 2188 #ifdef CONFIG_ARCH_HAS_HUGEPD 2189 static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end, 2190 unsigned long sz) 2191 { 2192 unsigned long __boundary = (addr + sz) & ~(sz-1); 2193 return (__boundary - 1 < end - 1) ? __boundary : end; 2194 } 2195 2196 static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, 2197 unsigned long end, unsigned int flags, 2198 struct page **pages, int *nr) 2199 { 2200 unsigned long pte_end; 2201 struct page *head, *page; 2202 pte_t pte; 2203 int refs; 2204 2205 pte_end = (addr + sz) & ~(sz-1); 2206 if (pte_end < end) 2207 end = pte_end; 2208 2209 pte = huge_ptep_get(ptep); 2210 2211 if (!pte_access_permitted(pte, flags & FOLL_WRITE)) 2212 return 0; 2213 2214 /* hugepages are never "special" */ 2215 VM_BUG_ON(!pfn_valid(pte_pfn(pte))); 2216 2217 head = pte_page(pte); 2218 page = head + ((addr & (sz-1)) >> PAGE_SHIFT); 2219 refs = record_subpages(page, addr, end, pages + *nr); 2220 2221 head = try_grab_compound_head(head, refs, flags); 2222 if (!head) 2223 return 0; 2224 2225 if (unlikely(pte_val(pte) != pte_val(*ptep))) { 2226 put_compound_head(head, refs, flags); 2227 return 0; 2228 } 2229 2230 *nr += refs; 2231 SetPageReferenced(head); 2232 return 1; 2233 } 2234 2235 static int gup_huge_pd(hugepd_t hugepd, unsigned long addr, 2236 unsigned int pdshift, unsigned long end, unsigned int flags, 2237 struct page **pages, int *nr) 2238 { 2239 pte_t *ptep; 2240 unsigned long sz = 1UL << hugepd_shift(hugepd); 2241 unsigned long next; 2242 2243 ptep = hugepte_offset(hugepd, addr, pdshift); 2244 do { 2245 next = hugepte_addr_end(addr, end, sz); 2246 if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr)) 2247 return 0; 2248 } while (ptep++, addr = next, addr != end); 2249 2250 return 1; 2251 } 2252 #else 2253 static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr, 2254 unsigned int pdshift, unsigned long end, unsigned int flags, 2255 struct page **pages, int *nr) 2256 { 2257 return 0; 2258 } 2259 #endif /* CONFIG_ARCH_HAS_HUGEPD */ 2260 2261 static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, 2262 unsigned long end, unsigned int flags, 2263 struct page **pages, int *nr) 2264 { 2265 struct page *head, *page; 2266 int refs; 2267 2268 if (!pmd_access_permitted(orig, flags & FOLL_WRITE)) 2269 return 0; 2270 2271 if (pmd_devmap(orig)) { 2272 if (unlikely(flags & FOLL_LONGTERM)) 2273 return 0; 2274 return __gup_device_huge_pmd(orig, pmdp, addr, end, flags, 2275 pages, nr); 2276 } 2277 2278 page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); 2279 refs = record_subpages(page, addr, end, pages + *nr); 2280 2281 head = try_grab_compound_head(pmd_page(orig), refs, flags); 2282 if (!head) 2283 return 0; 2284 2285 if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) { 2286 put_compound_head(head, refs, flags); 2287 return 0; 2288 } 2289 2290 *nr += refs; 2291 SetPageReferenced(head); 2292 return 1; 2293 } 2294 2295 static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, 2296 unsigned long end, unsigned int flags, 2297 struct page **pages, int *nr) 2298 { 2299 struct page *head, *page; 2300 int refs; 2301 2302 if (!pud_access_permitted(orig, flags & FOLL_WRITE)) 2303 return 0; 2304 2305 if (pud_devmap(orig)) { 2306 if (unlikely(flags & FOLL_LONGTERM)) 2307 return 0; 2308 return __gup_device_huge_pud(orig, pudp, addr, end, flags, 2309 pages, nr); 2310 } 2311 2312 page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); 2313 refs = record_subpages(page, addr, end, pages + *nr); 2314 2315 head = try_grab_compound_head(pud_page(orig), refs, flags); 2316 if (!head) 2317 return 0; 2318 2319 if (unlikely(pud_val(orig) != pud_val(*pudp))) { 2320 put_compound_head(head, refs, flags); 2321 return 0; 2322 } 2323 2324 *nr += refs; 2325 SetPageReferenced(head); 2326 return 1; 2327 } 2328 2329 static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr, 2330 unsigned long end, unsigned int flags, 2331 struct page **pages, int *nr) 2332 { 2333 int refs; 2334 struct page *head, *page; 2335 2336 if (!pgd_access_permitted(orig, flags & FOLL_WRITE)) 2337 return 0; 2338 2339 BUILD_BUG_ON(pgd_devmap(orig)); 2340 2341 page = pgd_page(orig) + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT); 2342 refs = record_subpages(page, addr, end, pages + *nr); 2343 2344 head = try_grab_compound_head(pgd_page(orig), refs, flags); 2345 if (!head) 2346 return 0; 2347 2348 if (unlikely(pgd_val(orig) != pgd_val(*pgdp))) { 2349 put_compound_head(head, refs, flags); 2350 return 0; 2351 } 2352 2353 *nr += refs; 2354 SetPageReferenced(head); 2355 return 1; 2356 } 2357 2358 static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned long end, 2359 unsigned int flags, struct page **pages, int *nr) 2360 { 2361 unsigned long next; 2362 pmd_t *pmdp; 2363 2364 pmdp = pmd_offset_lockless(pudp, pud, addr); 2365 do { 2366 pmd_t pmd = READ_ONCE(*pmdp); 2367 2368 next = pmd_addr_end(addr, end); 2369 if (!pmd_present(pmd)) 2370 return 0; 2371 2372 if (unlikely(pmd_trans_huge(pmd) || pmd_huge(pmd) || 2373 pmd_devmap(pmd))) { 2374 /* 2375 * NUMA hinting faults need to be handled in the GUP 2376 * slowpath for accounting purposes and so that they 2377 * can be serialised against THP migration. 2378 */ 2379 if (pmd_protnone(pmd)) 2380 return 0; 2381 2382 if (!gup_huge_pmd(pmd, pmdp, addr, next, flags, 2383 pages, nr)) 2384 return 0; 2385 2386 } else if (unlikely(is_hugepd(__hugepd(pmd_val(pmd))))) { 2387 /* 2388 * architecture have different format for hugetlbfs 2389 * pmd format and THP pmd format 2390 */ 2391 if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr, 2392 PMD_SHIFT, next, flags, pages, nr)) 2393 return 0; 2394 } else if (!gup_pte_range(pmd, addr, next, flags, pages, nr)) 2395 return 0; 2396 } while (pmdp++, addr = next, addr != end); 2397 2398 return 1; 2399 } 2400 2401 static int gup_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, unsigned long end, 2402 unsigned int flags, struct page **pages, int *nr) 2403 { 2404 unsigned long next; 2405 pud_t *pudp; 2406 2407 pudp = pud_offset_lockless(p4dp, p4d, addr); 2408 do { 2409 pud_t pud = READ_ONCE(*pudp); 2410 2411 next = pud_addr_end(addr, end); 2412 if (unlikely(!pud_present(pud))) 2413 return 0; 2414 if (unlikely(pud_huge(pud))) { 2415 if (!gup_huge_pud(pud, pudp, addr, next, flags, 2416 pages, nr)) 2417 return 0; 2418 } else if (unlikely(is_hugepd(__hugepd(pud_val(pud))))) { 2419 if (!gup_huge_pd(__hugepd(pud_val(pud)), addr, 2420 PUD_SHIFT, next, flags, pages, nr)) 2421 return 0; 2422 } else if (!gup_pmd_range(pudp, pud, addr, next, flags, pages, nr)) 2423 return 0; 2424 } while (pudp++, addr = next, addr != end); 2425 2426 return 1; 2427 } 2428 2429 static int gup_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, unsigned long end, 2430 unsigned int flags, struct page **pages, int *nr) 2431 { 2432 unsigned long next; 2433 p4d_t *p4dp; 2434 2435 p4dp = p4d_offset_lockless(pgdp, pgd, addr); 2436 do { 2437 p4d_t p4d = READ_ONCE(*p4dp); 2438 2439 next = p4d_addr_end(addr, end); 2440 if (p4d_none(p4d)) 2441 return 0; 2442 BUILD_BUG_ON(p4d_huge(p4d)); 2443 if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) { 2444 if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr, 2445 P4D_SHIFT, next, flags, pages, nr)) 2446 return 0; 2447 } else if (!gup_pud_range(p4dp, p4d, addr, next, flags, pages, nr)) 2448 return 0; 2449 } while (p4dp++, addr = next, addr != end); 2450 2451 return 1; 2452 } 2453 2454 static void gup_pgd_range(unsigned long addr, unsigned long end, 2455 unsigned int flags, struct page **pages, int *nr) 2456 { 2457 unsigned long next; 2458 pgd_t *pgdp; 2459 2460 pgdp = pgd_offset(current->mm, addr); 2461 do { 2462 pgd_t pgd = READ_ONCE(*pgdp); 2463 2464 next = pgd_addr_end(addr, end); 2465 if (pgd_none(pgd)) 2466 return; 2467 if (unlikely(pgd_huge(pgd))) { 2468 if (!gup_huge_pgd(pgd, pgdp, addr, next, flags, 2469 pages, nr)) 2470 return; 2471 } else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) { 2472 if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr, 2473 PGDIR_SHIFT, next, flags, pages, nr)) 2474 return; 2475 } else if (!gup_p4d_range(pgdp, pgd, addr, next, flags, pages, nr)) 2476 return; 2477 } while (pgdp++, addr = next, addr != end); 2478 } 2479 #else 2480 static inline void gup_pgd_range(unsigned long addr, unsigned long end, 2481 unsigned int flags, struct page **pages, int *nr) 2482 { 2483 } 2484 #endif /* CONFIG_HAVE_FAST_GUP */ 2485 2486 #ifndef gup_fast_permitted 2487 /* 2488 * Check if it's allowed to use get_user_pages_fast_only() for the range, or 2489 * we need to fall back to the slow version: 2490 */ 2491 static bool gup_fast_permitted(unsigned long start, unsigned long end) 2492 { 2493 return true; 2494 } 2495 #endif 2496 2497 static int __gup_longterm_unlocked(unsigned long start, int nr_pages, 2498 unsigned int gup_flags, struct page **pages) 2499 { 2500 int ret; 2501 2502 /* 2503 * FIXME: FOLL_LONGTERM does not work with 2504 * get_user_pages_unlocked() (see comments in that function) 2505 */ 2506 if (gup_flags & FOLL_LONGTERM) { 2507 mmap_read_lock(current->mm); 2508 ret = __gup_longterm_locked(current->mm, 2509 start, nr_pages, 2510 pages, NULL, gup_flags); 2511 mmap_read_unlock(current->mm); 2512 } else { 2513 ret = get_user_pages_unlocked(start, nr_pages, 2514 pages, gup_flags); 2515 } 2516 2517 return ret; 2518 } 2519 2520 static unsigned long lockless_pages_from_mm(unsigned long start, 2521 unsigned long end, 2522 unsigned int gup_flags, 2523 struct page **pages) 2524 { 2525 unsigned long flags; 2526 int nr_pinned = 0; 2527 unsigned seq; 2528 2529 if (!IS_ENABLED(CONFIG_HAVE_FAST_GUP) || 2530 !gup_fast_permitted(start, end)) 2531 return 0; 2532 2533 if (gup_flags & FOLL_PIN) { 2534 seq = raw_read_seqcount(¤t->mm->write_protect_seq); 2535 if (seq & 1) 2536 return 0; 2537 } 2538 2539 /* 2540 * Disable interrupts. The nested form is used, in order to allow full, 2541 * general purpose use of this routine. 2542 * 2543 * With interrupts disabled, we block page table pages from being freed 2544 * from under us. See struct mmu_table_batch comments in 2545 * include/asm-generic/tlb.h for more details. 2546 * 2547 * We do not adopt an rcu_read_lock() here as we also want to block IPIs 2548 * that come from THPs splitting. 2549 */ 2550 local_irq_save(flags); 2551 gup_pgd_range(start, end, gup_flags, pages, &nr_pinned); 2552 local_irq_restore(flags); 2553 2554 /* 2555 * When pinning pages for DMA there could be a concurrent write protect 2556 * from fork() via copy_page_range(), in this case always fail fast GUP. 2557 */ 2558 if (gup_flags & FOLL_PIN) { 2559 if (read_seqcount_retry(¤t->mm->write_protect_seq, seq)) { 2560 unpin_user_pages(pages, nr_pinned); 2561 return 0; 2562 } 2563 } 2564 return nr_pinned; 2565 } 2566 2567 static int internal_get_user_pages_fast(unsigned long start, 2568 unsigned long nr_pages, 2569 unsigned int gup_flags, 2570 struct page **pages) 2571 { 2572 unsigned long len, end; 2573 unsigned long nr_pinned; 2574 int ret; 2575 2576 if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM | 2577 FOLL_FORCE | FOLL_PIN | FOLL_GET | 2578 FOLL_FAST_ONLY))) 2579 return -EINVAL; 2580 2581 if (gup_flags & FOLL_PIN) 2582 atomic_set(¤t->mm->has_pinned, 1); 2583 2584 if (!(gup_flags & FOLL_FAST_ONLY)) 2585 might_lock_read(¤t->mm->mmap_lock); 2586 2587 start = untagged_addr(start) & PAGE_MASK; 2588 len = nr_pages << PAGE_SHIFT; 2589 if (check_add_overflow(start, len, &end)) 2590 return 0; 2591 if (unlikely(!access_ok((void __user *)start, len))) 2592 return -EFAULT; 2593 2594 nr_pinned = lockless_pages_from_mm(start, end, gup_flags, pages); 2595 if (nr_pinned == nr_pages || gup_flags & FOLL_FAST_ONLY) 2596 return nr_pinned; 2597 2598 /* Slow path: try to get the remaining pages with get_user_pages */ 2599 start += nr_pinned << PAGE_SHIFT; 2600 pages += nr_pinned; 2601 ret = __gup_longterm_unlocked(start, nr_pages - nr_pinned, gup_flags, 2602 pages); 2603 if (ret < 0) { 2604 /* 2605 * The caller has to unpin the pages we already pinned so 2606 * returning -errno is not an option 2607 */ 2608 if (nr_pinned) 2609 return nr_pinned; 2610 return ret; 2611 } 2612 return ret + nr_pinned; 2613 } 2614 2615 /** 2616 * get_user_pages_fast_only() - pin user pages in memory 2617 * @start: starting user address 2618 * @nr_pages: number of pages from start to pin 2619 * @gup_flags: flags modifying pin behaviour 2620 * @pages: array that receives pointers to the pages pinned. 2621 * Should be at least nr_pages long. 2622 * 2623 * Like get_user_pages_fast() except it's IRQ-safe in that it won't fall back to 2624 * the regular GUP. 2625 * Note a difference with get_user_pages_fast: this always returns the 2626 * number of pages pinned, 0 if no pages were pinned. 2627 * 2628 * If the architecture does not support this function, simply return with no 2629 * pages pinned. 2630 * 2631 * Careful, careful! COW breaking can go either way, so a non-write 2632 * access can get ambiguous page results. If you call this function without 2633 * 'write' set, you'd better be sure that you're ok with that ambiguity. 2634 */ 2635 int get_user_pages_fast_only(unsigned long start, int nr_pages, 2636 unsigned int gup_flags, struct page **pages) 2637 { 2638 int nr_pinned; 2639 /* 2640 * Internally (within mm/gup.c), gup fast variants must set FOLL_GET, 2641 * because gup fast is always a "pin with a +1 page refcount" request. 2642 * 2643 * FOLL_FAST_ONLY is required in order to match the API description of 2644 * this routine: no fall back to regular ("slow") GUP. 2645 */ 2646 gup_flags |= FOLL_GET | FOLL_FAST_ONLY; 2647 2648 nr_pinned = internal_get_user_pages_fast(start, nr_pages, gup_flags, 2649 pages); 2650 2651 /* 2652 * As specified in the API description above, this routine is not 2653 * allowed to return negative values. However, the common core 2654 * routine internal_get_user_pages_fast() *can* return -errno. 2655 * Therefore, correct for that here: 2656 */ 2657 if (nr_pinned < 0) 2658 nr_pinned = 0; 2659 2660 return nr_pinned; 2661 } 2662 EXPORT_SYMBOL_GPL(get_user_pages_fast_only); 2663 2664 /** 2665 * get_user_pages_fast() - pin user pages in memory 2666 * @start: starting user address 2667 * @nr_pages: number of pages from start to pin 2668 * @gup_flags: flags modifying pin behaviour 2669 * @pages: array that receives pointers to the pages pinned. 2670 * Should be at least nr_pages long. 2671 * 2672 * Attempt to pin user pages in memory without taking mm->mmap_lock. 2673 * If not successful, it will fall back to taking the lock and 2674 * calling get_user_pages(). 2675 * 2676 * Returns number of pages pinned. This may be fewer than the number requested. 2677 * If nr_pages is 0 or negative, returns 0. If no pages were pinned, returns 2678 * -errno. 2679 */ 2680 int get_user_pages_fast(unsigned long start, int nr_pages, 2681 unsigned int gup_flags, struct page **pages) 2682 { 2683 if (!is_valid_gup_flags(gup_flags)) 2684 return -EINVAL; 2685 2686 /* 2687 * The caller may or may not have explicitly set FOLL_GET; either way is 2688 * OK. However, internally (within mm/gup.c), gup fast variants must set 2689 * FOLL_GET, because gup fast is always a "pin with a +1 page refcount" 2690 * request. 2691 */ 2692 gup_flags |= FOLL_GET; 2693 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages); 2694 } 2695 EXPORT_SYMBOL_GPL(get_user_pages_fast); 2696 2697 /** 2698 * pin_user_pages_fast() - pin user pages in memory without taking locks 2699 * 2700 * @start: starting user address 2701 * @nr_pages: number of pages from start to pin 2702 * @gup_flags: flags modifying pin behaviour 2703 * @pages: array that receives pointers to the pages pinned. 2704 * Should be at least nr_pages long. 2705 * 2706 * Nearly the same as get_user_pages_fast(), except that FOLL_PIN is set. See 2707 * get_user_pages_fast() for documentation on the function arguments, because 2708 * the arguments here are identical. 2709 * 2710 * FOLL_PIN means that the pages must be released via unpin_user_page(). Please 2711 * see Documentation/core-api/pin_user_pages.rst for further details. 2712 */ 2713 int pin_user_pages_fast(unsigned long start, int nr_pages, 2714 unsigned int gup_flags, struct page **pages) 2715 { 2716 /* FOLL_GET and FOLL_PIN are mutually exclusive. */ 2717 if (WARN_ON_ONCE(gup_flags & FOLL_GET)) 2718 return -EINVAL; 2719 2720 gup_flags |= FOLL_PIN; 2721 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages); 2722 } 2723 EXPORT_SYMBOL_GPL(pin_user_pages_fast); 2724 2725 /* 2726 * This is the FOLL_PIN equivalent of get_user_pages_fast_only(). Behavior 2727 * is the same, except that this one sets FOLL_PIN instead of FOLL_GET. 2728 * 2729 * The API rules are the same, too: no negative values may be returned. 2730 */ 2731 int pin_user_pages_fast_only(unsigned long start, int nr_pages, 2732 unsigned int gup_flags, struct page **pages) 2733 { 2734 int nr_pinned; 2735 2736 /* 2737 * FOLL_GET and FOLL_PIN are mutually exclusive. Note that the API 2738 * rules require returning 0, rather than -errno: 2739 */ 2740 if (WARN_ON_ONCE(gup_flags & FOLL_GET)) 2741 return 0; 2742 /* 2743 * FOLL_FAST_ONLY is required in order to match the API description of 2744 * this routine: no fall back to regular ("slow") GUP. 2745 */ 2746 gup_flags |= (FOLL_PIN | FOLL_FAST_ONLY); 2747 nr_pinned = internal_get_user_pages_fast(start, nr_pages, gup_flags, 2748 pages); 2749 /* 2750 * This routine is not allowed to return negative values. However, 2751 * internal_get_user_pages_fast() *can* return -errno. Therefore, 2752 * correct for that here: 2753 */ 2754 if (nr_pinned < 0) 2755 nr_pinned = 0; 2756 2757 return nr_pinned; 2758 } 2759 EXPORT_SYMBOL_GPL(pin_user_pages_fast_only); 2760 2761 /** 2762 * pin_user_pages_remote() - pin pages of a remote process 2763 * 2764 * @mm: mm_struct of target mm 2765 * @start: starting user address 2766 * @nr_pages: number of pages from start to pin 2767 * @gup_flags: flags modifying lookup behaviour 2768 * @pages: array that receives pointers to the pages pinned. 2769 * Should be at least nr_pages long. Or NULL, if caller 2770 * only intends to ensure the pages are faulted in. 2771 * @vmas: array of pointers to vmas corresponding to each page. 2772 * Or NULL if the caller does not require them. 2773 * @locked: pointer to lock flag indicating whether lock is held and 2774 * subsequently whether VM_FAULT_RETRY functionality can be 2775 * utilised. Lock must initially be held. 2776 * 2777 * Nearly the same as get_user_pages_remote(), except that FOLL_PIN is set. See 2778 * get_user_pages_remote() for documentation on the function arguments, because 2779 * the arguments here are identical. 2780 * 2781 * FOLL_PIN means that the pages must be released via unpin_user_page(). Please 2782 * see Documentation/core-api/pin_user_pages.rst for details. 2783 */ 2784 long pin_user_pages_remote(struct mm_struct *mm, 2785 unsigned long start, unsigned long nr_pages, 2786 unsigned int gup_flags, struct page **pages, 2787 struct vm_area_struct **vmas, int *locked) 2788 { 2789 /* FOLL_GET and FOLL_PIN are mutually exclusive. */ 2790 if (WARN_ON_ONCE(gup_flags & FOLL_GET)) 2791 return -EINVAL; 2792 2793 gup_flags |= FOLL_PIN; 2794 return __get_user_pages_remote(mm, start, nr_pages, gup_flags, 2795 pages, vmas, locked); 2796 } 2797 EXPORT_SYMBOL(pin_user_pages_remote); 2798 2799 /** 2800 * pin_user_pages() - pin user pages in memory for use by other devices 2801 * 2802 * @start: starting user address 2803 * @nr_pages: number of pages from start to pin 2804 * @gup_flags: flags modifying lookup behaviour 2805 * @pages: array that receives pointers to the pages pinned. 2806 * Should be at least nr_pages long. Or NULL, if caller 2807 * only intends to ensure the pages are faulted in. 2808 * @vmas: array of pointers to vmas corresponding to each page. 2809 * Or NULL if the caller does not require them. 2810 * 2811 * Nearly the same as get_user_pages(), except that FOLL_TOUCH is not set, and 2812 * FOLL_PIN is set. 2813 * 2814 * FOLL_PIN means that the pages must be released via unpin_user_page(). Please 2815 * see Documentation/core-api/pin_user_pages.rst for details. 2816 */ 2817 long pin_user_pages(unsigned long start, unsigned long nr_pages, 2818 unsigned int gup_flags, struct page **pages, 2819 struct vm_area_struct **vmas) 2820 { 2821 /* FOLL_GET and FOLL_PIN are mutually exclusive. */ 2822 if (WARN_ON_ONCE(gup_flags & FOLL_GET)) 2823 return -EINVAL; 2824 2825 gup_flags |= FOLL_PIN; 2826 return __gup_longterm_locked(current->mm, start, nr_pages, 2827 pages, vmas, gup_flags); 2828 } 2829 EXPORT_SYMBOL(pin_user_pages); 2830 2831 /* 2832 * pin_user_pages_unlocked() is the FOLL_PIN variant of 2833 * get_user_pages_unlocked(). Behavior is the same, except that this one sets 2834 * FOLL_PIN and rejects FOLL_GET. 2835 */ 2836 long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages, 2837 struct page **pages, unsigned int gup_flags) 2838 { 2839 /* FOLL_GET and FOLL_PIN are mutually exclusive. */ 2840 if (WARN_ON_ONCE(gup_flags & FOLL_GET)) 2841 return -EINVAL; 2842 2843 gup_flags |= FOLL_PIN; 2844 return get_user_pages_unlocked(start, nr_pages, pages, gup_flags); 2845 } 2846 EXPORT_SYMBOL(pin_user_pages_unlocked); 2847 2848 /* 2849 * pin_user_pages_locked() is the FOLL_PIN variant of get_user_pages_locked(). 2850 * Behavior is the same, except that this one sets FOLL_PIN and rejects 2851 * FOLL_GET. 2852 */ 2853 long pin_user_pages_locked(unsigned long start, unsigned long nr_pages, 2854 unsigned int gup_flags, struct page **pages, 2855 int *locked) 2856 { 2857 /* 2858 * FIXME: Current FOLL_LONGTERM behavior is incompatible with 2859 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on 2860 * vmas. As there are no users of this flag in this call we simply 2861 * disallow this option for now. 2862 */ 2863 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM)) 2864 return -EINVAL; 2865 2866 /* FOLL_GET and FOLL_PIN are mutually exclusive. */ 2867 if (WARN_ON_ONCE(gup_flags & FOLL_GET)) 2868 return -EINVAL; 2869 2870 gup_flags |= FOLL_PIN; 2871 return __get_user_pages_locked(current->mm, start, nr_pages, 2872 pages, NULL, locked, 2873 gup_flags | FOLL_TOUCH); 2874 } 2875 EXPORT_SYMBOL(pin_user_pages_locked); 2876