gup.c (f4000fdf435b8301a11cf85237c561047f8c4c72) gup.c (eddb1c228f7951d399240a0cc57455dccc7f8777)
1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/kernel.h>
3#include <linux/errno.h>
4#include <linux/err.h>
5#include <linux/spinlock.h>
6
7#include <linux/mm.h>
8#include <linux/memremap.h>

--- 180 unchanged lines hidden (view full) ---

189 unsigned long address, pmd_t *pmd, unsigned int flags,
190 struct dev_pagemap **pgmap)
191{
192 struct mm_struct *mm = vma->vm_mm;
193 struct page *page;
194 spinlock_t *ptl;
195 pte_t *ptep, pte;
196
1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/kernel.h>
3#include <linux/errno.h>
4#include <linux/err.h>
5#include <linux/spinlock.h>
6
7#include <linux/mm.h>
8#include <linux/memremap.h>

--- 180 unchanged lines hidden (view full) ---

189 unsigned long address, pmd_t *pmd, unsigned int flags,
190 struct dev_pagemap **pgmap)
191{
192 struct mm_struct *mm = vma->vm_mm;
193 struct page *page;
194 spinlock_t *ptl;
195 pte_t *ptep, pte;
196
197 /* FOLL_GET and FOLL_PIN are mutually exclusive. */
198 if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) ==
199 (FOLL_PIN | FOLL_GET)))
200 return ERR_PTR(-EINVAL);
197retry:
198 if (unlikely(pmd_bad(*pmd)))
199 return no_page_table(vma, flags);
200
201 ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
202 pte = *ptep;
203 if (!pte_present(pte)) {
204 swp_entry_t entry;

--- 601 unchanged lines hidden (view full) ---

806 struct vm_area_struct *vma = NULL;
807 struct follow_page_context ctx = { NULL };
808
809 if (!nr_pages)
810 return 0;
811
812 start = untagged_addr(start);
813
201retry:
202 if (unlikely(pmd_bad(*pmd)))
203 return no_page_table(vma, flags);
204
205 ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
206 pte = *ptep;
207 if (!pte_present(pte)) {
208 swp_entry_t entry;

--- 601 unchanged lines hidden (view full) ---

810 struct vm_area_struct *vma = NULL;
811 struct follow_page_context ctx = { NULL };
812
813 if (!nr_pages)
814 return 0;
815
816 start = untagged_addr(start);
817
814 VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET));
818 VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN)));
815
816 /*
817 * If FOLL_FORCE is set then do not force a full fault as the hinting
818 * fault information is unrelated to the reference behaviour of a task
819 * using the address space
820 */
821 if (!(gup_flags & FOLL_FORCE))
822 gup_flags |= FOLL_NUMA;

--- 207 unchanged lines hidden (view full) ---

1030
1031 if (locked) {
1032 /* if VM_FAULT_RETRY can be returned, vmas become invalid */
1033 BUG_ON(vmas);
1034 /* check caller initialized locked */
1035 BUG_ON(*locked != 1);
1036 }
1037
819
820 /*
821 * If FOLL_FORCE is set then do not force a full fault as the hinting
822 * fault information is unrelated to the reference behaviour of a task
823 * using the address space
824 */
825 if (!(gup_flags & FOLL_FORCE))
826 gup_flags |= FOLL_NUMA;

--- 207 unchanged lines hidden (view full) ---

1034
1035 if (locked) {
1036 /* if VM_FAULT_RETRY can be returned, vmas become invalid */
1037 BUG_ON(vmas);
1038 /* check caller initialized locked */
1039 BUG_ON(*locked != 1);
1040 }
1041
1038 if (pages)
1042 /*
1043 * FOLL_PIN and FOLL_GET are mutually exclusive. Traditional behavior
1044 * is to set FOLL_GET if the caller wants pages[] filled in (but has
1045 * carelessly failed to specify FOLL_GET), so keep doing that, but only
1046 * for FOLL_GET, not for the newer FOLL_PIN.
1047 *
1048 * FOLL_PIN always expects pages to be non-null, but no need to assert
1049 * that here, as any failures will be obvious enough.
1050 */
1051 if (pages && !(flags & FOLL_PIN))
1039 flags |= FOLL_GET;
1040
1041 pages_done = 0;
1042 lock_dropped = false;
1043 for (;;) {
1044 ret = __get_user_pages(tsk, mm, start, nr_pages, flags, pages,
1045 vmas, locked);
1046 if (!locked)

--- 554 unchanged lines hidden (view full) ---

1601 *
1602 * See also get_user_pages_fast, for performance critical applications.
1603 *
1604 * get_user_pages should be phased out in favor of
1605 * get_user_pages_locked|unlocked or get_user_pages_fast. Nothing
1606 * should use get_user_pages because it cannot pass
1607 * FAULT_FLAG_ALLOW_RETRY to handle_mm_fault.
1608 */
1052 flags |= FOLL_GET;
1053
1054 pages_done = 0;
1055 lock_dropped = false;
1056 for (;;) {
1057 ret = __get_user_pages(tsk, mm, start, nr_pages, flags, pages,
1058 vmas, locked);
1059 if (!locked)

--- 554 unchanged lines hidden (view full) ---

1614 *
1615 * See also get_user_pages_fast, for performance critical applications.
1616 *
1617 * get_user_pages should be phased out in favor of
1618 * get_user_pages_locked|unlocked or get_user_pages_fast. Nothing
1619 * should use get_user_pages because it cannot pass
1620 * FAULT_FLAG_ALLOW_RETRY to handle_mm_fault.
1621 */
1622#ifdef CONFIG_MMU
1609long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
1610 unsigned long start, unsigned long nr_pages,
1611 unsigned int gup_flags, struct page **pages,
1612 struct vm_area_struct **vmas, int *locked)
1613{
1614 /*
1623long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
1624 unsigned long start, unsigned long nr_pages,
1625 unsigned int gup_flags, struct page **pages,
1626 struct vm_area_struct **vmas, int *locked)
1627{
1628 /*
1629 * FOLL_PIN must only be set internally by the pin_user_pages*() APIs,
1630 * never directly by the caller, so enforce that with an assertion:
1631 */
1632 if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
1633 return -EINVAL;
1634
1635 /*
1615 * Parts of FOLL_LONGTERM behavior are incompatible with
1616 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
1617 * vmas. However, this only comes up if locked is set, and there are
1618 * callers that do request FOLL_LONGTERM, but do not set locked. So,
1619 * allow what we can.
1620 */
1621 if (gup_flags & FOLL_LONGTERM) {
1622 if (WARN_ON_ONCE(locked))

--- 8 unchanged lines hidden (view full) ---

1631 }
1632
1633 return __get_user_pages_locked(tsk, mm, start, nr_pages, pages, vmas,
1634 locked,
1635 gup_flags | FOLL_TOUCH | FOLL_REMOTE);
1636}
1637EXPORT_SYMBOL(get_user_pages_remote);
1638
1636 * Parts of FOLL_LONGTERM behavior are incompatible with
1637 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
1638 * vmas. However, this only comes up if locked is set, and there are
1639 * callers that do request FOLL_LONGTERM, but do not set locked. So,
1640 * allow what we can.
1641 */
1642 if (gup_flags & FOLL_LONGTERM) {
1643 if (WARN_ON_ONCE(locked))

--- 8 unchanged lines hidden (view full) ---

1652 }
1653
1654 return __get_user_pages_locked(tsk, mm, start, nr_pages, pages, vmas,
1655 locked,
1656 gup_flags | FOLL_TOUCH | FOLL_REMOTE);
1657}
1658EXPORT_SYMBOL(get_user_pages_remote);
1659
1660#else /* CONFIG_MMU */
1661long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
1662 unsigned long start, unsigned long nr_pages,
1663 unsigned int gup_flags, struct page **pages,
1664 struct vm_area_struct **vmas, int *locked)
1665{
1666 return 0;
1667}
1668#endif /* !CONFIG_MMU */
1669
1639/*
1640 * This is the same as get_user_pages_remote(), just with a
1641 * less-flexible calling convention where we assume that the task
1642 * and mm being operated on are the current task's and don't allow
1643 * passing of a locked parameter. We also obviously don't pass
1644 * FOLL_REMOTE in here.
1645 */
1646long get_user_pages(unsigned long start, unsigned long nr_pages,
1647 unsigned int gup_flags, struct page **pages,
1648 struct vm_area_struct **vmas)
1649{
1670/*
1671 * This is the same as get_user_pages_remote(), just with a
1672 * less-flexible calling convention where we assume that the task
1673 * and mm being operated on are the current task's and don't allow
1674 * passing of a locked parameter. We also obviously don't pass
1675 * FOLL_REMOTE in here.
1676 */
1677long get_user_pages(unsigned long start, unsigned long nr_pages,
1678 unsigned int gup_flags, struct page **pages,
1679 struct vm_area_struct **vmas)
1680{
1681 /*
1682 * FOLL_PIN must only be set internally by the pin_user_pages*() APIs,
1683 * never directly by the caller, so enforce that with an assertion:
1684 */
1685 if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
1686 return -EINVAL;
1687
1650 return __gup_longterm_locked(current, current->mm, start, nr_pages,
1651 pages, vmas, gup_flags | FOLL_TOUCH);
1652}
1653EXPORT_SYMBOL(get_user_pages);
1654
1655/*
1656 * We can leverage the VM_FAULT_RETRY functionality in the page fault
1657 * paths better by using either get_user_pages_locked() or

--- 726 unchanged lines hidden (view full) ---

2384 } else {
2385 ret = get_user_pages_unlocked(start, nr_pages,
2386 pages, gup_flags);
2387 }
2388
2389 return ret;
2390}
2391
1688 return __gup_longterm_locked(current, current->mm, start, nr_pages,
1689 pages, vmas, gup_flags | FOLL_TOUCH);
1690}
1691EXPORT_SYMBOL(get_user_pages);
1692
1693/*
1694 * We can leverage the VM_FAULT_RETRY functionality in the page fault
1695 * paths better by using either get_user_pages_locked() or

--- 726 unchanged lines hidden (view full) ---

2422 } else {
2423 ret = get_user_pages_unlocked(start, nr_pages,
2424 pages, gup_flags);
2425 }
2426
2427 return ret;
2428}
2429
2392/**
2393 * get_user_pages_fast() - pin user pages in memory
2394 * @start: starting user address
2395 * @nr_pages: number of pages from start to pin
2396 * @gup_flags: flags modifying pin behaviour
2397 * @pages: array that receives pointers to the pages pinned.
2398 * Should be at least nr_pages long.
2399 *
2400 * Attempt to pin user pages in memory without taking mm->mmap_sem.
2401 * If not successful, it will fall back to taking the lock and
2402 * calling get_user_pages().
2403 *
2404 * Returns number of pages pinned. This may be fewer than the number
2405 * requested. If nr_pages is 0 or negative, returns 0. If no pages
2406 * were pinned, returns -errno.
2407 */
2408int get_user_pages_fast(unsigned long start, int nr_pages,
2409 unsigned int gup_flags, struct page **pages)
2430static int internal_get_user_pages_fast(unsigned long start, int nr_pages,
2431 unsigned int gup_flags,
2432 struct page **pages)
2410{
2411 unsigned long addr, len, end;
2412 int nr = 0, ret = 0;
2413
2414 if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM |
2433{
2434 unsigned long addr, len, end;
2435 int nr = 0, ret = 0;
2436
2437 if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM |
2415 FOLL_FORCE)))
2438 FOLL_FORCE | FOLL_PIN)))
2416 return -EINVAL;
2417
2418 start = untagged_addr(start) & PAGE_MASK;
2419 addr = start;
2420 len = (unsigned long) nr_pages << PAGE_SHIFT;
2421 end = start + len;
2422
2423 if (end <= start)

--- 23 unchanged lines hidden (view full) ---

2447 ret = nr;
2448 else
2449 ret += nr;
2450 }
2451 }
2452
2453 return ret;
2454}
2439 return -EINVAL;
2440
2441 start = untagged_addr(start) & PAGE_MASK;
2442 addr = start;
2443 len = (unsigned long) nr_pages << PAGE_SHIFT;
2444 end = start + len;
2445
2446 if (end <= start)

--- 23 unchanged lines hidden (view full) ---

2470 ret = nr;
2471 else
2472 ret += nr;
2473 }
2474 }
2475
2476 return ret;
2477}
2478
2479/**
2480 * get_user_pages_fast() - pin user pages in memory
2481 * @start: starting user address
2482 * @nr_pages: number of pages from start to pin
2483 * @gup_flags: flags modifying pin behaviour
2484 * @pages: array that receives pointers to the pages pinned.
2485 * Should be at least nr_pages long.
2486 *
2487 * Attempt to pin user pages in memory without taking mm->mmap_sem.
2488 * If not successful, it will fall back to taking the lock and
2489 * calling get_user_pages().
2490 *
2491 * Returns number of pages pinned. This may be fewer than the number requested.
2492 * If nr_pages is 0 or negative, returns 0. If no pages were pinned, returns
2493 * -errno.
2494 */
2495int get_user_pages_fast(unsigned long start, int nr_pages,
2496 unsigned int gup_flags, struct page **pages)
2497{
2498 /*
2499 * FOLL_PIN must only be set internally by the pin_user_pages*() APIs,
2500 * never directly by the caller, so enforce that:
2501 */
2502 if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
2503 return -EINVAL;
2504
2505 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
2506}
2455EXPORT_SYMBOL_GPL(get_user_pages_fast);
2507EXPORT_SYMBOL_GPL(get_user_pages_fast);
2508
2509/**
2510 * pin_user_pages_fast() - pin user pages in memory without taking locks
2511 *
2512 * For now, this is a placeholder function, until various call sites are
2513 * converted to use the correct get_user_pages*() or pin_user_pages*() API. So,
2514 * this is identical to get_user_pages_fast().
2515 *
2516 * This is intended for Case 1 (DIO) in Documentation/vm/pin_user_pages.rst. It
2517 * is NOT intended for Case 2 (RDMA: long-term pins).
2518 */
2519int pin_user_pages_fast(unsigned long start, int nr_pages,
2520 unsigned int gup_flags, struct page **pages)
2521{
2522 /*
2523 * This is a placeholder, until the pin functionality is activated.
2524 * Until then, just behave like the corresponding get_user_pages*()
2525 * routine.
2526 */
2527 return get_user_pages_fast(start, nr_pages, gup_flags, pages);
2528}
2529EXPORT_SYMBOL_GPL(pin_user_pages_fast);
2530
2531/**
2532 * pin_user_pages_remote() - pin pages of a remote process (task != current)
2533 *
2534 * For now, this is a placeholder function, until various call sites are
2535 * converted to use the correct get_user_pages*() or pin_user_pages*() API. So,
2536 * this is identical to get_user_pages_remote().
2537 *
2538 * This is intended for Case 1 (DIO) in Documentation/vm/pin_user_pages.rst. It
2539 * is NOT intended for Case 2 (RDMA: long-term pins).
2540 */
2541long pin_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
2542 unsigned long start, unsigned long nr_pages,
2543 unsigned int gup_flags, struct page **pages,
2544 struct vm_area_struct **vmas, int *locked)
2545{
2546 /*
2547 * This is a placeholder, until the pin functionality is activated.
2548 * Until then, just behave like the corresponding get_user_pages*()
2549 * routine.
2550 */
2551 return get_user_pages_remote(tsk, mm, start, nr_pages, gup_flags, pages,
2552 vmas, locked);
2553}
2554EXPORT_SYMBOL(pin_user_pages_remote);
2555
2556/**
2557 * pin_user_pages() - pin user pages in memory for use by other devices
2558 *
2559 * For now, this is a placeholder function, until various call sites are
2560 * converted to use the correct get_user_pages*() or pin_user_pages*() API. So,
2561 * this is identical to get_user_pages().
2562 *
2563 * This is intended for Case 1 (DIO) in Documentation/vm/pin_user_pages.rst. It
2564 * is NOT intended for Case 2 (RDMA: long-term pins).
2565 */
2566long pin_user_pages(unsigned long start, unsigned long nr_pages,
2567 unsigned int gup_flags, struct page **pages,
2568 struct vm_area_struct **vmas)
2569{
2570 /*
2571 * This is a placeholder, until the pin functionality is activated.
2572 * Until then, just behave like the corresponding get_user_pages*()
2573 * routine.
2574 */
2575 return get_user_pages(start, nr_pages, gup_flags, pages, vmas);
2576}
2577EXPORT_SYMBOL(pin_user_pages);