Lines Matching refs:pages
33 static inline void sanity_check_pinned_pages(struct page **pages, in sanity_check_pinned_pages() argument
51 for (; npages; npages--, pages++) { in sanity_check_pinned_pages()
52 struct page *page = *pages; in sanity_check_pinned_pages()
270 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages, in unpin_user_pages_dirty_lock() argument
278 unpin_user_pages(pages, npages); in unpin_user_pages_dirty_lock()
282 sanity_check_pinned_pages(pages, npages); in unpin_user_pages_dirty_lock()
284 folio = gup_folio_next(pages, npages, i, &nr); in unpin_user_pages_dirty_lock()
355 static void unpin_user_pages_lockless(struct page **pages, unsigned long npages) in unpin_user_pages_lockless() argument
367 folio = gup_folio_next(pages, npages, i, &nr); in unpin_user_pages_lockless()
381 void unpin_user_pages(struct page **pages, unsigned long npages) in unpin_user_pages() argument
395 sanity_check_pinned_pages(pages, npages); in unpin_user_pages()
397 folio = gup_folio_next(pages, npages, i, &nr); in unpin_user_pages()
1098 unsigned int gup_flags, struct page **pages, in __get_user_pages() argument
1110 VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN))); in __get_user_pages()
1139 pages ? &page : NULL); in __get_user_pages()
1189 if (pages) { in __get_user_pages()
1202 if (pages) { in __get_user_pages()
1238 pages[i + j] = subpage; in __get_user_pages()
1394 struct page **pages, in __get_user_pages_locked() argument
1426 if (pages && !(flags & FOLL_PIN)) in __get_user_pages_locked()
1431 ret = __get_user_pages(mm, start, nr_pages, flags, pages, in __get_user_pages_locked()
1464 if (likely(pages)) in __get_user_pages_locked()
1465 pages += ret; in __get_user_pages_locked()
1496 pages, locked); in __get_user_pages_locked()
1512 if (likely(pages)) in __get_user_pages_locked()
1513 pages++; in __get_user_pages_locked()
1713 unsigned long nr_pages, struct page **pages, in __get_user_pages_locked() argument
1753 if (pages) { in __get_user_pages_locked()
1754 pages[i] = virt_to_page((void *)start); in __get_user_pages_locked()
1755 if (pages[i]) in __get_user_pages_locked()
1756 get_page(pages[i]); in __get_user_pages_locked()
1952 struct page **pages) in collect_longterm_unpinnable_pages() argument
1959 struct folio *folio = page_folio(pages[i]); in collect_longterm_unpinnable_pages()
1999 struct page **pages) in migrate_longterm_unpinnable_pages() argument
2005 struct folio *folio = page_folio(pages[i]); in migrate_longterm_unpinnable_pages()
2012 pages[i] = NULL; in migrate_longterm_unpinnable_pages()
2031 unpin_user_page(pages[i]); in migrate_longterm_unpinnable_pages()
2032 pages[i] = NULL; in migrate_longterm_unpinnable_pages()
2055 if (pages[i]) in migrate_longterm_unpinnable_pages()
2056 unpin_user_page(pages[i]); in migrate_longterm_unpinnable_pages()
2079 struct page **pages) in check_and_migrate_movable_pages() argument
2083 collect_longterm_unpinnable_pages(&movable_page_list, nr_pages, pages); in check_and_migrate_movable_pages()
2088 pages); in check_and_migrate_movable_pages()
2092 struct page **pages) in check_and_migrate_movable_pages() argument
2105 struct page **pages, in __gup_longterm_locked() argument
2113 return __get_user_pages_locked(mm, start, nr_pages, pages, in __gup_longterm_locked()
2119 pages, locked, in __gup_longterm_locked()
2127 rc = check_and_migrate_movable_pages(nr_pinned_pages, pages); in __gup_longterm_locked()
2137 static bool is_valid_gup_args(struct page **pages, int *locked, in is_valid_gup_args() argument
2171 if (WARN_ON_ONCE((gup_flags & (FOLL_GET | FOLL_PIN)) && !pages)) in is_valid_gup_args()
2242 unsigned int gup_flags, struct page **pages, in get_user_pages_remote() argument
2247 if (!is_valid_gup_args(pages, locked, &gup_flags, in get_user_pages_remote()
2251 return __get_user_pages_locked(mm, start, nr_pages, pages, in get_user_pages_remote()
2260 unsigned int gup_flags, struct page **pages, in get_user_pages_remote() argument
2282 unsigned int gup_flags, struct page **pages) in get_user_pages() argument
2286 if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_TOUCH)) in get_user_pages()
2289 return __get_user_pages_locked(current->mm, start, nr_pages, pages, in get_user_pages()
2310 struct page **pages, unsigned int gup_flags) in get_user_pages_unlocked() argument
2314 if (!is_valid_gup_args(pages, NULL, &gup_flags, in get_user_pages_unlocked()
2318 return __get_user_pages_locked(current->mm, start, nr_pages, pages, in get_user_pages_unlocked()
2436 struct page **pages) in undo_dev_pagemap() argument
2439 struct page *page = pages[--(*nr)]; in undo_dev_pagemap()
2567 struct page **pages, int *nr) in gup_pte_range() argument
2600 undo_dev_pagemap(nr, nr_start, flags, pages); in gup_pte_range()
2648 pages[*nr] = page; in gup_pte_range()
2673 struct page **pages, int *nr) in gup_pte_range() argument
2682 struct page **pages, int *nr) in __gup_device_huge() argument
2692 undo_dev_pagemap(nr, nr_start, flags, pages); in __gup_device_huge()
2697 undo_dev_pagemap(nr, nr_start, flags, pages); in __gup_device_huge()
2702 pages[*nr] = page; in __gup_device_huge()
2704 undo_dev_pagemap(nr, nr_start, flags, pages); in __gup_device_huge()
2717 struct page **pages, int *nr) in __gup_device_huge_pmd() argument
2723 if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr)) in __gup_device_huge_pmd()
2727 undo_dev_pagemap(nr, nr_start, flags, pages); in __gup_device_huge_pmd()
2735 struct page **pages, int *nr) in __gup_device_huge_pud() argument
2741 if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr)) in __gup_device_huge_pud()
2745 undo_dev_pagemap(nr, nr_start, flags, pages); in __gup_device_huge_pud()
2753 struct page **pages, int *nr) in __gup_device_huge_pmd() argument
2761 struct page **pages, int *nr) in __gup_device_huge_pud() argument
2769 unsigned long end, struct page **pages) in record_subpages() argument
2774 pages[nr] = nth_page(page, nr); in record_subpages()
2789 struct page **pages, int *nr) in gup_hugepte() argument
2810 refs = record_subpages(page, addr, end, pages + *nr); in gup_hugepte()
2838 struct page **pages, int *nr) in gup_huge_pd() argument
2847 if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr)) in gup_huge_pd()
2856 struct page **pages, int *nr) in gup_huge_pd() argument
2864 struct page **pages, int *nr) in gup_huge_pmd() argument
2877 pages, nr); in gup_huge_pmd()
2881 refs = record_subpages(page, addr, end, pages + *nr); in gup_huge_pmd()
2908 struct page **pages, int *nr) in gup_huge_pud() argument
2921 pages, nr); in gup_huge_pud()
2925 refs = record_subpages(page, addr, end, pages + *nr); in gup_huge_pud()
2953 struct page **pages, int *nr) in gup_huge_pgd() argument
2965 refs = record_subpages(page, addr, end, pages + *nr); in gup_huge_pgd()
2992 unsigned int flags, struct page **pages, int *nr) in gup_pmd_range() argument
3012 pages, nr)) in gup_pmd_range()
3021 PMD_SHIFT, next, flags, pages, nr)) in gup_pmd_range()
3023 } else if (!gup_pte_range(pmd, pmdp, addr, next, flags, pages, nr)) in gup_pmd_range()
3031 unsigned int flags, struct page **pages, int *nr) in gup_pud_range() argument
3045 pages, nr)) in gup_pud_range()
3049 PUD_SHIFT, next, flags, pages, nr)) in gup_pud_range()
3051 } else if (!gup_pmd_range(pudp, pud, addr, next, flags, pages, nr)) in gup_pud_range()
3059 unsigned int flags, struct page **pages, int *nr) in gup_p4d_range() argument
3074 P4D_SHIFT, next, flags, pages, nr)) in gup_p4d_range()
3076 } else if (!gup_pud_range(p4dp, p4d, addr, next, flags, pages, nr)) in gup_p4d_range()
3084 unsigned int flags, struct page **pages, int *nr) in gup_pgd_range() argument
3098 pages, nr)) in gup_pgd_range()
3102 PGDIR_SHIFT, next, flags, pages, nr)) in gup_pgd_range()
3104 } else if (!gup_p4d_range(pgdp, pgd, addr, next, flags, pages, nr)) in gup_pgd_range()
3110 unsigned int flags, struct page **pages, int *nr) in gup_pgd_range() argument
3129 struct page **pages) in lockless_pages_from_mm() argument
3157 gup_pgd_range(start, end, gup_flags, pages, &nr_pinned); in lockless_pages_from_mm()
3166 unpin_user_pages_lockless(pages, nr_pinned); in lockless_pages_from_mm()
3169 sanity_check_pinned_pages(pages, nr_pinned); in lockless_pages_from_mm()
3178 struct page **pages) in internal_get_user_pages_fast() argument
3206 nr_pinned = lockless_pages_from_mm(start, end, gup_flags, pages); in internal_get_user_pages_fast()
3212 pages += nr_pinned; in internal_get_user_pages_fast()
3214 pages, &locked, in internal_get_user_pages_fast()
3247 unsigned int gup_flags, struct page **pages) in get_user_pages_fast_only() argument
3256 if (!is_valid_gup_args(pages, NULL, &gup_flags, in get_user_pages_fast_only()
3260 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages); in get_user_pages_fast_only()
3281 unsigned int gup_flags, struct page **pages) in get_user_pages_fast() argument
3289 if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_GET)) in get_user_pages_fast()
3291 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages); in get_user_pages_fast()
3315 unsigned int gup_flags, struct page **pages) in pin_user_pages_fast() argument
3317 if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_PIN)) in pin_user_pages_fast()
3319 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages); in pin_user_pages_fast()
3348 unsigned int gup_flags, struct page **pages, in pin_user_pages_remote() argument
3353 if (!is_valid_gup_args(pages, locked, &gup_flags, in pin_user_pages_remote()
3356 return __gup_longterm_locked(mm, start, nr_pages, pages, in pin_user_pages_remote()
3381 unsigned int gup_flags, struct page **pages) in pin_user_pages() argument
3385 if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_PIN)) in pin_user_pages()
3388 pages, &locked, gup_flags); in pin_user_pages()
3401 struct page **pages, unsigned int gup_flags) in pin_user_pages_unlocked() argument
3405 if (!is_valid_gup_args(pages, NULL, &gup_flags, in pin_user_pages_unlocked()
3409 return __gup_longterm_locked(current->mm, start, nr_pages, pages, in pin_user_pages_unlocked()