33 static inline void sanity_check_pinned_pages(struct page **pages,  in sanity_check_pinned_pages()  argument
40 	 * We only pin anonymous pages if they are exclusive. Once pinned, we  in sanity_check_pinned_pages()
44 	 * We'd like to verify that our pinned anonymous pages are still mapped  in sanity_check_pinned_pages()
51 	for (; npages; npages--, pages++) {  in sanity_check_pinned_pages()
52 		struct page *page = *pages;  in sanity_check_pinned_pages()
179  * Pages that were pinned via pin_user_pages*() must be released via either
181  * that such pages can be separately tracked and uniquely handled. In
249  * unpin_user_pages_dirty_lock() - release and optionally dirty gup-pinned pages
250  * @pages:  array of pages to be maybe marked dirty, and definitely released.
251  * @npages: number of pages in the @pages array.
252  * @make_dirty: whether to mark the pages dirty
257  * For each page in the @pages array, make that page (or its head page, if a
259  * listed as clean. In any case, releases all pages using unpin_user_page(),
270 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,  in unpin_user_pages_dirty_lock()  argument
278 		unpin_user_pages(pages, npages);  in unpin_user_pages_dirty_lock()
282 	sanity_check_pinned_pages(pages, npages);  in unpin_user_pages_dirty_lock()
284 		folio = gup_folio_next(pages, npages, i, &nr);  in unpin_user_pages_dirty_lock()
320  * @npages: number of consecutive pages to release.
321  * @make_dirty: whether to mark the pages dirty
323  * "gup-pinned page range" refers to a range of pages that has had one of the
327  * its head pages, if a compound page) dirty, if @make_dirty is true, and if the
355 static void unpin_user_pages_lockless(struct page **pages, unsigned long npages)  in unpin_user_pages_lockless()  argument
363 	 * fork() and some anonymous pages might now actually be shared --  in unpin_user_pages_lockless()
367 		folio = gup_folio_next(pages, npages, i, &nr);  in unpin_user_pages_lockless()
373  * unpin_user_pages() - release an array of gup-pinned pages.
374  * @pages:  array of pages to be marked dirty and released.
375  * @npages: number of pages in the @pages array.
377  * For each page in the @pages array, release the page using unpin_user_page().
381 void unpin_user_pages(struct page **pages, unsigned long npages)  in unpin_user_pages()  argument
388 	 * If this WARN_ON() fires, then the system *might* be leaking pages (by  in unpin_user_pages()
395 	sanity_check_pinned_pages(pages, npages);  in unpin_user_pages()
397 		folio = gup_folio_next(pages, npages, i, &nr);  in unpin_user_pages()
420 	 * has touched so far, we don't want to allocate unnecessary pages or  in no_page_table()
518 	 * We only care about anon pages in can_follow_write_pte() and don't  in follow_page_pte()
529 		 * Only return device mapping pages in the FOLL_GET or FOLL_PIN  in follow_page_pte()
540 			/* Avoid special (like zero) pages in core dumps */  in follow_page_pte()
707  * When getting pages from ZONE_DEVICE memory, the @ctx->pgmap caches
761 	 * to fail on PROT_NONE-mapped pages.  in follow_page()
781 	/* user gate pages are read-only */  in get_gate_page()
978 			 * Anon pages in shared mappings are surprising: now  in check_vma_flags()
1043  * __get_user_pages() - pin user pages in memory
1046  * @nr_pages:	number of pages from start to pin
1048  * @pages:	array that receives pointers to the pages pinned.
1050  *		only intends to ensure the pages are faulted in.
1053  * Returns either number of pages pinned (which may be less than the
1057  * -- If nr_pages is >0, but no pages were pinned, returns -errno.
1058  * -- If nr_pages is >0, and some pages were pinned, returns the number of
1059  *    pages pinned. Again, this may be less than nr_pages.
1062  * The caller is responsible for releasing returned @pages, via put_page().
1098 		unsigned int gup_flags, struct page **pages,  in __get_user_pages()  argument
1110 	VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN)));  in __get_user_pages()
1139 						pages ? &page : NULL);  in __get_user_pages()
1156 		 * If we have a pending SIGKILL, don't keep faulting pages and  in __get_user_pages()
1185 			 * struct page. If the caller expects **pages to be  in __get_user_pages()
1189 			if (pages) {  in __get_user_pages()
1202 		if (pages) {  in __get_user_pages()
1214 			 * pages.  in __get_user_pages()
1238 				pages[i + j] = subpage;  in __get_user_pages()
1394 						struct page **pages,  in __get_user_pages_locked()  argument
1419 	 * is to set FOLL_GET if the caller wants pages[] filled in (but has  in __get_user_pages_locked()
1423 	 * FOLL_PIN always expects pages to be non-null, but no need to assert  in __get_user_pages_locked()
1426 	if (pages && !(flags & FOLL_PIN))  in __get_user_pages_locked()
1431 		ret = __get_user_pages(mm, start, nr_pages, flags, pages,  in __get_user_pages_locked()
1462 		 * For the prefault case (!pages) we only update counts.  in __get_user_pages_locked()
1464 		if (likely(pages))  in __get_user_pages_locked()
1465 			pages += ret;  in __get_user_pages_locked()
1496 				       pages, locked);  in __get_user_pages_locked()
1512 		if (likely(pages))  in __get_user_pages_locked()
1513 			pages++;  in __get_user_pages_locked()
1529  * populate_vma_page_range() -  populate a range of pages in the vma.
1535  * This takes care of mlocking the pages too if VM_LOCKED is set.
1537  * Return either number of pages pinned in the vma, or a negative error
1603  * This takes care of mlocking the pages, too, if VM_LOCKED is set.
1611  * Returns either number of processed pages in the MM, or a negative error
1652  * __mm_populate - populate and/or mlock pages within a range of address space.
1670 		 * We want to fault in pages for [nstart; end) address range.  in __mm_populate()
1692 		 * Now fault in a range of pages. populate_vma_page_range()  in __mm_populate()
1693 		 * double checks the vma flags, so that it won't mlock pages  in __mm_populate()
1713 		unsigned long nr_pages, struct page **pages,  in __get_user_pages_locked()  argument
1753 		if (pages) {  in __get_user_pages_locked()
1754 			pages[i] = virt_to_page((void *)start);  in __get_user_pages_locked()
1755 			if (pages[i])  in __get_user_pages_locked()
1756 				get_page(pages[i]);  in __get_user_pages_locked()
1842  * already know that some or all of the pages in the address range aren't in
1847  * Note that we don't pin or otherwise hold the pages referenced that we fault
1947  * Returns the number of collected pages. Return value is always >= 0.
1952 					struct page **pages)  in collect_longterm_unpinnable_pages()  argument
1959 		struct folio *folio = page_folio(pages[i]);  in collect_longterm_unpinnable_pages()
1992  * Unpins all pages and migrates device coherent pages and movable_page_list.
1993  * Returns -EAGAIN if all pages were successfully migrated or -errno for failure
1999 					struct page **pages)  in migrate_longterm_unpinnable_pages()  argument
2005 		struct folio *folio = page_folio(pages[i]);  in migrate_longterm_unpinnable_pages()
2012 			pages[i] = NULL;  in migrate_longterm_unpinnable_pages()
2025 		 * We can't migrate pages with unexpected references, so drop  in migrate_longterm_unpinnable_pages()
2027 		 * Migrating pages have been added to movable_page_list after  in migrate_longterm_unpinnable_pages()
2031 		unpin_user_page(pages[i]);  in migrate_longterm_unpinnable_pages()
2032 		pages[i] = NULL;  in migrate_longterm_unpinnable_pages()
2055 		if (pages[i])  in migrate_longterm_unpinnable_pages()
2056 			unpin_user_page(pages[i]);  in migrate_longterm_unpinnable_pages()
2063  * Check whether all pages are *allowed* to be pinned. Rather confusingly, all
2064  * pages in the range are required to be pinned via FOLL_PIN, before calling
2067  * If any pages in the range are not allowed to be pinned, then this routine
2068  * will migrate those pages away, unpin all the pages in the range and return
2075  * If everything is OK and all pages in the range are allowed to be pinned, then
2076  * this routine leaves all pages pinned and returns zero for success.
2079 					    struct page **pages)  in check_and_migrate_movable_pages()  argument
2083 	collect_longterm_unpinnable_pages(&movable_page_list, nr_pages, pages);  in check_and_migrate_movable_pages()
2088 						pages);  in check_and_migrate_movable_pages()
2092 					    struct page **pages)  in check_and_migrate_movable_pages()  argument
2105 				  struct page **pages,  in __gup_longterm_locked()  argument
2113 		return __get_user_pages_locked(mm, start, nr_pages, pages,  in __gup_longterm_locked()
2119 							  pages, locked,  in __gup_longterm_locked()
2127 		rc = check_and_migrate_movable_pages(nr_pinned_pages, pages);  in __gup_longterm_locked()
2137 static bool is_valid_gup_args(struct page **pages, int *locked,  in is_valid_gup_args()  argument
2170 	/* Pages input must be given if using GET/PIN */  in is_valid_gup_args()
2171 	if (WARN_ON_ONCE((gup_flags & (FOLL_GET | FOLL_PIN)) && !pages))  in is_valid_gup_args()
2185  * get_user_pages_remote() - pin user pages in memory
2188  * @nr_pages:	number of pages from start to pin
2190  * @pages:	array that receives pointers to the pages pinned.
2192  *		only intends to ensure the pages are faulted in.
2197  * Returns either number of pages pinned (which may be less than the
2201  * -- If nr_pages is >0, but no pages were pinned, returns -errno.
2202  * -- If nr_pages is >0, and some pages were pinned, returns the number of
2203  *    pages pinned. Again, this may be less than nr_pages.
2205  * The caller is responsible for releasing returned @pages, via put_page().
2229  * via the user virtual addresses. The pages may be submitted for
2242 		unsigned int gup_flags, struct page **pages,  in get_user_pages_remote()  argument
2247 	if (!is_valid_gup_args(pages, locked, &gup_flags,  in get_user_pages_remote()
2251 	return __get_user_pages_locked(mm, start, nr_pages, pages,  in get_user_pages_remote()
2260 			   unsigned int gup_flags, struct page **pages,  in get_user_pages_remote()  argument
2268  * get_user_pages() - pin user pages in memory
2270  * @nr_pages:   number of pages from start to pin
2272  * @pages:      array that receives pointers to the pages pinned.
2274  *              only intends to ensure the pages are faulted in.
2282 		    unsigned int gup_flags, struct page **pages)  in get_user_pages()  argument
2286 	if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_TOUCH))  in get_user_pages()
2289 	return __get_user_pages_locked(current->mm, start, nr_pages, pages,  in get_user_pages()
2298  *      get_user_pages(mm, ..., pages, NULL);
2303  *      get_user_pages_unlocked(mm, ..., pages);
2310 			     struct page **pages, unsigned int gup_flags)  in get_user_pages_unlocked()  argument
2314 	if (!is_valid_gup_args(pages, NULL, &gup_flags,  in get_user_pages_unlocked()
2318 	return __get_user_pages_locked(current->mm, start, nr_pages, pages,  in get_user_pages_unlocked()
2326  * get_user_pages_fast attempts to pin user pages by walking the page
2328  * protected from page table pages being freed from under it, and should
2333  * pages are freed. This is unsuitable for architectures that do not need
2336  * Another way to achieve this is to batch up page table containing pages
2338  * pages. Disabling interrupts will allow the fast_gup walker to both block
2346  *  free pages containing page tables or TLB flushing requires IPI broadcast.
2436 					    struct page **pages)  in undo_dev_pagemap()  argument
2439 		struct page *page = pages[--(*nr)];  in undo_dev_pagemap()
2567 			 struct page **pages, int *nr)  in gup_pte_range()  argument
2582 		 * Always fallback to ordinary GUP on PROT_NONE-mapped pages:  in gup_pte_range()
2583 		 * pte_access_permitted() better should reject these pages  in gup_pte_range()
2600 				undo_dev_pagemap(nr, nr_start, flags, pages);  in gup_pte_range()
2648 		pages[*nr] = page;  in gup_pte_range()
2668  * get_user_pages_fast_only implementation that can pin pages. Thus it's still
2673 			 struct page **pages, int *nr)  in gup_pte_range()  argument
2682 			     struct page **pages, int *nr)  in __gup_device_huge()  argument
2692 			undo_dev_pagemap(nr, nr_start, flags, pages);  in __gup_device_huge()
2697 			undo_dev_pagemap(nr, nr_start, flags, pages);  in __gup_device_huge()
2702 		pages[*nr] = page;  in __gup_device_huge()
2704 			undo_dev_pagemap(nr, nr_start, flags, pages);  in __gup_device_huge()
2717 				 struct page **pages, int *nr)  in __gup_device_huge_pmd()  argument
2723 	if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))  in __gup_device_huge_pmd()
2727 		undo_dev_pagemap(nr, nr_start, flags, pages);  in __gup_device_huge_pmd()
2735 				 struct page **pages, int *nr)  in __gup_device_huge_pud()  argument
2741 	if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))  in __gup_device_huge_pud()
2745 		undo_dev_pagemap(nr, nr_start, flags, pages);  in __gup_device_huge_pud()
2753 				 struct page **pages, int *nr)  in __gup_device_huge_pmd()  argument
2761 				 struct page **pages, int *nr)  in __gup_device_huge_pud()  argument
2769 			   unsigned long end, struct page **pages)  in record_subpages()  argument
2774 		pages[nr] = nth_page(page, nr);  in record_subpages()
2789 		       struct page **pages, int *nr)  in gup_hugepte()  argument
2810 	refs = record_subpages(page, addr, end, pages + *nr);  in gup_hugepte()
2838 		struct page **pages, int *nr)  in gup_huge_pd()  argument
2847 		if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr))  in gup_huge_pd()
2856 		struct page **pages, int *nr)  in gup_huge_pd()  argument
2864 			struct page **pages, int *nr)  in gup_huge_pmd()  argument
2877 					     pages, nr);  in gup_huge_pmd()
2881 	refs = record_subpages(page, addr, end, pages + *nr);  in gup_huge_pmd()
2908 			struct page **pages, int *nr)  in gup_huge_pud()  argument
2921 					     pages, nr);  in gup_huge_pud()
2925 	refs = record_subpages(page, addr, end, pages + *nr);  in gup_huge_pud()
2953 			struct page **pages, int *nr)  in gup_huge_pgd()  argument
2965 	refs = record_subpages(page, addr, end, pages + *nr);  in gup_huge_pgd()
2992 		unsigned int flags, struct page **pages, int *nr)  in gup_pmd_range()  argument
3012 				pages, nr))  in gup_pmd_range()
3021 					 PMD_SHIFT, next, flags, pages, nr))  in gup_pmd_range()
3023 		} else if (!gup_pte_range(pmd, pmdp, addr, next, flags, pages, nr))  in gup_pmd_range()
3031 			 unsigned int flags, struct page **pages, int *nr)  in gup_pud_range()  argument
3045 					  pages, nr))  in gup_pud_range()
3049 					 PUD_SHIFT, next, flags, pages, nr))  in gup_pud_range()
3051 		} else if (!gup_pmd_range(pudp, pud, addr, next, flags, pages, nr))  in gup_pud_range()
3059 			 unsigned int flags, struct page **pages, int *nr)  in gup_p4d_range()  argument
3074 					 P4D_SHIFT, next, flags, pages, nr))  in gup_p4d_range()
3076 		} else if (!gup_pud_range(p4dp, p4d, addr, next, flags, pages, nr))  in gup_p4d_range()
3084 		unsigned int flags, struct page **pages, int *nr)  in gup_pgd_range()  argument
3098 					  pages, nr))  in gup_pgd_range()
3102 					 PGDIR_SHIFT, next, flags, pages, nr))  in gup_pgd_range()
3104 		} else if (!gup_p4d_range(pgdp, pgd, addr, next, flags, pages, nr))  in gup_pgd_range()
3110 		unsigned int flags, struct page **pages, int *nr)  in gup_pgd_range()  argument
3129 					    struct page **pages)  in lockless_pages_from_mm()  argument
3149 	 * With interrupts disabled, we block page table pages from being freed  in lockless_pages_from_mm()
3157 	gup_pgd_range(start, end, gup_flags, pages, &nr_pinned);  in lockless_pages_from_mm()
3161 	 * When pinning pages for DMA there could be a concurrent write protect  in lockless_pages_from_mm()
3166 			unpin_user_pages_lockless(pages, nr_pinned);  in lockless_pages_from_mm()
3169 			sanity_check_pinned_pages(pages, nr_pinned);  in lockless_pages_from_mm()
3178 					struct page **pages)  in internal_get_user_pages_fast()  argument
3206 	nr_pinned = lockless_pages_from_mm(start, end, gup_flags, pages);  in internal_get_user_pages_fast()
3210 	/* Slow path: try to get the remaining pages with get_user_pages */  in internal_get_user_pages_fast()
3212 	pages += nr_pinned;  in internal_get_user_pages_fast()
3214 				    pages, &locked,  in internal_get_user_pages_fast()
3218 		 * The caller has to unpin the pages we already pinned so  in internal_get_user_pages_fast()
3229  * get_user_pages_fast_only() - pin user pages in memory
3231  * @nr_pages:   number of pages from start to pin
3233  * @pages:      array that receives pointers to the pages pinned.
3240  * pages pinned.
3247 			     unsigned int gup_flags, struct page **pages)  in get_user_pages_fast_only()  argument
3256 	if (!is_valid_gup_args(pages, NULL, &gup_flags,  in get_user_pages_fast_only()
3260 	return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);  in get_user_pages_fast_only()
3265  * get_user_pages_fast() - pin user pages in memory
3267  * @nr_pages:   number of pages from start to pin
3269  * @pages:      array that receives pointers to the pages pinned.
3272  * Attempt to pin user pages in memory without taking mm->mmap_lock.
3276  * Returns number of pages pinned. This may be fewer than the number requested.
3277  * If nr_pages is 0 or negative, returns 0. If no pages were pinned, returns
3281 			unsigned int gup_flags, struct page **pages)  in get_user_pages_fast()  argument
3289 	if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_GET))  in get_user_pages_fast()
3291 	return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);  in get_user_pages_fast()
3296  * pin_user_pages_fast() - pin user pages in memory without taking locks
3299  * @nr_pages:   number of pages from start to pin
3301  * @pages:      array that receives pointers to the pages pinned.
3308  * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
3311  * Note that if a zero_page is amongst the returned pages, it will not have
3315 			unsigned int gup_flags, struct page **pages)  in pin_user_pages_fast()  argument
3317 	if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_PIN))  in pin_user_pages_fast()
3319 	return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);  in pin_user_pages_fast()
3324  * pin_user_pages_remote() - pin pages of a remote process
3328  * @nr_pages:	number of pages from start to pin
3330  * @pages:	array that receives pointers to the pages pinned.
3340  * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
3343  * Note that if a zero_page is amongst the returned pages, it will not have
3348 			   unsigned int gup_flags, struct page **pages,  in pin_user_pages_remote()  argument
3353 	if (!is_valid_gup_args(pages, locked, &gup_flags,  in pin_user_pages_remote()
3356 	return __gup_longterm_locked(mm, start, nr_pages, pages,  in pin_user_pages_remote()
3363  * pin_user_pages() - pin user pages in memory for use by other devices
3366  * @nr_pages:	number of pages from start to pin
3368  * @pages:	array that receives pointers to the pages pinned.
3374  * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
3377  * Note that if a zero_page is amongst the returned pages, it will not have
3381 		    unsigned int gup_flags, struct page **pages)  in pin_user_pages()  argument
3385 	if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_PIN))  in pin_user_pages()
3388 				     pages, &locked, gup_flags);  in pin_user_pages()
3397  * Note that if a zero_page is amongst the returned pages, it will not have
3401 			     struct page **pages, unsigned int gup_flags)  in pin_user_pages_unlocked()  argument
3405 	if (!is_valid_gup_args(pages, NULL, &gup_flags,  in pin_user_pages_unlocked()
3409 	return __gup_longterm_locked(current->mm, start, nr_pages, pages,  in pin_user_pages_unlocked()