1 #include <linux/kernel.h> 2 #include <linux/errno.h> 3 #include <linux/err.h> 4 #include <linux/spinlock.h> 5 6 #include <linux/mm.h> 7 #include <linux/memremap.h> 8 #include <linux/pagemap.h> 9 #include <linux/rmap.h> 10 #include <linux/swap.h> 11 #include <linux/swapops.h> 12 13 #include <linux/sched.h> 14 #include <linux/rwsem.h> 15 #include <linux/hugetlb.h> 16 17 #include <asm/mmu_context.h> 18 #include <asm/pgtable.h> 19 #include <asm/tlbflush.h> 20 21 #include "internal.h" 22 23 static struct page *no_page_table(struct vm_area_struct *vma, 24 unsigned int flags) 25 { 26 /* 27 * When core dumping an enormous anonymous area that nobody 28 * has touched so far, we don't want to allocate unnecessary pages or 29 * page tables. Return error instead of NULL to skip handle_mm_fault, 30 * then get_dump_page() will return NULL to leave a hole in the dump. 31 * But we can only make this optimization where a hole would surely 32 * be zero-filled if handle_mm_fault() actually did handle it. 33 */ 34 if ((flags & FOLL_DUMP) && (!vma->vm_ops || !vma->vm_ops->fault)) 35 return ERR_PTR(-EFAULT); 36 return NULL; 37 } 38 39 static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address, 40 pte_t *pte, unsigned int flags) 41 { 42 /* No page to get reference */ 43 if (flags & FOLL_GET) 44 return -EFAULT; 45 46 if (flags & FOLL_TOUCH) { 47 pte_t entry = *pte; 48 49 if (flags & FOLL_WRITE) 50 entry = pte_mkdirty(entry); 51 entry = pte_mkyoung(entry); 52 53 if (!pte_same(*pte, entry)) { 54 set_pte_at(vma->vm_mm, address, pte, entry); 55 update_mmu_cache(vma, address, pte); 56 } 57 } 58 59 /* Proper page table entry exists, but no corresponding struct page */ 60 return -EEXIST; 61 } 62 63 /* 64 * FOLL_FORCE can write to even unwritable pte's, but only 65 * after we've gone through a COW cycle and they are dirty. 66 */ 67 static inline bool can_follow_write_pte(pte_t pte, unsigned int flags) 68 { 69 return pte_write(pte) || 70 ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pte_dirty(pte)); 71 } 72 73 static struct page *follow_page_pte(struct vm_area_struct *vma, 74 unsigned long address, pmd_t *pmd, unsigned int flags) 75 { 76 struct mm_struct *mm = vma->vm_mm; 77 struct dev_pagemap *pgmap = NULL; 78 struct page *page; 79 spinlock_t *ptl; 80 pte_t *ptep, pte; 81 82 retry: 83 if (unlikely(pmd_bad(*pmd))) 84 return no_page_table(vma, flags); 85 86 ptep = pte_offset_map_lock(mm, pmd, address, &ptl); 87 pte = *ptep; 88 if (!pte_present(pte)) { 89 swp_entry_t entry; 90 /* 91 * KSM's break_ksm() relies upon recognizing a ksm page 92 * even while it is being migrated, so for that case we 93 * need migration_entry_wait(). 94 */ 95 if (likely(!(flags & FOLL_MIGRATION))) 96 goto no_page; 97 if (pte_none(pte)) 98 goto no_page; 99 entry = pte_to_swp_entry(pte); 100 if (!is_migration_entry(entry)) 101 goto no_page; 102 pte_unmap_unlock(ptep, ptl); 103 migration_entry_wait(mm, pmd, address); 104 goto retry; 105 } 106 if ((flags & FOLL_NUMA) && pte_protnone(pte)) 107 goto no_page; 108 if ((flags & FOLL_WRITE) && !can_follow_write_pte(pte, flags)) { 109 pte_unmap_unlock(ptep, ptl); 110 return NULL; 111 } 112 113 page = vm_normal_page(vma, address, pte); 114 if (!page && pte_devmap(pte) && (flags & FOLL_GET)) { 115 /* 116 * Only return device mapping pages in the FOLL_GET case since 117 * they are only valid while holding the pgmap reference. 118 */ 119 pgmap = get_dev_pagemap(pte_pfn(pte), NULL); 120 if (pgmap) 121 page = pte_page(pte); 122 else 123 goto no_page; 124 } else if (unlikely(!page)) { 125 if (flags & FOLL_DUMP) { 126 /* Avoid special (like zero) pages in core dumps */ 127 page = ERR_PTR(-EFAULT); 128 goto out; 129 } 130 131 if (is_zero_pfn(pte_pfn(pte))) { 132 page = pte_page(pte); 133 } else { 134 int ret; 135 136 ret = follow_pfn_pte(vma, address, ptep, flags); 137 page = ERR_PTR(ret); 138 goto out; 139 } 140 } 141 142 if (flags & FOLL_SPLIT && PageTransCompound(page)) { 143 int ret; 144 get_page(page); 145 pte_unmap_unlock(ptep, ptl); 146 lock_page(page); 147 ret = split_huge_page(page); 148 unlock_page(page); 149 put_page(page); 150 if (ret) 151 return ERR_PTR(ret); 152 goto retry; 153 } 154 155 if (flags & FOLL_GET) { 156 get_page(page); 157 158 /* drop the pgmap reference now that we hold the page */ 159 if (pgmap) { 160 put_dev_pagemap(pgmap); 161 pgmap = NULL; 162 } 163 } 164 if (flags & FOLL_TOUCH) { 165 if ((flags & FOLL_WRITE) && 166 !pte_dirty(pte) && !PageDirty(page)) 167 set_page_dirty(page); 168 /* 169 * pte_mkyoung() would be more correct here, but atomic care 170 * is needed to avoid losing the dirty bit: it is easier to use 171 * mark_page_accessed(). 172 */ 173 mark_page_accessed(page); 174 } 175 if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) { 176 /* Do not mlock pte-mapped THP */ 177 if (PageTransCompound(page)) 178 goto out; 179 180 /* 181 * The preliminary mapping check is mainly to avoid the 182 * pointless overhead of lock_page on the ZERO_PAGE 183 * which might bounce very badly if there is contention. 184 * 185 * If the page is already locked, we don't need to 186 * handle it now - vmscan will handle it later if and 187 * when it attempts to reclaim the page. 188 */ 189 if (page->mapping && trylock_page(page)) { 190 lru_add_drain(); /* push cached pages to LRU */ 191 /* 192 * Because we lock page here, and migration is 193 * blocked by the pte's page reference, and we 194 * know the page is still mapped, we don't even 195 * need to check for file-cache page truncation. 196 */ 197 mlock_vma_page(page); 198 unlock_page(page); 199 } 200 } 201 out: 202 pte_unmap_unlock(ptep, ptl); 203 return page; 204 no_page: 205 pte_unmap_unlock(ptep, ptl); 206 if (!pte_none(pte)) 207 return NULL; 208 return no_page_table(vma, flags); 209 } 210 211 /** 212 * follow_page_mask - look up a page descriptor from a user-virtual address 213 * @vma: vm_area_struct mapping @address 214 * @address: virtual address to look up 215 * @flags: flags modifying lookup behaviour 216 * @page_mask: on output, *page_mask is set according to the size of the page 217 * 218 * @flags can have FOLL_ flags set, defined in <linux/mm.h> 219 * 220 * Returns the mapped (struct page *), %NULL if no mapping exists, or 221 * an error pointer if there is a mapping to something not represented 222 * by a page descriptor (see also vm_normal_page()). 223 */ 224 struct page *follow_page_mask(struct vm_area_struct *vma, 225 unsigned long address, unsigned int flags, 226 unsigned int *page_mask) 227 { 228 pgd_t *pgd; 229 pud_t *pud; 230 pmd_t *pmd; 231 spinlock_t *ptl; 232 struct page *page; 233 struct mm_struct *mm = vma->vm_mm; 234 235 *page_mask = 0; 236 237 page = follow_huge_addr(mm, address, flags & FOLL_WRITE); 238 if (!IS_ERR(page)) { 239 BUG_ON(flags & FOLL_GET); 240 return page; 241 } 242 243 pgd = pgd_offset(mm, address); 244 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) 245 return no_page_table(vma, flags); 246 247 pud = pud_offset(pgd, address); 248 if (pud_none(*pud)) 249 return no_page_table(vma, flags); 250 if (pud_huge(*pud) && vma->vm_flags & VM_HUGETLB) { 251 page = follow_huge_pud(mm, address, pud, flags); 252 if (page) 253 return page; 254 return no_page_table(vma, flags); 255 } 256 if (unlikely(pud_bad(*pud))) 257 return no_page_table(vma, flags); 258 259 pmd = pmd_offset(pud, address); 260 if (pmd_none(*pmd)) 261 return no_page_table(vma, flags); 262 if (pmd_huge(*pmd) && vma->vm_flags & VM_HUGETLB) { 263 page = follow_huge_pmd(mm, address, pmd, flags); 264 if (page) 265 return page; 266 return no_page_table(vma, flags); 267 } 268 if ((flags & FOLL_NUMA) && pmd_protnone(*pmd)) 269 return no_page_table(vma, flags); 270 if (pmd_devmap(*pmd)) { 271 ptl = pmd_lock(mm, pmd); 272 page = follow_devmap_pmd(vma, address, pmd, flags); 273 spin_unlock(ptl); 274 if (page) 275 return page; 276 } 277 if (likely(!pmd_trans_huge(*pmd))) 278 return follow_page_pte(vma, address, pmd, flags); 279 280 ptl = pmd_lock(mm, pmd); 281 if (unlikely(!pmd_trans_huge(*pmd))) { 282 spin_unlock(ptl); 283 return follow_page_pte(vma, address, pmd, flags); 284 } 285 if (flags & FOLL_SPLIT) { 286 int ret; 287 page = pmd_page(*pmd); 288 if (is_huge_zero_page(page)) { 289 spin_unlock(ptl); 290 ret = 0; 291 split_huge_pmd(vma, pmd, address); 292 if (pmd_trans_unstable(pmd)) 293 ret = -EBUSY; 294 } else { 295 get_page(page); 296 spin_unlock(ptl); 297 lock_page(page); 298 ret = split_huge_page(page); 299 unlock_page(page); 300 put_page(page); 301 if (pmd_none(*pmd)) 302 return no_page_table(vma, flags); 303 } 304 305 return ret ? ERR_PTR(ret) : 306 follow_page_pte(vma, address, pmd, flags); 307 } 308 309 page = follow_trans_huge_pmd(vma, address, pmd, flags); 310 spin_unlock(ptl); 311 *page_mask = HPAGE_PMD_NR - 1; 312 return page; 313 } 314 315 static int get_gate_page(struct mm_struct *mm, unsigned long address, 316 unsigned int gup_flags, struct vm_area_struct **vma, 317 struct page **page) 318 { 319 pgd_t *pgd; 320 pud_t *pud; 321 pmd_t *pmd; 322 pte_t *pte; 323 int ret = -EFAULT; 324 325 /* user gate pages are read-only */ 326 if (gup_flags & FOLL_WRITE) 327 return -EFAULT; 328 if (address > TASK_SIZE) 329 pgd = pgd_offset_k(address); 330 else 331 pgd = pgd_offset_gate(mm, address); 332 BUG_ON(pgd_none(*pgd)); 333 pud = pud_offset(pgd, address); 334 BUG_ON(pud_none(*pud)); 335 pmd = pmd_offset(pud, address); 336 if (pmd_none(*pmd)) 337 return -EFAULT; 338 VM_BUG_ON(pmd_trans_huge(*pmd)); 339 pte = pte_offset_map(pmd, address); 340 if (pte_none(*pte)) 341 goto unmap; 342 *vma = get_gate_vma(mm); 343 if (!page) 344 goto out; 345 *page = vm_normal_page(*vma, address, *pte); 346 if (!*page) { 347 if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(*pte))) 348 goto unmap; 349 *page = pte_page(*pte); 350 } 351 get_page(*page); 352 out: 353 ret = 0; 354 unmap: 355 pte_unmap(pte); 356 return ret; 357 } 358 359 /* 360 * mmap_sem must be held on entry. If @nonblocking != NULL and 361 * *@flags does not include FOLL_NOWAIT, the mmap_sem may be released. 362 * If it is, *@nonblocking will be set to 0 and -EBUSY returned. 363 */ 364 static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma, 365 unsigned long address, unsigned int *flags, int *nonblocking) 366 { 367 unsigned int fault_flags = 0; 368 int ret; 369 370 /* mlock all present pages, but do not fault in new pages */ 371 if ((*flags & (FOLL_POPULATE | FOLL_MLOCK)) == FOLL_MLOCK) 372 return -ENOENT; 373 /* For mm_populate(), just skip the stack guard page. */ 374 if ((*flags & FOLL_POPULATE) && 375 (stack_guard_page_start(vma, address) || 376 stack_guard_page_end(vma, address + PAGE_SIZE))) 377 return -ENOENT; 378 if (*flags & FOLL_WRITE) 379 fault_flags |= FAULT_FLAG_WRITE; 380 if (*flags & FOLL_REMOTE) 381 fault_flags |= FAULT_FLAG_REMOTE; 382 if (nonblocking) 383 fault_flags |= FAULT_FLAG_ALLOW_RETRY; 384 if (*flags & FOLL_NOWAIT) 385 fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT; 386 if (*flags & FOLL_TRIED) { 387 VM_WARN_ON_ONCE(fault_flags & FAULT_FLAG_ALLOW_RETRY); 388 fault_flags |= FAULT_FLAG_TRIED; 389 } 390 391 ret = handle_mm_fault(vma, address, fault_flags); 392 if (ret & VM_FAULT_ERROR) { 393 if (ret & VM_FAULT_OOM) 394 return -ENOMEM; 395 if (ret & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE)) 396 return *flags & FOLL_HWPOISON ? -EHWPOISON : -EFAULT; 397 if (ret & (VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV)) 398 return -EFAULT; 399 BUG(); 400 } 401 402 if (tsk) { 403 if (ret & VM_FAULT_MAJOR) 404 tsk->maj_flt++; 405 else 406 tsk->min_flt++; 407 } 408 409 if (ret & VM_FAULT_RETRY) { 410 if (nonblocking) 411 *nonblocking = 0; 412 return -EBUSY; 413 } 414 415 /* 416 * The VM_FAULT_WRITE bit tells us that do_wp_page has broken COW when 417 * necessary, even if maybe_mkwrite decided not to set pte_write. We 418 * can thus safely do subsequent page lookups as if they were reads. 419 * But only do so when looping for pte_write is futile: in some cases 420 * userspace may also be wanting to write to the gotten user page, 421 * which a read fault here might prevent (a readonly page might get 422 * reCOWed by userspace write). 423 */ 424 if ((ret & VM_FAULT_WRITE) && !(vma->vm_flags & VM_WRITE)) 425 *flags |= FOLL_COW; 426 return 0; 427 } 428 429 static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) 430 { 431 vm_flags_t vm_flags = vma->vm_flags; 432 int write = (gup_flags & FOLL_WRITE); 433 int foreign = (gup_flags & FOLL_REMOTE); 434 435 if (vm_flags & (VM_IO | VM_PFNMAP)) 436 return -EFAULT; 437 438 if (write) { 439 if (!(vm_flags & VM_WRITE)) { 440 if (!(gup_flags & FOLL_FORCE)) 441 return -EFAULT; 442 /* 443 * We used to let the write,force case do COW in a 444 * VM_MAYWRITE VM_SHARED !VM_WRITE vma, so ptrace could 445 * set a breakpoint in a read-only mapping of an 446 * executable, without corrupting the file (yet only 447 * when that file had been opened for writing!). 448 * Anon pages in shared mappings are surprising: now 449 * just reject it. 450 */ 451 if (!is_cow_mapping(vm_flags)) 452 return -EFAULT; 453 } 454 } else if (!(vm_flags & VM_READ)) { 455 if (!(gup_flags & FOLL_FORCE)) 456 return -EFAULT; 457 /* 458 * Is there actually any vma we can reach here which does not 459 * have VM_MAYREAD set? 460 */ 461 if (!(vm_flags & VM_MAYREAD)) 462 return -EFAULT; 463 } 464 /* 465 * gups are always data accesses, not instruction 466 * fetches, so execute=false here 467 */ 468 if (!arch_vma_access_permitted(vma, write, false, foreign)) 469 return -EFAULT; 470 return 0; 471 } 472 473 /** 474 * __get_user_pages() - pin user pages in memory 475 * @tsk: task_struct of target task 476 * @mm: mm_struct of target mm 477 * @start: starting user address 478 * @nr_pages: number of pages from start to pin 479 * @gup_flags: flags modifying pin behaviour 480 * @pages: array that receives pointers to the pages pinned. 481 * Should be at least nr_pages long. Or NULL, if caller 482 * only intends to ensure the pages are faulted in. 483 * @vmas: array of pointers to vmas corresponding to each page. 484 * Or NULL if the caller does not require them. 485 * @nonblocking: whether waiting for disk IO or mmap_sem contention 486 * 487 * Returns number of pages pinned. This may be fewer than the number 488 * requested. If nr_pages is 0 or negative, returns 0. If no pages 489 * were pinned, returns -errno. Each page returned must be released 490 * with a put_page() call when it is finished with. vmas will only 491 * remain valid while mmap_sem is held. 492 * 493 * Must be called with mmap_sem held. It may be released. See below. 494 * 495 * __get_user_pages walks a process's page tables and takes a reference to 496 * each struct page that each user address corresponds to at a given 497 * instant. That is, it takes the page that would be accessed if a user 498 * thread accesses the given user virtual address at that instant. 499 * 500 * This does not guarantee that the page exists in the user mappings when 501 * __get_user_pages returns, and there may even be a completely different 502 * page there in some cases (eg. if mmapped pagecache has been invalidated 503 * and subsequently re faulted). However it does guarantee that the page 504 * won't be freed completely. And mostly callers simply care that the page 505 * contains data that was valid *at some point in time*. Typically, an IO 506 * or similar operation cannot guarantee anything stronger anyway because 507 * locks can't be held over the syscall boundary. 508 * 509 * If @gup_flags & FOLL_WRITE == 0, the page must not be written to. If 510 * the page is written to, set_page_dirty (or set_page_dirty_lock, as 511 * appropriate) must be called after the page is finished with, and 512 * before put_page is called. 513 * 514 * If @nonblocking != NULL, __get_user_pages will not wait for disk IO 515 * or mmap_sem contention, and if waiting is needed to pin all pages, 516 * *@nonblocking will be set to 0. Further, if @gup_flags does not 517 * include FOLL_NOWAIT, the mmap_sem will be released via up_read() in 518 * this case. 519 * 520 * A caller using such a combination of @nonblocking and @gup_flags 521 * must therefore hold the mmap_sem for reading only, and recognize 522 * when it's been released. Otherwise, it must be held for either 523 * reading or writing and will not be released. 524 * 525 * In most cases, get_user_pages or get_user_pages_fast should be used 526 * instead of __get_user_pages. __get_user_pages should be used only if 527 * you need some special @gup_flags. 528 */ 529 static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 530 unsigned long start, unsigned long nr_pages, 531 unsigned int gup_flags, struct page **pages, 532 struct vm_area_struct **vmas, int *nonblocking) 533 { 534 long i = 0; 535 unsigned int page_mask; 536 struct vm_area_struct *vma = NULL; 537 538 if (!nr_pages) 539 return 0; 540 541 VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET)); 542 543 /* 544 * If FOLL_FORCE is set then do not force a full fault as the hinting 545 * fault information is unrelated to the reference behaviour of a task 546 * using the address space 547 */ 548 if (!(gup_flags & FOLL_FORCE)) 549 gup_flags |= FOLL_NUMA; 550 551 do { 552 struct page *page; 553 unsigned int foll_flags = gup_flags; 554 unsigned int page_increm; 555 556 /* first iteration or cross vma bound */ 557 if (!vma || start >= vma->vm_end) { 558 vma = find_extend_vma(mm, start); 559 if (!vma && in_gate_area(mm, start)) { 560 int ret; 561 ret = get_gate_page(mm, start & PAGE_MASK, 562 gup_flags, &vma, 563 pages ? &pages[i] : NULL); 564 if (ret) 565 return i ? : ret; 566 page_mask = 0; 567 goto next_page; 568 } 569 570 if (!vma || check_vma_flags(vma, gup_flags)) 571 return i ? : -EFAULT; 572 if (is_vm_hugetlb_page(vma)) { 573 i = follow_hugetlb_page(mm, vma, pages, vmas, 574 &start, &nr_pages, i, 575 gup_flags); 576 continue; 577 } 578 } 579 retry: 580 /* 581 * If we have a pending SIGKILL, don't keep faulting pages and 582 * potentially allocating memory. 583 */ 584 if (unlikely(fatal_signal_pending(current))) 585 return i ? i : -ERESTARTSYS; 586 cond_resched(); 587 page = follow_page_mask(vma, start, foll_flags, &page_mask); 588 if (!page) { 589 int ret; 590 ret = faultin_page(tsk, vma, start, &foll_flags, 591 nonblocking); 592 switch (ret) { 593 case 0: 594 goto retry; 595 case -EFAULT: 596 case -ENOMEM: 597 case -EHWPOISON: 598 return i ? i : ret; 599 case -EBUSY: 600 return i; 601 case -ENOENT: 602 goto next_page; 603 } 604 BUG(); 605 } else if (PTR_ERR(page) == -EEXIST) { 606 /* 607 * Proper page table entry exists, but no corresponding 608 * struct page. 609 */ 610 goto next_page; 611 } else if (IS_ERR(page)) { 612 return i ? i : PTR_ERR(page); 613 } 614 if (pages) { 615 pages[i] = page; 616 flush_anon_page(vma, page, start); 617 flush_dcache_page(page); 618 page_mask = 0; 619 } 620 next_page: 621 if (vmas) { 622 vmas[i] = vma; 623 page_mask = 0; 624 } 625 page_increm = 1 + (~(start >> PAGE_SHIFT) & page_mask); 626 if (page_increm > nr_pages) 627 page_increm = nr_pages; 628 i += page_increm; 629 start += page_increm * PAGE_SIZE; 630 nr_pages -= page_increm; 631 } while (nr_pages); 632 return i; 633 } 634 635 static bool vma_permits_fault(struct vm_area_struct *vma, 636 unsigned int fault_flags) 637 { 638 bool write = !!(fault_flags & FAULT_FLAG_WRITE); 639 bool foreign = !!(fault_flags & FAULT_FLAG_REMOTE); 640 vm_flags_t vm_flags = write ? VM_WRITE : VM_READ; 641 642 if (!(vm_flags & vma->vm_flags)) 643 return false; 644 645 /* 646 * The architecture might have a hardware protection 647 * mechanism other than read/write that can deny access. 648 * 649 * gup always represents data access, not instruction 650 * fetches, so execute=false here: 651 */ 652 if (!arch_vma_access_permitted(vma, write, false, foreign)) 653 return false; 654 655 return true; 656 } 657 658 /* 659 * fixup_user_fault() - manually resolve a user page fault 660 * @tsk: the task_struct to use for page fault accounting, or 661 * NULL if faults are not to be recorded. 662 * @mm: mm_struct of target mm 663 * @address: user address 664 * @fault_flags:flags to pass down to handle_mm_fault() 665 * @unlocked: did we unlock the mmap_sem while retrying, maybe NULL if caller 666 * does not allow retry 667 * 668 * This is meant to be called in the specific scenario where for locking reasons 669 * we try to access user memory in atomic context (within a pagefault_disable() 670 * section), this returns -EFAULT, and we want to resolve the user fault before 671 * trying again. 672 * 673 * Typically this is meant to be used by the futex code. 674 * 675 * The main difference with get_user_pages() is that this function will 676 * unconditionally call handle_mm_fault() which will in turn perform all the 677 * necessary SW fixup of the dirty and young bits in the PTE, while 678 * get_user_pages() only guarantees to update these in the struct page. 679 * 680 * This is important for some architectures where those bits also gate the 681 * access permission to the page because they are maintained in software. On 682 * such architectures, gup() will not be enough to make a subsequent access 683 * succeed. 684 * 685 * This function will not return with an unlocked mmap_sem. So it has not the 686 * same semantics wrt the @mm->mmap_sem as does filemap_fault(). 687 */ 688 int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm, 689 unsigned long address, unsigned int fault_flags, 690 bool *unlocked) 691 { 692 struct vm_area_struct *vma; 693 int ret, major = 0; 694 695 if (unlocked) 696 fault_flags |= FAULT_FLAG_ALLOW_RETRY; 697 698 retry: 699 vma = find_extend_vma(mm, address); 700 if (!vma || address < vma->vm_start) 701 return -EFAULT; 702 703 if (!vma_permits_fault(vma, fault_flags)) 704 return -EFAULT; 705 706 ret = handle_mm_fault(vma, address, fault_flags); 707 major |= ret & VM_FAULT_MAJOR; 708 if (ret & VM_FAULT_ERROR) { 709 if (ret & VM_FAULT_OOM) 710 return -ENOMEM; 711 if (ret & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE)) 712 return -EHWPOISON; 713 if (ret & (VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV)) 714 return -EFAULT; 715 BUG(); 716 } 717 718 if (ret & VM_FAULT_RETRY) { 719 down_read(&mm->mmap_sem); 720 if (!(fault_flags & FAULT_FLAG_TRIED)) { 721 *unlocked = true; 722 fault_flags &= ~FAULT_FLAG_ALLOW_RETRY; 723 fault_flags |= FAULT_FLAG_TRIED; 724 goto retry; 725 } 726 } 727 728 if (tsk) { 729 if (major) 730 tsk->maj_flt++; 731 else 732 tsk->min_flt++; 733 } 734 return 0; 735 } 736 EXPORT_SYMBOL_GPL(fixup_user_fault); 737 738 static __always_inline long __get_user_pages_locked(struct task_struct *tsk, 739 struct mm_struct *mm, 740 unsigned long start, 741 unsigned long nr_pages, 742 struct page **pages, 743 struct vm_area_struct **vmas, 744 int *locked, bool notify_drop, 745 unsigned int flags) 746 { 747 long ret, pages_done; 748 bool lock_dropped; 749 750 if (locked) { 751 /* if VM_FAULT_RETRY can be returned, vmas become invalid */ 752 BUG_ON(vmas); 753 /* check caller initialized locked */ 754 BUG_ON(*locked != 1); 755 } 756 757 if (pages) 758 flags |= FOLL_GET; 759 760 pages_done = 0; 761 lock_dropped = false; 762 for (;;) { 763 ret = __get_user_pages(tsk, mm, start, nr_pages, flags, pages, 764 vmas, locked); 765 if (!locked) 766 /* VM_FAULT_RETRY couldn't trigger, bypass */ 767 return ret; 768 769 /* VM_FAULT_RETRY cannot return errors */ 770 if (!*locked) { 771 BUG_ON(ret < 0); 772 BUG_ON(ret >= nr_pages); 773 } 774 775 if (!pages) 776 /* If it's a prefault don't insist harder */ 777 return ret; 778 779 if (ret > 0) { 780 nr_pages -= ret; 781 pages_done += ret; 782 if (!nr_pages) 783 break; 784 } 785 if (*locked) { 786 /* VM_FAULT_RETRY didn't trigger */ 787 if (!pages_done) 788 pages_done = ret; 789 break; 790 } 791 /* VM_FAULT_RETRY triggered, so seek to the faulting offset */ 792 pages += ret; 793 start += ret << PAGE_SHIFT; 794 795 /* 796 * Repeat on the address that fired VM_FAULT_RETRY 797 * without FAULT_FLAG_ALLOW_RETRY but with 798 * FAULT_FLAG_TRIED. 799 */ 800 *locked = 1; 801 lock_dropped = true; 802 down_read(&mm->mmap_sem); 803 ret = __get_user_pages(tsk, mm, start, 1, flags | FOLL_TRIED, 804 pages, NULL, NULL); 805 if (ret != 1) { 806 BUG_ON(ret > 1); 807 if (!pages_done) 808 pages_done = ret; 809 break; 810 } 811 nr_pages--; 812 pages_done++; 813 if (!nr_pages) 814 break; 815 pages++; 816 start += PAGE_SIZE; 817 } 818 if (notify_drop && lock_dropped && *locked) { 819 /* 820 * We must let the caller know we temporarily dropped the lock 821 * and so the critical section protected by it was lost. 822 */ 823 up_read(&mm->mmap_sem); 824 *locked = 0; 825 } 826 return pages_done; 827 } 828 829 /* 830 * We can leverage the VM_FAULT_RETRY functionality in the page fault 831 * paths better by using either get_user_pages_locked() or 832 * get_user_pages_unlocked(). 833 * 834 * get_user_pages_locked() is suitable to replace the form: 835 * 836 * down_read(&mm->mmap_sem); 837 * do_something() 838 * get_user_pages(tsk, mm, ..., pages, NULL); 839 * up_read(&mm->mmap_sem); 840 * 841 * to: 842 * 843 * int locked = 1; 844 * down_read(&mm->mmap_sem); 845 * do_something() 846 * get_user_pages_locked(tsk, mm, ..., pages, &locked); 847 * if (locked) 848 * up_read(&mm->mmap_sem); 849 */ 850 long get_user_pages_locked(unsigned long start, unsigned long nr_pages, 851 unsigned int gup_flags, struct page **pages, 852 int *locked) 853 { 854 return __get_user_pages_locked(current, current->mm, start, nr_pages, 855 pages, NULL, locked, true, 856 gup_flags | FOLL_TOUCH); 857 } 858 EXPORT_SYMBOL(get_user_pages_locked); 859 860 /* 861 * Same as get_user_pages_unlocked(...., FOLL_TOUCH) but it allows for 862 * tsk, mm to be specified. 863 * 864 * NOTE: here FOLL_TOUCH is not set implicitly and must be set by the 865 * caller if required (just like with __get_user_pages). "FOLL_GET" 866 * is set implicitly if "pages" is non-NULL. 867 */ 868 static __always_inline long __get_user_pages_unlocked(struct task_struct *tsk, 869 struct mm_struct *mm, unsigned long start, 870 unsigned long nr_pages, struct page **pages, 871 unsigned int gup_flags) 872 { 873 long ret; 874 int locked = 1; 875 876 down_read(&mm->mmap_sem); 877 ret = __get_user_pages_locked(tsk, mm, start, nr_pages, pages, NULL, 878 &locked, false, gup_flags); 879 if (locked) 880 up_read(&mm->mmap_sem); 881 return ret; 882 } 883 884 /* 885 * get_user_pages_unlocked() is suitable to replace the form: 886 * 887 * down_read(&mm->mmap_sem); 888 * get_user_pages(tsk, mm, ..., pages, NULL); 889 * up_read(&mm->mmap_sem); 890 * 891 * with: 892 * 893 * get_user_pages_unlocked(tsk, mm, ..., pages); 894 * 895 * It is functionally equivalent to get_user_pages_fast so 896 * get_user_pages_fast should be used instead if specific gup_flags 897 * (e.g. FOLL_FORCE) are not required. 898 */ 899 long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, 900 struct page **pages, unsigned int gup_flags) 901 { 902 return __get_user_pages_unlocked(current, current->mm, start, nr_pages, 903 pages, gup_flags | FOLL_TOUCH); 904 } 905 EXPORT_SYMBOL(get_user_pages_unlocked); 906 907 /* 908 * get_user_pages_remote() - pin user pages in memory 909 * @tsk: the task_struct to use for page fault accounting, or 910 * NULL if faults are not to be recorded. 911 * @mm: mm_struct of target mm 912 * @start: starting user address 913 * @nr_pages: number of pages from start to pin 914 * @gup_flags: flags modifying lookup behaviour 915 * @pages: array that receives pointers to the pages pinned. 916 * Should be at least nr_pages long. Or NULL, if caller 917 * only intends to ensure the pages are faulted in. 918 * @vmas: array of pointers to vmas corresponding to each page. 919 * Or NULL if the caller does not require them. 920 * @locked: pointer to lock flag indicating whether lock is held and 921 * subsequently whether VM_FAULT_RETRY functionality can be 922 * utilised. Lock must initially be held. 923 * 924 * Returns number of pages pinned. This may be fewer than the number 925 * requested. If nr_pages is 0 or negative, returns 0. If no pages 926 * were pinned, returns -errno. Each page returned must be released 927 * with a put_page() call when it is finished with. vmas will only 928 * remain valid while mmap_sem is held. 929 * 930 * Must be called with mmap_sem held for read or write. 931 * 932 * get_user_pages walks a process's page tables and takes a reference to 933 * each struct page that each user address corresponds to at a given 934 * instant. That is, it takes the page that would be accessed if a user 935 * thread accesses the given user virtual address at that instant. 936 * 937 * This does not guarantee that the page exists in the user mappings when 938 * get_user_pages returns, and there may even be a completely different 939 * page there in some cases (eg. if mmapped pagecache has been invalidated 940 * and subsequently re faulted). However it does guarantee that the page 941 * won't be freed completely. And mostly callers simply care that the page 942 * contains data that was valid *at some point in time*. Typically, an IO 943 * or similar operation cannot guarantee anything stronger anyway because 944 * locks can't be held over the syscall boundary. 945 * 946 * If gup_flags & FOLL_WRITE == 0, the page must not be written to. If the page 947 * is written to, set_page_dirty (or set_page_dirty_lock, as appropriate) must 948 * be called after the page is finished with, and before put_page is called. 949 * 950 * get_user_pages is typically used for fewer-copy IO operations, to get a 951 * handle on the memory by some means other than accesses via the user virtual 952 * addresses. The pages may be submitted for DMA to devices or accessed via 953 * their kernel linear mapping (via the kmap APIs). Care should be taken to 954 * use the correct cache flushing APIs. 955 * 956 * See also get_user_pages_fast, for performance critical applications. 957 * 958 * get_user_pages should be phased out in favor of 959 * get_user_pages_locked|unlocked or get_user_pages_fast. Nothing 960 * should use get_user_pages because it cannot pass 961 * FAULT_FLAG_ALLOW_RETRY to handle_mm_fault. 962 */ 963 long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, 964 unsigned long start, unsigned long nr_pages, 965 unsigned int gup_flags, struct page **pages, 966 struct vm_area_struct **vmas, int *locked) 967 { 968 return __get_user_pages_locked(tsk, mm, start, nr_pages, pages, vmas, 969 locked, true, 970 gup_flags | FOLL_TOUCH | FOLL_REMOTE); 971 } 972 EXPORT_SYMBOL(get_user_pages_remote); 973 974 /* 975 * This is the same as get_user_pages_remote(), just with a 976 * less-flexible calling convention where we assume that the task 977 * and mm being operated on are the current task's and don't allow 978 * passing of a locked parameter. We also obviously don't pass 979 * FOLL_REMOTE in here. 980 */ 981 long get_user_pages(unsigned long start, unsigned long nr_pages, 982 unsigned int gup_flags, struct page **pages, 983 struct vm_area_struct **vmas) 984 { 985 return __get_user_pages_locked(current, current->mm, start, nr_pages, 986 pages, vmas, NULL, false, 987 gup_flags | FOLL_TOUCH); 988 } 989 EXPORT_SYMBOL(get_user_pages); 990 991 /** 992 * populate_vma_page_range() - populate a range of pages in the vma. 993 * @vma: target vma 994 * @start: start address 995 * @end: end address 996 * @nonblocking: 997 * 998 * This takes care of mlocking the pages too if VM_LOCKED is set. 999 * 1000 * return 0 on success, negative error code on error. 1001 * 1002 * vma->vm_mm->mmap_sem must be held. 1003 * 1004 * If @nonblocking is NULL, it may be held for read or write and will 1005 * be unperturbed. 1006 * 1007 * If @nonblocking is non-NULL, it must held for read only and may be 1008 * released. If it's released, *@nonblocking will be set to 0. 1009 */ 1010 long populate_vma_page_range(struct vm_area_struct *vma, 1011 unsigned long start, unsigned long end, int *nonblocking) 1012 { 1013 struct mm_struct *mm = vma->vm_mm; 1014 unsigned long nr_pages = (end - start) / PAGE_SIZE; 1015 int gup_flags; 1016 1017 VM_BUG_ON(start & ~PAGE_MASK); 1018 VM_BUG_ON(end & ~PAGE_MASK); 1019 VM_BUG_ON_VMA(start < vma->vm_start, vma); 1020 VM_BUG_ON_VMA(end > vma->vm_end, vma); 1021 VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_sem), mm); 1022 1023 gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK; 1024 if (vma->vm_flags & VM_LOCKONFAULT) 1025 gup_flags &= ~FOLL_POPULATE; 1026 /* 1027 * We want to touch writable mappings with a write fault in order 1028 * to break COW, except for shared mappings because these don't COW 1029 * and we would not want to dirty them for nothing. 1030 */ 1031 if ((vma->vm_flags & (VM_WRITE | VM_SHARED)) == VM_WRITE) 1032 gup_flags |= FOLL_WRITE; 1033 1034 /* 1035 * We want mlock to succeed for regions that have any permissions 1036 * other than PROT_NONE. 1037 */ 1038 if (vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC)) 1039 gup_flags |= FOLL_FORCE; 1040 1041 /* 1042 * We made sure addr is within a VMA, so the following will 1043 * not result in a stack expansion that recurses back here. 1044 */ 1045 return __get_user_pages(current, mm, start, nr_pages, gup_flags, 1046 NULL, NULL, nonblocking); 1047 } 1048 1049 /* 1050 * __mm_populate - populate and/or mlock pages within a range of address space. 1051 * 1052 * This is used to implement mlock() and the MAP_POPULATE / MAP_LOCKED mmap 1053 * flags. VMAs must be already marked with the desired vm_flags, and 1054 * mmap_sem must not be held. 1055 */ 1056 int __mm_populate(unsigned long start, unsigned long len, int ignore_errors) 1057 { 1058 struct mm_struct *mm = current->mm; 1059 unsigned long end, nstart, nend; 1060 struct vm_area_struct *vma = NULL; 1061 int locked = 0; 1062 long ret = 0; 1063 1064 VM_BUG_ON(start & ~PAGE_MASK); 1065 VM_BUG_ON(len != PAGE_ALIGN(len)); 1066 end = start + len; 1067 1068 for (nstart = start; nstart < end; nstart = nend) { 1069 /* 1070 * We want to fault in pages for [nstart; end) address range. 1071 * Find first corresponding VMA. 1072 */ 1073 if (!locked) { 1074 locked = 1; 1075 down_read(&mm->mmap_sem); 1076 vma = find_vma(mm, nstart); 1077 } else if (nstart >= vma->vm_end) 1078 vma = vma->vm_next; 1079 if (!vma || vma->vm_start >= end) 1080 break; 1081 /* 1082 * Set [nstart; nend) to intersection of desired address 1083 * range with the first VMA. Also, skip undesirable VMA types. 1084 */ 1085 nend = min(end, vma->vm_end); 1086 if (vma->vm_flags & (VM_IO | VM_PFNMAP)) 1087 continue; 1088 if (nstart < vma->vm_start) 1089 nstart = vma->vm_start; 1090 /* 1091 * Now fault in a range of pages. populate_vma_page_range() 1092 * double checks the vma flags, so that it won't mlock pages 1093 * if the vma was already munlocked. 1094 */ 1095 ret = populate_vma_page_range(vma, nstart, nend, &locked); 1096 if (ret < 0) { 1097 if (ignore_errors) { 1098 ret = 0; 1099 continue; /* continue at next VMA */ 1100 } 1101 break; 1102 } 1103 nend = nstart + ret * PAGE_SIZE; 1104 ret = 0; 1105 } 1106 if (locked) 1107 up_read(&mm->mmap_sem); 1108 return ret; /* 0 or negative error code */ 1109 } 1110 1111 /** 1112 * get_dump_page() - pin user page in memory while writing it to core dump 1113 * @addr: user address 1114 * 1115 * Returns struct page pointer of user page pinned for dump, 1116 * to be freed afterwards by put_page(). 1117 * 1118 * Returns NULL on any kind of failure - a hole must then be inserted into 1119 * the corefile, to preserve alignment with its headers; and also returns 1120 * NULL wherever the ZERO_PAGE, or an anonymous pte_none, has been found - 1121 * allowing a hole to be left in the corefile to save diskspace. 1122 * 1123 * Called without mmap_sem, but after all other threads have been killed. 1124 */ 1125 #ifdef CONFIG_ELF_CORE 1126 struct page *get_dump_page(unsigned long addr) 1127 { 1128 struct vm_area_struct *vma; 1129 struct page *page; 1130 1131 if (__get_user_pages(current, current->mm, addr, 1, 1132 FOLL_FORCE | FOLL_DUMP | FOLL_GET, &page, &vma, 1133 NULL) < 1) 1134 return NULL; 1135 flush_cache_page(vma, addr, page_to_pfn(page)); 1136 return page; 1137 } 1138 #endif /* CONFIG_ELF_CORE */ 1139 1140 /* 1141 * Generic RCU Fast GUP 1142 * 1143 * get_user_pages_fast attempts to pin user pages by walking the page 1144 * tables directly and avoids taking locks. Thus the walker needs to be 1145 * protected from page table pages being freed from under it, and should 1146 * block any THP splits. 1147 * 1148 * One way to achieve this is to have the walker disable interrupts, and 1149 * rely on IPIs from the TLB flushing code blocking before the page table 1150 * pages are freed. This is unsuitable for architectures that do not need 1151 * to broadcast an IPI when invalidating TLBs. 1152 * 1153 * Another way to achieve this is to batch up page table containing pages 1154 * belonging to more than one mm_user, then rcu_sched a callback to free those 1155 * pages. Disabling interrupts will allow the fast_gup walker to both block 1156 * the rcu_sched callback, and an IPI that we broadcast for splitting THPs 1157 * (which is a relatively rare event). The code below adopts this strategy. 1158 * 1159 * Before activating this code, please be aware that the following assumptions 1160 * are currently made: 1161 * 1162 * *) HAVE_RCU_TABLE_FREE is enabled, and tlb_remove_table is used to free 1163 * pages containing page tables. 1164 * 1165 * *) ptes can be read atomically by the architecture. 1166 * 1167 * *) access_ok is sufficient to validate userspace address ranges. 1168 * 1169 * The last two assumptions can be relaxed by the addition of helper functions. 1170 * 1171 * This code is based heavily on the PowerPC implementation by Nick Piggin. 1172 */ 1173 #ifdef CONFIG_HAVE_GENERIC_RCU_GUP 1174 1175 #ifdef __HAVE_ARCH_PTE_SPECIAL 1176 static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, 1177 int write, struct page **pages, int *nr) 1178 { 1179 pte_t *ptep, *ptem; 1180 int ret = 0; 1181 1182 ptem = ptep = pte_offset_map(&pmd, addr); 1183 do { 1184 /* 1185 * In the line below we are assuming that the pte can be read 1186 * atomically. If this is not the case for your architecture, 1187 * please wrap this in a helper function! 1188 * 1189 * for an example see gup_get_pte in arch/x86/mm/gup.c 1190 */ 1191 pte_t pte = READ_ONCE(*ptep); 1192 struct page *head, *page; 1193 1194 /* 1195 * Similar to the PMD case below, NUMA hinting must take slow 1196 * path using the pte_protnone check. 1197 */ 1198 if (!pte_present(pte) || pte_special(pte) || 1199 pte_protnone(pte) || (write && !pte_write(pte))) 1200 goto pte_unmap; 1201 1202 if (!arch_pte_access_permitted(pte, write)) 1203 goto pte_unmap; 1204 1205 VM_BUG_ON(!pfn_valid(pte_pfn(pte))); 1206 page = pte_page(pte); 1207 head = compound_head(page); 1208 1209 if (!page_cache_get_speculative(head)) 1210 goto pte_unmap; 1211 1212 if (unlikely(pte_val(pte) != pte_val(*ptep))) { 1213 put_page(head); 1214 goto pte_unmap; 1215 } 1216 1217 VM_BUG_ON_PAGE(compound_head(page) != head, page); 1218 pages[*nr] = page; 1219 (*nr)++; 1220 1221 } while (ptep++, addr += PAGE_SIZE, addr != end); 1222 1223 ret = 1; 1224 1225 pte_unmap: 1226 pte_unmap(ptem); 1227 return ret; 1228 } 1229 #else 1230 1231 /* 1232 * If we can't determine whether or not a pte is special, then fail immediately 1233 * for ptes. Note, we can still pin HugeTLB and THP as these are guaranteed not 1234 * to be special. 1235 * 1236 * For a futex to be placed on a THP tail page, get_futex_key requires a 1237 * __get_user_pages_fast implementation that can pin pages. Thus it's still 1238 * useful to have gup_huge_pmd even if we can't operate on ptes. 1239 */ 1240 static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, 1241 int write, struct page **pages, int *nr) 1242 { 1243 return 0; 1244 } 1245 #endif /* __HAVE_ARCH_PTE_SPECIAL */ 1246 1247 static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, 1248 unsigned long end, int write, struct page **pages, int *nr) 1249 { 1250 struct page *head, *page; 1251 int refs; 1252 1253 if (write && !pmd_write(orig)) 1254 return 0; 1255 1256 refs = 0; 1257 head = pmd_page(orig); 1258 page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT); 1259 do { 1260 VM_BUG_ON_PAGE(compound_head(page) != head, page); 1261 pages[*nr] = page; 1262 (*nr)++; 1263 page++; 1264 refs++; 1265 } while (addr += PAGE_SIZE, addr != end); 1266 1267 if (!page_cache_add_speculative(head, refs)) { 1268 *nr -= refs; 1269 return 0; 1270 } 1271 1272 if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) { 1273 *nr -= refs; 1274 while (refs--) 1275 put_page(head); 1276 return 0; 1277 } 1278 1279 return 1; 1280 } 1281 1282 static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, 1283 unsigned long end, int write, struct page **pages, int *nr) 1284 { 1285 struct page *head, *page; 1286 int refs; 1287 1288 if (write && !pud_write(orig)) 1289 return 0; 1290 1291 refs = 0; 1292 head = pud_page(orig); 1293 page = head + ((addr & ~PUD_MASK) >> PAGE_SHIFT); 1294 do { 1295 VM_BUG_ON_PAGE(compound_head(page) != head, page); 1296 pages[*nr] = page; 1297 (*nr)++; 1298 page++; 1299 refs++; 1300 } while (addr += PAGE_SIZE, addr != end); 1301 1302 if (!page_cache_add_speculative(head, refs)) { 1303 *nr -= refs; 1304 return 0; 1305 } 1306 1307 if (unlikely(pud_val(orig) != pud_val(*pudp))) { 1308 *nr -= refs; 1309 while (refs--) 1310 put_page(head); 1311 return 0; 1312 } 1313 1314 return 1; 1315 } 1316 1317 static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr, 1318 unsigned long end, int write, 1319 struct page **pages, int *nr) 1320 { 1321 int refs; 1322 struct page *head, *page; 1323 1324 if (write && !pgd_write(orig)) 1325 return 0; 1326 1327 refs = 0; 1328 head = pgd_page(orig); 1329 page = head + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT); 1330 do { 1331 VM_BUG_ON_PAGE(compound_head(page) != head, page); 1332 pages[*nr] = page; 1333 (*nr)++; 1334 page++; 1335 refs++; 1336 } while (addr += PAGE_SIZE, addr != end); 1337 1338 if (!page_cache_add_speculative(head, refs)) { 1339 *nr -= refs; 1340 return 0; 1341 } 1342 1343 if (unlikely(pgd_val(orig) != pgd_val(*pgdp))) { 1344 *nr -= refs; 1345 while (refs--) 1346 put_page(head); 1347 return 0; 1348 } 1349 1350 return 1; 1351 } 1352 1353 static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end, 1354 int write, struct page **pages, int *nr) 1355 { 1356 unsigned long next; 1357 pmd_t *pmdp; 1358 1359 pmdp = pmd_offset(&pud, addr); 1360 do { 1361 pmd_t pmd = READ_ONCE(*pmdp); 1362 1363 next = pmd_addr_end(addr, end); 1364 if (pmd_none(pmd)) 1365 return 0; 1366 1367 if (unlikely(pmd_trans_huge(pmd) || pmd_huge(pmd))) { 1368 /* 1369 * NUMA hinting faults need to be handled in the GUP 1370 * slowpath for accounting purposes and so that they 1371 * can be serialised against THP migration. 1372 */ 1373 if (pmd_protnone(pmd)) 1374 return 0; 1375 1376 if (!gup_huge_pmd(pmd, pmdp, addr, next, write, 1377 pages, nr)) 1378 return 0; 1379 1380 } else if (unlikely(is_hugepd(__hugepd(pmd_val(pmd))))) { 1381 /* 1382 * architecture have different format for hugetlbfs 1383 * pmd format and THP pmd format 1384 */ 1385 if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr, 1386 PMD_SHIFT, next, write, pages, nr)) 1387 return 0; 1388 } else if (!gup_pte_range(pmd, addr, next, write, pages, nr)) 1389 return 0; 1390 } while (pmdp++, addr = next, addr != end); 1391 1392 return 1; 1393 } 1394 1395 static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end, 1396 int write, struct page **pages, int *nr) 1397 { 1398 unsigned long next; 1399 pud_t *pudp; 1400 1401 pudp = pud_offset(&pgd, addr); 1402 do { 1403 pud_t pud = READ_ONCE(*pudp); 1404 1405 next = pud_addr_end(addr, end); 1406 if (pud_none(pud)) 1407 return 0; 1408 if (unlikely(pud_huge(pud))) { 1409 if (!gup_huge_pud(pud, pudp, addr, next, write, 1410 pages, nr)) 1411 return 0; 1412 } else if (unlikely(is_hugepd(__hugepd(pud_val(pud))))) { 1413 if (!gup_huge_pd(__hugepd(pud_val(pud)), addr, 1414 PUD_SHIFT, next, write, pages, nr)) 1415 return 0; 1416 } else if (!gup_pmd_range(pud, addr, next, write, pages, nr)) 1417 return 0; 1418 } while (pudp++, addr = next, addr != end); 1419 1420 return 1; 1421 } 1422 1423 /* 1424 * Like get_user_pages_fast() except it's IRQ-safe in that it won't fall back to 1425 * the regular GUP. It will only return non-negative values. 1426 */ 1427 int __get_user_pages_fast(unsigned long start, int nr_pages, int write, 1428 struct page **pages) 1429 { 1430 struct mm_struct *mm = current->mm; 1431 unsigned long addr, len, end; 1432 unsigned long next, flags; 1433 pgd_t *pgdp; 1434 int nr = 0; 1435 1436 start &= PAGE_MASK; 1437 addr = start; 1438 len = (unsigned long) nr_pages << PAGE_SHIFT; 1439 end = start + len; 1440 1441 if (unlikely(!access_ok(write ? VERIFY_WRITE : VERIFY_READ, 1442 start, len))) 1443 return 0; 1444 1445 /* 1446 * Disable interrupts. We use the nested form as we can already have 1447 * interrupts disabled by get_futex_key. 1448 * 1449 * With interrupts disabled, we block page table pages from being 1450 * freed from under us. See mmu_gather_tlb in asm-generic/tlb.h 1451 * for more details. 1452 * 1453 * We do not adopt an rcu_read_lock(.) here as we also want to 1454 * block IPIs that come from THPs splitting. 1455 */ 1456 1457 local_irq_save(flags); 1458 pgdp = pgd_offset(mm, addr); 1459 do { 1460 pgd_t pgd = READ_ONCE(*pgdp); 1461 1462 next = pgd_addr_end(addr, end); 1463 if (pgd_none(pgd)) 1464 break; 1465 if (unlikely(pgd_huge(pgd))) { 1466 if (!gup_huge_pgd(pgd, pgdp, addr, next, write, 1467 pages, &nr)) 1468 break; 1469 } else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) { 1470 if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr, 1471 PGDIR_SHIFT, next, write, pages, &nr)) 1472 break; 1473 } else if (!gup_pud_range(pgd, addr, next, write, pages, &nr)) 1474 break; 1475 } while (pgdp++, addr = next, addr != end); 1476 local_irq_restore(flags); 1477 1478 return nr; 1479 } 1480 1481 /** 1482 * get_user_pages_fast() - pin user pages in memory 1483 * @start: starting user address 1484 * @nr_pages: number of pages from start to pin 1485 * @write: whether pages will be written to 1486 * @pages: array that receives pointers to the pages pinned. 1487 * Should be at least nr_pages long. 1488 * 1489 * Attempt to pin user pages in memory without taking mm->mmap_sem. 1490 * If not successful, it will fall back to taking the lock and 1491 * calling get_user_pages(). 1492 * 1493 * Returns number of pages pinned. This may be fewer than the number 1494 * requested. If nr_pages is 0 or negative, returns 0. If no pages 1495 * were pinned, returns -errno. 1496 */ 1497 int get_user_pages_fast(unsigned long start, int nr_pages, int write, 1498 struct page **pages) 1499 { 1500 int nr, ret; 1501 1502 start &= PAGE_MASK; 1503 nr = __get_user_pages_fast(start, nr_pages, write, pages); 1504 ret = nr; 1505 1506 if (nr < nr_pages) { 1507 /* Try to get the remaining pages with get_user_pages */ 1508 start += nr << PAGE_SHIFT; 1509 pages += nr; 1510 1511 ret = get_user_pages_unlocked(start, nr_pages - nr, pages, 1512 write ? FOLL_WRITE : 0); 1513 1514 /* Have to be a bit careful with return values */ 1515 if (nr > 0) { 1516 if (ret < 0) 1517 ret = nr; 1518 else 1519 ret += nr; 1520 } 1521 } 1522 1523 return ret; 1524 } 1525 1526 #endif /* CONFIG_HAVE_GENERIC_RCU_GUP */ 1527