1 /* 2 * linux/mm/vmalloc.c 3 * 4 * Copyright (C) 1993 Linus Torvalds 5 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 6 * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000 7 * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002 8 * Numa awareness, Christoph Lameter, SGI, June 2005 9 */ 10 11 #include <linux/vmalloc.h> 12 #include <linux/mm.h> 13 #include <linux/module.h> 14 #include <linux/highmem.h> 15 #include <linux/sched.h> 16 #include <linux/slab.h> 17 #include <linux/spinlock.h> 18 #include <linux/interrupt.h> 19 #include <linux/proc_fs.h> 20 #include <linux/seq_file.h> 21 #include <linux/debugobjects.h> 22 #include <linux/kallsyms.h> 23 #include <linux/list.h> 24 #include <linux/rbtree.h> 25 #include <linux/radix-tree.h> 26 #include <linux/rcupdate.h> 27 #include <linux/pfn.h> 28 #include <linux/kmemleak.h> 29 #include <linux/atomic.h> 30 #include <linux/compiler.h> 31 #include <linux/llist.h> 32 #include <linux/bitops.h> 33 34 #include <asm/uaccess.h> 35 #include <asm/tlbflush.h> 36 #include <asm/shmparam.h> 37 38 #include "internal.h" 39 40 struct vfree_deferred { 41 struct llist_head list; 42 struct work_struct wq; 43 }; 44 static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred); 45 46 static void __vunmap(const void *, int); 47 48 static void free_work(struct work_struct *w) 49 { 50 struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq); 51 struct llist_node *llnode = llist_del_all(&p->list); 52 while (llnode) { 53 void *p = llnode; 54 llnode = llist_next(llnode); 55 __vunmap(p, 1); 56 } 57 } 58 59 /*** Page table manipulation functions ***/ 60 61 static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end) 62 { 63 pte_t *pte; 64 65 pte = pte_offset_kernel(pmd, addr); 66 do { 67 pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte); 68 WARN_ON(!pte_none(ptent) && !pte_present(ptent)); 69 } while (pte++, addr += PAGE_SIZE, addr != end); 70 } 71 72 static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end) 73 { 74 pmd_t *pmd; 75 unsigned long next; 76 77 pmd = pmd_offset(pud, addr); 78 do { 79 next = pmd_addr_end(addr, end); 80 if (pmd_clear_huge(pmd)) 81 continue; 82 if (pmd_none_or_clear_bad(pmd)) 83 continue; 84 vunmap_pte_range(pmd, addr, next); 85 } while (pmd++, addr = next, addr != end); 86 } 87 88 static void vunmap_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end) 89 { 90 pud_t *pud; 91 unsigned long next; 92 93 pud = pud_offset(pgd, addr); 94 do { 95 next = pud_addr_end(addr, end); 96 if (pud_clear_huge(pud)) 97 continue; 98 if (pud_none_or_clear_bad(pud)) 99 continue; 100 vunmap_pmd_range(pud, addr, next); 101 } while (pud++, addr = next, addr != end); 102 } 103 104 static void vunmap_page_range(unsigned long addr, unsigned long end) 105 { 106 pgd_t *pgd; 107 unsigned long next; 108 109 BUG_ON(addr >= end); 110 pgd = pgd_offset_k(addr); 111 do { 112 next = pgd_addr_end(addr, end); 113 if (pgd_none_or_clear_bad(pgd)) 114 continue; 115 vunmap_pud_range(pgd, addr, next); 116 } while (pgd++, addr = next, addr != end); 117 } 118 119 static int vmap_pte_range(pmd_t *pmd, unsigned long addr, 120 unsigned long end, pgprot_t prot, struct page **pages, int *nr) 121 { 122 pte_t *pte; 123 124 /* 125 * nr is a running index into the array which helps higher level 126 * callers keep track of where we're up to. 127 */ 128 129 pte = pte_alloc_kernel(pmd, addr); 130 if (!pte) 131 return -ENOMEM; 132 do { 133 struct page *page = pages[*nr]; 134 135 if (WARN_ON(!pte_none(*pte))) 136 return -EBUSY; 137 if (WARN_ON(!page)) 138 return -ENOMEM; 139 set_pte_at(&init_mm, addr, pte, mk_pte(page, prot)); 140 (*nr)++; 141 } while (pte++, addr += PAGE_SIZE, addr != end); 142 return 0; 143 } 144 145 static int vmap_pmd_range(pud_t *pud, unsigned long addr, 146 unsigned long end, pgprot_t prot, struct page **pages, int *nr) 147 { 148 pmd_t *pmd; 149 unsigned long next; 150 151 pmd = pmd_alloc(&init_mm, pud, addr); 152 if (!pmd) 153 return -ENOMEM; 154 do { 155 next = pmd_addr_end(addr, end); 156 if (vmap_pte_range(pmd, addr, next, prot, pages, nr)) 157 return -ENOMEM; 158 } while (pmd++, addr = next, addr != end); 159 return 0; 160 } 161 162 static int vmap_pud_range(pgd_t *pgd, unsigned long addr, 163 unsigned long end, pgprot_t prot, struct page **pages, int *nr) 164 { 165 pud_t *pud; 166 unsigned long next; 167 168 pud = pud_alloc(&init_mm, pgd, addr); 169 if (!pud) 170 return -ENOMEM; 171 do { 172 next = pud_addr_end(addr, end); 173 if (vmap_pmd_range(pud, addr, next, prot, pages, nr)) 174 return -ENOMEM; 175 } while (pud++, addr = next, addr != end); 176 return 0; 177 } 178 179 /* 180 * Set up page tables in kva (addr, end). The ptes shall have prot "prot", and 181 * will have pfns corresponding to the "pages" array. 182 * 183 * Ie. pte at addr+N*PAGE_SIZE shall point to pfn corresponding to pages[N] 184 */ 185 static int vmap_page_range_noflush(unsigned long start, unsigned long end, 186 pgprot_t prot, struct page **pages) 187 { 188 pgd_t *pgd; 189 unsigned long next; 190 unsigned long addr = start; 191 int err = 0; 192 int nr = 0; 193 194 BUG_ON(addr >= end); 195 pgd = pgd_offset_k(addr); 196 do { 197 next = pgd_addr_end(addr, end); 198 err = vmap_pud_range(pgd, addr, next, prot, pages, &nr); 199 if (err) 200 return err; 201 } while (pgd++, addr = next, addr != end); 202 203 return nr; 204 } 205 206 static int vmap_page_range(unsigned long start, unsigned long end, 207 pgprot_t prot, struct page **pages) 208 { 209 int ret; 210 211 ret = vmap_page_range_noflush(start, end, prot, pages); 212 flush_cache_vmap(start, end); 213 return ret; 214 } 215 216 int is_vmalloc_or_module_addr(const void *x) 217 { 218 /* 219 * ARM, x86-64 and sparc64 put modules in a special place, 220 * and fall back on vmalloc() if that fails. Others 221 * just put it in the vmalloc space. 222 */ 223 #if defined(CONFIG_MODULES) && defined(MODULES_VADDR) 224 unsigned long addr = (unsigned long)x; 225 if (addr >= MODULES_VADDR && addr < MODULES_END) 226 return 1; 227 #endif 228 return is_vmalloc_addr(x); 229 } 230 231 /* 232 * Walk a vmap address to the struct page it maps. 233 */ 234 struct page *vmalloc_to_page(const void *vmalloc_addr) 235 { 236 unsigned long addr = (unsigned long) vmalloc_addr; 237 struct page *page = NULL; 238 pgd_t *pgd = pgd_offset_k(addr); 239 240 /* 241 * XXX we might need to change this if we add VIRTUAL_BUG_ON for 242 * architectures that do not vmalloc module space 243 */ 244 VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr)); 245 246 if (!pgd_none(*pgd)) { 247 pud_t *pud = pud_offset(pgd, addr); 248 if (!pud_none(*pud)) { 249 pmd_t *pmd = pmd_offset(pud, addr); 250 if (!pmd_none(*pmd)) { 251 pte_t *ptep, pte; 252 253 ptep = pte_offset_map(pmd, addr); 254 pte = *ptep; 255 if (pte_present(pte)) 256 page = pte_page(pte); 257 pte_unmap(ptep); 258 } 259 } 260 } 261 return page; 262 } 263 EXPORT_SYMBOL(vmalloc_to_page); 264 265 /* 266 * Map a vmalloc()-space virtual address to the physical page frame number. 267 */ 268 unsigned long vmalloc_to_pfn(const void *vmalloc_addr) 269 { 270 return page_to_pfn(vmalloc_to_page(vmalloc_addr)); 271 } 272 EXPORT_SYMBOL(vmalloc_to_pfn); 273 274 275 /*** Global kva allocator ***/ 276 277 #define VM_LAZY_FREE 0x01 278 #define VM_LAZY_FREEING 0x02 279 #define VM_VM_AREA 0x04 280 281 static DEFINE_SPINLOCK(vmap_area_lock); 282 /* Export for kexec only */ 283 LIST_HEAD(vmap_area_list); 284 static struct rb_root vmap_area_root = RB_ROOT; 285 286 /* The vmap cache globals are protected by vmap_area_lock */ 287 static struct rb_node *free_vmap_cache; 288 static unsigned long cached_hole_size; 289 static unsigned long cached_vstart; 290 static unsigned long cached_align; 291 292 static unsigned long vmap_area_pcpu_hole; 293 294 static struct vmap_area *__find_vmap_area(unsigned long addr) 295 { 296 struct rb_node *n = vmap_area_root.rb_node; 297 298 while (n) { 299 struct vmap_area *va; 300 301 va = rb_entry(n, struct vmap_area, rb_node); 302 if (addr < va->va_start) 303 n = n->rb_left; 304 else if (addr >= va->va_end) 305 n = n->rb_right; 306 else 307 return va; 308 } 309 310 return NULL; 311 } 312 313 static void __insert_vmap_area(struct vmap_area *va) 314 { 315 struct rb_node **p = &vmap_area_root.rb_node; 316 struct rb_node *parent = NULL; 317 struct rb_node *tmp; 318 319 while (*p) { 320 struct vmap_area *tmp_va; 321 322 parent = *p; 323 tmp_va = rb_entry(parent, struct vmap_area, rb_node); 324 if (va->va_start < tmp_va->va_end) 325 p = &(*p)->rb_left; 326 else if (va->va_end > tmp_va->va_start) 327 p = &(*p)->rb_right; 328 else 329 BUG(); 330 } 331 332 rb_link_node(&va->rb_node, parent, p); 333 rb_insert_color(&va->rb_node, &vmap_area_root); 334 335 /* address-sort this list */ 336 tmp = rb_prev(&va->rb_node); 337 if (tmp) { 338 struct vmap_area *prev; 339 prev = rb_entry(tmp, struct vmap_area, rb_node); 340 list_add_rcu(&va->list, &prev->list); 341 } else 342 list_add_rcu(&va->list, &vmap_area_list); 343 } 344 345 static void purge_vmap_area_lazy(void); 346 347 /* 348 * Allocate a region of KVA of the specified size and alignment, within the 349 * vstart and vend. 350 */ 351 static struct vmap_area *alloc_vmap_area(unsigned long size, 352 unsigned long align, 353 unsigned long vstart, unsigned long vend, 354 int node, gfp_t gfp_mask) 355 { 356 struct vmap_area *va; 357 struct rb_node *n; 358 unsigned long addr; 359 int purged = 0; 360 struct vmap_area *first; 361 362 BUG_ON(!size); 363 BUG_ON(offset_in_page(size)); 364 BUG_ON(!is_power_of_2(align)); 365 366 va = kmalloc_node(sizeof(struct vmap_area), 367 gfp_mask & GFP_RECLAIM_MASK, node); 368 if (unlikely(!va)) 369 return ERR_PTR(-ENOMEM); 370 371 /* 372 * Only scan the relevant parts containing pointers to other objects 373 * to avoid false negatives. 374 */ 375 kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask & GFP_RECLAIM_MASK); 376 377 retry: 378 spin_lock(&vmap_area_lock); 379 /* 380 * Invalidate cache if we have more permissive parameters. 381 * cached_hole_size notes the largest hole noticed _below_ 382 * the vmap_area cached in free_vmap_cache: if size fits 383 * into that hole, we want to scan from vstart to reuse 384 * the hole instead of allocating above free_vmap_cache. 385 * Note that __free_vmap_area may update free_vmap_cache 386 * without updating cached_hole_size or cached_align. 387 */ 388 if (!free_vmap_cache || 389 size < cached_hole_size || 390 vstart < cached_vstart || 391 align < cached_align) { 392 nocache: 393 cached_hole_size = 0; 394 free_vmap_cache = NULL; 395 } 396 /* record if we encounter less permissive parameters */ 397 cached_vstart = vstart; 398 cached_align = align; 399 400 /* find starting point for our search */ 401 if (free_vmap_cache) { 402 first = rb_entry(free_vmap_cache, struct vmap_area, rb_node); 403 addr = ALIGN(first->va_end, align); 404 if (addr < vstart) 405 goto nocache; 406 if (addr + size < addr) 407 goto overflow; 408 409 } else { 410 addr = ALIGN(vstart, align); 411 if (addr + size < addr) 412 goto overflow; 413 414 n = vmap_area_root.rb_node; 415 first = NULL; 416 417 while (n) { 418 struct vmap_area *tmp; 419 tmp = rb_entry(n, struct vmap_area, rb_node); 420 if (tmp->va_end >= addr) { 421 first = tmp; 422 if (tmp->va_start <= addr) 423 break; 424 n = n->rb_left; 425 } else 426 n = n->rb_right; 427 } 428 429 if (!first) 430 goto found; 431 } 432 433 /* from the starting point, walk areas until a suitable hole is found */ 434 while (addr + size > first->va_start && addr + size <= vend) { 435 if (addr + cached_hole_size < first->va_start) 436 cached_hole_size = first->va_start - addr; 437 addr = ALIGN(first->va_end, align); 438 if (addr + size < addr) 439 goto overflow; 440 441 if (list_is_last(&first->list, &vmap_area_list)) 442 goto found; 443 444 first = list_next_entry(first, list); 445 } 446 447 found: 448 if (addr + size > vend) 449 goto overflow; 450 451 va->va_start = addr; 452 va->va_end = addr + size; 453 va->flags = 0; 454 __insert_vmap_area(va); 455 free_vmap_cache = &va->rb_node; 456 spin_unlock(&vmap_area_lock); 457 458 BUG_ON(!IS_ALIGNED(va->va_start, align)); 459 BUG_ON(va->va_start < vstart); 460 BUG_ON(va->va_end > vend); 461 462 return va; 463 464 overflow: 465 spin_unlock(&vmap_area_lock); 466 if (!purged) { 467 purge_vmap_area_lazy(); 468 purged = 1; 469 goto retry; 470 } 471 if (printk_ratelimit()) 472 pr_warn("vmap allocation for size %lu failed: " 473 "use vmalloc=<size> to increase size.\n", size); 474 kfree(va); 475 return ERR_PTR(-EBUSY); 476 } 477 478 static void __free_vmap_area(struct vmap_area *va) 479 { 480 BUG_ON(RB_EMPTY_NODE(&va->rb_node)); 481 482 if (free_vmap_cache) { 483 if (va->va_end < cached_vstart) { 484 free_vmap_cache = NULL; 485 } else { 486 struct vmap_area *cache; 487 cache = rb_entry(free_vmap_cache, struct vmap_area, rb_node); 488 if (va->va_start <= cache->va_start) { 489 free_vmap_cache = rb_prev(&va->rb_node); 490 /* 491 * We don't try to update cached_hole_size or 492 * cached_align, but it won't go very wrong. 493 */ 494 } 495 } 496 } 497 rb_erase(&va->rb_node, &vmap_area_root); 498 RB_CLEAR_NODE(&va->rb_node); 499 list_del_rcu(&va->list); 500 501 /* 502 * Track the highest possible candidate for pcpu area 503 * allocation. Areas outside of vmalloc area can be returned 504 * here too, consider only end addresses which fall inside 505 * vmalloc area proper. 506 */ 507 if (va->va_end > VMALLOC_START && va->va_end <= VMALLOC_END) 508 vmap_area_pcpu_hole = max(vmap_area_pcpu_hole, va->va_end); 509 510 kfree_rcu(va, rcu_head); 511 } 512 513 /* 514 * Free a region of KVA allocated by alloc_vmap_area 515 */ 516 static void free_vmap_area(struct vmap_area *va) 517 { 518 spin_lock(&vmap_area_lock); 519 __free_vmap_area(va); 520 spin_unlock(&vmap_area_lock); 521 } 522 523 /* 524 * Clear the pagetable entries of a given vmap_area 525 */ 526 static void unmap_vmap_area(struct vmap_area *va) 527 { 528 vunmap_page_range(va->va_start, va->va_end); 529 } 530 531 static void vmap_debug_free_range(unsigned long start, unsigned long end) 532 { 533 /* 534 * Unmap page tables and force a TLB flush immediately if 535 * CONFIG_DEBUG_PAGEALLOC is set. This catches use after free 536 * bugs similarly to those in linear kernel virtual address 537 * space after a page has been freed. 538 * 539 * All the lazy freeing logic is still retained, in order to 540 * minimise intrusiveness of this debugging feature. 541 * 542 * This is going to be *slow* (linear kernel virtual address 543 * debugging doesn't do a broadcast TLB flush so it is a lot 544 * faster). 545 */ 546 #ifdef CONFIG_DEBUG_PAGEALLOC 547 vunmap_page_range(start, end); 548 flush_tlb_kernel_range(start, end); 549 #endif 550 } 551 552 /* 553 * lazy_max_pages is the maximum amount of virtual address space we gather up 554 * before attempting to purge with a TLB flush. 555 * 556 * There is a tradeoff here: a larger number will cover more kernel page tables 557 * and take slightly longer to purge, but it will linearly reduce the number of 558 * global TLB flushes that must be performed. It would seem natural to scale 559 * this number up linearly with the number of CPUs (because vmapping activity 560 * could also scale linearly with the number of CPUs), however it is likely 561 * that in practice, workloads might be constrained in other ways that mean 562 * vmap activity will not scale linearly with CPUs. Also, I want to be 563 * conservative and not introduce a big latency on huge systems, so go with 564 * a less aggressive log scale. It will still be an improvement over the old 565 * code, and it will be simple to change the scale factor if we find that it 566 * becomes a problem on bigger systems. 567 */ 568 static unsigned long lazy_max_pages(void) 569 { 570 unsigned int log; 571 572 log = fls(num_online_cpus()); 573 574 return log * (32UL * 1024 * 1024 / PAGE_SIZE); 575 } 576 577 static atomic_t vmap_lazy_nr = ATOMIC_INIT(0); 578 579 /* for per-CPU blocks */ 580 static void purge_fragmented_blocks_allcpus(void); 581 582 /* 583 * called before a call to iounmap() if the caller wants vm_area_struct's 584 * immediately freed. 585 */ 586 void set_iounmap_nonlazy(void) 587 { 588 atomic_set(&vmap_lazy_nr, lazy_max_pages()+1); 589 } 590 591 /* 592 * Purges all lazily-freed vmap areas. 593 * 594 * If sync is 0 then don't purge if there is already a purge in progress. 595 * If force_flush is 1, then flush kernel TLBs between *start and *end even 596 * if we found no lazy vmap areas to unmap (callers can use this to optimise 597 * their own TLB flushing). 598 * Returns with *start = min(*start, lowest purged address) 599 * *end = max(*end, highest purged address) 600 */ 601 static void __purge_vmap_area_lazy(unsigned long *start, unsigned long *end, 602 int sync, int force_flush) 603 { 604 static DEFINE_SPINLOCK(purge_lock); 605 LIST_HEAD(valist); 606 struct vmap_area *va; 607 struct vmap_area *n_va; 608 int nr = 0; 609 610 /* 611 * If sync is 0 but force_flush is 1, we'll go sync anyway but callers 612 * should not expect such behaviour. This just simplifies locking for 613 * the case that isn't actually used at the moment anyway. 614 */ 615 if (!sync && !force_flush) { 616 if (!spin_trylock(&purge_lock)) 617 return; 618 } else 619 spin_lock(&purge_lock); 620 621 if (sync) 622 purge_fragmented_blocks_allcpus(); 623 624 rcu_read_lock(); 625 list_for_each_entry_rcu(va, &vmap_area_list, list) { 626 if (va->flags & VM_LAZY_FREE) { 627 if (va->va_start < *start) 628 *start = va->va_start; 629 if (va->va_end > *end) 630 *end = va->va_end; 631 nr += (va->va_end - va->va_start) >> PAGE_SHIFT; 632 list_add_tail(&va->purge_list, &valist); 633 va->flags |= VM_LAZY_FREEING; 634 va->flags &= ~VM_LAZY_FREE; 635 } 636 } 637 rcu_read_unlock(); 638 639 if (nr) 640 atomic_sub(nr, &vmap_lazy_nr); 641 642 if (nr || force_flush) 643 flush_tlb_kernel_range(*start, *end); 644 645 if (nr) { 646 spin_lock(&vmap_area_lock); 647 list_for_each_entry_safe(va, n_va, &valist, purge_list) 648 __free_vmap_area(va); 649 spin_unlock(&vmap_area_lock); 650 } 651 spin_unlock(&purge_lock); 652 } 653 654 /* 655 * Kick off a purge of the outstanding lazy areas. Don't bother if somebody 656 * is already purging. 657 */ 658 static void try_purge_vmap_area_lazy(void) 659 { 660 unsigned long start = ULONG_MAX, end = 0; 661 662 __purge_vmap_area_lazy(&start, &end, 0, 0); 663 } 664 665 /* 666 * Kick off a purge of the outstanding lazy areas. 667 */ 668 static void purge_vmap_area_lazy(void) 669 { 670 unsigned long start = ULONG_MAX, end = 0; 671 672 __purge_vmap_area_lazy(&start, &end, 1, 0); 673 } 674 675 /* 676 * Free a vmap area, caller ensuring that the area has been unmapped 677 * and flush_cache_vunmap had been called for the correct range 678 * previously. 679 */ 680 static void free_vmap_area_noflush(struct vmap_area *va) 681 { 682 va->flags |= VM_LAZY_FREE; 683 atomic_add((va->va_end - va->va_start) >> PAGE_SHIFT, &vmap_lazy_nr); 684 if (unlikely(atomic_read(&vmap_lazy_nr) > lazy_max_pages())) 685 try_purge_vmap_area_lazy(); 686 } 687 688 /* 689 * Free and unmap a vmap area, caller ensuring flush_cache_vunmap had been 690 * called for the correct range previously. 691 */ 692 static void free_unmap_vmap_area_noflush(struct vmap_area *va) 693 { 694 unmap_vmap_area(va); 695 free_vmap_area_noflush(va); 696 } 697 698 /* 699 * Free and unmap a vmap area 700 */ 701 static void free_unmap_vmap_area(struct vmap_area *va) 702 { 703 flush_cache_vunmap(va->va_start, va->va_end); 704 free_unmap_vmap_area_noflush(va); 705 } 706 707 static struct vmap_area *find_vmap_area(unsigned long addr) 708 { 709 struct vmap_area *va; 710 711 spin_lock(&vmap_area_lock); 712 va = __find_vmap_area(addr); 713 spin_unlock(&vmap_area_lock); 714 715 return va; 716 } 717 718 static void free_unmap_vmap_area_addr(unsigned long addr) 719 { 720 struct vmap_area *va; 721 722 va = find_vmap_area(addr); 723 BUG_ON(!va); 724 free_unmap_vmap_area(va); 725 } 726 727 728 /*** Per cpu kva allocator ***/ 729 730 /* 731 * vmap space is limited especially on 32 bit architectures. Ensure there is 732 * room for at least 16 percpu vmap blocks per CPU. 733 */ 734 /* 735 * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able 736 * to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess 737 * instead (we just need a rough idea) 738 */ 739 #if BITS_PER_LONG == 32 740 #define VMALLOC_SPACE (128UL*1024*1024) 741 #else 742 #define VMALLOC_SPACE (128UL*1024*1024*1024) 743 #endif 744 745 #define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE) 746 #define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */ 747 #define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */ 748 #define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2) 749 #define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */ 750 #define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */ 751 #define VMAP_BBMAP_BITS \ 752 VMAP_MIN(VMAP_BBMAP_BITS_MAX, \ 753 VMAP_MAX(VMAP_BBMAP_BITS_MIN, \ 754 VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16)) 755 756 #define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE) 757 758 static bool vmap_initialized __read_mostly = false; 759 760 struct vmap_block_queue { 761 spinlock_t lock; 762 struct list_head free; 763 }; 764 765 struct vmap_block { 766 spinlock_t lock; 767 struct vmap_area *va; 768 unsigned long free, dirty; 769 unsigned long dirty_min, dirty_max; /*< dirty range */ 770 struct list_head free_list; 771 struct rcu_head rcu_head; 772 struct list_head purge; 773 }; 774 775 /* Queue of free and dirty vmap blocks, for allocation and flushing purposes */ 776 static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue); 777 778 /* 779 * Radix tree of vmap blocks, indexed by address, to quickly find a vmap block 780 * in the free path. Could get rid of this if we change the API to return a 781 * "cookie" from alloc, to be passed to free. But no big deal yet. 782 */ 783 static DEFINE_SPINLOCK(vmap_block_tree_lock); 784 static RADIX_TREE(vmap_block_tree, GFP_ATOMIC); 785 786 /* 787 * We should probably have a fallback mechanism to allocate virtual memory 788 * out of partially filled vmap blocks. However vmap block sizing should be 789 * fairly reasonable according to the vmalloc size, so it shouldn't be a 790 * big problem. 791 */ 792 793 static unsigned long addr_to_vb_idx(unsigned long addr) 794 { 795 addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1); 796 addr /= VMAP_BLOCK_SIZE; 797 return addr; 798 } 799 800 static void *vmap_block_vaddr(unsigned long va_start, unsigned long pages_off) 801 { 802 unsigned long addr; 803 804 addr = va_start + (pages_off << PAGE_SHIFT); 805 BUG_ON(addr_to_vb_idx(addr) != addr_to_vb_idx(va_start)); 806 return (void *)addr; 807 } 808 809 /** 810 * new_vmap_block - allocates new vmap_block and occupies 2^order pages in this 811 * block. Of course pages number can't exceed VMAP_BBMAP_BITS 812 * @order: how many 2^order pages should be occupied in newly allocated block 813 * @gfp_mask: flags for the page level allocator 814 * 815 * Returns: virtual address in a newly allocated block or ERR_PTR(-errno) 816 */ 817 static void *new_vmap_block(unsigned int order, gfp_t gfp_mask) 818 { 819 struct vmap_block_queue *vbq; 820 struct vmap_block *vb; 821 struct vmap_area *va; 822 unsigned long vb_idx; 823 int node, err; 824 void *vaddr; 825 826 node = numa_node_id(); 827 828 vb = kmalloc_node(sizeof(struct vmap_block), 829 gfp_mask & GFP_RECLAIM_MASK, node); 830 if (unlikely(!vb)) 831 return ERR_PTR(-ENOMEM); 832 833 va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE, 834 VMALLOC_START, VMALLOC_END, 835 node, gfp_mask); 836 if (IS_ERR(va)) { 837 kfree(vb); 838 return ERR_CAST(va); 839 } 840 841 err = radix_tree_preload(gfp_mask); 842 if (unlikely(err)) { 843 kfree(vb); 844 free_vmap_area(va); 845 return ERR_PTR(err); 846 } 847 848 vaddr = vmap_block_vaddr(va->va_start, 0); 849 spin_lock_init(&vb->lock); 850 vb->va = va; 851 /* At least something should be left free */ 852 BUG_ON(VMAP_BBMAP_BITS <= (1UL << order)); 853 vb->free = VMAP_BBMAP_BITS - (1UL << order); 854 vb->dirty = 0; 855 vb->dirty_min = VMAP_BBMAP_BITS; 856 vb->dirty_max = 0; 857 INIT_LIST_HEAD(&vb->free_list); 858 859 vb_idx = addr_to_vb_idx(va->va_start); 860 spin_lock(&vmap_block_tree_lock); 861 err = radix_tree_insert(&vmap_block_tree, vb_idx, vb); 862 spin_unlock(&vmap_block_tree_lock); 863 BUG_ON(err); 864 radix_tree_preload_end(); 865 866 vbq = &get_cpu_var(vmap_block_queue); 867 spin_lock(&vbq->lock); 868 list_add_tail_rcu(&vb->free_list, &vbq->free); 869 spin_unlock(&vbq->lock); 870 put_cpu_var(vmap_block_queue); 871 872 return vaddr; 873 } 874 875 static void free_vmap_block(struct vmap_block *vb) 876 { 877 struct vmap_block *tmp; 878 unsigned long vb_idx; 879 880 vb_idx = addr_to_vb_idx(vb->va->va_start); 881 spin_lock(&vmap_block_tree_lock); 882 tmp = radix_tree_delete(&vmap_block_tree, vb_idx); 883 spin_unlock(&vmap_block_tree_lock); 884 BUG_ON(tmp != vb); 885 886 free_vmap_area_noflush(vb->va); 887 kfree_rcu(vb, rcu_head); 888 } 889 890 static void purge_fragmented_blocks(int cpu) 891 { 892 LIST_HEAD(purge); 893 struct vmap_block *vb; 894 struct vmap_block *n_vb; 895 struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); 896 897 rcu_read_lock(); 898 list_for_each_entry_rcu(vb, &vbq->free, free_list) { 899 900 if (!(vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS)) 901 continue; 902 903 spin_lock(&vb->lock); 904 if (vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS) { 905 vb->free = 0; /* prevent further allocs after releasing lock */ 906 vb->dirty = VMAP_BBMAP_BITS; /* prevent purging it again */ 907 vb->dirty_min = 0; 908 vb->dirty_max = VMAP_BBMAP_BITS; 909 spin_lock(&vbq->lock); 910 list_del_rcu(&vb->free_list); 911 spin_unlock(&vbq->lock); 912 spin_unlock(&vb->lock); 913 list_add_tail(&vb->purge, &purge); 914 } else 915 spin_unlock(&vb->lock); 916 } 917 rcu_read_unlock(); 918 919 list_for_each_entry_safe(vb, n_vb, &purge, purge) { 920 list_del(&vb->purge); 921 free_vmap_block(vb); 922 } 923 } 924 925 static void purge_fragmented_blocks_allcpus(void) 926 { 927 int cpu; 928 929 for_each_possible_cpu(cpu) 930 purge_fragmented_blocks(cpu); 931 } 932 933 static void *vb_alloc(unsigned long size, gfp_t gfp_mask) 934 { 935 struct vmap_block_queue *vbq; 936 struct vmap_block *vb; 937 void *vaddr = NULL; 938 unsigned int order; 939 940 BUG_ON(offset_in_page(size)); 941 BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); 942 if (WARN_ON(size == 0)) { 943 /* 944 * Allocating 0 bytes isn't what caller wants since 945 * get_order(0) returns funny result. Just warn and terminate 946 * early. 947 */ 948 return NULL; 949 } 950 order = get_order(size); 951 952 rcu_read_lock(); 953 vbq = &get_cpu_var(vmap_block_queue); 954 list_for_each_entry_rcu(vb, &vbq->free, free_list) { 955 unsigned long pages_off; 956 957 spin_lock(&vb->lock); 958 if (vb->free < (1UL << order)) { 959 spin_unlock(&vb->lock); 960 continue; 961 } 962 963 pages_off = VMAP_BBMAP_BITS - vb->free; 964 vaddr = vmap_block_vaddr(vb->va->va_start, pages_off); 965 vb->free -= 1UL << order; 966 if (vb->free == 0) { 967 spin_lock(&vbq->lock); 968 list_del_rcu(&vb->free_list); 969 spin_unlock(&vbq->lock); 970 } 971 972 spin_unlock(&vb->lock); 973 break; 974 } 975 976 put_cpu_var(vmap_block_queue); 977 rcu_read_unlock(); 978 979 /* Allocate new block if nothing was found */ 980 if (!vaddr) 981 vaddr = new_vmap_block(order, gfp_mask); 982 983 return vaddr; 984 } 985 986 static void vb_free(const void *addr, unsigned long size) 987 { 988 unsigned long offset; 989 unsigned long vb_idx; 990 unsigned int order; 991 struct vmap_block *vb; 992 993 BUG_ON(offset_in_page(size)); 994 BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); 995 996 flush_cache_vunmap((unsigned long)addr, (unsigned long)addr + size); 997 998 order = get_order(size); 999 1000 offset = (unsigned long)addr & (VMAP_BLOCK_SIZE - 1); 1001 offset >>= PAGE_SHIFT; 1002 1003 vb_idx = addr_to_vb_idx((unsigned long)addr); 1004 rcu_read_lock(); 1005 vb = radix_tree_lookup(&vmap_block_tree, vb_idx); 1006 rcu_read_unlock(); 1007 BUG_ON(!vb); 1008 1009 vunmap_page_range((unsigned long)addr, (unsigned long)addr + size); 1010 1011 spin_lock(&vb->lock); 1012 1013 /* Expand dirty range */ 1014 vb->dirty_min = min(vb->dirty_min, offset); 1015 vb->dirty_max = max(vb->dirty_max, offset + (1UL << order)); 1016 1017 vb->dirty += 1UL << order; 1018 if (vb->dirty == VMAP_BBMAP_BITS) { 1019 BUG_ON(vb->free); 1020 spin_unlock(&vb->lock); 1021 free_vmap_block(vb); 1022 } else 1023 spin_unlock(&vb->lock); 1024 } 1025 1026 /** 1027 * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer 1028 * 1029 * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily 1030 * to amortize TLB flushing overheads. What this means is that any page you 1031 * have now, may, in a former life, have been mapped into kernel virtual 1032 * address by the vmap layer and so there might be some CPUs with TLB entries 1033 * still referencing that page (additional to the regular 1:1 kernel mapping). 1034 * 1035 * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can 1036 * be sure that none of the pages we have control over will have any aliases 1037 * from the vmap layer. 1038 */ 1039 void vm_unmap_aliases(void) 1040 { 1041 unsigned long start = ULONG_MAX, end = 0; 1042 int cpu; 1043 int flush = 0; 1044 1045 if (unlikely(!vmap_initialized)) 1046 return; 1047 1048 for_each_possible_cpu(cpu) { 1049 struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); 1050 struct vmap_block *vb; 1051 1052 rcu_read_lock(); 1053 list_for_each_entry_rcu(vb, &vbq->free, free_list) { 1054 spin_lock(&vb->lock); 1055 if (vb->dirty) { 1056 unsigned long va_start = vb->va->va_start; 1057 unsigned long s, e; 1058 1059 s = va_start + (vb->dirty_min << PAGE_SHIFT); 1060 e = va_start + (vb->dirty_max << PAGE_SHIFT); 1061 1062 start = min(s, start); 1063 end = max(e, end); 1064 1065 flush = 1; 1066 } 1067 spin_unlock(&vb->lock); 1068 } 1069 rcu_read_unlock(); 1070 } 1071 1072 __purge_vmap_area_lazy(&start, &end, 1, flush); 1073 } 1074 EXPORT_SYMBOL_GPL(vm_unmap_aliases); 1075 1076 /** 1077 * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram 1078 * @mem: the pointer returned by vm_map_ram 1079 * @count: the count passed to that vm_map_ram call (cannot unmap partial) 1080 */ 1081 void vm_unmap_ram(const void *mem, unsigned int count) 1082 { 1083 unsigned long size = count << PAGE_SHIFT; 1084 unsigned long addr = (unsigned long)mem; 1085 1086 BUG_ON(!addr); 1087 BUG_ON(addr < VMALLOC_START); 1088 BUG_ON(addr > VMALLOC_END); 1089 BUG_ON(!IS_ALIGNED(addr, PAGE_SIZE)); 1090 1091 debug_check_no_locks_freed(mem, size); 1092 vmap_debug_free_range(addr, addr+size); 1093 1094 if (likely(count <= VMAP_MAX_ALLOC)) 1095 vb_free(mem, size); 1096 else 1097 free_unmap_vmap_area_addr(addr); 1098 } 1099 EXPORT_SYMBOL(vm_unmap_ram); 1100 1101 /** 1102 * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space) 1103 * @pages: an array of pointers to the pages to be mapped 1104 * @count: number of pages 1105 * @node: prefer to allocate data structures on this node 1106 * @prot: memory protection to use. PAGE_KERNEL for regular RAM 1107 * 1108 * If you use this function for less than VMAP_MAX_ALLOC pages, it could be 1109 * faster than vmap so it's good. But if you mix long-life and short-life 1110 * objects with vm_map_ram(), it could consume lots of address space through 1111 * fragmentation (especially on a 32bit machine). You could see failures in 1112 * the end. Please use this function for short-lived objects. 1113 * 1114 * Returns: a pointer to the address that has been mapped, or %NULL on failure 1115 */ 1116 void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot) 1117 { 1118 unsigned long size = count << PAGE_SHIFT; 1119 unsigned long addr; 1120 void *mem; 1121 1122 if (likely(count <= VMAP_MAX_ALLOC)) { 1123 mem = vb_alloc(size, GFP_KERNEL); 1124 if (IS_ERR(mem)) 1125 return NULL; 1126 addr = (unsigned long)mem; 1127 } else { 1128 struct vmap_area *va; 1129 va = alloc_vmap_area(size, PAGE_SIZE, 1130 VMALLOC_START, VMALLOC_END, node, GFP_KERNEL); 1131 if (IS_ERR(va)) 1132 return NULL; 1133 1134 addr = va->va_start; 1135 mem = (void *)addr; 1136 } 1137 if (vmap_page_range(addr, addr + size, prot, pages) < 0) { 1138 vm_unmap_ram(mem, count); 1139 return NULL; 1140 } 1141 return mem; 1142 } 1143 EXPORT_SYMBOL(vm_map_ram); 1144 1145 static struct vm_struct *vmlist __initdata; 1146 /** 1147 * vm_area_add_early - add vmap area early during boot 1148 * @vm: vm_struct to add 1149 * 1150 * This function is used to add fixed kernel vm area to vmlist before 1151 * vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags 1152 * should contain proper values and the other fields should be zero. 1153 * 1154 * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. 1155 */ 1156 void __init vm_area_add_early(struct vm_struct *vm) 1157 { 1158 struct vm_struct *tmp, **p; 1159 1160 BUG_ON(vmap_initialized); 1161 for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) { 1162 if (tmp->addr >= vm->addr) { 1163 BUG_ON(tmp->addr < vm->addr + vm->size); 1164 break; 1165 } else 1166 BUG_ON(tmp->addr + tmp->size > vm->addr); 1167 } 1168 vm->next = *p; 1169 *p = vm; 1170 } 1171 1172 /** 1173 * vm_area_register_early - register vmap area early during boot 1174 * @vm: vm_struct to register 1175 * @align: requested alignment 1176 * 1177 * This function is used to register kernel vm area before 1178 * vmalloc_init() is called. @vm->size and @vm->flags should contain 1179 * proper values on entry and other fields should be zero. On return, 1180 * vm->addr contains the allocated address. 1181 * 1182 * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. 1183 */ 1184 void __init vm_area_register_early(struct vm_struct *vm, size_t align) 1185 { 1186 static size_t vm_init_off __initdata; 1187 unsigned long addr; 1188 1189 addr = ALIGN(VMALLOC_START + vm_init_off, align); 1190 vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START; 1191 1192 vm->addr = (void *)addr; 1193 1194 vm_area_add_early(vm); 1195 } 1196 1197 void __init vmalloc_init(void) 1198 { 1199 struct vmap_area *va; 1200 struct vm_struct *tmp; 1201 int i; 1202 1203 for_each_possible_cpu(i) { 1204 struct vmap_block_queue *vbq; 1205 struct vfree_deferred *p; 1206 1207 vbq = &per_cpu(vmap_block_queue, i); 1208 spin_lock_init(&vbq->lock); 1209 INIT_LIST_HEAD(&vbq->free); 1210 p = &per_cpu(vfree_deferred, i); 1211 init_llist_head(&p->list); 1212 INIT_WORK(&p->wq, free_work); 1213 } 1214 1215 /* Import existing vmlist entries. */ 1216 for (tmp = vmlist; tmp; tmp = tmp->next) { 1217 va = kzalloc(sizeof(struct vmap_area), GFP_NOWAIT); 1218 va->flags = VM_VM_AREA; 1219 va->va_start = (unsigned long)tmp->addr; 1220 va->va_end = va->va_start + tmp->size; 1221 va->vm = tmp; 1222 __insert_vmap_area(va); 1223 } 1224 1225 vmap_area_pcpu_hole = VMALLOC_END; 1226 1227 vmap_initialized = true; 1228 } 1229 1230 /** 1231 * map_kernel_range_noflush - map kernel VM area with the specified pages 1232 * @addr: start of the VM area to map 1233 * @size: size of the VM area to map 1234 * @prot: page protection flags to use 1235 * @pages: pages to map 1236 * 1237 * Map PFN_UP(@size) pages at @addr. The VM area @addr and @size 1238 * specify should have been allocated using get_vm_area() and its 1239 * friends. 1240 * 1241 * NOTE: 1242 * This function does NOT do any cache flushing. The caller is 1243 * responsible for calling flush_cache_vmap() on to-be-mapped areas 1244 * before calling this function. 1245 * 1246 * RETURNS: 1247 * The number of pages mapped on success, -errno on failure. 1248 */ 1249 int map_kernel_range_noflush(unsigned long addr, unsigned long size, 1250 pgprot_t prot, struct page **pages) 1251 { 1252 return vmap_page_range_noflush(addr, addr + size, prot, pages); 1253 } 1254 1255 /** 1256 * unmap_kernel_range_noflush - unmap kernel VM area 1257 * @addr: start of the VM area to unmap 1258 * @size: size of the VM area to unmap 1259 * 1260 * Unmap PFN_UP(@size) pages at @addr. The VM area @addr and @size 1261 * specify should have been allocated using get_vm_area() and its 1262 * friends. 1263 * 1264 * NOTE: 1265 * This function does NOT do any cache flushing. The caller is 1266 * responsible for calling flush_cache_vunmap() on to-be-mapped areas 1267 * before calling this function and flush_tlb_kernel_range() after. 1268 */ 1269 void unmap_kernel_range_noflush(unsigned long addr, unsigned long size) 1270 { 1271 vunmap_page_range(addr, addr + size); 1272 } 1273 EXPORT_SYMBOL_GPL(unmap_kernel_range_noflush); 1274 1275 /** 1276 * unmap_kernel_range - unmap kernel VM area and flush cache and TLB 1277 * @addr: start of the VM area to unmap 1278 * @size: size of the VM area to unmap 1279 * 1280 * Similar to unmap_kernel_range_noflush() but flushes vcache before 1281 * the unmapping and tlb after. 1282 */ 1283 void unmap_kernel_range(unsigned long addr, unsigned long size) 1284 { 1285 unsigned long end = addr + size; 1286 1287 flush_cache_vunmap(addr, end); 1288 vunmap_page_range(addr, end); 1289 flush_tlb_kernel_range(addr, end); 1290 } 1291 EXPORT_SYMBOL_GPL(unmap_kernel_range); 1292 1293 int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page **pages) 1294 { 1295 unsigned long addr = (unsigned long)area->addr; 1296 unsigned long end = addr + get_vm_area_size(area); 1297 int err; 1298 1299 err = vmap_page_range(addr, end, prot, pages); 1300 1301 return err > 0 ? 0 : err; 1302 } 1303 EXPORT_SYMBOL_GPL(map_vm_area); 1304 1305 static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va, 1306 unsigned long flags, const void *caller) 1307 { 1308 spin_lock(&vmap_area_lock); 1309 vm->flags = flags; 1310 vm->addr = (void *)va->va_start; 1311 vm->size = va->va_end - va->va_start; 1312 vm->caller = caller; 1313 va->vm = vm; 1314 va->flags |= VM_VM_AREA; 1315 spin_unlock(&vmap_area_lock); 1316 } 1317 1318 static void clear_vm_uninitialized_flag(struct vm_struct *vm) 1319 { 1320 /* 1321 * Before removing VM_UNINITIALIZED, 1322 * we should make sure that vm has proper values. 1323 * Pair with smp_rmb() in show_numa_info(). 1324 */ 1325 smp_wmb(); 1326 vm->flags &= ~VM_UNINITIALIZED; 1327 } 1328 1329 static struct vm_struct *__get_vm_area_node(unsigned long size, 1330 unsigned long align, unsigned long flags, unsigned long start, 1331 unsigned long end, int node, gfp_t gfp_mask, const void *caller) 1332 { 1333 struct vmap_area *va; 1334 struct vm_struct *area; 1335 1336 BUG_ON(in_interrupt()); 1337 if (flags & VM_IOREMAP) 1338 align = 1ul << clamp_t(int, fls_long(size), 1339 PAGE_SHIFT, IOREMAP_MAX_ORDER); 1340 1341 size = PAGE_ALIGN(size); 1342 if (unlikely(!size)) 1343 return NULL; 1344 1345 area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node); 1346 if (unlikely(!area)) 1347 return NULL; 1348 1349 if (!(flags & VM_NO_GUARD)) 1350 size += PAGE_SIZE; 1351 1352 va = alloc_vmap_area(size, align, start, end, node, gfp_mask); 1353 if (IS_ERR(va)) { 1354 kfree(area); 1355 return NULL; 1356 } 1357 1358 setup_vmalloc_vm(area, va, flags, caller); 1359 1360 return area; 1361 } 1362 1363 struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags, 1364 unsigned long start, unsigned long end) 1365 { 1366 return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE, 1367 GFP_KERNEL, __builtin_return_address(0)); 1368 } 1369 EXPORT_SYMBOL_GPL(__get_vm_area); 1370 1371 struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags, 1372 unsigned long start, unsigned long end, 1373 const void *caller) 1374 { 1375 return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE, 1376 GFP_KERNEL, caller); 1377 } 1378 1379 /** 1380 * get_vm_area - reserve a contiguous kernel virtual area 1381 * @size: size of the area 1382 * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC 1383 * 1384 * Search an area of @size in the kernel virtual mapping area, 1385 * and reserved it for out purposes. Returns the area descriptor 1386 * on success or %NULL on failure. 1387 */ 1388 struct vm_struct *get_vm_area(unsigned long size, unsigned long flags) 1389 { 1390 return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END, 1391 NUMA_NO_NODE, GFP_KERNEL, 1392 __builtin_return_address(0)); 1393 } 1394 1395 struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags, 1396 const void *caller) 1397 { 1398 return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END, 1399 NUMA_NO_NODE, GFP_KERNEL, caller); 1400 } 1401 1402 /** 1403 * find_vm_area - find a continuous kernel virtual area 1404 * @addr: base address 1405 * 1406 * Search for the kernel VM area starting at @addr, and return it. 1407 * It is up to the caller to do all required locking to keep the returned 1408 * pointer valid. 1409 */ 1410 struct vm_struct *find_vm_area(const void *addr) 1411 { 1412 struct vmap_area *va; 1413 1414 va = find_vmap_area((unsigned long)addr); 1415 if (va && va->flags & VM_VM_AREA) 1416 return va->vm; 1417 1418 return NULL; 1419 } 1420 1421 /** 1422 * remove_vm_area - find and remove a continuous kernel virtual area 1423 * @addr: base address 1424 * 1425 * Search for the kernel VM area starting at @addr, and remove it. 1426 * This function returns the found VM area, but using it is NOT safe 1427 * on SMP machines, except for its size or flags. 1428 */ 1429 struct vm_struct *remove_vm_area(const void *addr) 1430 { 1431 struct vmap_area *va; 1432 1433 va = find_vmap_area((unsigned long)addr); 1434 if (va && va->flags & VM_VM_AREA) { 1435 struct vm_struct *vm = va->vm; 1436 1437 spin_lock(&vmap_area_lock); 1438 va->vm = NULL; 1439 va->flags &= ~VM_VM_AREA; 1440 spin_unlock(&vmap_area_lock); 1441 1442 vmap_debug_free_range(va->va_start, va->va_end); 1443 kasan_free_shadow(vm); 1444 free_unmap_vmap_area(va); 1445 1446 return vm; 1447 } 1448 return NULL; 1449 } 1450 1451 static void __vunmap(const void *addr, int deallocate_pages) 1452 { 1453 struct vm_struct *area; 1454 1455 if (!addr) 1456 return; 1457 1458 if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n", 1459 addr)) 1460 return; 1461 1462 area = remove_vm_area(addr); 1463 if (unlikely(!area)) { 1464 WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n", 1465 addr); 1466 return; 1467 } 1468 1469 debug_check_no_locks_freed(addr, get_vm_area_size(area)); 1470 debug_check_no_obj_freed(addr, get_vm_area_size(area)); 1471 1472 if (deallocate_pages) { 1473 int i; 1474 1475 for (i = 0; i < area->nr_pages; i++) { 1476 struct page *page = area->pages[i]; 1477 1478 BUG_ON(!page); 1479 __free_kmem_pages(page, 0); 1480 } 1481 1482 kvfree(area->pages); 1483 } 1484 1485 kfree(area); 1486 return; 1487 } 1488 1489 /** 1490 * vfree - release memory allocated by vmalloc() 1491 * @addr: memory base address 1492 * 1493 * Free the virtually continuous memory area starting at @addr, as 1494 * obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is 1495 * NULL, no operation is performed. 1496 * 1497 * Must not be called in NMI context (strictly speaking, only if we don't 1498 * have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling 1499 * conventions for vfree() arch-depenedent would be a really bad idea) 1500 * 1501 * NOTE: assumes that the object at *addr has a size >= sizeof(llist_node) 1502 */ 1503 void vfree(const void *addr) 1504 { 1505 BUG_ON(in_nmi()); 1506 1507 kmemleak_free(addr); 1508 1509 if (!addr) 1510 return; 1511 if (unlikely(in_interrupt())) { 1512 struct vfree_deferred *p = this_cpu_ptr(&vfree_deferred); 1513 if (llist_add((struct llist_node *)addr, &p->list)) 1514 schedule_work(&p->wq); 1515 } else 1516 __vunmap(addr, 1); 1517 } 1518 EXPORT_SYMBOL(vfree); 1519 1520 /** 1521 * vunmap - release virtual mapping obtained by vmap() 1522 * @addr: memory base address 1523 * 1524 * Free the virtually contiguous memory area starting at @addr, 1525 * which was created from the page array passed to vmap(). 1526 * 1527 * Must not be called in interrupt context. 1528 */ 1529 void vunmap(const void *addr) 1530 { 1531 BUG_ON(in_interrupt()); 1532 might_sleep(); 1533 if (addr) 1534 __vunmap(addr, 0); 1535 } 1536 EXPORT_SYMBOL(vunmap); 1537 1538 /** 1539 * vmap - map an array of pages into virtually contiguous space 1540 * @pages: array of page pointers 1541 * @count: number of pages to map 1542 * @flags: vm_area->flags 1543 * @prot: page protection for the mapping 1544 * 1545 * Maps @count pages from @pages into contiguous kernel virtual 1546 * space. 1547 */ 1548 void *vmap(struct page **pages, unsigned int count, 1549 unsigned long flags, pgprot_t prot) 1550 { 1551 struct vm_struct *area; 1552 1553 might_sleep(); 1554 1555 if (count > totalram_pages) 1556 return NULL; 1557 1558 area = get_vm_area_caller((count << PAGE_SHIFT), flags, 1559 __builtin_return_address(0)); 1560 if (!area) 1561 return NULL; 1562 1563 if (map_vm_area(area, prot, pages)) { 1564 vunmap(area->addr); 1565 return NULL; 1566 } 1567 1568 return area->addr; 1569 } 1570 EXPORT_SYMBOL(vmap); 1571 1572 static void *__vmalloc_node(unsigned long size, unsigned long align, 1573 gfp_t gfp_mask, pgprot_t prot, 1574 int node, const void *caller); 1575 static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, 1576 pgprot_t prot, int node) 1577 { 1578 const int order = 0; 1579 struct page **pages; 1580 unsigned int nr_pages, array_size, i; 1581 const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO; 1582 const gfp_t alloc_mask = gfp_mask | __GFP_NOWARN; 1583 1584 nr_pages = get_vm_area_size(area) >> PAGE_SHIFT; 1585 array_size = (nr_pages * sizeof(struct page *)); 1586 1587 area->nr_pages = nr_pages; 1588 /* Please note that the recursion is strictly bounded. */ 1589 if (array_size > PAGE_SIZE) { 1590 pages = __vmalloc_node(array_size, 1, nested_gfp|__GFP_HIGHMEM, 1591 PAGE_KERNEL, node, area->caller); 1592 } else { 1593 pages = kmalloc_node(array_size, nested_gfp, node); 1594 } 1595 area->pages = pages; 1596 if (!area->pages) { 1597 remove_vm_area(area->addr); 1598 kfree(area); 1599 return NULL; 1600 } 1601 1602 for (i = 0; i < area->nr_pages; i++) { 1603 struct page *page; 1604 1605 if (node == NUMA_NO_NODE) 1606 page = alloc_kmem_pages(alloc_mask, order); 1607 else 1608 page = alloc_kmem_pages_node(node, alloc_mask, order); 1609 1610 if (unlikely(!page)) { 1611 /* Successfully allocated i pages, free them in __vunmap() */ 1612 area->nr_pages = i; 1613 goto fail; 1614 } 1615 area->pages[i] = page; 1616 if (gfpflags_allow_blocking(gfp_mask)) 1617 cond_resched(); 1618 } 1619 1620 if (map_vm_area(area, prot, pages)) 1621 goto fail; 1622 return area->addr; 1623 1624 fail: 1625 warn_alloc_failed(gfp_mask, order, 1626 "vmalloc: allocation failure, allocated %ld of %ld bytes\n", 1627 (area->nr_pages*PAGE_SIZE), area->size); 1628 vfree(area->addr); 1629 return NULL; 1630 } 1631 1632 /** 1633 * __vmalloc_node_range - allocate virtually contiguous memory 1634 * @size: allocation size 1635 * @align: desired alignment 1636 * @start: vm area range start 1637 * @end: vm area range end 1638 * @gfp_mask: flags for the page level allocator 1639 * @prot: protection mask for the allocated pages 1640 * @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD) 1641 * @node: node to use for allocation or NUMA_NO_NODE 1642 * @caller: caller's return address 1643 * 1644 * Allocate enough pages to cover @size from the page level 1645 * allocator with @gfp_mask flags. Map them into contiguous 1646 * kernel virtual space, using a pagetable protection of @prot. 1647 */ 1648 void *__vmalloc_node_range(unsigned long size, unsigned long align, 1649 unsigned long start, unsigned long end, gfp_t gfp_mask, 1650 pgprot_t prot, unsigned long vm_flags, int node, 1651 const void *caller) 1652 { 1653 struct vm_struct *area; 1654 void *addr; 1655 unsigned long real_size = size; 1656 1657 size = PAGE_ALIGN(size); 1658 if (!size || (size >> PAGE_SHIFT) > totalram_pages) 1659 goto fail; 1660 1661 area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNINITIALIZED | 1662 vm_flags, start, end, node, gfp_mask, caller); 1663 if (!area) 1664 goto fail; 1665 1666 addr = __vmalloc_area_node(area, gfp_mask, prot, node); 1667 if (!addr) 1668 return NULL; 1669 1670 /* 1671 * In this function, newly allocated vm_struct has VM_UNINITIALIZED 1672 * flag. It means that vm_struct is not fully initialized. 1673 * Now, it is fully initialized, so remove this flag here. 1674 */ 1675 clear_vm_uninitialized_flag(area); 1676 1677 /* 1678 * A ref_count = 2 is needed because vm_struct allocated in 1679 * __get_vm_area_node() contains a reference to the virtual address of 1680 * the vmalloc'ed block. 1681 */ 1682 kmemleak_alloc(addr, real_size, 2, gfp_mask); 1683 1684 return addr; 1685 1686 fail: 1687 warn_alloc_failed(gfp_mask, 0, 1688 "vmalloc: allocation failure: %lu bytes\n", 1689 real_size); 1690 return NULL; 1691 } 1692 1693 /** 1694 * __vmalloc_node - allocate virtually contiguous memory 1695 * @size: allocation size 1696 * @align: desired alignment 1697 * @gfp_mask: flags for the page level allocator 1698 * @prot: protection mask for the allocated pages 1699 * @node: node to use for allocation or NUMA_NO_NODE 1700 * @caller: caller's return address 1701 * 1702 * Allocate enough pages to cover @size from the page level 1703 * allocator with @gfp_mask flags. Map them into contiguous 1704 * kernel virtual space, using a pagetable protection of @prot. 1705 */ 1706 static void *__vmalloc_node(unsigned long size, unsigned long align, 1707 gfp_t gfp_mask, pgprot_t prot, 1708 int node, const void *caller) 1709 { 1710 return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END, 1711 gfp_mask, prot, 0, node, caller); 1712 } 1713 1714 void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot) 1715 { 1716 return __vmalloc_node(size, 1, gfp_mask, prot, NUMA_NO_NODE, 1717 __builtin_return_address(0)); 1718 } 1719 EXPORT_SYMBOL(__vmalloc); 1720 1721 static inline void *__vmalloc_node_flags(unsigned long size, 1722 int node, gfp_t flags) 1723 { 1724 return __vmalloc_node(size, 1, flags, PAGE_KERNEL, 1725 node, __builtin_return_address(0)); 1726 } 1727 1728 /** 1729 * vmalloc - allocate virtually contiguous memory 1730 * @size: allocation size 1731 * Allocate enough pages to cover @size from the page level 1732 * allocator and map them into contiguous kernel virtual space. 1733 * 1734 * For tight control over page level allocator and protection flags 1735 * use __vmalloc() instead. 1736 */ 1737 void *vmalloc(unsigned long size) 1738 { 1739 return __vmalloc_node_flags(size, NUMA_NO_NODE, 1740 GFP_KERNEL | __GFP_HIGHMEM); 1741 } 1742 EXPORT_SYMBOL(vmalloc); 1743 1744 /** 1745 * vzalloc - allocate virtually contiguous memory with zero fill 1746 * @size: allocation size 1747 * Allocate enough pages to cover @size from the page level 1748 * allocator and map them into contiguous kernel virtual space. 1749 * The memory allocated is set to zero. 1750 * 1751 * For tight control over page level allocator and protection flags 1752 * use __vmalloc() instead. 1753 */ 1754 void *vzalloc(unsigned long size) 1755 { 1756 return __vmalloc_node_flags(size, NUMA_NO_NODE, 1757 GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO); 1758 } 1759 EXPORT_SYMBOL(vzalloc); 1760 1761 /** 1762 * vmalloc_user - allocate zeroed virtually contiguous memory for userspace 1763 * @size: allocation size 1764 * 1765 * The resulting memory area is zeroed so it can be mapped to userspace 1766 * without leaking data. 1767 */ 1768 void *vmalloc_user(unsigned long size) 1769 { 1770 struct vm_struct *area; 1771 void *ret; 1772 1773 ret = __vmalloc_node(size, SHMLBA, 1774 GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, 1775 PAGE_KERNEL, NUMA_NO_NODE, 1776 __builtin_return_address(0)); 1777 if (ret) { 1778 area = find_vm_area(ret); 1779 area->flags |= VM_USERMAP; 1780 } 1781 return ret; 1782 } 1783 EXPORT_SYMBOL(vmalloc_user); 1784 1785 /** 1786 * vmalloc_node - allocate memory on a specific node 1787 * @size: allocation size 1788 * @node: numa node 1789 * 1790 * Allocate enough pages to cover @size from the page level 1791 * allocator and map them into contiguous kernel virtual space. 1792 * 1793 * For tight control over page level allocator and protection flags 1794 * use __vmalloc() instead. 1795 */ 1796 void *vmalloc_node(unsigned long size, int node) 1797 { 1798 return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL, 1799 node, __builtin_return_address(0)); 1800 } 1801 EXPORT_SYMBOL(vmalloc_node); 1802 1803 /** 1804 * vzalloc_node - allocate memory on a specific node with zero fill 1805 * @size: allocation size 1806 * @node: numa node 1807 * 1808 * Allocate enough pages to cover @size from the page level 1809 * allocator and map them into contiguous kernel virtual space. 1810 * The memory allocated is set to zero. 1811 * 1812 * For tight control over page level allocator and protection flags 1813 * use __vmalloc_node() instead. 1814 */ 1815 void *vzalloc_node(unsigned long size, int node) 1816 { 1817 return __vmalloc_node_flags(size, node, 1818 GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO); 1819 } 1820 EXPORT_SYMBOL(vzalloc_node); 1821 1822 #ifndef PAGE_KERNEL_EXEC 1823 # define PAGE_KERNEL_EXEC PAGE_KERNEL 1824 #endif 1825 1826 /** 1827 * vmalloc_exec - allocate virtually contiguous, executable memory 1828 * @size: allocation size 1829 * 1830 * Kernel-internal function to allocate enough pages to cover @size 1831 * the page level allocator and map them into contiguous and 1832 * executable kernel virtual space. 1833 * 1834 * For tight control over page level allocator and protection flags 1835 * use __vmalloc() instead. 1836 */ 1837 1838 void *vmalloc_exec(unsigned long size) 1839 { 1840 return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC, 1841 NUMA_NO_NODE, __builtin_return_address(0)); 1842 } 1843 1844 #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32) 1845 #define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL 1846 #elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA) 1847 #define GFP_VMALLOC32 GFP_DMA | GFP_KERNEL 1848 #else 1849 #define GFP_VMALLOC32 GFP_KERNEL 1850 #endif 1851 1852 /** 1853 * vmalloc_32 - allocate virtually contiguous memory (32bit addressable) 1854 * @size: allocation size 1855 * 1856 * Allocate enough 32bit PA addressable pages to cover @size from the 1857 * page level allocator and map them into contiguous kernel virtual space. 1858 */ 1859 void *vmalloc_32(unsigned long size) 1860 { 1861 return __vmalloc_node(size, 1, GFP_VMALLOC32, PAGE_KERNEL, 1862 NUMA_NO_NODE, __builtin_return_address(0)); 1863 } 1864 EXPORT_SYMBOL(vmalloc_32); 1865 1866 /** 1867 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory 1868 * @size: allocation size 1869 * 1870 * The resulting memory area is 32bit addressable and zeroed so it can be 1871 * mapped to userspace without leaking data. 1872 */ 1873 void *vmalloc_32_user(unsigned long size) 1874 { 1875 struct vm_struct *area; 1876 void *ret; 1877 1878 ret = __vmalloc_node(size, 1, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL, 1879 NUMA_NO_NODE, __builtin_return_address(0)); 1880 if (ret) { 1881 area = find_vm_area(ret); 1882 area->flags |= VM_USERMAP; 1883 } 1884 return ret; 1885 } 1886 EXPORT_SYMBOL(vmalloc_32_user); 1887 1888 /* 1889 * small helper routine , copy contents to buf from addr. 1890 * If the page is not present, fill zero. 1891 */ 1892 1893 static int aligned_vread(char *buf, char *addr, unsigned long count) 1894 { 1895 struct page *p; 1896 int copied = 0; 1897 1898 while (count) { 1899 unsigned long offset, length; 1900 1901 offset = offset_in_page(addr); 1902 length = PAGE_SIZE - offset; 1903 if (length > count) 1904 length = count; 1905 p = vmalloc_to_page(addr); 1906 /* 1907 * To do safe access to this _mapped_ area, we need 1908 * lock. But adding lock here means that we need to add 1909 * overhead of vmalloc()/vfree() calles for this _debug_ 1910 * interface, rarely used. Instead of that, we'll use 1911 * kmap() and get small overhead in this access function. 1912 */ 1913 if (p) { 1914 /* 1915 * we can expect USER0 is not used (see vread/vwrite's 1916 * function description) 1917 */ 1918 void *map = kmap_atomic(p); 1919 memcpy(buf, map + offset, length); 1920 kunmap_atomic(map); 1921 } else 1922 memset(buf, 0, length); 1923 1924 addr += length; 1925 buf += length; 1926 copied += length; 1927 count -= length; 1928 } 1929 return copied; 1930 } 1931 1932 static int aligned_vwrite(char *buf, char *addr, unsigned long count) 1933 { 1934 struct page *p; 1935 int copied = 0; 1936 1937 while (count) { 1938 unsigned long offset, length; 1939 1940 offset = offset_in_page(addr); 1941 length = PAGE_SIZE - offset; 1942 if (length > count) 1943 length = count; 1944 p = vmalloc_to_page(addr); 1945 /* 1946 * To do safe access to this _mapped_ area, we need 1947 * lock. But adding lock here means that we need to add 1948 * overhead of vmalloc()/vfree() calles for this _debug_ 1949 * interface, rarely used. Instead of that, we'll use 1950 * kmap() and get small overhead in this access function. 1951 */ 1952 if (p) { 1953 /* 1954 * we can expect USER0 is not used (see vread/vwrite's 1955 * function description) 1956 */ 1957 void *map = kmap_atomic(p); 1958 memcpy(map + offset, buf, length); 1959 kunmap_atomic(map); 1960 } 1961 addr += length; 1962 buf += length; 1963 copied += length; 1964 count -= length; 1965 } 1966 return copied; 1967 } 1968 1969 /** 1970 * vread() - read vmalloc area in a safe way. 1971 * @buf: buffer for reading data 1972 * @addr: vm address. 1973 * @count: number of bytes to be read. 1974 * 1975 * Returns # of bytes which addr and buf should be increased. 1976 * (same number to @count). Returns 0 if [addr...addr+count) doesn't 1977 * includes any intersect with alive vmalloc area. 1978 * 1979 * This function checks that addr is a valid vmalloc'ed area, and 1980 * copy data from that area to a given buffer. If the given memory range 1981 * of [addr...addr+count) includes some valid address, data is copied to 1982 * proper area of @buf. If there are memory holes, they'll be zero-filled. 1983 * IOREMAP area is treated as memory hole and no copy is done. 1984 * 1985 * If [addr...addr+count) doesn't includes any intersects with alive 1986 * vm_struct area, returns 0. @buf should be kernel's buffer. 1987 * 1988 * Note: In usual ops, vread() is never necessary because the caller 1989 * should know vmalloc() area is valid and can use memcpy(). 1990 * This is for routines which have to access vmalloc area without 1991 * any informaion, as /dev/kmem. 1992 * 1993 */ 1994 1995 long vread(char *buf, char *addr, unsigned long count) 1996 { 1997 struct vmap_area *va; 1998 struct vm_struct *vm; 1999 char *vaddr, *buf_start = buf; 2000 unsigned long buflen = count; 2001 unsigned long n; 2002 2003 /* Don't allow overflow */ 2004 if ((unsigned long) addr + count < count) 2005 count = -(unsigned long) addr; 2006 2007 spin_lock(&vmap_area_lock); 2008 list_for_each_entry(va, &vmap_area_list, list) { 2009 if (!count) 2010 break; 2011 2012 if (!(va->flags & VM_VM_AREA)) 2013 continue; 2014 2015 vm = va->vm; 2016 vaddr = (char *) vm->addr; 2017 if (addr >= vaddr + get_vm_area_size(vm)) 2018 continue; 2019 while (addr < vaddr) { 2020 if (count == 0) 2021 goto finished; 2022 *buf = '\0'; 2023 buf++; 2024 addr++; 2025 count--; 2026 } 2027 n = vaddr + get_vm_area_size(vm) - addr; 2028 if (n > count) 2029 n = count; 2030 if (!(vm->flags & VM_IOREMAP)) 2031 aligned_vread(buf, addr, n); 2032 else /* IOREMAP area is treated as memory hole */ 2033 memset(buf, 0, n); 2034 buf += n; 2035 addr += n; 2036 count -= n; 2037 } 2038 finished: 2039 spin_unlock(&vmap_area_lock); 2040 2041 if (buf == buf_start) 2042 return 0; 2043 /* zero-fill memory holes */ 2044 if (buf != buf_start + buflen) 2045 memset(buf, 0, buflen - (buf - buf_start)); 2046 2047 return buflen; 2048 } 2049 2050 /** 2051 * vwrite() - write vmalloc area in a safe way. 2052 * @buf: buffer for source data 2053 * @addr: vm address. 2054 * @count: number of bytes to be read. 2055 * 2056 * Returns # of bytes which addr and buf should be incresed. 2057 * (same number to @count). 2058 * If [addr...addr+count) doesn't includes any intersect with valid 2059 * vmalloc area, returns 0. 2060 * 2061 * This function checks that addr is a valid vmalloc'ed area, and 2062 * copy data from a buffer to the given addr. If specified range of 2063 * [addr...addr+count) includes some valid address, data is copied from 2064 * proper area of @buf. If there are memory holes, no copy to hole. 2065 * IOREMAP area is treated as memory hole and no copy is done. 2066 * 2067 * If [addr...addr+count) doesn't includes any intersects with alive 2068 * vm_struct area, returns 0. @buf should be kernel's buffer. 2069 * 2070 * Note: In usual ops, vwrite() is never necessary because the caller 2071 * should know vmalloc() area is valid and can use memcpy(). 2072 * This is for routines which have to access vmalloc area without 2073 * any informaion, as /dev/kmem. 2074 */ 2075 2076 long vwrite(char *buf, char *addr, unsigned long count) 2077 { 2078 struct vmap_area *va; 2079 struct vm_struct *vm; 2080 char *vaddr; 2081 unsigned long n, buflen; 2082 int copied = 0; 2083 2084 /* Don't allow overflow */ 2085 if ((unsigned long) addr + count < count) 2086 count = -(unsigned long) addr; 2087 buflen = count; 2088 2089 spin_lock(&vmap_area_lock); 2090 list_for_each_entry(va, &vmap_area_list, list) { 2091 if (!count) 2092 break; 2093 2094 if (!(va->flags & VM_VM_AREA)) 2095 continue; 2096 2097 vm = va->vm; 2098 vaddr = (char *) vm->addr; 2099 if (addr >= vaddr + get_vm_area_size(vm)) 2100 continue; 2101 while (addr < vaddr) { 2102 if (count == 0) 2103 goto finished; 2104 buf++; 2105 addr++; 2106 count--; 2107 } 2108 n = vaddr + get_vm_area_size(vm) - addr; 2109 if (n > count) 2110 n = count; 2111 if (!(vm->flags & VM_IOREMAP)) { 2112 aligned_vwrite(buf, addr, n); 2113 copied++; 2114 } 2115 buf += n; 2116 addr += n; 2117 count -= n; 2118 } 2119 finished: 2120 spin_unlock(&vmap_area_lock); 2121 if (!copied) 2122 return 0; 2123 return buflen; 2124 } 2125 2126 /** 2127 * remap_vmalloc_range_partial - map vmalloc pages to userspace 2128 * @vma: vma to cover 2129 * @uaddr: target user address to start at 2130 * @kaddr: virtual address of vmalloc kernel memory 2131 * @size: size of map area 2132 * 2133 * Returns: 0 for success, -Exxx on failure 2134 * 2135 * This function checks that @kaddr is a valid vmalloc'ed area, 2136 * and that it is big enough to cover the range starting at 2137 * @uaddr in @vma. Will return failure if that criteria isn't 2138 * met. 2139 * 2140 * Similar to remap_pfn_range() (see mm/memory.c) 2141 */ 2142 int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr, 2143 void *kaddr, unsigned long size) 2144 { 2145 struct vm_struct *area; 2146 2147 size = PAGE_ALIGN(size); 2148 2149 if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr)) 2150 return -EINVAL; 2151 2152 area = find_vm_area(kaddr); 2153 if (!area) 2154 return -EINVAL; 2155 2156 if (!(area->flags & VM_USERMAP)) 2157 return -EINVAL; 2158 2159 if (kaddr + size > area->addr + area->size) 2160 return -EINVAL; 2161 2162 do { 2163 struct page *page = vmalloc_to_page(kaddr); 2164 int ret; 2165 2166 ret = vm_insert_page(vma, uaddr, page); 2167 if (ret) 2168 return ret; 2169 2170 uaddr += PAGE_SIZE; 2171 kaddr += PAGE_SIZE; 2172 size -= PAGE_SIZE; 2173 } while (size > 0); 2174 2175 vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP; 2176 2177 return 0; 2178 } 2179 EXPORT_SYMBOL(remap_vmalloc_range_partial); 2180 2181 /** 2182 * remap_vmalloc_range - map vmalloc pages to userspace 2183 * @vma: vma to cover (map full range of vma) 2184 * @addr: vmalloc memory 2185 * @pgoff: number of pages into addr before first page to map 2186 * 2187 * Returns: 0 for success, -Exxx on failure 2188 * 2189 * This function checks that addr is a valid vmalloc'ed area, and 2190 * that it is big enough to cover the vma. Will return failure if 2191 * that criteria isn't met. 2192 * 2193 * Similar to remap_pfn_range() (see mm/memory.c) 2194 */ 2195 int remap_vmalloc_range(struct vm_area_struct *vma, void *addr, 2196 unsigned long pgoff) 2197 { 2198 return remap_vmalloc_range_partial(vma, vma->vm_start, 2199 addr + (pgoff << PAGE_SHIFT), 2200 vma->vm_end - vma->vm_start); 2201 } 2202 EXPORT_SYMBOL(remap_vmalloc_range); 2203 2204 /* 2205 * Implement a stub for vmalloc_sync_all() if the architecture chose not to 2206 * have one. 2207 */ 2208 void __weak vmalloc_sync_all(void) 2209 { 2210 } 2211 2212 2213 static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data) 2214 { 2215 pte_t ***p = data; 2216 2217 if (p) { 2218 *(*p) = pte; 2219 (*p)++; 2220 } 2221 return 0; 2222 } 2223 2224 /** 2225 * alloc_vm_area - allocate a range of kernel address space 2226 * @size: size of the area 2227 * @ptes: returns the PTEs for the address space 2228 * 2229 * Returns: NULL on failure, vm_struct on success 2230 * 2231 * This function reserves a range of kernel address space, and 2232 * allocates pagetables to map that range. No actual mappings 2233 * are created. 2234 * 2235 * If @ptes is non-NULL, pointers to the PTEs (in init_mm) 2236 * allocated for the VM area are returned. 2237 */ 2238 struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes) 2239 { 2240 struct vm_struct *area; 2241 2242 area = get_vm_area_caller(size, VM_IOREMAP, 2243 __builtin_return_address(0)); 2244 if (area == NULL) 2245 return NULL; 2246 2247 /* 2248 * This ensures that page tables are constructed for this region 2249 * of kernel virtual address space and mapped into init_mm. 2250 */ 2251 if (apply_to_page_range(&init_mm, (unsigned long)area->addr, 2252 size, f, ptes ? &ptes : NULL)) { 2253 free_vm_area(area); 2254 return NULL; 2255 } 2256 2257 return area; 2258 } 2259 EXPORT_SYMBOL_GPL(alloc_vm_area); 2260 2261 void free_vm_area(struct vm_struct *area) 2262 { 2263 struct vm_struct *ret; 2264 ret = remove_vm_area(area->addr); 2265 BUG_ON(ret != area); 2266 kfree(area); 2267 } 2268 EXPORT_SYMBOL_GPL(free_vm_area); 2269 2270 #ifdef CONFIG_SMP 2271 static struct vmap_area *node_to_va(struct rb_node *n) 2272 { 2273 return n ? rb_entry(n, struct vmap_area, rb_node) : NULL; 2274 } 2275 2276 /** 2277 * pvm_find_next_prev - find the next and prev vmap_area surrounding @end 2278 * @end: target address 2279 * @pnext: out arg for the next vmap_area 2280 * @pprev: out arg for the previous vmap_area 2281 * 2282 * Returns: %true if either or both of next and prev are found, 2283 * %false if no vmap_area exists 2284 * 2285 * Find vmap_areas end addresses of which enclose @end. ie. if not 2286 * NULL, *pnext->va_end > @end and *pprev->va_end <= @end. 2287 */ 2288 static bool pvm_find_next_prev(unsigned long end, 2289 struct vmap_area **pnext, 2290 struct vmap_area **pprev) 2291 { 2292 struct rb_node *n = vmap_area_root.rb_node; 2293 struct vmap_area *va = NULL; 2294 2295 while (n) { 2296 va = rb_entry(n, struct vmap_area, rb_node); 2297 if (end < va->va_end) 2298 n = n->rb_left; 2299 else if (end > va->va_end) 2300 n = n->rb_right; 2301 else 2302 break; 2303 } 2304 2305 if (!va) 2306 return false; 2307 2308 if (va->va_end > end) { 2309 *pnext = va; 2310 *pprev = node_to_va(rb_prev(&(*pnext)->rb_node)); 2311 } else { 2312 *pprev = va; 2313 *pnext = node_to_va(rb_next(&(*pprev)->rb_node)); 2314 } 2315 return true; 2316 } 2317 2318 /** 2319 * pvm_determine_end - find the highest aligned address between two vmap_areas 2320 * @pnext: in/out arg for the next vmap_area 2321 * @pprev: in/out arg for the previous vmap_area 2322 * @align: alignment 2323 * 2324 * Returns: determined end address 2325 * 2326 * Find the highest aligned address between *@pnext and *@pprev below 2327 * VMALLOC_END. *@pnext and *@pprev are adjusted so that the aligned 2328 * down address is between the end addresses of the two vmap_areas. 2329 * 2330 * Please note that the address returned by this function may fall 2331 * inside *@pnext vmap_area. The caller is responsible for checking 2332 * that. 2333 */ 2334 static unsigned long pvm_determine_end(struct vmap_area **pnext, 2335 struct vmap_area **pprev, 2336 unsigned long align) 2337 { 2338 const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); 2339 unsigned long addr; 2340 2341 if (*pnext) 2342 addr = min((*pnext)->va_start & ~(align - 1), vmalloc_end); 2343 else 2344 addr = vmalloc_end; 2345 2346 while (*pprev && (*pprev)->va_end > addr) { 2347 *pnext = *pprev; 2348 *pprev = node_to_va(rb_prev(&(*pnext)->rb_node)); 2349 } 2350 2351 return addr; 2352 } 2353 2354 /** 2355 * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator 2356 * @offsets: array containing offset of each area 2357 * @sizes: array containing size of each area 2358 * @nr_vms: the number of areas to allocate 2359 * @align: alignment, all entries in @offsets and @sizes must be aligned to this 2360 * 2361 * Returns: kmalloc'd vm_struct pointer array pointing to allocated 2362 * vm_structs on success, %NULL on failure 2363 * 2364 * Percpu allocator wants to use congruent vm areas so that it can 2365 * maintain the offsets among percpu areas. This function allocates 2366 * congruent vmalloc areas for it with GFP_KERNEL. These areas tend to 2367 * be scattered pretty far, distance between two areas easily going up 2368 * to gigabytes. To avoid interacting with regular vmallocs, these 2369 * areas are allocated from top. 2370 * 2371 * Despite its complicated look, this allocator is rather simple. It 2372 * does everything top-down and scans areas from the end looking for 2373 * matching slot. While scanning, if any of the areas overlaps with 2374 * existing vmap_area, the base address is pulled down to fit the 2375 * area. Scanning is repeated till all the areas fit and then all 2376 * necessary data structres are inserted and the result is returned. 2377 */ 2378 struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, 2379 const size_t *sizes, int nr_vms, 2380 size_t align) 2381 { 2382 const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align); 2383 const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); 2384 struct vmap_area **vas, *prev, *next; 2385 struct vm_struct **vms; 2386 int area, area2, last_area, term_area; 2387 unsigned long base, start, end, last_end; 2388 bool purged = false; 2389 2390 /* verify parameters and allocate data structures */ 2391 BUG_ON(offset_in_page(align) || !is_power_of_2(align)); 2392 for (last_area = 0, area = 0; area < nr_vms; area++) { 2393 start = offsets[area]; 2394 end = start + sizes[area]; 2395 2396 /* is everything aligned properly? */ 2397 BUG_ON(!IS_ALIGNED(offsets[area], align)); 2398 BUG_ON(!IS_ALIGNED(sizes[area], align)); 2399 2400 /* detect the area with the highest address */ 2401 if (start > offsets[last_area]) 2402 last_area = area; 2403 2404 for (area2 = 0; area2 < nr_vms; area2++) { 2405 unsigned long start2 = offsets[area2]; 2406 unsigned long end2 = start2 + sizes[area2]; 2407 2408 if (area2 == area) 2409 continue; 2410 2411 BUG_ON(start2 >= start && start2 < end); 2412 BUG_ON(end2 <= end && end2 > start); 2413 } 2414 } 2415 last_end = offsets[last_area] + sizes[last_area]; 2416 2417 if (vmalloc_end - vmalloc_start < last_end) { 2418 WARN_ON(true); 2419 return NULL; 2420 } 2421 2422 vms = kcalloc(nr_vms, sizeof(vms[0]), GFP_KERNEL); 2423 vas = kcalloc(nr_vms, sizeof(vas[0]), GFP_KERNEL); 2424 if (!vas || !vms) 2425 goto err_free2; 2426 2427 for (area = 0; area < nr_vms; area++) { 2428 vas[area] = kzalloc(sizeof(struct vmap_area), GFP_KERNEL); 2429 vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL); 2430 if (!vas[area] || !vms[area]) 2431 goto err_free; 2432 } 2433 retry: 2434 spin_lock(&vmap_area_lock); 2435 2436 /* start scanning - we scan from the top, begin with the last area */ 2437 area = term_area = last_area; 2438 start = offsets[area]; 2439 end = start + sizes[area]; 2440 2441 if (!pvm_find_next_prev(vmap_area_pcpu_hole, &next, &prev)) { 2442 base = vmalloc_end - last_end; 2443 goto found; 2444 } 2445 base = pvm_determine_end(&next, &prev, align) - end; 2446 2447 while (true) { 2448 BUG_ON(next && next->va_end <= base + end); 2449 BUG_ON(prev && prev->va_end > base + end); 2450 2451 /* 2452 * base might have underflowed, add last_end before 2453 * comparing. 2454 */ 2455 if (base + last_end < vmalloc_start + last_end) { 2456 spin_unlock(&vmap_area_lock); 2457 if (!purged) { 2458 purge_vmap_area_lazy(); 2459 purged = true; 2460 goto retry; 2461 } 2462 goto err_free; 2463 } 2464 2465 /* 2466 * If next overlaps, move base downwards so that it's 2467 * right below next and then recheck. 2468 */ 2469 if (next && next->va_start < base + end) { 2470 base = pvm_determine_end(&next, &prev, align) - end; 2471 term_area = area; 2472 continue; 2473 } 2474 2475 /* 2476 * If prev overlaps, shift down next and prev and move 2477 * base so that it's right below new next and then 2478 * recheck. 2479 */ 2480 if (prev && prev->va_end > base + start) { 2481 next = prev; 2482 prev = node_to_va(rb_prev(&next->rb_node)); 2483 base = pvm_determine_end(&next, &prev, align) - end; 2484 term_area = area; 2485 continue; 2486 } 2487 2488 /* 2489 * This area fits, move on to the previous one. If 2490 * the previous one is the terminal one, we're done. 2491 */ 2492 area = (area + nr_vms - 1) % nr_vms; 2493 if (area == term_area) 2494 break; 2495 start = offsets[area]; 2496 end = start + sizes[area]; 2497 pvm_find_next_prev(base + end, &next, &prev); 2498 } 2499 found: 2500 /* we've found a fitting base, insert all va's */ 2501 for (area = 0; area < nr_vms; area++) { 2502 struct vmap_area *va = vas[area]; 2503 2504 va->va_start = base + offsets[area]; 2505 va->va_end = va->va_start + sizes[area]; 2506 __insert_vmap_area(va); 2507 } 2508 2509 vmap_area_pcpu_hole = base + offsets[last_area]; 2510 2511 spin_unlock(&vmap_area_lock); 2512 2513 /* insert all vm's */ 2514 for (area = 0; area < nr_vms; area++) 2515 setup_vmalloc_vm(vms[area], vas[area], VM_ALLOC, 2516 pcpu_get_vm_areas); 2517 2518 kfree(vas); 2519 return vms; 2520 2521 err_free: 2522 for (area = 0; area < nr_vms; area++) { 2523 kfree(vas[area]); 2524 kfree(vms[area]); 2525 } 2526 err_free2: 2527 kfree(vas); 2528 kfree(vms); 2529 return NULL; 2530 } 2531 2532 /** 2533 * pcpu_free_vm_areas - free vmalloc areas for percpu allocator 2534 * @vms: vm_struct pointer array returned by pcpu_get_vm_areas() 2535 * @nr_vms: the number of allocated areas 2536 * 2537 * Free vm_structs and the array allocated by pcpu_get_vm_areas(). 2538 */ 2539 void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms) 2540 { 2541 int i; 2542 2543 for (i = 0; i < nr_vms; i++) 2544 free_vm_area(vms[i]); 2545 kfree(vms); 2546 } 2547 #endif /* CONFIG_SMP */ 2548 2549 #ifdef CONFIG_PROC_FS 2550 static void *s_start(struct seq_file *m, loff_t *pos) 2551 __acquires(&vmap_area_lock) 2552 { 2553 loff_t n = *pos; 2554 struct vmap_area *va; 2555 2556 spin_lock(&vmap_area_lock); 2557 va = list_first_entry(&vmap_area_list, typeof(*va), list); 2558 while (n > 0 && &va->list != &vmap_area_list) { 2559 n--; 2560 va = list_next_entry(va, list); 2561 } 2562 if (!n && &va->list != &vmap_area_list) 2563 return va; 2564 2565 return NULL; 2566 2567 } 2568 2569 static void *s_next(struct seq_file *m, void *p, loff_t *pos) 2570 { 2571 struct vmap_area *va = p, *next; 2572 2573 ++*pos; 2574 next = list_next_entry(va, list); 2575 if (&next->list != &vmap_area_list) 2576 return next; 2577 2578 return NULL; 2579 } 2580 2581 static void s_stop(struct seq_file *m, void *p) 2582 __releases(&vmap_area_lock) 2583 { 2584 spin_unlock(&vmap_area_lock); 2585 } 2586 2587 static void show_numa_info(struct seq_file *m, struct vm_struct *v) 2588 { 2589 if (IS_ENABLED(CONFIG_NUMA)) { 2590 unsigned int nr, *counters = m->private; 2591 2592 if (!counters) 2593 return; 2594 2595 if (v->flags & VM_UNINITIALIZED) 2596 return; 2597 /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */ 2598 smp_rmb(); 2599 2600 memset(counters, 0, nr_node_ids * sizeof(unsigned int)); 2601 2602 for (nr = 0; nr < v->nr_pages; nr++) 2603 counters[page_to_nid(v->pages[nr])]++; 2604 2605 for_each_node_state(nr, N_HIGH_MEMORY) 2606 if (counters[nr]) 2607 seq_printf(m, " N%u=%u", nr, counters[nr]); 2608 } 2609 } 2610 2611 static int s_show(struct seq_file *m, void *p) 2612 { 2613 struct vmap_area *va = p; 2614 struct vm_struct *v; 2615 2616 /* 2617 * s_show can encounter race with remove_vm_area, !VM_VM_AREA on 2618 * behalf of vmap area is being tear down or vm_map_ram allocation. 2619 */ 2620 if (!(va->flags & VM_VM_AREA)) 2621 return 0; 2622 2623 v = va->vm; 2624 2625 seq_printf(m, "0x%pK-0x%pK %7ld", 2626 v->addr, v->addr + v->size, v->size); 2627 2628 if (v->caller) 2629 seq_printf(m, " %pS", v->caller); 2630 2631 if (v->nr_pages) 2632 seq_printf(m, " pages=%d", v->nr_pages); 2633 2634 if (v->phys_addr) 2635 seq_printf(m, " phys=%llx", (unsigned long long)v->phys_addr); 2636 2637 if (v->flags & VM_IOREMAP) 2638 seq_puts(m, " ioremap"); 2639 2640 if (v->flags & VM_ALLOC) 2641 seq_puts(m, " vmalloc"); 2642 2643 if (v->flags & VM_MAP) 2644 seq_puts(m, " vmap"); 2645 2646 if (v->flags & VM_USERMAP) 2647 seq_puts(m, " user"); 2648 2649 if (is_vmalloc_addr(v->pages)) 2650 seq_puts(m, " vpages"); 2651 2652 show_numa_info(m, v); 2653 seq_putc(m, '\n'); 2654 return 0; 2655 } 2656 2657 static const struct seq_operations vmalloc_op = { 2658 .start = s_start, 2659 .next = s_next, 2660 .stop = s_stop, 2661 .show = s_show, 2662 }; 2663 2664 static int vmalloc_open(struct inode *inode, struct file *file) 2665 { 2666 if (IS_ENABLED(CONFIG_NUMA)) 2667 return seq_open_private(file, &vmalloc_op, 2668 nr_node_ids * sizeof(unsigned int)); 2669 else 2670 return seq_open(file, &vmalloc_op); 2671 } 2672 2673 static const struct file_operations proc_vmalloc_operations = { 2674 .open = vmalloc_open, 2675 .read = seq_read, 2676 .llseek = seq_lseek, 2677 .release = seq_release_private, 2678 }; 2679 2680 static int __init proc_vmalloc_init(void) 2681 { 2682 proc_create("vmallocinfo", S_IRUSR, NULL, &proc_vmalloc_operations); 2683 return 0; 2684 } 2685 module_init(proc_vmalloc_init); 2686 2687 #endif 2688 2689