Lines Matching +full:vm +full:- +full:map
1 // SPDX-License-Identifier: GPL-2.0-only
5 * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
50 #include "pgalloc-track.h"
53 static unsigned int __ro_after_init ioremap_max_page_shift = BITS_PER_LONG - 1;
104 return -ENOMEM;
136 if ((end - addr) != PMD_SIZE)
160 return -ENOMEM;
171 return -ENOMEM;
172 } while (pmd++, phys_addr += (next - addr), addr = next, addr != end);
186 if ((end - addr) != PUD_SIZE)
210 return -ENOMEM;
222 return -ENOMEM;
223 } while (pud++, phys_addr += (next - addr), addr = next, addr != end);
237 if ((end - addr) != P4D_SIZE)
261 return -ENOMEM;
273 return -ENOMEM;
274 } while (p4d++, phys_addr += (next - addr), addr = next, addr != end);
299 } while (pgd++, phys_addr += (next - addr), addr = next, addr != end);
409 * or be re-mapped for something else, if TLB flushes are being delayed or
443 * vunmap_range - unmap kernel virtual addresses
444 * @addr: start of the VM area to unmap
445 * @end: end of the VM area to unmap (non-inclusive)
448 * caches. Any subsequent access to the address before it has been re-mapped
471 return -ENOMEM;
476 return -EBUSY;
478 return -ENOMEM;
480 return -EINVAL;
498 return -ENOMEM;
502 return -ENOMEM;
516 return -ENOMEM;
520 return -ENOMEM;
534 return -ENOMEM;
538 return -ENOMEM;
582 unsigned int i, nr = (end - addr) >> PAGE_SHIFT;
590 for (i = 0; i < nr; i += 1U << (page_shift - PAGE_SHIFT)) {
617 * vmap_pages_range - map pages to a kernel virtual address
618 * @addr: start of the VM area to map
619 * @end: end of the VM area to map (non-inclusive)
621 * @pages: pages to map (always PAGE_SIZE pages)
626 * 0 on success, -errno on failure.
641 * ARM, x86-64 and sparc64 put modules in a special place,
716 * Map a vmalloc()-space virtual address to the physical page frame number.
757 * This augment red-black tree represents the free vmap space.
758 * All vmap_area objects in this tree are sorted by va->va_start
763 * of its sub-tree, right or left. Therefore it is possible to
778 return (va->va_end - va->va_start);
787 return va ? va->subtree_max_size : 0;
817 if (tmp->va_end > addr) {
819 if (tmp->va_start <= addr)
822 n = n->rb_left;
824 n = n->rb_right;
832 struct rb_node *n = root->rb_node;
840 if (addr < va->va_start)
841 n = n->rb_left;
842 else if (addr >= va->va_end)
843 n = n->rb_right;
868 link = &root->rb_node;
880 * it link, where the new va->rb_node will be attached to.
890 if (va->va_end <= tmp_va->va_start)
891 link = &(*link)->rb_left;
892 else if (va->va_start >= tmp_va->va_end)
893 link = &(*link)->rb_right;
895 WARN(1, "vmalloc bug: 0x%lx-0x%lx overlaps with 0x%lx-0x%lx\n",
896 va->va_start, va->va_end, tmp_va->va_start, tmp_va->va_end);
902 *parent = &tmp_va->rb_node;
913 * The red-black tree where we try to find VA neighbors
920 list = &rb_entry(parent, struct vmap_area, rb_node)->list;
921 return (&parent->rb_right == link ? list->next : list);
934 head = &rb_entry(parent, struct vmap_area, rb_node)->list;
935 if (&parent->rb_right != link)
936 head = head->prev;
939 /* Insert to the rb-tree */
940 rb_link_node(&va->rb_node, parent, link);
944 * to the tree. We do not set va->subtree_max_size to
953 rb_insert_augmented(&va->rb_node,
955 va->subtree_max_size = 0;
957 rb_insert_color(&va->rb_node, root);
960 /* Address-sort this list */
961 list_add(&va->list, head);
983 if (WARN_ON(RB_EMPTY_NODE(&va->rb_node)))
987 rb_erase_augmented(&va->rb_node,
990 rb_erase(&va->rb_node, root);
992 list_del_init(&va->list);
993 RB_CLEAR_NODE(&va->rb_node);
1016 get_subtree_max_size(va->rb_node.rb_left),
1017 get_subtree_max_size(va->rb_node.rb_right));
1028 if (computed_size != va->subtree_max_size)
1030 va_size(va), va->subtree_max_size);
1042 * - After VA has been inserted to the tree(free path);
1043 * - After VA has been shrunk(allocation path);
1044 * - After VA has been increased(merging path).
1050 * 4--8
1054 * 2--2 8--8
1060 * node becomes 4--6.
1070 free_vmap_area_rb_augment_cb_propagate(&va->rb_node, NULL);
1109 * Merge de-allocated chunk of VA memory with previous
1147 * |<------VA------>|<-----Next----->|
1153 if (sibling->va_start == va->va_end) {
1154 sibling->va_start = va->va_start;
1168 * |<-----Prev----->|<------VA------>|
1172 if (next->prev != head) {
1173 sibling = list_entry(next->prev, struct vmap_area, list);
1174 if (sibling->va_end == va->va_start) {
1185 sibling->va_end = va->va_end;
1227 if (va->va_start > vstart)
1228 nva_start_addr = ALIGN(va->va_start, align);
1237 return (nva_start_addr + size <= va->va_end);
1256 node = root->rb_node;
1259 length = adjust_search_size ? size + align - 1 : size;
1264 if (get_subtree_max_size(node->rb_left) >= length &&
1265 vstart < va->va_start) {
1266 node = node->rb_left;
1273 * sub-tree if it does not have a free block that is
1276 if (get_subtree_max_size(node->rb_right) >= length) {
1277 node = node->rb_right;
1282 * OK. We roll back and find the first right sub-tree,
1292 if (get_subtree_max_size(node->rb_right) >= length &&
1293 vstart <= va->va_start) {
1297 * to enter same sub-tree after it has already been checked
1300 vstart = va->va_start + 1;
1301 node = node->rb_right;
1365 if (nva_start_addr < va->va_start ||
1366 nva_start_addr + size > va->va_end)
1370 if (va->va_start == nva_start_addr) {
1371 if (va->va_end == nva_start_addr + size)
1375 } else if (va->va_end == nva_start_addr + size) {
1398 * |---------------|
1408 * |-------|-------|
1410 va->va_start += size;
1417 * |-------|-------|
1419 va->va_end = nva_start_addr;
1426 * |---|-------|---|
1431 * For percpu allocator we do not do any pre-allocation
1457 return -1;
1463 lva->va_start = va->va_start;
1464 lva->va_end = nva_start_addr;
1469 va->va_start = nva_start_addr + size;
1471 return -1;
1478 insert_vmap_area_augment(lva, &va->rb_node, root, head);
1507 if (align <= PAGE_SIZE || (align > PAGE_SIZE && (vend - vstart) == size))
1514 if (va->va_start > vstart)
1515 nva_start_addr = ALIGN(va->va_start, align);
1565 * We do it in non-atomic context, thus it allows us to use more
1595 return ERR_PTR(-EINVAL);
1598 return ERR_PTR(-EBUSY);
1605 return ERR_PTR(-ENOMEM);
1611 kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask);
1628 va->va_start = addr;
1629 va->va_end = addr + size;
1630 va->vm = NULL;
1631 va->flags = va_flags;
1637 BUG_ON(!IS_ALIGNED(va->va_start, align));
1638 BUG_ON(va->va_start < vstart);
1639 BUG_ON(va->va_end > vend);
1669 return ERR_PTR(-EBUSY);
1718 /* for per-CPU blocks */
1722 * Purges all lazily-freed vmap areas.
1743 struct vmap_area, list)->va_start);
1747 struct vmap_area, list)->va_end);
1754 unsigned long nr = (va->va_end - va->va_start) >> PAGE_SHIFT;
1755 unsigned long orig_start = va->va_start;
1756 unsigned long orig_end = va->va_end;
1759 * Finally insert or merge lazily-freed area. It is
1771 va->va_start, va->va_end);
1820 unsigned long va_start = va->va_start;
1823 if (WARN_ON_ONCE(!list_empty(&va->list)))
1826 nr_lazy = atomic_long_add_return((va->va_end - va->va_start) >>
1849 flush_cache_vunmap(va->va_start, va->va_end);
1850 vunmap_range_noflush(va->va_start, va->va_end);
1852 flush_tlb_kernel_range(va->va_start, va->va_end);
1889 * to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess
1913 * regular operations: Purge if vb->free is less than 1/4 of the capacity.
1918 #define VMAP_BLOCK 0x2 /* mark out the vmap_block sub-type*/
1927 * be allocated. If it is an issue, we can use rb-tree
1952 * A per-cpu vmap_block_queue is used in both ways, to serialize
1955 * overload it, since we already have the per-cpu array which is
1969 * |------|------|------|------|------|------|...<vmap address space>
1972 * - CPU_1 invokes vm_unmap_ram(6), 6 belongs to CPU0 zone, thus
1973 * it access: CPU0/INDEX0 -> vmap_blocks -> xa_lock;
1975 * - CPU_2 invokes vm_unmap_ram(11), 11 belongs to CPU1 zone, thus
1976 * it access: CPU1/INDEX1 -> vmap_blocks -> xa_lock;
1978 * - CPU_0 invokes vm_unmap_ram(20), 20 belongs to CPU2 zone, thus
1979 * it access: CPU2/INDEX2 -> vmap_blocks -> xa_lock.
1992 * if an index points on it which is nr_cpu_ids - 1.
2009 addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1);
2024 * new_vmap_block - allocates new vmap_block and occupies 2^order pages in this
2029 * Return: virtual address in a newly allocated block or ERR_PTR(-errno)
2046 return ERR_PTR(-ENOMEM);
2057 vaddr = vmap_block_vaddr(va->va_start, 0);
2058 spin_lock_init(&vb->lock);
2059 vb->va = va;
2062 bitmap_zero(vb->used_map, VMAP_BBMAP_BITS);
2063 vb->free = VMAP_BBMAP_BITS - (1UL << order);
2064 vb->dirty = 0;
2065 vb->dirty_min = VMAP_BBMAP_BITS;
2066 vb->dirty_max = 0;
2067 bitmap_set(vb->used_map, 0, (1UL << order));
2068 INIT_LIST_HEAD(&vb->free_list);
2069 vb->cpu = raw_smp_processor_id();
2071 xa = addr_to_vb_xa(va->va_start);
2072 vb_idx = addr_to_vb_idx(va->va_start);
2081 * rather than vb->cpu due to task migration, which
2086 vbq = per_cpu_ptr(&vmap_block_queue, vb->cpu);
2087 spin_lock(&vbq->lock);
2088 list_add_tail_rcu(&vb->free_list, &vbq->free);
2089 spin_unlock(&vbq->lock);
2099 xa = addr_to_vb_xa(vb->va->va_start);
2100 tmp = xa_erase(xa, addr_to_vb_idx(vb->va->va_start));
2104 unlink_va(vb->va, &vmap_area_root);
2107 free_vmap_area_noflush(vb->va);
2114 struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, vb->cpu);
2116 if (vb->free + vb->dirty != VMAP_BBMAP_BITS ||
2117 vb->dirty == VMAP_BBMAP_BITS)
2121 if (!(force_purge || vb->free < VMAP_PURGE_THRESHOLD))
2125 WRITE_ONCE(vb->free, 0);
2127 WRITE_ONCE(vb->dirty, VMAP_BBMAP_BITS);
2128 vb->dirty_min = 0;
2129 vb->dirty_max = VMAP_BBMAP_BITS;
2130 spin_lock(&vbq->lock);
2131 list_del_rcu(&vb->free_list);
2132 spin_unlock(&vbq->lock);
2133 list_add_tail(&vb->purge, purge_list);
2142 list_del(&vb->purge);
2154 list_for_each_entry_rcu(vb, &vbq->free, free_list) {
2155 unsigned long free = READ_ONCE(vb->free);
2156 unsigned long dirty = READ_ONCE(vb->dirty);
2162 spin_lock(&vb->lock);
2164 spin_unlock(&vb->lock);
2199 list_for_each_entry_rcu(vb, &vbq->free, free_list) {
2202 if (READ_ONCE(vb->free) < (1UL << order))
2205 spin_lock(&vb->lock);
2206 if (vb->free < (1UL << order)) {
2207 spin_unlock(&vb->lock);
2211 pages_off = VMAP_BBMAP_BITS - vb->free;
2212 vaddr = vmap_block_vaddr(vb->va->va_start, pages_off);
2213 WRITE_ONCE(vb->free, vb->free - (1UL << order));
2214 bitmap_set(vb->used_map, pages_off, (1UL << order));
2215 if (vb->free == 0) {
2216 spin_lock(&vbq->lock);
2217 list_del_rcu(&vb->free_list);
2218 spin_unlock(&vbq->lock);
2221 spin_unlock(&vb->lock);
2247 offset = (addr & (VMAP_BLOCK_SIZE - 1)) >> PAGE_SHIFT;
2252 spin_lock(&vb->lock);
2253 bitmap_clear(vb->used_map, offset, (1UL << order));
2254 spin_unlock(&vb->lock);
2261 spin_lock(&vb->lock);
2264 vb->dirty_min = min(vb->dirty_min, offset);
2265 vb->dirty_max = max(vb->dirty_max, offset + (1UL << order));
2267 WRITE_ONCE(vb->dirty, vb->dirty + (1UL << order));
2268 if (vb->dirty == VMAP_BBMAP_BITS) {
2269 BUG_ON(vb->free);
2270 spin_unlock(&vb->lock);
2273 spin_unlock(&vb->lock);
2292 xa_for_each(&vbq->vmap_blocks, idx, vb) {
2293 spin_lock(&vb->lock);
2301 vb->dirty_max && vb->dirty != VMAP_BBMAP_BITS) {
2302 unsigned long va_start = vb->va->va_start;
2305 s = va_start + (vb->dirty_min << PAGE_SHIFT);
2306 e = va_start + (vb->dirty_max << PAGE_SHIFT);
2312 vb->dirty_min = VMAP_BBMAP_BITS;
2313 vb->dirty_max = 0;
2317 spin_unlock(&vb->lock);
2329 * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer
2351 * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram
2379 debug_check_no_locks_freed((void *)va->va_start,
2380 (va->va_end - va->va_start));
2386 * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space)
2392 * faster than vmap so it's good. But if you mix long-life and short-life
2395 * the end. Please use this function for short-lived objects.
2418 addr = va->va_start;
2430 * With hardware tag-based KASAN, marking is skipped for
2431 * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc().
2441 static inline unsigned int vm_area_page_order(struct vm_struct *vm)
2444 return vm->page_order;
2450 static inline void set_vm_area_page_order(struct vm_struct *vm, unsigned int order)
2453 vm->page_order = order;
2460 * vm_area_add_early - add vmap area early during boot
2461 * @vm: vm_struct to add
2463 * This function is used to add fixed kernel vm area to vmlist before
2464 * vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags
2469 void __init vm_area_add_early(struct vm_struct *vm)
2474 for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) {
2475 if (tmp->addr >= vm->addr) {
2476 BUG_ON(tmp->addr < vm->addr + vm->size);
2479 BUG_ON(tmp->addr + tmp->size > vm->addr);
2481 vm->next = *p;
2482 *p = vm;
2486 * vm_area_register_early - register vmap area early during boot
2487 * @vm: vm_struct to register
2490 * This function is used to register kernel vm area before
2491 * vmalloc_init() is called. @vm->size and @vm->flags should contain
2493 * vm->addr contains the allocated address.
2497 void __init vm_area_register_early(struct vm_struct *vm, size_t align)
2504 for (p = &vmlist; (cur = *p) != NULL; p = &cur->next) {
2505 if ((unsigned long)cur->addr - addr >= vm->size)
2507 addr = ALIGN((unsigned long)cur->addr + cur->size, align);
2510 BUG_ON(addr > VMALLOC_END - vm->size);
2511 vm->addr = (void *)addr;
2512 vm->next = *p;
2513 *p = vm;
2514 kasan_populate_early_vm_area_shadow(vm->addr, vm->size);
2525 * -|-----|.....|-----|-----|-----|.....|-
2527 * |<--------------------------------->|
2530 if (busy->va_start - vmap_start > 0) {
2533 free->va_start = vmap_start;
2534 free->va_end = busy->va_start;
2542 vmap_start = busy->va_end;
2545 if (vmap_end - vmap_start > 0) {
2548 free->va_start = vmap_start;
2549 free->va_end = vmap_end;
2558 static inline void setup_vmalloc_vm_locked(struct vm_struct *vm,
2561 vm->flags = flags;
2562 vm->addr = (void *)va->va_start;
2563 vm->size = va->va_end - va->va_start;
2564 vm->caller = caller;
2565 va->vm = vm;
2568 static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
2572 setup_vmalloc_vm_locked(vm, va, flags, caller);
2576 static void clear_vm_uninitialized_flag(struct vm_struct *vm)
2580 * we should make sure that vm has proper values.
2584 vm->flags &= ~VM_UNINITIALIZED;
2621 * Mark pages for non-VM_ALLOC mappings as accessible. Do it now as a
2622 * best-effort approach, as they can be mapped outside of vmalloc code.
2625 * With hardware tag-based KASAN, marking is skipped for
2626 * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc().
2629 area->addr = kasan_unpoison_vmalloc(area->addr, requested_size,
2644 * get_vm_area - reserve a contiguous kernel virtual area
2671 * find_vm_area - find a continuous kernel virtual area
2674 * Search for the kernel VM area starting at @addr, and return it.
2688 return va->vm;
2692 * remove_vm_area - find and remove a continuous kernel virtual area
2695 * Search for the kernel VM area starting at @addr, and remove it.
2696 * This function returns the found VM area, but using it is NOT safe
2704 struct vm_struct *vm;
2713 if (!va || !va->vm)
2715 vm = va->vm;
2717 debug_check_no_locks_freed(vm->addr, get_vm_area_size(vm));
2718 debug_check_no_obj_freed(vm->addr, get_vm_area_size(vm));
2719 kasan_free_module_shadow(vm);
2720 kasan_poison_vmalloc(vm->addr, get_vm_area_size(vm));
2723 return vm;
2732 for (i = 0; i < area->nr_pages; i++)
2733 if (page_address(area->pages[i]))
2734 set_direct_map(area->pages[i]);
2738 * Flush the vm mapping and reset the direct map.
2749 * the vm_unmap_aliases() flush includes the direct map.
2751 for (i = 0; i < area->nr_pages; i += 1U << page_order) {
2752 unsigned long addr = (unsigned long)page_address(area->pages[i]);
2765 * Set direct map to something invalid so that it won't be cached if
2767 * reset the direct map permissions to the default.
2779 llist_for_each_safe(llnode, t, llist_del_all(&p->list))
2784 * vfree_atomic - release memory allocated by vmalloc()
2803 if (addr && llist_add((struct llist_node *)addr, &p->list))
2804 schedule_work(&p->wq);
2808 * vfree - Release memory allocated by vmalloc()
2822 * conventions for vfree() arch-dependent would be a really bad idea).
2826 struct vm_struct *vm;
2841 vm = remove_vm_area(addr);
2842 if (unlikely(!vm)) {
2843 WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
2848 if (unlikely(vm->flags & VM_FLUSH_RESET_PERMS))
2849 vm_reset_perms(vm);
2850 for (i = 0; i < vm->nr_pages; i++) {
2851 struct page *page = vm->pages[i];
2854 if (!(vm->flags & VM_MAP_PUT_PAGES))
2855 mod_memcg_page_state(page, MEMCG_VMALLOC, -1);
2857 * High-order allocs for huge vmallocs are split, so
2858 * can be freed as an array of order-0 allocations
2863 if (!(vm->flags & VM_MAP_PUT_PAGES))
2864 atomic_long_sub(vm->nr_pages, &nr_vmalloc_pages);
2865 kvfree(vm->pages);
2866 kfree(vm);
2871 * vunmap - release virtual mapping obtained by vmap()
2881 struct vm_struct *vm;
2888 vm = remove_vm_area(addr);
2889 if (unlikely(!vm)) {
2890 WARN(1, KERN_ERR "Trying to vunmap() nonexistent vm area (%p)\n",
2894 kfree(vm);
2899 * vmap - map an array of pages into virtually contiguous space
2901 * @count: number of pages to map
2902 * @flags: vm_area->flags
2940 addr = (unsigned long)area->addr;
2943 vunmap(area->addr);
2948 area->pages = pages;
2949 area->nr_pages = count;
2951 return area->addr;
2965 unsigned long pfn = data->pfns[data->idx];
2969 return -EINVAL;
2971 ptent = pte_mkspecial(pfn_pte(pfn, data->prot));
2974 data->idx++;
2979 * vmap_pfn - map an array of PFNs into virtually contiguous space
2981 * @count: number of pages to map
2996 if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
3002 flush_cache_vmap((unsigned long)area->addr,
3003 (unsigned long)area->addr + count * PAGE_SIZE);
3005 return area->addr;
3021 * For order-0 pages we make use of bulk allocator, if
3034 * A maximum allowed request is hard-coded and is 100
3036 * long preemption off scenario in the bulk-allocator
3039 nr_pages_request = min(100U, nr_pages - nr_allocated);
3069 * potentially dangerous (pre-mature OOM, disruptive reclaim
3075 /* High-order pages or fallback path if "bulk" fails. */
3090 * small-page vmallocs). Some drivers do their own refcounting
3091 * on vmalloc_to_page() pages, some use page->mapping,
3092 * page->lru, etc.
3098 * Careful, we allocate and map page-order pages, but
3118 unsigned long addr = (unsigned long)area->addr;
3133 area->pages = __vmalloc_node(array_size, 1, nested_gfp, node,
3134 area->caller);
3136 area->pages = kmalloc_node(array_size, nested_gfp, node);
3139 if (!area->pages) {
3147 set_vm_area_page_order(area, page_shift - PAGE_SHIFT);
3150 area->nr_pages = vm_area_alloc_pages(gfp_mask | __GFP_NOWARN,
3151 node, page_order, nr_small_pages, area->pages);
3153 atomic_long_add(area->nr_pages, &nr_vmalloc_pages);
3157 for (i = 0; i < area->nr_pages; i++)
3158 mod_memcg_page_state(area->pages[i], MEMCG_VMALLOC, 1);
3165 if (area->nr_pages != nr_small_pages) {
3168 * also:-
3170 * - a pending fatal signal
3171 * - insufficient huge page-order pages
3173 * Since we always retry allocations at order-0 in the huge page
3179 area->nr_pages * PAGE_SIZE);
3193 ret = vmap_pages_range(addr, addr + size, prot, area->pages,
3206 "vmalloc error: size %lu, failed to map pages",
3207 area->nr_pages * PAGE_SIZE);
3211 return area->addr;
3214 vfree(area->addr);
3219 * __vmalloc_node_range - allocate virtually contiguous memory
3222 * @start: vm area range start
3223 * @end: vm area range end
3226 * @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD)
3241 * Map them into contiguous kernel virtual space, using a pagetable
3330 /* Allocate physical pages and map them into vmalloc space. */
3340 * Tag-based KASAN modes only assign tags to normal non-executable
3348 area->addr = kasan_unpoison_vmalloc(area->addr, real_size, kasan_flags);
3361 return area->addr;
3375 * __vmalloc_node - allocate virtually contiguous memory
3383 * @gfp_mask flags. Map them into contiguous kernel virtual space.
3385 * Reclaim modifiers in @gfp_mask - __GFP_NORETRY, __GFP_RETRY_MAYFAIL
3416 * vmalloc - allocate virtually contiguous memory
3420 * allocator and map them into contiguous kernel virtual space.
3435 * vmalloc_huge - allocate virtually contiguous memory, allow huge pages
3440 * allocator and map them into contiguous kernel virtual space.
3455 * vzalloc - allocate virtually contiguous memory with zero fill
3459 * allocator and map them into contiguous kernel virtual space.
3475 * vmalloc_user - allocate zeroed virtually contiguous memory for userspace
3493 * vmalloc_node - allocate memory on a specific node
3498 * allocator and map them into contiguous kernel virtual space.
3513 * vzalloc_node - allocate memory on a specific node with zero fill
3518 * allocator and map them into contiguous kernel virtual space.
3543 * vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
3547 * page level allocator and map them into contiguous kernel virtual space.
3559 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
3590 remains -= copied;
3596 return count - remains;
3616 length = PAGE_SIZE - offset;
3635 remains -= copied;
3641 return count - remains;
3678 spin_lock(&vb->lock);
3679 if (bitmap_empty(vb->used_map, VMAP_BBMAP_BITS)) {
3680 spin_unlock(&vb->lock);
3684 for_each_set_bitrange(rs, re, vb->used_map, VMAP_BBMAP_BITS) {
3690 start = vmap_block_vaddr(vb->va->va_start, rs);
3693 size_t to_zero = min_t(size_t, start - addr, remains);
3697 remains -= zeroed;
3705 n = ((re - rs + 1) << PAGE_SHIFT) - offset;
3712 remains -= copied;
3718 spin_unlock(&vb->lock);
3721 /* zero-fill the left dirty or free regions */
3722 return count - remains + zero_iter(iter, remains);
3725 spin_unlock(&vb->lock);
3726 return count - remains;
3730 * vread_iter() - read vmalloc area in a safe way to an iterator.
3732 * @addr: vm address.
3738 * proper area of @buf. If there are memory holes, they'll be zero-filled.
3756 struct vm_struct *vm;
3764 count = -(unsigned long) addr;
3774 if ((unsigned long)addr + remains <= va->va_start)
3783 vm = va->vm;
3784 flags = va->flags & VMAP_FLAGS_MASK;
3786 * VMAP_BLOCK indicates a sub-type of vm_map_ram area, need
3791 if (!vm && !flags)
3794 if (vm && (vm->flags & VM_UNINITIALIZED))
3800 vaddr = (char *) va->va_start;
3801 size = vm ? get_vm_area_size(vm) : va_size(va);
3807 size_t to_zero = min_t(size_t, vaddr - addr, remains);
3811 remains -= zeroed;
3817 n = vaddr + size - addr;
3823 else if (!(vm->flags & VM_IOREMAP))
3829 remains -= copied;
3837 /* zero-fill memory holes */
3838 return count - remains + zero_iter(iter, remains);
3843 return count - remains;
3847 * remap_vmalloc_range_partial - map vmalloc pages to userspace
3852 * @size: size of map area
3854 * Returns: 0 for success, -Exxx on failure
3872 return -EINVAL;
3877 return -EINVAL;
3881 return -EINVAL;
3883 if (!(area->flags & (VM_USERMAP | VM_DMA_COHERENT)))
3884 return -EINVAL;
3888 return -EINVAL;
3901 size -= PAGE_SIZE;
3910 * remap_vmalloc_range - map vmalloc pages to userspace
3911 * @vma: vma to cover (map full range of vma)
3913 * @pgoff: number of pages into addr before first page to map
3915 * Returns: 0 for success, -Exxx on failure
3926 return remap_vmalloc_range_partial(vma, vma->vm_start,
3928 vma->vm_end - vma->vm_start);
3935 ret = remove_vm_area(area->addr);
3948 * pvm_find_va_enclose_addr - find the vmap_area @addr belongs to
3953 * i.e. va->va_start < addr && va->va_end < addr or NULL
3967 if (tmp->va_start <= addr) {
3969 if (tmp->va_end >= addr)
3972 n = n->rb_right;
3974 n = n->rb_left;
3982 * pvm_determine_end_from_reverse - find the highest aligned address
3985 * in - the VA we start the search(reverse order);
3986 * out - the VA with the highest aligned end address.
3994 unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
4000 addr = min((*va)->va_end & ~(align - 1), vmalloc_end);
4001 if ((*va)->va_start < addr)
4010 * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator
4019 * Percpu allocator wants to use congruent vm areas so that it can
4027 * does everything top-down and scans free blocks from the end looking
4038 const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
4068 if (vmalloc_end - vmalloc_start < last_end) {
4087 /* start scanning - we scan from the top, begin with the last area */
4093 base = pvm_determine_end_from_reverse(&va, align) - end;
4113 if (base + end > va->va_end) {
4114 base = pvm_determine_end_from_reverse(&va, align) - end;
4122 if (base + start < va->va_start) {
4123 va = node_to_va(rb_prev(&va->rb_node));
4124 base = pvm_determine_end_from_reverse(&va, align) - end;
4133 area = (area + nr_vms - 1) % nr_vms;
4163 va->va_start = start;
4164 va->va_end = start + size;
4171 if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area]))
4175 /* insert all vm's */
4186 * Mark allocated areas as accessible. Do it now as a best-effort
4188 * With hardware tag-based KASAN, marking is skipped for
4189 * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc().
4192 vms[area]->addr = kasan_unpoison_vmalloc(vms[area]->addr,
4193 vms[area]->size, KASAN_VMALLOC_PROT_NORMAL);
4205 while (area--) {
4206 orig_start = vas[area]->va_start;
4207 orig_end = vas[area]->va_end;
4212 va->va_start, va->va_end);
4256 orig_start = vas[area]->va_start;
4257 orig_end = vas[area]->va_end;
4262 va->va_start, va->va_end);
4273 * pcpu_free_vm_areas - free vmalloc areas for percpu allocator
4294 struct vm_struct *vm;
4307 vm = va->vm;
4308 if (!vm) {
4312 addr = (unsigned long)vm->addr;
4313 caller = vm->caller;
4314 nr_pages = vm->nr_pages;
4316 pr_cont(" %u-page vmalloc region starting at %#lx allocated at %pS\n",
4349 unsigned int nr, *counters = m->private;
4355 if (v->flags & VM_UNINITIALIZED)
4362 for (nr = 0; nr < v->nr_pages; nr += step)
4363 counters[page_to_nid(v->pages[nr])] += step;
4376 seq_printf(m, "0x%pK-0x%pK %7ld unpurged vm_area\n",
4377 (void *)va->va_start, (void *)va->va_end,
4378 va->va_end - va->va_start);
4390 if (!va->vm) {
4391 if (va->flags & VMAP_RAM)
4392 seq_printf(m, "0x%pK-0x%pK %7ld vm_map_ram\n",
4393 (void *)va->va_start, (void *)va->va_end,
4394 va->va_end - va->va_start);
4399 v = va->vm;
4401 seq_printf(m, "0x%pK-0x%pK %7ld",
4402 v->addr, v->addr + v->size, v->size);
4404 if (v->caller)
4405 seq_printf(m, " %pS", v->caller);
4407 if (v->nr_pages)
4408 seq_printf(m, " pages=%d", v->nr_pages);
4410 if (v->phys_addr)
4411 seq_printf(m, " phys=%pa", &v->phys_addr);
4413 if (v->flags & VM_IOREMAP)
4416 if (v->flags & VM_ALLOC)
4419 if (v->flags & VM_MAP)
4422 if (v->flags & VM_USERMAP)
4425 if (v->flags & VM_DMA_COHERENT)
4426 seq_puts(m, " dma-coherent");
4428 if (is_vmalloc_addr(v->pages))
4438 if (list_is_last(&va->list, &vmap_area_list))
4481 spin_lock_init(&vbq->lock);
4482 INIT_LIST_HEAD(&vbq->free);
4484 init_llist_head(&p->list);
4485 INIT_WORK(&p->wq, delayed_vfree_work);
4486 xa_init(&vbq->vmap_blocks);
4490 for (tmp = vmlist; tmp; tmp = tmp->next) {
4495 va->va_start = (unsigned long)tmp->addr;
4496 va->va_end = va->va_start + tmp->size;
4497 va->vm = tmp;