1 /* 2 * Virtual Memory Map support 3 * 4 * (C) 2007 sgi. Christoph Lameter <clameter@sgi.com>. 5 * 6 * Virtual memory maps allow VM primitives pfn_to_page, page_to_pfn, 7 * virt_to_page, page_address() to be implemented as a base offset 8 * calculation without memory access. 9 * 10 * However, virtual mappings need a page table and TLBs. Many Linux 11 * architectures already map their physical space using 1-1 mappings 12 * via TLBs. For those arches the virtual memmory map is essentially 13 * for free if we use the same page size as the 1-1 mappings. In that 14 * case the overhead consists of a few additional pages that are 15 * allocated to create a view of memory for vmemmap. 16 * 17 * Special Kconfig settings: 18 * 19 * CONFIG_ARCH_POPULATES_SPARSEMEM_VMEMMAP 20 * 21 * The architecture has its own functions to populate the memory 22 * map and provides a vmemmap_populate function. 23 * 24 * CONFIG_ARCH_POPULATES_SPARSEMEM_VMEMMAP_PMD 25 * 26 * The architecture provides functions to populate the pmd level 27 * of the vmemmap mappings. Allowing mappings using large pages 28 * where available. 29 * 30 * If neither are set then PAGE_SIZE mappings are generated which 31 * require one PTE/TLB per PAGE_SIZE chunk of the virtual memory map. 32 */ 33 #include <linux/mm.h> 34 #include <linux/mmzone.h> 35 #include <linux/bootmem.h> 36 #include <linux/highmem.h> 37 #include <linux/module.h> 38 #include <linux/spinlock.h> 39 #include <linux/vmalloc.h> 40 #include <asm/dma.h> 41 #include <asm/pgalloc.h> 42 #include <asm/pgtable.h> 43 44 /* 45 * Allocate a block of memory to be used to back the virtual memory map 46 * or to back the page tables that are used to create the mapping. 47 * Uses the main allocators if they are available, else bootmem. 48 */ 49 void * __meminit vmemmap_alloc_block(unsigned long size, int node) 50 { 51 /* If the main allocator is up use that, fallback to bootmem. */ 52 if (slab_is_available()) { 53 struct page *page = alloc_pages_node(node, 54 GFP_KERNEL | __GFP_ZERO, get_order(size)); 55 if (page) 56 return page_address(page); 57 return NULL; 58 } else 59 return __alloc_bootmem_node(NODE_DATA(node), size, size, 60 __pa(MAX_DMA_ADDRESS)); 61 } 62 63 #ifndef CONFIG_ARCH_POPULATES_SPARSEMEM_VMEMMAP 64 void __meminit vmemmap_verify(pte_t *pte, int node, 65 unsigned long start, unsigned long end) 66 { 67 unsigned long pfn = pte_pfn(*pte); 68 int actual_node = early_pfn_to_nid(pfn); 69 70 if (actual_node != node) 71 printk(KERN_WARNING "[%lx-%lx] potential offnode " 72 "page_structs\n", start, end - 1); 73 } 74 75 #ifndef CONFIG_ARCH_POPULATES_SPARSEMEM_VMEMMAP_PMD 76 static int __meminit vmemmap_populate_pte(pmd_t *pmd, unsigned long addr, 77 unsigned long end, int node) 78 { 79 pte_t *pte; 80 81 for (pte = pte_offset_kernel(pmd, addr); addr < end; 82 pte++, addr += PAGE_SIZE) 83 if (pte_none(*pte)) { 84 pte_t entry; 85 void *p = vmemmap_alloc_block(PAGE_SIZE, node); 86 if (!p) 87 return -ENOMEM; 88 89 entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL); 90 set_pte(pte, entry); 91 92 } else 93 vmemmap_verify(pte, node, addr + PAGE_SIZE, end); 94 95 return 0; 96 } 97 98 int __meminit vmemmap_populate_pmd(pud_t *pud, unsigned long addr, 99 unsigned long end, int node) 100 { 101 pmd_t *pmd; 102 int error = 0; 103 unsigned long next; 104 105 for (pmd = pmd_offset(pud, addr); addr < end && !error; 106 pmd++, addr = next) { 107 if (pmd_none(*pmd)) { 108 void *p = vmemmap_alloc_block(PAGE_SIZE, node); 109 if (!p) 110 return -ENOMEM; 111 112 pmd_populate_kernel(&init_mm, pmd, p); 113 } else 114 vmemmap_verify((pte_t *)pmd, node, 115 pmd_addr_end(addr, end), end); 116 next = pmd_addr_end(addr, end); 117 error = vmemmap_populate_pte(pmd, addr, next, node); 118 } 119 return error; 120 } 121 #endif /* CONFIG_ARCH_POPULATES_SPARSEMEM_VMEMMAP_PMD */ 122 123 static int __meminit vmemmap_populate_pud(pgd_t *pgd, unsigned long addr, 124 unsigned long end, int node) 125 { 126 pud_t *pud; 127 int error = 0; 128 unsigned long next; 129 130 for (pud = pud_offset(pgd, addr); addr < end && !error; 131 pud++, addr = next) { 132 if (pud_none(*pud)) { 133 void *p = vmemmap_alloc_block(PAGE_SIZE, node); 134 if (!p) 135 return -ENOMEM; 136 137 pud_populate(&init_mm, pud, p); 138 } 139 next = pud_addr_end(addr, end); 140 error = vmemmap_populate_pmd(pud, addr, next, node); 141 } 142 return error; 143 } 144 145 int __meminit vmemmap_populate(struct page *start_page, 146 unsigned long nr, int node) 147 { 148 pgd_t *pgd; 149 unsigned long addr = (unsigned long)start_page; 150 unsigned long end = (unsigned long)(start_page + nr); 151 unsigned long next; 152 int error = 0; 153 154 printk(KERN_DEBUG "[%lx-%lx] Virtual memory section" 155 " (%ld pages) node %d\n", addr, end - 1, nr, node); 156 157 for (pgd = pgd_offset_k(addr); addr < end && !error; 158 pgd++, addr = next) { 159 if (pgd_none(*pgd)) { 160 void *p = vmemmap_alloc_block(PAGE_SIZE, node); 161 if (!p) 162 return -ENOMEM; 163 164 pgd_populate(&init_mm, pgd, p); 165 } 166 next = pgd_addr_end(addr,end); 167 error = vmemmap_populate_pud(pgd, addr, next, node); 168 } 169 return error; 170 } 171 #endif /* !CONFIG_ARCH_POPULATES_SPARSEMEM_VMEMMAP */ 172 173 struct page __init *sparse_early_mem_map_populate(unsigned long pnum, int nid) 174 { 175 struct page *map = pfn_to_page(pnum * PAGES_PER_SECTION); 176 int error = vmemmap_populate(map, PAGES_PER_SECTION, nid); 177 if (error) 178 return NULL; 179 180 return map; 181 } 182