1 #include <linux/gfp.h> 2 #include <linux/initrd.h> 3 #include <linux/ioport.h> 4 #include <linux/swap.h> 5 #include <linux/memblock.h> 6 #include <linux/bootmem.h> /* for max_low_pfn */ 7 8 #include <asm/cacheflush.h> 9 #include <asm/e820.h> 10 #include <asm/init.h> 11 #include <asm/page.h> 12 #include <asm/page_types.h> 13 #include <asm/sections.h> 14 #include <asm/setup.h> 15 #include <asm/tlbflush.h> 16 #include <asm/tlb.h> 17 #include <asm/proto.h> 18 #include <asm/dma.h> /* for MAX_DMA_PFN */ 19 20 unsigned long __initdata pgt_buf_start; 21 unsigned long __meminitdata pgt_buf_end; 22 unsigned long __meminitdata pgt_buf_top; 23 24 int after_bootmem; 25 26 int direct_gbpages 27 #ifdef CONFIG_DIRECT_GBPAGES 28 = 1 29 #endif 30 ; 31 32 struct map_range { 33 unsigned long start; 34 unsigned long end; 35 unsigned page_size_mask; 36 }; 37 38 /* 39 * First calculate space needed for kernel direct mapping page tables to cover 40 * mr[0].start to mr[nr_range - 1].end, while accounting for possible 2M and 1GB 41 * pages. Then find enough contiguous space for those page tables. 42 */ 43 static void __init find_early_table_space(struct map_range *mr, int nr_range) 44 { 45 int i; 46 unsigned long puds = 0, pmds = 0, ptes = 0, tables; 47 unsigned long start = 0, good_end; 48 phys_addr_t base; 49 50 for (i = 0; i < nr_range; i++) { 51 unsigned long range, extra; 52 53 range = mr[i].end - mr[i].start; 54 puds += (range + PUD_SIZE - 1) >> PUD_SHIFT; 55 56 if (mr[i].page_size_mask & (1 << PG_LEVEL_1G)) { 57 extra = range - ((range >> PUD_SHIFT) << PUD_SHIFT); 58 pmds += (extra + PMD_SIZE - 1) >> PMD_SHIFT; 59 } else { 60 pmds += (range + PMD_SIZE - 1) >> PMD_SHIFT; 61 } 62 63 if (mr[i].page_size_mask & (1 << PG_LEVEL_2M)) { 64 extra = range - ((range >> PMD_SHIFT) << PMD_SHIFT); 65 #ifdef CONFIG_X86_32 66 extra += PMD_SIZE; 67 #endif 68 ptes += (extra + PAGE_SIZE - 1) >> PAGE_SHIFT; 69 } else { 70 ptes += (range + PAGE_SIZE - 1) >> PAGE_SHIFT; 71 } 72 } 73 74 tables = roundup(puds * sizeof(pud_t), PAGE_SIZE); 75 tables += roundup(pmds * sizeof(pmd_t), PAGE_SIZE); 76 tables += roundup(ptes * sizeof(pte_t), PAGE_SIZE); 77 78 #ifdef CONFIG_X86_32 79 /* for fixmap */ 80 tables += roundup(__end_of_fixed_addresses * sizeof(pte_t), PAGE_SIZE); 81 #endif 82 good_end = max_pfn_mapped << PAGE_SHIFT; 83 84 base = memblock_find_in_range(start, good_end, tables, PAGE_SIZE); 85 if (!base) 86 panic("Cannot find space for the kernel page tables"); 87 88 pgt_buf_start = base >> PAGE_SHIFT; 89 pgt_buf_end = pgt_buf_start; 90 pgt_buf_top = pgt_buf_start + (tables >> PAGE_SHIFT); 91 92 printk(KERN_DEBUG "kernel direct mapping tables up to %#lx @ [mem %#010lx-%#010lx]\n", 93 mr[nr_range - 1].end - 1, pgt_buf_start << PAGE_SHIFT, 94 (pgt_buf_top << PAGE_SHIFT) - 1); 95 } 96 97 void __init native_pagetable_reserve(u64 start, u64 end) 98 { 99 memblock_reserve(start, end - start); 100 } 101 102 #ifdef CONFIG_X86_32 103 #define NR_RANGE_MR 3 104 #else /* CONFIG_X86_64 */ 105 #define NR_RANGE_MR 5 106 #endif 107 108 static int __meminit save_mr(struct map_range *mr, int nr_range, 109 unsigned long start_pfn, unsigned long end_pfn, 110 unsigned long page_size_mask) 111 { 112 if (start_pfn < end_pfn) { 113 if (nr_range >= NR_RANGE_MR) 114 panic("run out of range for init_memory_mapping\n"); 115 mr[nr_range].start = start_pfn<<PAGE_SHIFT; 116 mr[nr_range].end = end_pfn<<PAGE_SHIFT; 117 mr[nr_range].page_size_mask = page_size_mask; 118 nr_range++; 119 } 120 121 return nr_range; 122 } 123 124 /* 125 * Setup the direct mapping of the physical memory at PAGE_OFFSET. 126 * This runs before bootmem is initialized and gets pages directly from 127 * the physical memory. To access them they are temporarily mapped. 128 */ 129 unsigned long __init_refok init_memory_mapping(unsigned long start, 130 unsigned long end) 131 { 132 unsigned long page_size_mask = 0; 133 unsigned long start_pfn, end_pfn; 134 unsigned long ret = 0; 135 unsigned long pos; 136 137 struct map_range mr[NR_RANGE_MR]; 138 int nr_range, i; 139 int use_pse, use_gbpages; 140 141 printk(KERN_INFO "init_memory_mapping: [mem %#010lx-%#010lx]\n", 142 start, end - 1); 143 144 #if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KMEMCHECK) 145 /* 146 * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages. 147 * This will simplify cpa(), which otherwise needs to support splitting 148 * large pages into small in interrupt context, etc. 149 */ 150 use_pse = use_gbpages = 0; 151 #else 152 use_pse = cpu_has_pse; 153 use_gbpages = direct_gbpages; 154 #endif 155 156 /* Enable PSE if available */ 157 if (cpu_has_pse) 158 set_in_cr4(X86_CR4_PSE); 159 160 /* Enable PGE if available */ 161 if (cpu_has_pge) { 162 set_in_cr4(X86_CR4_PGE); 163 __supported_pte_mask |= _PAGE_GLOBAL; 164 } 165 166 if (use_gbpages) 167 page_size_mask |= 1 << PG_LEVEL_1G; 168 if (use_pse) 169 page_size_mask |= 1 << PG_LEVEL_2M; 170 171 memset(mr, 0, sizeof(mr)); 172 nr_range = 0; 173 174 /* head if not big page alignment ? */ 175 start_pfn = start >> PAGE_SHIFT; 176 pos = start_pfn << PAGE_SHIFT; 177 #ifdef CONFIG_X86_32 178 /* 179 * Don't use a large page for the first 2/4MB of memory 180 * because there are often fixed size MTRRs in there 181 * and overlapping MTRRs into large pages can cause 182 * slowdowns. 183 */ 184 if (pos == 0) 185 end_pfn = 1<<(PMD_SHIFT - PAGE_SHIFT); 186 else 187 end_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT) 188 << (PMD_SHIFT - PAGE_SHIFT); 189 #else /* CONFIG_X86_64 */ 190 end_pfn = ((pos + (PMD_SIZE - 1)) >> PMD_SHIFT) 191 << (PMD_SHIFT - PAGE_SHIFT); 192 #endif 193 if (end_pfn > (end >> PAGE_SHIFT)) 194 end_pfn = end >> PAGE_SHIFT; 195 if (start_pfn < end_pfn) { 196 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); 197 pos = end_pfn << PAGE_SHIFT; 198 } 199 200 /* big page (2M) range */ 201 start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT) 202 << (PMD_SHIFT - PAGE_SHIFT); 203 #ifdef CONFIG_X86_32 204 end_pfn = (end>>PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT); 205 #else /* CONFIG_X86_64 */ 206 end_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT) 207 << (PUD_SHIFT - PAGE_SHIFT); 208 if (end_pfn > ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT))) 209 end_pfn = ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT)); 210 #endif 211 212 if (start_pfn < end_pfn) { 213 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 214 page_size_mask & (1<<PG_LEVEL_2M)); 215 pos = end_pfn << PAGE_SHIFT; 216 } 217 218 #ifdef CONFIG_X86_64 219 /* big page (1G) range */ 220 start_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT) 221 << (PUD_SHIFT - PAGE_SHIFT); 222 end_pfn = (end >> PUD_SHIFT) << (PUD_SHIFT - PAGE_SHIFT); 223 if (start_pfn < end_pfn) { 224 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 225 page_size_mask & 226 ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G))); 227 pos = end_pfn << PAGE_SHIFT; 228 } 229 230 /* tail is not big page (1G) alignment */ 231 start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT) 232 << (PMD_SHIFT - PAGE_SHIFT); 233 end_pfn = (end >> PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT); 234 if (start_pfn < end_pfn) { 235 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 236 page_size_mask & (1<<PG_LEVEL_2M)); 237 pos = end_pfn << PAGE_SHIFT; 238 } 239 #endif 240 241 /* tail is not big page (2M) alignment */ 242 start_pfn = pos>>PAGE_SHIFT; 243 end_pfn = end>>PAGE_SHIFT; 244 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); 245 246 /* try to merge same page size and continuous */ 247 for (i = 0; nr_range > 1 && i < nr_range - 1; i++) { 248 unsigned long old_start; 249 if (mr[i].end != mr[i+1].start || 250 mr[i].page_size_mask != mr[i+1].page_size_mask) 251 continue; 252 /* move it */ 253 old_start = mr[i].start; 254 memmove(&mr[i], &mr[i+1], 255 (nr_range - 1 - i) * sizeof(struct map_range)); 256 mr[i--].start = old_start; 257 nr_range--; 258 } 259 260 for (i = 0; i < nr_range; i++) 261 printk(KERN_DEBUG " [mem %#010lx-%#010lx] page %s\n", 262 mr[i].start, mr[i].end - 1, 263 (mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":( 264 (mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k")); 265 266 /* 267 * Find space for the kernel direct mapping tables. 268 * 269 * Later we should allocate these tables in the local node of the 270 * memory mapped. Unfortunately this is done currently before the 271 * nodes are discovered. 272 */ 273 if (!after_bootmem) 274 find_early_table_space(mr, nr_range); 275 276 for (i = 0; i < nr_range; i++) 277 ret = kernel_physical_mapping_init(mr[i].start, mr[i].end, 278 mr[i].page_size_mask); 279 280 #ifdef CONFIG_X86_32 281 early_ioremap_page_table_range_init(); 282 283 load_cr3(swapper_pg_dir); 284 #endif 285 286 __flush_tlb_all(); 287 288 /* 289 * Reserve the kernel pagetable pages we used (pgt_buf_start - 290 * pgt_buf_end) and free the other ones (pgt_buf_end - pgt_buf_top) 291 * so that they can be reused for other purposes. 292 * 293 * On native it just means calling memblock_reserve, on Xen it also 294 * means marking RW the pagetable pages that we allocated before 295 * but that haven't been used. 296 * 297 * In fact on xen we mark RO the whole range pgt_buf_start - 298 * pgt_buf_top, because we have to make sure that when 299 * init_memory_mapping reaches the pagetable pages area, it maps 300 * RO all the pagetable pages, including the ones that are beyond 301 * pgt_buf_end at that time. 302 */ 303 if (!after_bootmem && pgt_buf_end > pgt_buf_start) 304 x86_init.mapping.pagetable_reserve(PFN_PHYS(pgt_buf_start), 305 PFN_PHYS(pgt_buf_end)); 306 307 if (!after_bootmem) 308 early_memtest(start, end); 309 310 return ret >> PAGE_SHIFT; 311 } 312 313 314 /* 315 * devmem_is_allowed() checks to see if /dev/mem access to a certain address 316 * is valid. The argument is a physical page number. 317 * 318 * 319 * On x86, access has to be given to the first megabyte of ram because that area 320 * contains bios code and data regions used by X and dosemu and similar apps. 321 * Access has to be given to non-kernel-ram areas as well, these contain the PCI 322 * mmio resources as well as potential bios/acpi data regions. 323 */ 324 int devmem_is_allowed(unsigned long pagenr) 325 { 326 if (pagenr < 256) 327 return 1; 328 if (iomem_is_exclusive(pagenr << PAGE_SHIFT)) 329 return 0; 330 if (!page_is_ram(pagenr)) 331 return 1; 332 return 0; 333 } 334 335 void free_init_pages(char *what, unsigned long begin, unsigned long end) 336 { 337 unsigned long addr; 338 unsigned long begin_aligned, end_aligned; 339 340 /* Make sure boundaries are page aligned */ 341 begin_aligned = PAGE_ALIGN(begin); 342 end_aligned = end & PAGE_MASK; 343 344 if (WARN_ON(begin_aligned != begin || end_aligned != end)) { 345 begin = begin_aligned; 346 end = end_aligned; 347 } 348 349 if (begin >= end) 350 return; 351 352 addr = begin; 353 354 /* 355 * If debugging page accesses then do not free this memory but 356 * mark them not present - any buggy init-section access will 357 * create a kernel page fault: 358 */ 359 #ifdef CONFIG_DEBUG_PAGEALLOC 360 printk(KERN_INFO "debug: unmapping init [mem %#010lx-%#010lx]\n", 361 begin, end - 1); 362 set_memory_np(begin, (end - begin) >> PAGE_SHIFT); 363 #else 364 /* 365 * We just marked the kernel text read only above, now that 366 * we are going to free part of that, we need to make that 367 * writeable and non-executable first. 368 */ 369 set_memory_nx(begin, (end - begin) >> PAGE_SHIFT); 370 set_memory_rw(begin, (end - begin) >> PAGE_SHIFT); 371 372 printk(KERN_INFO "Freeing %s: %luk freed\n", what, (end - begin) >> 10); 373 374 for (; addr < end; addr += PAGE_SIZE) { 375 ClearPageReserved(virt_to_page(addr)); 376 init_page_count(virt_to_page(addr)); 377 memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE); 378 free_page(addr); 379 totalram_pages++; 380 } 381 #endif 382 } 383 384 void free_initmem(void) 385 { 386 free_init_pages("unused kernel memory", 387 (unsigned long)(&__init_begin), 388 (unsigned long)(&__init_end)); 389 } 390 391 #ifdef CONFIG_BLK_DEV_INITRD 392 void __init free_initrd_mem(unsigned long start, unsigned long end) 393 { 394 /* 395 * end could be not aligned, and We can not align that, 396 * decompresser could be confused by aligned initrd_end 397 * We already reserve the end partial page before in 398 * - i386_start_kernel() 399 * - x86_64_start_kernel() 400 * - relocate_initrd() 401 * So here We can do PAGE_ALIGN() safely to get partial page to be freed 402 */ 403 free_init_pages("initrd memory", start, PAGE_ALIGN(end)); 404 } 405 #endif 406 407 void __init zone_sizes_init(void) 408 { 409 unsigned long max_zone_pfns[MAX_NR_ZONES]; 410 411 memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); 412 413 #ifdef CONFIG_ZONE_DMA 414 max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN; 415 #endif 416 #ifdef CONFIG_ZONE_DMA32 417 max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN; 418 #endif 419 max_zone_pfns[ZONE_NORMAL] = max_low_pfn; 420 #ifdef CONFIG_HIGHMEM 421 max_zone_pfns[ZONE_HIGHMEM] = max_pfn; 422 #endif 423 424 free_area_init_nodes(max_zone_pfns); 425 } 426 427