1 #include <linux/gfp.h> 2 #include <linux/initrd.h> 3 #include <linux/ioport.h> 4 #include <linux/swap.h> 5 #include <linux/memblock.h> 6 #include <linux/bootmem.h> /* for max_low_pfn */ 7 8 #include <asm/cacheflush.h> 9 #include <asm/e820.h> 10 #include <asm/init.h> 11 #include <asm/page.h> 12 #include <asm/page_types.h> 13 #include <asm/sections.h> 14 #include <asm/setup.h> 15 #include <asm/tlbflush.h> 16 #include <asm/tlb.h> 17 #include <asm/proto.h> 18 #include <asm/dma.h> /* for MAX_DMA_PFN */ 19 #include <asm/microcode.h> 20 21 #include "mm_internal.h" 22 23 static unsigned long __initdata pgt_buf_start; 24 static unsigned long __initdata pgt_buf_end; 25 static unsigned long __initdata pgt_buf_top; 26 27 static unsigned long min_pfn_mapped; 28 29 static bool __initdata can_use_brk_pgt = true; 30 31 /* 32 * Pages returned are already directly mapped. 33 * 34 * Changing that is likely to break Xen, see commit: 35 * 36 * 279b706 x86,xen: introduce x86_init.mapping.pagetable_reserve 37 * 38 * for detailed information. 39 */ 40 __ref void *alloc_low_pages(unsigned int num) 41 { 42 unsigned long pfn; 43 int i; 44 45 if (after_bootmem) { 46 unsigned int order; 47 48 order = get_order((unsigned long)num << PAGE_SHIFT); 49 return (void *)__get_free_pages(GFP_ATOMIC | __GFP_NOTRACK | 50 __GFP_ZERO, order); 51 } 52 53 if ((pgt_buf_end + num) > pgt_buf_top || !can_use_brk_pgt) { 54 unsigned long ret; 55 if (min_pfn_mapped >= max_pfn_mapped) 56 panic("alloc_low_page: ran out of memory"); 57 ret = memblock_find_in_range(min_pfn_mapped << PAGE_SHIFT, 58 max_pfn_mapped << PAGE_SHIFT, 59 PAGE_SIZE * num , PAGE_SIZE); 60 if (!ret) 61 panic("alloc_low_page: can not alloc memory"); 62 memblock_reserve(ret, PAGE_SIZE * num); 63 pfn = ret >> PAGE_SHIFT; 64 } else { 65 pfn = pgt_buf_end; 66 pgt_buf_end += num; 67 printk(KERN_DEBUG "BRK [%#010lx, %#010lx] PGTABLE\n", 68 pfn << PAGE_SHIFT, (pgt_buf_end << PAGE_SHIFT) - 1); 69 } 70 71 for (i = 0; i < num; i++) { 72 void *adr; 73 74 adr = __va((pfn + i) << PAGE_SHIFT); 75 clear_page(adr); 76 } 77 78 return __va(pfn << PAGE_SHIFT); 79 } 80 81 /* need 4 4k for initial PMD_SIZE, 4k for 0-ISA_END_ADDRESS */ 82 #define INIT_PGT_BUF_SIZE (5 * PAGE_SIZE) 83 RESERVE_BRK(early_pgt_alloc, INIT_PGT_BUF_SIZE); 84 void __init early_alloc_pgt_buf(void) 85 { 86 unsigned long tables = INIT_PGT_BUF_SIZE; 87 phys_addr_t base; 88 89 base = __pa(extend_brk(tables, PAGE_SIZE)); 90 91 pgt_buf_start = base >> PAGE_SHIFT; 92 pgt_buf_end = pgt_buf_start; 93 pgt_buf_top = pgt_buf_start + (tables >> PAGE_SHIFT); 94 } 95 96 int after_bootmem; 97 98 int direct_gbpages 99 #ifdef CONFIG_DIRECT_GBPAGES 100 = 1 101 #endif 102 ; 103 104 static void __init init_gbpages(void) 105 { 106 #ifdef CONFIG_X86_64 107 if (direct_gbpages && cpu_has_gbpages) 108 printk(KERN_INFO "Using GB pages for direct mapping\n"); 109 else 110 direct_gbpages = 0; 111 #endif 112 } 113 114 struct map_range { 115 unsigned long start; 116 unsigned long end; 117 unsigned page_size_mask; 118 }; 119 120 static int page_size_mask; 121 122 static void __init probe_page_size_mask(void) 123 { 124 init_gbpages(); 125 126 #if !defined(CONFIG_DEBUG_PAGEALLOC) && !defined(CONFIG_KMEMCHECK) 127 /* 128 * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages. 129 * This will simplify cpa(), which otherwise needs to support splitting 130 * large pages into small in interrupt context, etc. 131 */ 132 if (direct_gbpages) 133 page_size_mask |= 1 << PG_LEVEL_1G; 134 if (cpu_has_pse) 135 page_size_mask |= 1 << PG_LEVEL_2M; 136 #endif 137 138 /* Enable PSE if available */ 139 if (cpu_has_pse) 140 set_in_cr4(X86_CR4_PSE); 141 142 /* Enable PGE if available */ 143 if (cpu_has_pge) { 144 set_in_cr4(X86_CR4_PGE); 145 __supported_pte_mask |= _PAGE_GLOBAL; 146 } 147 } 148 149 #ifdef CONFIG_X86_32 150 #define NR_RANGE_MR 3 151 #else /* CONFIG_X86_64 */ 152 #define NR_RANGE_MR 5 153 #endif 154 155 static int __meminit save_mr(struct map_range *mr, int nr_range, 156 unsigned long start_pfn, unsigned long end_pfn, 157 unsigned long page_size_mask) 158 { 159 if (start_pfn < end_pfn) { 160 if (nr_range >= NR_RANGE_MR) 161 panic("run out of range for init_memory_mapping\n"); 162 mr[nr_range].start = start_pfn<<PAGE_SHIFT; 163 mr[nr_range].end = end_pfn<<PAGE_SHIFT; 164 mr[nr_range].page_size_mask = page_size_mask; 165 nr_range++; 166 } 167 168 return nr_range; 169 } 170 171 /* 172 * adjust the page_size_mask for small range to go with 173 * big page size instead small one if nearby are ram too. 174 */ 175 static void __init_refok adjust_range_page_size_mask(struct map_range *mr, 176 int nr_range) 177 { 178 int i; 179 180 for (i = 0; i < nr_range; i++) { 181 if ((page_size_mask & (1<<PG_LEVEL_2M)) && 182 !(mr[i].page_size_mask & (1<<PG_LEVEL_2M))) { 183 unsigned long start = round_down(mr[i].start, PMD_SIZE); 184 unsigned long end = round_up(mr[i].end, PMD_SIZE); 185 186 #ifdef CONFIG_X86_32 187 if ((end >> PAGE_SHIFT) > max_low_pfn) 188 continue; 189 #endif 190 191 if (memblock_is_region_memory(start, end - start)) 192 mr[i].page_size_mask |= 1<<PG_LEVEL_2M; 193 } 194 if ((page_size_mask & (1<<PG_LEVEL_1G)) && 195 !(mr[i].page_size_mask & (1<<PG_LEVEL_1G))) { 196 unsigned long start = round_down(mr[i].start, PUD_SIZE); 197 unsigned long end = round_up(mr[i].end, PUD_SIZE); 198 199 if (memblock_is_region_memory(start, end - start)) 200 mr[i].page_size_mask |= 1<<PG_LEVEL_1G; 201 } 202 } 203 } 204 205 static int __meminit split_mem_range(struct map_range *mr, int nr_range, 206 unsigned long start, 207 unsigned long end) 208 { 209 unsigned long start_pfn, end_pfn, limit_pfn; 210 unsigned long pfn; 211 int i; 212 213 limit_pfn = PFN_DOWN(end); 214 215 /* head if not big page alignment ? */ 216 pfn = start_pfn = PFN_DOWN(start); 217 #ifdef CONFIG_X86_32 218 /* 219 * Don't use a large page for the first 2/4MB of memory 220 * because there are often fixed size MTRRs in there 221 * and overlapping MTRRs into large pages can cause 222 * slowdowns. 223 */ 224 if (pfn == 0) 225 end_pfn = PFN_DOWN(PMD_SIZE); 226 else 227 end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE)); 228 #else /* CONFIG_X86_64 */ 229 end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE)); 230 #endif 231 if (end_pfn > limit_pfn) 232 end_pfn = limit_pfn; 233 if (start_pfn < end_pfn) { 234 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); 235 pfn = end_pfn; 236 } 237 238 /* big page (2M) range */ 239 start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE)); 240 #ifdef CONFIG_X86_32 241 end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE)); 242 #else /* CONFIG_X86_64 */ 243 end_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE)); 244 if (end_pfn > round_down(limit_pfn, PFN_DOWN(PMD_SIZE))) 245 end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE)); 246 #endif 247 248 if (start_pfn < end_pfn) { 249 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 250 page_size_mask & (1<<PG_LEVEL_2M)); 251 pfn = end_pfn; 252 } 253 254 #ifdef CONFIG_X86_64 255 /* big page (1G) range */ 256 start_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE)); 257 end_pfn = round_down(limit_pfn, PFN_DOWN(PUD_SIZE)); 258 if (start_pfn < end_pfn) { 259 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 260 page_size_mask & 261 ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G))); 262 pfn = end_pfn; 263 } 264 265 /* tail is not big page (1G) alignment */ 266 start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE)); 267 end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE)); 268 if (start_pfn < end_pfn) { 269 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 270 page_size_mask & (1<<PG_LEVEL_2M)); 271 pfn = end_pfn; 272 } 273 #endif 274 275 /* tail is not big page (2M) alignment */ 276 start_pfn = pfn; 277 end_pfn = limit_pfn; 278 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); 279 280 /* try to merge same page size and continuous */ 281 for (i = 0; nr_range > 1 && i < nr_range - 1; i++) { 282 unsigned long old_start; 283 if (mr[i].end != mr[i+1].start || 284 mr[i].page_size_mask != mr[i+1].page_size_mask) 285 continue; 286 /* move it */ 287 old_start = mr[i].start; 288 memmove(&mr[i], &mr[i+1], 289 (nr_range - 1 - i) * sizeof(struct map_range)); 290 mr[i--].start = old_start; 291 nr_range--; 292 } 293 294 if (!after_bootmem) 295 adjust_range_page_size_mask(mr, nr_range); 296 297 for (i = 0; i < nr_range; i++) 298 printk(KERN_DEBUG " [mem %#010lx-%#010lx] page %s\n", 299 mr[i].start, mr[i].end - 1, 300 (mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":( 301 (mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k")); 302 303 return nr_range; 304 } 305 306 struct range pfn_mapped[E820_X_MAX]; 307 int nr_pfn_mapped; 308 309 static void add_pfn_range_mapped(unsigned long start_pfn, unsigned long end_pfn) 310 { 311 nr_pfn_mapped = add_range_with_merge(pfn_mapped, E820_X_MAX, 312 nr_pfn_mapped, start_pfn, end_pfn); 313 nr_pfn_mapped = clean_sort_range(pfn_mapped, E820_X_MAX); 314 315 max_pfn_mapped = max(max_pfn_mapped, end_pfn); 316 317 if (start_pfn < (1UL<<(32-PAGE_SHIFT))) 318 max_low_pfn_mapped = max(max_low_pfn_mapped, 319 min(end_pfn, 1UL<<(32-PAGE_SHIFT))); 320 } 321 322 bool pfn_range_is_mapped(unsigned long start_pfn, unsigned long end_pfn) 323 { 324 int i; 325 326 for (i = 0; i < nr_pfn_mapped; i++) 327 if ((start_pfn >= pfn_mapped[i].start) && 328 (end_pfn <= pfn_mapped[i].end)) 329 return true; 330 331 return false; 332 } 333 334 /* 335 * Setup the direct mapping of the physical memory at PAGE_OFFSET. 336 * This runs before bootmem is initialized and gets pages directly from 337 * the physical memory. To access them they are temporarily mapped. 338 */ 339 unsigned long __init_refok init_memory_mapping(unsigned long start, 340 unsigned long end) 341 { 342 struct map_range mr[NR_RANGE_MR]; 343 unsigned long ret = 0; 344 int nr_range, i; 345 346 pr_info("init_memory_mapping: [mem %#010lx-%#010lx]\n", 347 start, end - 1); 348 349 memset(mr, 0, sizeof(mr)); 350 nr_range = split_mem_range(mr, 0, start, end); 351 352 for (i = 0; i < nr_range; i++) 353 ret = kernel_physical_mapping_init(mr[i].start, mr[i].end, 354 mr[i].page_size_mask); 355 356 add_pfn_range_mapped(start >> PAGE_SHIFT, ret >> PAGE_SHIFT); 357 358 return ret >> PAGE_SHIFT; 359 } 360 361 /* 362 * would have hole in the middle or ends, and only ram parts will be mapped. 363 */ 364 static unsigned long __init init_range_memory_mapping( 365 unsigned long r_start, 366 unsigned long r_end) 367 { 368 unsigned long start_pfn, end_pfn; 369 unsigned long mapped_ram_size = 0; 370 int i; 371 372 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) { 373 u64 start = clamp_val(PFN_PHYS(start_pfn), r_start, r_end); 374 u64 end = clamp_val(PFN_PHYS(end_pfn), r_start, r_end); 375 if (start >= end) 376 continue; 377 378 /* 379 * if it is overlapping with brk pgt, we need to 380 * alloc pgt buf from memblock instead. 381 */ 382 can_use_brk_pgt = max(start, (u64)pgt_buf_end<<PAGE_SHIFT) >= 383 min(end, (u64)pgt_buf_top<<PAGE_SHIFT); 384 init_memory_mapping(start, end); 385 mapped_ram_size += end - start; 386 can_use_brk_pgt = true; 387 } 388 389 return mapped_ram_size; 390 } 391 392 /* (PUD_SHIFT-PMD_SHIFT)/2 */ 393 #define STEP_SIZE_SHIFT 5 394 void __init init_mem_mapping(void) 395 { 396 unsigned long end, real_end, start, last_start; 397 unsigned long step_size; 398 unsigned long addr; 399 unsigned long mapped_ram_size = 0; 400 unsigned long new_mapped_ram_size; 401 402 probe_page_size_mask(); 403 404 #ifdef CONFIG_X86_64 405 end = max_pfn << PAGE_SHIFT; 406 #else 407 end = max_low_pfn << PAGE_SHIFT; 408 #endif 409 410 /* the ISA range is always mapped regardless of memory holes */ 411 init_memory_mapping(0, ISA_END_ADDRESS); 412 413 /* xen has big range in reserved near end of ram, skip it at first.*/ 414 addr = memblock_find_in_range(ISA_END_ADDRESS, end, PMD_SIZE, PMD_SIZE); 415 real_end = addr + PMD_SIZE; 416 417 /* step_size need to be small so pgt_buf from BRK could cover it */ 418 step_size = PMD_SIZE; 419 max_pfn_mapped = 0; /* will get exact value next */ 420 min_pfn_mapped = real_end >> PAGE_SHIFT; 421 last_start = start = real_end; 422 while (last_start > ISA_END_ADDRESS) { 423 if (last_start > step_size) { 424 start = round_down(last_start - 1, step_size); 425 if (start < ISA_END_ADDRESS) 426 start = ISA_END_ADDRESS; 427 } else 428 start = ISA_END_ADDRESS; 429 new_mapped_ram_size = init_range_memory_mapping(start, 430 last_start); 431 last_start = start; 432 min_pfn_mapped = last_start >> PAGE_SHIFT; 433 /* only increase step_size after big range get mapped */ 434 if (new_mapped_ram_size > mapped_ram_size) 435 step_size <<= STEP_SIZE_SHIFT; 436 mapped_ram_size += new_mapped_ram_size; 437 } 438 439 if (real_end < end) 440 init_range_memory_mapping(real_end, end); 441 442 #ifdef CONFIG_X86_64 443 if (max_pfn > max_low_pfn) { 444 /* can we preseve max_low_pfn ?*/ 445 max_low_pfn = max_pfn; 446 } 447 #else 448 early_ioremap_page_table_range_init(); 449 #endif 450 451 load_cr3(swapper_pg_dir); 452 __flush_tlb_all(); 453 454 early_memtest(0, max_pfn_mapped << PAGE_SHIFT); 455 } 456 457 /* 458 * devmem_is_allowed() checks to see if /dev/mem access to a certain address 459 * is valid. The argument is a physical page number. 460 * 461 * 462 * On x86, access has to be given to the first megabyte of ram because that area 463 * contains bios code and data regions used by X and dosemu and similar apps. 464 * Access has to be given to non-kernel-ram areas as well, these contain the PCI 465 * mmio resources as well as potential bios/acpi data regions. 466 */ 467 int devmem_is_allowed(unsigned long pagenr) 468 { 469 if (pagenr < 256) 470 return 1; 471 if (iomem_is_exclusive(pagenr << PAGE_SHIFT)) 472 return 0; 473 if (!page_is_ram(pagenr)) 474 return 1; 475 return 0; 476 } 477 478 void free_init_pages(char *what, unsigned long begin, unsigned long end) 479 { 480 unsigned long addr; 481 unsigned long begin_aligned, end_aligned; 482 483 /* Make sure boundaries are page aligned */ 484 begin_aligned = PAGE_ALIGN(begin); 485 end_aligned = end & PAGE_MASK; 486 487 if (WARN_ON(begin_aligned != begin || end_aligned != end)) { 488 begin = begin_aligned; 489 end = end_aligned; 490 } 491 492 if (begin >= end) 493 return; 494 495 addr = begin; 496 497 /* 498 * If debugging page accesses then do not free this memory but 499 * mark them not present - any buggy init-section access will 500 * create a kernel page fault: 501 */ 502 #ifdef CONFIG_DEBUG_PAGEALLOC 503 printk(KERN_INFO "debug: unmapping init [mem %#010lx-%#010lx]\n", 504 begin, end - 1); 505 set_memory_np(begin, (end - begin) >> PAGE_SHIFT); 506 #else 507 /* 508 * We just marked the kernel text read only above, now that 509 * we are going to free part of that, we need to make that 510 * writeable and non-executable first. 511 */ 512 set_memory_nx(begin, (end - begin) >> PAGE_SHIFT); 513 set_memory_rw(begin, (end - begin) >> PAGE_SHIFT); 514 515 printk(KERN_INFO "Freeing %s: %luk freed\n", what, (end - begin) >> 10); 516 517 for (; addr < end; addr += PAGE_SIZE) { 518 memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE); 519 free_reserved_page(virt_to_page(addr)); 520 } 521 #endif 522 } 523 524 void free_initmem(void) 525 { 526 free_init_pages("unused kernel memory", 527 (unsigned long)(&__init_begin), 528 (unsigned long)(&__init_end)); 529 } 530 531 #ifdef CONFIG_BLK_DEV_INITRD 532 void __init free_initrd_mem(unsigned long start, unsigned long end) 533 { 534 #ifdef CONFIG_MICROCODE_EARLY 535 /* 536 * Remember, initrd memory may contain microcode or other useful things. 537 * Before we lose initrd mem, we need to find a place to hold them 538 * now that normal virtual memory is enabled. 539 */ 540 save_microcode_in_initrd(); 541 #endif 542 543 /* 544 * end could be not aligned, and We can not align that, 545 * decompresser could be confused by aligned initrd_end 546 * We already reserve the end partial page before in 547 * - i386_start_kernel() 548 * - x86_64_start_kernel() 549 * - relocate_initrd() 550 * So here We can do PAGE_ALIGN() safely to get partial page to be freed 551 */ 552 free_init_pages("initrd memory", start, PAGE_ALIGN(end)); 553 } 554 #endif 555 556 void __init zone_sizes_init(void) 557 { 558 unsigned long max_zone_pfns[MAX_NR_ZONES]; 559 560 memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); 561 562 #ifdef CONFIG_ZONE_DMA 563 max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN; 564 #endif 565 #ifdef CONFIG_ZONE_DMA32 566 max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN; 567 #endif 568 max_zone_pfns[ZONE_NORMAL] = max_low_pfn; 569 #ifdef CONFIG_HIGHMEM 570 max_zone_pfns[ZONE_HIGHMEM] = max_pfn; 571 #endif 572 573 free_area_init_nodes(max_zone_pfns); 574 } 575 576