1 #include <linux/gfp.h> 2 #include <linux/initrd.h> 3 #include <linux/ioport.h> 4 #include <linux/swap.h> 5 #include <linux/memblock.h> 6 #include <linux/bootmem.h> /* for max_low_pfn */ 7 8 #include <asm/cacheflush.h> 9 #include <asm/e820.h> 10 #include <asm/init.h> 11 #include <asm/page.h> 12 #include <asm/page_types.h> 13 #include <asm/sections.h> 14 #include <asm/setup.h> 15 #include <asm/tlbflush.h> 16 #include <asm/tlb.h> 17 #include <asm/proto.h> 18 #include <asm/dma.h> /* for MAX_DMA_PFN */ 19 #include <asm/microcode.h> 20 21 #include "mm_internal.h" 22 23 static unsigned long __initdata pgt_buf_start; 24 static unsigned long __initdata pgt_buf_end; 25 static unsigned long __initdata pgt_buf_top; 26 27 static unsigned long min_pfn_mapped; 28 29 static bool __initdata can_use_brk_pgt = true; 30 31 /* 32 * Pages returned are already directly mapped. 33 * 34 * Changing that is likely to break Xen, see commit: 35 * 36 * 279b706 x86,xen: introduce x86_init.mapping.pagetable_reserve 37 * 38 * for detailed information. 39 */ 40 __ref void *alloc_low_pages(unsigned int num) 41 { 42 unsigned long pfn; 43 int i; 44 45 if (after_bootmem) { 46 unsigned int order; 47 48 order = get_order((unsigned long)num << PAGE_SHIFT); 49 return (void *)__get_free_pages(GFP_ATOMIC | __GFP_NOTRACK | 50 __GFP_ZERO, order); 51 } 52 53 if ((pgt_buf_end + num) > pgt_buf_top || !can_use_brk_pgt) { 54 unsigned long ret; 55 if (min_pfn_mapped >= max_pfn_mapped) 56 panic("alloc_low_page: ran out of memory"); 57 ret = memblock_find_in_range(min_pfn_mapped << PAGE_SHIFT, 58 max_pfn_mapped << PAGE_SHIFT, 59 PAGE_SIZE * num , PAGE_SIZE); 60 if (!ret) 61 panic("alloc_low_page: can not alloc memory"); 62 memblock_reserve(ret, PAGE_SIZE * num); 63 pfn = ret >> PAGE_SHIFT; 64 } else { 65 pfn = pgt_buf_end; 66 pgt_buf_end += num; 67 printk(KERN_DEBUG "BRK [%#010lx, %#010lx] PGTABLE\n", 68 pfn << PAGE_SHIFT, (pgt_buf_end << PAGE_SHIFT) - 1); 69 } 70 71 for (i = 0; i < num; i++) { 72 void *adr; 73 74 adr = __va((pfn + i) << PAGE_SHIFT); 75 clear_page(adr); 76 } 77 78 return __va(pfn << PAGE_SHIFT); 79 } 80 81 /* need 3 4k for initial PMD_SIZE, 3 4k for 0-ISA_END_ADDRESS */ 82 #define INIT_PGT_BUF_SIZE (6 * PAGE_SIZE) 83 RESERVE_BRK(early_pgt_alloc, INIT_PGT_BUF_SIZE); 84 void __init early_alloc_pgt_buf(void) 85 { 86 unsigned long tables = INIT_PGT_BUF_SIZE; 87 phys_addr_t base; 88 89 base = __pa(extend_brk(tables, PAGE_SIZE)); 90 91 pgt_buf_start = base >> PAGE_SHIFT; 92 pgt_buf_end = pgt_buf_start; 93 pgt_buf_top = pgt_buf_start + (tables >> PAGE_SHIFT); 94 } 95 96 int after_bootmem; 97 98 int direct_gbpages 99 #ifdef CONFIG_DIRECT_GBPAGES 100 = 1 101 #endif 102 ; 103 104 static void __init init_gbpages(void) 105 { 106 #ifdef CONFIG_X86_64 107 if (direct_gbpages && cpu_has_gbpages) 108 printk(KERN_INFO "Using GB pages for direct mapping\n"); 109 else 110 direct_gbpages = 0; 111 #endif 112 } 113 114 struct map_range { 115 unsigned long start; 116 unsigned long end; 117 unsigned page_size_mask; 118 }; 119 120 static int page_size_mask; 121 122 static void __init probe_page_size_mask(void) 123 { 124 init_gbpages(); 125 126 #if !defined(CONFIG_DEBUG_PAGEALLOC) && !defined(CONFIG_KMEMCHECK) 127 /* 128 * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages. 129 * This will simplify cpa(), which otherwise needs to support splitting 130 * large pages into small in interrupt context, etc. 131 */ 132 if (direct_gbpages) 133 page_size_mask |= 1 << PG_LEVEL_1G; 134 if (cpu_has_pse) 135 page_size_mask |= 1 << PG_LEVEL_2M; 136 #endif 137 138 /* Enable PSE if available */ 139 if (cpu_has_pse) 140 set_in_cr4(X86_CR4_PSE); 141 142 /* Enable PGE if available */ 143 if (cpu_has_pge) { 144 set_in_cr4(X86_CR4_PGE); 145 __supported_pte_mask |= _PAGE_GLOBAL; 146 } 147 } 148 149 #ifdef CONFIG_X86_32 150 #define NR_RANGE_MR 3 151 #else /* CONFIG_X86_64 */ 152 #define NR_RANGE_MR 5 153 #endif 154 155 static int __meminit save_mr(struct map_range *mr, int nr_range, 156 unsigned long start_pfn, unsigned long end_pfn, 157 unsigned long page_size_mask) 158 { 159 if (start_pfn < end_pfn) { 160 if (nr_range >= NR_RANGE_MR) 161 panic("run out of range for init_memory_mapping\n"); 162 mr[nr_range].start = start_pfn<<PAGE_SHIFT; 163 mr[nr_range].end = end_pfn<<PAGE_SHIFT; 164 mr[nr_range].page_size_mask = page_size_mask; 165 nr_range++; 166 } 167 168 return nr_range; 169 } 170 171 /* 172 * adjust the page_size_mask for small range to go with 173 * big page size instead small one if nearby are ram too. 174 */ 175 static void __init_refok adjust_range_page_size_mask(struct map_range *mr, 176 int nr_range) 177 { 178 int i; 179 180 for (i = 0; i < nr_range; i++) { 181 if ((page_size_mask & (1<<PG_LEVEL_2M)) && 182 !(mr[i].page_size_mask & (1<<PG_LEVEL_2M))) { 183 unsigned long start = round_down(mr[i].start, PMD_SIZE); 184 unsigned long end = round_up(mr[i].end, PMD_SIZE); 185 186 #ifdef CONFIG_X86_32 187 if ((end >> PAGE_SHIFT) > max_low_pfn) 188 continue; 189 #endif 190 191 if (memblock_is_region_memory(start, end - start)) 192 mr[i].page_size_mask |= 1<<PG_LEVEL_2M; 193 } 194 if ((page_size_mask & (1<<PG_LEVEL_1G)) && 195 !(mr[i].page_size_mask & (1<<PG_LEVEL_1G))) { 196 unsigned long start = round_down(mr[i].start, PUD_SIZE); 197 unsigned long end = round_up(mr[i].end, PUD_SIZE); 198 199 if (memblock_is_region_memory(start, end - start)) 200 mr[i].page_size_mask |= 1<<PG_LEVEL_1G; 201 } 202 } 203 } 204 205 static int __meminit split_mem_range(struct map_range *mr, int nr_range, 206 unsigned long start, 207 unsigned long end) 208 { 209 unsigned long start_pfn, end_pfn, limit_pfn; 210 unsigned long pfn; 211 int i; 212 213 limit_pfn = PFN_DOWN(end); 214 215 /* head if not big page alignment ? */ 216 pfn = start_pfn = PFN_DOWN(start); 217 #ifdef CONFIG_X86_32 218 /* 219 * Don't use a large page for the first 2/4MB of memory 220 * because there are often fixed size MTRRs in there 221 * and overlapping MTRRs into large pages can cause 222 * slowdowns. 223 */ 224 if (pfn == 0) 225 end_pfn = PFN_DOWN(PMD_SIZE); 226 else 227 end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE)); 228 #else /* CONFIG_X86_64 */ 229 end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE)); 230 #endif 231 if (end_pfn > limit_pfn) 232 end_pfn = limit_pfn; 233 if (start_pfn < end_pfn) { 234 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); 235 pfn = end_pfn; 236 } 237 238 /* big page (2M) range */ 239 start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE)); 240 #ifdef CONFIG_X86_32 241 end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE)); 242 #else /* CONFIG_X86_64 */ 243 end_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE)); 244 if (end_pfn > round_down(limit_pfn, PFN_DOWN(PMD_SIZE))) 245 end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE)); 246 #endif 247 248 if (start_pfn < end_pfn) { 249 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 250 page_size_mask & (1<<PG_LEVEL_2M)); 251 pfn = end_pfn; 252 } 253 254 #ifdef CONFIG_X86_64 255 /* big page (1G) range */ 256 start_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE)); 257 end_pfn = round_down(limit_pfn, PFN_DOWN(PUD_SIZE)); 258 if (start_pfn < end_pfn) { 259 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 260 page_size_mask & 261 ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G))); 262 pfn = end_pfn; 263 } 264 265 /* tail is not big page (1G) alignment */ 266 start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE)); 267 end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE)); 268 if (start_pfn < end_pfn) { 269 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 270 page_size_mask & (1<<PG_LEVEL_2M)); 271 pfn = end_pfn; 272 } 273 #endif 274 275 /* tail is not big page (2M) alignment */ 276 start_pfn = pfn; 277 end_pfn = limit_pfn; 278 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); 279 280 if (!after_bootmem) 281 adjust_range_page_size_mask(mr, nr_range); 282 283 /* try to merge same page size and continuous */ 284 for (i = 0; nr_range > 1 && i < nr_range - 1; i++) { 285 unsigned long old_start; 286 if (mr[i].end != mr[i+1].start || 287 mr[i].page_size_mask != mr[i+1].page_size_mask) 288 continue; 289 /* move it */ 290 old_start = mr[i].start; 291 memmove(&mr[i], &mr[i+1], 292 (nr_range - 1 - i) * sizeof(struct map_range)); 293 mr[i--].start = old_start; 294 nr_range--; 295 } 296 297 for (i = 0; i < nr_range; i++) 298 printk(KERN_DEBUG " [mem %#010lx-%#010lx] page %s\n", 299 mr[i].start, mr[i].end - 1, 300 (mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":( 301 (mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k")); 302 303 return nr_range; 304 } 305 306 struct range pfn_mapped[E820_X_MAX]; 307 int nr_pfn_mapped; 308 309 static void add_pfn_range_mapped(unsigned long start_pfn, unsigned long end_pfn) 310 { 311 nr_pfn_mapped = add_range_with_merge(pfn_mapped, E820_X_MAX, 312 nr_pfn_mapped, start_pfn, end_pfn); 313 nr_pfn_mapped = clean_sort_range(pfn_mapped, E820_X_MAX); 314 315 max_pfn_mapped = max(max_pfn_mapped, end_pfn); 316 317 if (start_pfn < (1UL<<(32-PAGE_SHIFT))) 318 max_low_pfn_mapped = max(max_low_pfn_mapped, 319 min(end_pfn, 1UL<<(32-PAGE_SHIFT))); 320 } 321 322 bool pfn_range_is_mapped(unsigned long start_pfn, unsigned long end_pfn) 323 { 324 int i; 325 326 for (i = 0; i < nr_pfn_mapped; i++) 327 if ((start_pfn >= pfn_mapped[i].start) && 328 (end_pfn <= pfn_mapped[i].end)) 329 return true; 330 331 return false; 332 } 333 334 /* 335 * Setup the direct mapping of the physical memory at PAGE_OFFSET. 336 * This runs before bootmem is initialized and gets pages directly from 337 * the physical memory. To access them they are temporarily mapped. 338 */ 339 unsigned long __init_refok init_memory_mapping(unsigned long start, 340 unsigned long end) 341 { 342 struct map_range mr[NR_RANGE_MR]; 343 unsigned long ret = 0; 344 int nr_range, i; 345 346 pr_info("init_memory_mapping: [mem %#010lx-%#010lx]\n", 347 start, end - 1); 348 349 memset(mr, 0, sizeof(mr)); 350 nr_range = split_mem_range(mr, 0, start, end); 351 352 for (i = 0; i < nr_range; i++) 353 ret = kernel_physical_mapping_init(mr[i].start, mr[i].end, 354 mr[i].page_size_mask); 355 356 add_pfn_range_mapped(start >> PAGE_SHIFT, ret >> PAGE_SHIFT); 357 358 return ret >> PAGE_SHIFT; 359 } 360 361 /* 362 * We need to iterate through the E820 memory map and create direct mappings 363 * for only E820_RAM and E820_KERN_RESERVED regions. We cannot simply 364 * create direct mappings for all pfns from [0 to max_low_pfn) and 365 * [4GB to max_pfn) because of possible memory holes in high addresses 366 * that cannot be marked as UC by fixed/variable range MTRRs. 367 * Depending on the alignment of E820 ranges, this may possibly result 368 * in using smaller size (i.e. 4K instead of 2M or 1G) page tables. 369 * 370 * init_mem_mapping() calls init_range_memory_mapping() with big range. 371 * That range would have hole in the middle or ends, and only ram parts 372 * will be mapped in init_range_memory_mapping(). 373 */ 374 static unsigned long __init init_range_memory_mapping( 375 unsigned long r_start, 376 unsigned long r_end) 377 { 378 unsigned long start_pfn, end_pfn; 379 unsigned long mapped_ram_size = 0; 380 int i; 381 382 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) { 383 u64 start = clamp_val(PFN_PHYS(start_pfn), r_start, r_end); 384 u64 end = clamp_val(PFN_PHYS(end_pfn), r_start, r_end); 385 if (start >= end) 386 continue; 387 388 /* 389 * if it is overlapping with brk pgt, we need to 390 * alloc pgt buf from memblock instead. 391 */ 392 can_use_brk_pgt = max(start, (u64)pgt_buf_end<<PAGE_SHIFT) >= 393 min(end, (u64)pgt_buf_top<<PAGE_SHIFT); 394 init_memory_mapping(start, end); 395 mapped_ram_size += end - start; 396 can_use_brk_pgt = true; 397 } 398 399 return mapped_ram_size; 400 } 401 402 /* (PUD_SHIFT-PMD_SHIFT)/2 */ 403 #define STEP_SIZE_SHIFT 5 404 void __init init_mem_mapping(void) 405 { 406 unsigned long end, real_end, start, last_start; 407 unsigned long step_size; 408 unsigned long addr; 409 unsigned long mapped_ram_size = 0; 410 unsigned long new_mapped_ram_size; 411 412 probe_page_size_mask(); 413 414 #ifdef CONFIG_X86_64 415 end = max_pfn << PAGE_SHIFT; 416 #else 417 end = max_low_pfn << PAGE_SHIFT; 418 #endif 419 420 /* the ISA range is always mapped regardless of memory holes */ 421 init_memory_mapping(0, ISA_END_ADDRESS); 422 423 /* xen has big range in reserved near end of ram, skip it at first.*/ 424 addr = memblock_find_in_range(ISA_END_ADDRESS, end, PMD_SIZE, PMD_SIZE); 425 real_end = addr + PMD_SIZE; 426 427 /* step_size need to be small so pgt_buf from BRK could cover it */ 428 step_size = PMD_SIZE; 429 max_pfn_mapped = 0; /* will get exact value next */ 430 min_pfn_mapped = real_end >> PAGE_SHIFT; 431 last_start = start = real_end; 432 433 /* 434 * We start from the top (end of memory) and go to the bottom. 435 * The memblock_find_in_range() gets us a block of RAM from the 436 * end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages 437 * for page table. 438 */ 439 while (last_start > ISA_END_ADDRESS) { 440 if (last_start > step_size) { 441 start = round_down(last_start - 1, step_size); 442 if (start < ISA_END_ADDRESS) 443 start = ISA_END_ADDRESS; 444 } else 445 start = ISA_END_ADDRESS; 446 new_mapped_ram_size = init_range_memory_mapping(start, 447 last_start); 448 last_start = start; 449 min_pfn_mapped = last_start >> PAGE_SHIFT; 450 /* only increase step_size after big range get mapped */ 451 if (new_mapped_ram_size > mapped_ram_size) 452 step_size <<= STEP_SIZE_SHIFT; 453 mapped_ram_size += new_mapped_ram_size; 454 } 455 456 if (real_end < end) 457 init_range_memory_mapping(real_end, end); 458 459 #ifdef CONFIG_X86_64 460 if (max_pfn > max_low_pfn) { 461 /* can we preseve max_low_pfn ?*/ 462 max_low_pfn = max_pfn; 463 } 464 #else 465 early_ioremap_page_table_range_init(); 466 #endif 467 468 load_cr3(swapper_pg_dir); 469 __flush_tlb_all(); 470 471 early_memtest(0, max_pfn_mapped << PAGE_SHIFT); 472 } 473 474 /* 475 * devmem_is_allowed() checks to see if /dev/mem access to a certain address 476 * is valid. The argument is a physical page number. 477 * 478 * 479 * On x86, access has to be given to the first megabyte of ram because that area 480 * contains bios code and data regions used by X and dosemu and similar apps. 481 * Access has to be given to non-kernel-ram areas as well, these contain the PCI 482 * mmio resources as well as potential bios/acpi data regions. 483 */ 484 int devmem_is_allowed(unsigned long pagenr) 485 { 486 if (pagenr < 256) 487 return 1; 488 if (iomem_is_exclusive(pagenr << PAGE_SHIFT)) 489 return 0; 490 if (!page_is_ram(pagenr)) 491 return 1; 492 return 0; 493 } 494 495 void free_init_pages(char *what, unsigned long begin, unsigned long end) 496 { 497 unsigned long begin_aligned, end_aligned; 498 499 /* Make sure boundaries are page aligned */ 500 begin_aligned = PAGE_ALIGN(begin); 501 end_aligned = end & PAGE_MASK; 502 503 if (WARN_ON(begin_aligned != begin || end_aligned != end)) { 504 begin = begin_aligned; 505 end = end_aligned; 506 } 507 508 if (begin >= end) 509 return; 510 511 /* 512 * If debugging page accesses then do not free this memory but 513 * mark them not present - any buggy init-section access will 514 * create a kernel page fault: 515 */ 516 #ifdef CONFIG_DEBUG_PAGEALLOC 517 printk(KERN_INFO "debug: unmapping init [mem %#010lx-%#010lx]\n", 518 begin, end - 1); 519 set_memory_np(begin, (end - begin) >> PAGE_SHIFT); 520 #else 521 /* 522 * We just marked the kernel text read only above, now that 523 * we are going to free part of that, we need to make that 524 * writeable and non-executable first. 525 */ 526 set_memory_nx(begin, (end - begin) >> PAGE_SHIFT); 527 set_memory_rw(begin, (end - begin) >> PAGE_SHIFT); 528 529 free_reserved_area((void *)begin, (void *)end, POISON_FREE_INITMEM, what); 530 #endif 531 } 532 533 void free_initmem(void) 534 { 535 free_init_pages("unused kernel", 536 (unsigned long)(&__init_begin), 537 (unsigned long)(&__init_end)); 538 } 539 540 #ifdef CONFIG_BLK_DEV_INITRD 541 void __init free_initrd_mem(unsigned long start, unsigned long end) 542 { 543 #ifdef CONFIG_MICROCODE_EARLY 544 /* 545 * Remember, initrd memory may contain microcode or other useful things. 546 * Before we lose initrd mem, we need to find a place to hold them 547 * now that normal virtual memory is enabled. 548 */ 549 save_microcode_in_initrd(); 550 #endif 551 552 /* 553 * end could be not aligned, and We can not align that, 554 * decompresser could be confused by aligned initrd_end 555 * We already reserve the end partial page before in 556 * - i386_start_kernel() 557 * - x86_64_start_kernel() 558 * - relocate_initrd() 559 * So here We can do PAGE_ALIGN() safely to get partial page to be freed 560 */ 561 free_init_pages("initrd", start, PAGE_ALIGN(end)); 562 } 563 #endif 564 565 void __init zone_sizes_init(void) 566 { 567 unsigned long max_zone_pfns[MAX_NR_ZONES]; 568 569 memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); 570 571 #ifdef CONFIG_ZONE_DMA 572 max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN; 573 #endif 574 #ifdef CONFIG_ZONE_DMA32 575 max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN; 576 #endif 577 max_zone_pfns[ZONE_NORMAL] = max_low_pfn; 578 #ifdef CONFIG_HIGHMEM 579 max_zone_pfns[ZONE_HIGHMEM] = max_pfn; 580 #endif 581 582 free_area_init_nodes(max_zone_pfns); 583 } 584 585