1 /* 2 * linux/arch/arm/mm/init.c 3 * 4 * Copyright (C) 1995-2005 Russell King 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 #include <linux/config.h> 11 #include <linux/kernel.h> 12 #include <linux/errno.h> 13 #include <linux/ptrace.h> 14 #include <linux/swap.h> 15 #include <linux/init.h> 16 #include <linux/bootmem.h> 17 #include <linux/mman.h> 18 #include <linux/nodemask.h> 19 #include <linux/initrd.h> 20 21 #include <asm/mach-types.h> 22 #include <asm/hardware.h> 23 #include <asm/setup.h> 24 #include <asm/tlb.h> 25 26 #include <asm/mach/arch.h> 27 #include <asm/mach/map.h> 28 29 #define TABLE_SIZE (2 * PTRS_PER_PTE * sizeof(pte_t)) 30 31 DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); 32 33 extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; 34 extern void _stext, _text, _etext, __data_start, _end, __init_begin, __init_end; 35 extern unsigned long phys_initrd_start; 36 extern unsigned long phys_initrd_size; 37 38 /* 39 * The sole use of this is to pass memory configuration 40 * data from paging_init to mem_init. 41 */ 42 static struct meminfo meminfo __initdata = { 0, }; 43 44 /* 45 * empty_zero_page is a special page that is used for 46 * zero-initialized data and COW. 47 */ 48 struct page *empty_zero_page; 49 50 void show_mem(void) 51 { 52 int free = 0, total = 0, reserved = 0; 53 int shared = 0, cached = 0, slab = 0, node; 54 55 printk("Mem-info:\n"); 56 show_free_areas(); 57 printk("Free swap: %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10)); 58 59 for_each_online_node(node) { 60 struct page *page, *end; 61 62 page = NODE_MEM_MAP(node); 63 end = page + NODE_DATA(node)->node_spanned_pages; 64 65 do { 66 total++; 67 if (PageReserved(page)) 68 reserved++; 69 else if (PageSwapCache(page)) 70 cached++; 71 else if (PageSlab(page)) 72 slab++; 73 else if (!page_count(page)) 74 free++; 75 else 76 shared += page_count(page) - 1; 77 page++; 78 } while (page < end); 79 } 80 81 printk("%d pages of RAM\n", total); 82 printk("%d free pages\n", free); 83 printk("%d reserved pages\n", reserved); 84 printk("%d slab pages\n", slab); 85 printk("%d pages shared\n", shared); 86 printk("%d pages swap cached\n", cached); 87 } 88 89 static inline pmd_t *pmd_off(pgd_t *pgd, unsigned long virt) 90 { 91 return pmd_offset(pgd, virt); 92 } 93 94 static inline pmd_t *pmd_off_k(unsigned long virt) 95 { 96 return pmd_off(pgd_offset_k(virt), virt); 97 } 98 99 #define for_each_nodebank(iter,mi,no) \ 100 for (iter = 0; iter < mi->nr_banks; iter++) \ 101 if (mi->bank[iter].node == no) 102 103 /* 104 * FIXME: We really want to avoid allocating the bootmap bitmap 105 * over the top of the initrd. Hopefully, this is located towards 106 * the start of a bank, so if we allocate the bootmap bitmap at 107 * the end, we won't clash. 108 */ 109 static unsigned int __init 110 find_bootmap_pfn(int node, struct meminfo *mi, unsigned int bootmap_pages) 111 { 112 unsigned int start_pfn, bank, bootmap_pfn; 113 114 start_pfn = PAGE_ALIGN(__pa(&_end)) >> PAGE_SHIFT; 115 bootmap_pfn = 0; 116 117 for_each_nodebank(bank, mi, node) { 118 unsigned int start, end; 119 120 start = mi->bank[bank].start >> PAGE_SHIFT; 121 end = (mi->bank[bank].size + 122 mi->bank[bank].start) >> PAGE_SHIFT; 123 124 if (end < start_pfn) 125 continue; 126 127 if (start < start_pfn) 128 start = start_pfn; 129 130 if (end <= start) 131 continue; 132 133 if (end - start >= bootmap_pages) { 134 bootmap_pfn = start; 135 break; 136 } 137 } 138 139 if (bootmap_pfn == 0) 140 BUG(); 141 142 return bootmap_pfn; 143 } 144 145 static int __init check_initrd(struct meminfo *mi) 146 { 147 int initrd_node = -2; 148 #ifdef CONFIG_BLK_DEV_INITRD 149 unsigned long end = phys_initrd_start + phys_initrd_size; 150 151 /* 152 * Make sure that the initrd is within a valid area of 153 * memory. 154 */ 155 if (phys_initrd_size) { 156 unsigned int i; 157 158 initrd_node = -1; 159 160 for (i = 0; i < mi->nr_banks; i++) { 161 unsigned long bank_end; 162 163 bank_end = mi->bank[i].start + mi->bank[i].size; 164 165 if (mi->bank[i].start <= phys_initrd_start && 166 end <= bank_end) 167 initrd_node = mi->bank[i].node; 168 } 169 } 170 171 if (initrd_node == -1) { 172 printk(KERN_ERR "initrd (0x%08lx - 0x%08lx) extends beyond " 173 "physical memory - disabling initrd\n", 174 phys_initrd_start, end); 175 phys_initrd_start = phys_initrd_size = 0; 176 } 177 #endif 178 179 return initrd_node; 180 } 181 182 /* 183 * Reserve the various regions of node 0 184 */ 185 static __init void reserve_node_zero(pg_data_t *pgdat) 186 { 187 unsigned long res_size = 0; 188 189 /* 190 * Register the kernel text and data with bootmem. 191 * Note that this can only be in node 0. 192 */ 193 #ifdef CONFIG_XIP_KERNEL 194 reserve_bootmem_node(pgdat, __pa(&__data_start), &_end - &__data_start); 195 #else 196 reserve_bootmem_node(pgdat, __pa(&_stext), &_end - &_stext); 197 #endif 198 199 /* 200 * Reserve the page tables. These are already in use, 201 * and can only be in node 0. 202 */ 203 reserve_bootmem_node(pgdat, __pa(swapper_pg_dir), 204 PTRS_PER_PGD * sizeof(pgd_t)); 205 206 /* 207 * Hmm... This should go elsewhere, but we really really need to 208 * stop things allocating the low memory; ideally we need a better 209 * implementation of GFP_DMA which does not assume that DMA-able 210 * memory starts at zero. 211 */ 212 if (machine_is_integrator() || machine_is_cintegrator()) 213 res_size = __pa(swapper_pg_dir) - PHYS_OFFSET; 214 215 /* 216 * These should likewise go elsewhere. They pre-reserve the 217 * screen memory region at the start of main system memory. 218 */ 219 if (machine_is_edb7211()) 220 res_size = 0x00020000; 221 if (machine_is_p720t()) 222 res_size = 0x00014000; 223 224 #ifdef CONFIG_SA1111 225 /* 226 * Because of the SA1111 DMA bug, we want to preserve our 227 * precious DMA-able memory... 228 */ 229 res_size = __pa(swapper_pg_dir) - PHYS_OFFSET; 230 #endif 231 if (res_size) 232 reserve_bootmem_node(pgdat, PHYS_OFFSET, res_size); 233 } 234 235 void __init build_mem_type_table(void); 236 void __init create_mapping(struct map_desc *md); 237 238 static unsigned long __init 239 bootmem_init_node(int node, int initrd_node, struct meminfo *mi) 240 { 241 unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES]; 242 unsigned long start_pfn, end_pfn, boot_pfn; 243 unsigned int boot_pages; 244 pg_data_t *pgdat; 245 int i; 246 247 start_pfn = -1UL; 248 end_pfn = 0; 249 250 /* 251 * Calculate the pfn range, and map the memory banks for this node. 252 */ 253 for_each_nodebank(i, mi, node) { 254 unsigned long start, end; 255 struct map_desc map; 256 257 start = mi->bank[i].start >> PAGE_SHIFT; 258 end = (mi->bank[i].start + mi->bank[i].size) >> PAGE_SHIFT; 259 260 if (start_pfn > start) 261 start_pfn = start; 262 if (end_pfn < end) 263 end_pfn = end; 264 265 map.pfn = __phys_to_pfn(mi->bank[i].start); 266 map.virtual = __phys_to_virt(mi->bank[i].start); 267 map.length = mi->bank[i].size; 268 map.type = MT_MEMORY; 269 270 create_mapping(&map); 271 } 272 273 /* 274 * If there is no memory in this node, ignore it. 275 */ 276 if (end_pfn == 0) 277 return end_pfn; 278 279 /* 280 * Allocate the bootmem bitmap page. 281 */ 282 boot_pages = bootmem_bootmap_pages(end_pfn - start_pfn); 283 boot_pfn = find_bootmap_pfn(node, mi, boot_pages); 284 285 /* 286 * Initialise the bootmem allocator for this node, handing the 287 * memory banks over to bootmem. 288 */ 289 node_set_online(node); 290 pgdat = NODE_DATA(node); 291 init_bootmem_node(pgdat, boot_pfn, start_pfn, end_pfn); 292 293 for_each_nodebank(i, mi, node) 294 free_bootmem_node(pgdat, mi->bank[i].start, mi->bank[i].size); 295 296 /* 297 * Reserve the bootmem bitmap for this node. 298 */ 299 reserve_bootmem_node(pgdat, boot_pfn << PAGE_SHIFT, 300 boot_pages << PAGE_SHIFT); 301 302 #ifdef CONFIG_BLK_DEV_INITRD 303 /* 304 * If the initrd is in this node, reserve its memory. 305 */ 306 if (node == initrd_node) { 307 reserve_bootmem_node(pgdat, phys_initrd_start, 308 phys_initrd_size); 309 initrd_start = __phys_to_virt(phys_initrd_start); 310 initrd_end = initrd_start + phys_initrd_size; 311 } 312 #endif 313 314 /* 315 * Finally, reserve any node zero regions. 316 */ 317 if (node == 0) 318 reserve_node_zero(pgdat); 319 320 /* 321 * initialise the zones within this node. 322 */ 323 memset(zone_size, 0, sizeof(zone_size)); 324 memset(zhole_size, 0, sizeof(zhole_size)); 325 326 /* 327 * The size of this node has already been determined. If we need 328 * to do anything fancy with the allocation of this memory to the 329 * zones, now is the time to do it. 330 */ 331 zone_size[0] = end_pfn - start_pfn; 332 333 /* 334 * For each bank in this node, calculate the size of the holes. 335 * holes = node_size - sum(bank_sizes_in_node) 336 */ 337 zhole_size[0] = zone_size[0]; 338 for_each_nodebank(i, mi, node) 339 zhole_size[0] -= mi->bank[i].size >> PAGE_SHIFT; 340 341 /* 342 * Adjust the sizes according to any special requirements for 343 * this machine type. 344 */ 345 arch_adjust_zones(node, zone_size, zhole_size); 346 347 free_area_init_node(node, pgdat, zone_size, start_pfn, zhole_size); 348 349 return end_pfn; 350 } 351 352 static void __init bootmem_init(struct meminfo *mi) 353 { 354 unsigned long addr, memend_pfn = 0; 355 int node, initrd_node, i; 356 357 /* 358 * Invalidate the node number for empty or invalid memory banks 359 */ 360 for (i = 0; i < mi->nr_banks; i++) 361 if (mi->bank[i].size == 0 || mi->bank[i].node >= MAX_NUMNODES) 362 mi->bank[i].node = -1; 363 364 memcpy(&meminfo, mi, sizeof(meminfo)); 365 366 /* 367 * Clear out all the mappings below the kernel image. 368 */ 369 for (addr = 0; addr < MODULE_START; addr += PGDIR_SIZE) 370 pmd_clear(pmd_off_k(addr)); 371 #ifdef CONFIG_XIP_KERNEL 372 /* The XIP kernel is mapped in the module area -- skip over it */ 373 addr = ((unsigned long)&_etext + PGDIR_SIZE - 1) & PGDIR_MASK; 374 #endif 375 for ( ; addr < PAGE_OFFSET; addr += PGDIR_SIZE) 376 pmd_clear(pmd_off_k(addr)); 377 378 /* 379 * Clear out all the kernel space mappings, except for the first 380 * memory bank, up to the end of the vmalloc region. 381 */ 382 for (addr = __phys_to_virt(mi->bank[0].start + mi->bank[0].size); 383 addr < VMALLOC_END; addr += PGDIR_SIZE) 384 pmd_clear(pmd_off_k(addr)); 385 386 /* 387 * Locate which node contains the ramdisk image, if any. 388 */ 389 initrd_node = check_initrd(mi); 390 391 /* 392 * Run through each node initialising the bootmem allocator. 393 */ 394 for_each_node(node) { 395 unsigned long end_pfn; 396 397 end_pfn = bootmem_init_node(node, initrd_node, mi); 398 399 /* 400 * Remember the highest memory PFN. 401 */ 402 if (end_pfn > memend_pfn) 403 memend_pfn = end_pfn; 404 } 405 406 high_memory = __va(memend_pfn << PAGE_SHIFT); 407 408 /* 409 * This doesn't seem to be used by the Linux memory manager any 410 * more, but is used by ll_rw_block. If we can get rid of it, we 411 * also get rid of some of the stuff above as well. 412 * 413 * Note: max_low_pfn and max_pfn reflect the number of _pages_ in 414 * the system, not the maximum PFN. 415 */ 416 max_pfn = max_low_pfn = memend_pfn - PHYS_PFN_OFFSET; 417 } 418 419 /* 420 * Set up device the mappings. Since we clear out the page tables for all 421 * mappings above VMALLOC_END, we will remove any debug device mappings. 422 * This means you have to be careful how you debug this function, or any 423 * called function. (Do it by code inspection!) 424 */ 425 static void __init devicemaps_init(struct machine_desc *mdesc) 426 { 427 struct map_desc map; 428 unsigned long addr; 429 void *vectors; 430 431 for (addr = VMALLOC_END; addr; addr += PGDIR_SIZE) 432 pmd_clear(pmd_off_k(addr)); 433 434 /* 435 * Map the kernel if it is XIP. 436 * It is always first in the modulearea. 437 */ 438 #ifdef CONFIG_XIP_KERNEL 439 map.pfn = __phys_to_pfn(CONFIG_XIP_PHYS_ADDR & PGDIR_MASK); 440 map.virtual = MODULE_START; 441 map.length = ((unsigned long)&_etext - map.virtual + ~PGDIR_MASK) & PGDIR_MASK; 442 map.type = MT_ROM; 443 create_mapping(&map); 444 #endif 445 446 /* 447 * Map the cache flushing regions. 448 */ 449 #ifdef FLUSH_BASE 450 map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS); 451 map.virtual = FLUSH_BASE; 452 map.length = PGDIR_SIZE; 453 map.type = MT_CACHECLEAN; 454 create_mapping(&map); 455 #endif 456 #ifdef FLUSH_BASE_MINICACHE 457 map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS + PGDIR_SIZE); 458 map.virtual = FLUSH_BASE_MINICACHE; 459 map.length = PGDIR_SIZE; 460 map.type = MT_MINICLEAN; 461 create_mapping(&map); 462 #endif 463 464 flush_cache_all(); 465 local_flush_tlb_all(); 466 467 vectors = alloc_bootmem_low_pages(PAGE_SIZE); 468 BUG_ON(!vectors); 469 470 /* 471 * Create a mapping for the machine vectors at the high-vectors 472 * location (0xffff0000). If we aren't using high-vectors, also 473 * create a mapping at the low-vectors virtual address. 474 */ 475 map.pfn = __phys_to_pfn(virt_to_phys(vectors)); 476 map.virtual = 0xffff0000; 477 map.length = PAGE_SIZE; 478 map.type = MT_HIGH_VECTORS; 479 create_mapping(&map); 480 481 if (!vectors_high()) { 482 map.virtual = 0; 483 map.type = MT_LOW_VECTORS; 484 create_mapping(&map); 485 } 486 487 /* 488 * Ask the machine support to map in the statically mapped devices. 489 */ 490 if (mdesc->map_io) 491 mdesc->map_io(); 492 493 /* 494 * Finally flush the tlb again - this ensures that we're in a 495 * consistent state wrt the writebuffer if the writebuffer needs 496 * draining. After this point, we can start to touch devices 497 * again. 498 */ 499 local_flush_tlb_all(); 500 } 501 502 /* 503 * paging_init() sets up the page tables, initialises the zone memory 504 * maps, and sets up the zero page, bad page and bad page tables. 505 */ 506 void __init paging_init(struct meminfo *mi, struct machine_desc *mdesc) 507 { 508 void *zero_page; 509 510 build_mem_type_table(); 511 bootmem_init(mi); 512 devicemaps_init(mdesc); 513 514 top_pmd = pmd_off_k(0xffff0000); 515 516 /* 517 * allocate the zero page. Note that we count on this going ok. 518 */ 519 zero_page = alloc_bootmem_low_pages(PAGE_SIZE); 520 memzero(zero_page, PAGE_SIZE); 521 empty_zero_page = virt_to_page(zero_page); 522 flush_dcache_page(empty_zero_page); 523 } 524 525 static inline void free_area(unsigned long addr, unsigned long end, char *s) 526 { 527 unsigned int size = (end - addr) >> 10; 528 529 for (; addr < end; addr += PAGE_SIZE) { 530 struct page *page = virt_to_page(addr); 531 ClearPageReserved(page); 532 set_page_count(page, 1); 533 free_page(addr); 534 totalram_pages++; 535 } 536 537 if (size && s) 538 printk(KERN_INFO "Freeing %s memory: %dK\n", s, size); 539 } 540 541 static inline void 542 free_memmap(int node, unsigned long start_pfn, unsigned long end_pfn) 543 { 544 struct page *start_pg, *end_pg; 545 unsigned long pg, pgend; 546 547 /* 548 * Convert start_pfn/end_pfn to a struct page pointer. 549 */ 550 start_pg = pfn_to_page(start_pfn); 551 end_pg = pfn_to_page(end_pfn); 552 553 /* 554 * Convert to physical addresses, and 555 * round start upwards and end downwards. 556 */ 557 pg = PAGE_ALIGN(__pa(start_pg)); 558 pgend = __pa(end_pg) & PAGE_MASK; 559 560 /* 561 * If there are free pages between these, 562 * free the section of the memmap array. 563 */ 564 if (pg < pgend) 565 free_bootmem_node(NODE_DATA(node), pg, pgend - pg); 566 } 567 568 /* 569 * The mem_map array can get very big. Free the unused area of the memory map. 570 */ 571 static void __init free_unused_memmap_node(int node, struct meminfo *mi) 572 { 573 unsigned long bank_start, prev_bank_end = 0; 574 unsigned int i; 575 576 /* 577 * [FIXME] This relies on each bank being in address order. This 578 * may not be the case, especially if the user has provided the 579 * information on the command line. 580 */ 581 for_each_nodebank(i, mi, node) { 582 bank_start = mi->bank[i].start >> PAGE_SHIFT; 583 if (bank_start < prev_bank_end) { 584 printk(KERN_ERR "MEM: unordered memory banks. " 585 "Not freeing memmap.\n"); 586 break; 587 } 588 589 /* 590 * If we had a previous bank, and there is a space 591 * between the current bank and the previous, free it. 592 */ 593 if (prev_bank_end && prev_bank_end != bank_start) 594 free_memmap(node, prev_bank_end, bank_start); 595 596 prev_bank_end = (mi->bank[i].start + 597 mi->bank[i].size) >> PAGE_SHIFT; 598 } 599 } 600 601 /* 602 * mem_init() marks the free areas in the mem_map and tells us how much 603 * memory is free. This is done after various parts of the system have 604 * claimed their memory after the kernel image. 605 */ 606 void __init mem_init(void) 607 { 608 unsigned int codepages, datapages, initpages; 609 int i, node; 610 611 codepages = &_etext - &_text; 612 datapages = &_end - &__data_start; 613 initpages = &__init_end - &__init_begin; 614 615 #ifndef CONFIG_DISCONTIGMEM 616 max_mapnr = virt_to_page(high_memory) - mem_map; 617 #endif 618 619 /* this will put all unused low memory onto the freelists */ 620 for_each_online_node(node) { 621 pg_data_t *pgdat = NODE_DATA(node); 622 623 free_unused_memmap_node(node, &meminfo); 624 625 if (pgdat->node_spanned_pages != 0) 626 totalram_pages += free_all_bootmem_node(pgdat); 627 } 628 629 #ifdef CONFIG_SA1111 630 /* now that our DMA memory is actually so designated, we can free it */ 631 free_area(PAGE_OFFSET, (unsigned long)swapper_pg_dir, NULL); 632 #endif 633 634 /* 635 * Since our memory may not be contiguous, calculate the 636 * real number of pages we have in this system 637 */ 638 printk(KERN_INFO "Memory:"); 639 640 num_physpages = 0; 641 for (i = 0; i < meminfo.nr_banks; i++) { 642 num_physpages += meminfo.bank[i].size >> PAGE_SHIFT; 643 printk(" %ldMB", meminfo.bank[i].size >> 20); 644 } 645 646 printk(" = %luMB total\n", num_physpages >> (20 - PAGE_SHIFT)); 647 printk(KERN_NOTICE "Memory: %luKB available (%dK code, " 648 "%dK data, %dK init)\n", 649 (unsigned long) nr_free_pages() << (PAGE_SHIFT-10), 650 codepages >> 10, datapages >> 10, initpages >> 10); 651 652 if (PAGE_SIZE >= 16384 && num_physpages <= 128) { 653 extern int sysctl_overcommit_memory; 654 /* 655 * On a machine this small we won't get 656 * anywhere without overcommit, so turn 657 * it on by default. 658 */ 659 sysctl_overcommit_memory = OVERCOMMIT_ALWAYS; 660 } 661 } 662 663 void free_initmem(void) 664 { 665 if (!machine_is_integrator() && !machine_is_cintegrator()) { 666 free_area((unsigned long)(&__init_begin), 667 (unsigned long)(&__init_end), 668 "init"); 669 } 670 } 671 672 #ifdef CONFIG_BLK_DEV_INITRD 673 674 static int keep_initrd; 675 676 void free_initrd_mem(unsigned long start, unsigned long end) 677 { 678 if (!keep_initrd) 679 free_area(start, end, "initrd"); 680 } 681 682 static int __init keepinitrd_setup(char *__unused) 683 { 684 keep_initrd = 1; 685 return 1; 686 } 687 688 __setup("keepinitrd", keepinitrd_setup); 689 #endif 690