1 /* 2 * linux/arch/arm/mm/init.c 3 * 4 * Copyright (C) 1995-2005 Russell King 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 #include <linux/config.h> 11 #include <linux/kernel.h> 12 #include <linux/errno.h> 13 #include <linux/ptrace.h> 14 #include <linux/swap.h> 15 #include <linux/init.h> 16 #include <linux/bootmem.h> 17 #include <linux/mman.h> 18 #include <linux/nodemask.h> 19 #include <linux/initrd.h> 20 21 #include <asm/mach-types.h> 22 #include <asm/hardware.h> 23 #include <asm/setup.h> 24 #include <asm/tlb.h> 25 26 #include <asm/mach/arch.h> 27 #include <asm/mach/map.h> 28 29 #define TABLE_SIZE (2 * PTRS_PER_PTE * sizeof(pte_t)) 30 31 DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); 32 33 extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; 34 extern void _stext, _text, _etext, __data_start, _end, __init_begin, __init_end; 35 extern unsigned long phys_initrd_start; 36 extern unsigned long phys_initrd_size; 37 38 /* 39 * The sole use of this is to pass memory configuration 40 * data from paging_init to mem_init. 41 */ 42 static struct meminfo meminfo __initdata = { 0, }; 43 44 /* 45 * empty_zero_page is a special page that is used for 46 * zero-initialized data and COW. 47 */ 48 struct page *empty_zero_page; 49 50 void show_mem(void) 51 { 52 int free = 0, total = 0, reserved = 0; 53 int shared = 0, cached = 0, slab = 0, node; 54 55 printk("Mem-info:\n"); 56 show_free_areas(); 57 printk("Free swap: %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10)); 58 59 for_each_online_node(node) { 60 struct page *page, *end; 61 62 page = NODE_MEM_MAP(node); 63 end = page + NODE_DATA(node)->node_spanned_pages; 64 65 do { 66 total++; 67 if (PageReserved(page)) 68 reserved++; 69 else if (PageSwapCache(page)) 70 cached++; 71 else if (PageSlab(page)) 72 slab++; 73 else if (!page_count(page)) 74 free++; 75 else 76 shared += page_count(page) - 1; 77 page++; 78 } while (page < end); 79 } 80 81 printk("%d pages of RAM\n", total); 82 printk("%d free pages\n", free); 83 printk("%d reserved pages\n", reserved); 84 printk("%d slab pages\n", slab); 85 printk("%d pages shared\n", shared); 86 printk("%d pages swap cached\n", cached); 87 } 88 89 static inline pmd_t *pmd_off(pgd_t *pgd, unsigned long virt) 90 { 91 return pmd_offset(pgd, virt); 92 } 93 94 static inline pmd_t *pmd_off_k(unsigned long virt) 95 { 96 return pmd_off(pgd_offset_k(virt), virt); 97 } 98 99 #define for_each_nodebank(iter,mi,no) \ 100 for (iter = 0; iter < mi->nr_banks; iter++) \ 101 if (mi->bank[iter].node == no) 102 103 /* 104 * FIXME: We really want to avoid allocating the bootmap bitmap 105 * over the top of the initrd. Hopefully, this is located towards 106 * the start of a bank, so if we allocate the bootmap bitmap at 107 * the end, we won't clash. 108 */ 109 static unsigned int __init 110 find_bootmap_pfn(int node, struct meminfo *mi, unsigned int bootmap_pages) 111 { 112 unsigned int start_pfn, bank, bootmap_pfn; 113 114 start_pfn = PAGE_ALIGN(__pa(&_end)) >> PAGE_SHIFT; 115 bootmap_pfn = 0; 116 117 for_each_nodebank(bank, mi, node) { 118 unsigned int start, end; 119 120 start = mi->bank[bank].start >> PAGE_SHIFT; 121 end = (mi->bank[bank].size + 122 mi->bank[bank].start) >> PAGE_SHIFT; 123 124 if (end < start_pfn) 125 continue; 126 127 if (start < start_pfn) 128 start = start_pfn; 129 130 if (end <= start) 131 continue; 132 133 if (end - start >= bootmap_pages) { 134 bootmap_pfn = start; 135 break; 136 } 137 } 138 139 if (bootmap_pfn == 0) 140 BUG(); 141 142 return bootmap_pfn; 143 } 144 145 static int __init check_initrd(struct meminfo *mi) 146 { 147 int initrd_node = -2; 148 #ifdef CONFIG_BLK_DEV_INITRD 149 unsigned long end = phys_initrd_start + phys_initrd_size; 150 151 /* 152 * Make sure that the initrd is within a valid area of 153 * memory. 154 */ 155 if (phys_initrd_size) { 156 unsigned int i; 157 158 initrd_node = -1; 159 160 for (i = 0; i < mi->nr_banks; i++) { 161 unsigned long bank_end; 162 163 bank_end = mi->bank[i].start + mi->bank[i].size; 164 165 if (mi->bank[i].start <= phys_initrd_start && 166 end <= bank_end) 167 initrd_node = mi->bank[i].node; 168 } 169 } 170 171 if (initrd_node == -1) { 172 printk(KERN_ERR "initrd (0x%08lx - 0x%08lx) extends beyond " 173 "physical memory - disabling initrd\n", 174 phys_initrd_start, end); 175 phys_initrd_start = phys_initrd_size = 0; 176 } 177 #endif 178 179 return initrd_node; 180 } 181 182 /* 183 * Reserve the various regions of node 0 184 */ 185 static __init void reserve_node_zero(pg_data_t *pgdat) 186 { 187 unsigned long res_size = 0; 188 189 /* 190 * Register the kernel text and data with bootmem. 191 * Note that this can only be in node 0. 192 */ 193 #ifdef CONFIG_XIP_KERNEL 194 reserve_bootmem_node(pgdat, __pa(&__data_start), &_end - &__data_start); 195 #else 196 reserve_bootmem_node(pgdat, __pa(&_stext), &_end - &_stext); 197 #endif 198 199 /* 200 * Reserve the page tables. These are already in use, 201 * and can only be in node 0. 202 */ 203 reserve_bootmem_node(pgdat, __pa(swapper_pg_dir), 204 PTRS_PER_PGD * sizeof(pgd_t)); 205 206 /* 207 * Hmm... This should go elsewhere, but we really really need to 208 * stop things allocating the low memory; ideally we need a better 209 * implementation of GFP_DMA which does not assume that DMA-able 210 * memory starts at zero. 211 */ 212 if (machine_is_integrator() || machine_is_cintegrator()) 213 res_size = __pa(swapper_pg_dir) - PHYS_OFFSET; 214 215 /* 216 * These should likewise go elsewhere. They pre-reserve the 217 * screen memory region at the start of main system memory. 218 */ 219 if (machine_is_edb7211()) 220 res_size = 0x00020000; 221 if (machine_is_p720t()) 222 res_size = 0x00014000; 223 224 #ifdef CONFIG_SA1111 225 /* 226 * Because of the SA1111 DMA bug, we want to preserve our 227 * precious DMA-able memory... 228 */ 229 res_size = __pa(swapper_pg_dir) - PHYS_OFFSET; 230 #endif 231 if (res_size) 232 reserve_bootmem_node(pgdat, PHYS_OFFSET, res_size); 233 } 234 235 void __init build_mem_type_table(void); 236 void __init create_mapping(struct map_desc *md); 237 238 static unsigned long __init 239 bootmem_init_node(int node, int initrd_node, struct meminfo *mi) 240 { 241 unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES]; 242 unsigned long start_pfn, end_pfn, boot_pfn; 243 unsigned int boot_pages; 244 pg_data_t *pgdat; 245 int i; 246 247 start_pfn = -1UL; 248 end_pfn = 0; 249 250 /* 251 * Calculate the pfn range, and map the memory banks for this node. 252 */ 253 for_each_nodebank(i, mi, node) { 254 unsigned long start, end; 255 struct map_desc map; 256 257 start = mi->bank[i].start >> PAGE_SHIFT; 258 end = (mi->bank[i].start + mi->bank[i].size) >> PAGE_SHIFT; 259 260 if (start_pfn > start) 261 start_pfn = start; 262 if (end_pfn < end) 263 end_pfn = end; 264 265 map.pfn = __phys_to_pfn(mi->bank[i].start); 266 map.virtual = __phys_to_virt(mi->bank[i].start); 267 map.length = mi->bank[i].size; 268 map.type = MT_MEMORY; 269 270 create_mapping(&map); 271 } 272 273 /* 274 * If there is no memory in this node, ignore it. 275 */ 276 if (end_pfn == 0) 277 return end_pfn; 278 279 /* 280 * Allocate the bootmem bitmap page. 281 */ 282 boot_pages = bootmem_bootmap_pages(end_pfn - start_pfn); 283 boot_pfn = find_bootmap_pfn(node, mi, boot_pages); 284 285 /* 286 * Initialise the bootmem allocator for this node, handing the 287 * memory banks over to bootmem. 288 */ 289 node_set_online(node); 290 pgdat = NODE_DATA(node); 291 init_bootmem_node(pgdat, boot_pfn, start_pfn, end_pfn); 292 293 for_each_nodebank(i, mi, node) 294 free_bootmem_node(pgdat, mi->bank[i].start, mi->bank[i].size); 295 296 /* 297 * Reserve the bootmem bitmap for this node. 298 */ 299 reserve_bootmem_node(pgdat, boot_pfn << PAGE_SHIFT, 300 boot_pages << PAGE_SHIFT); 301 302 #ifdef CONFIG_BLK_DEV_INITRD 303 /* 304 * If the initrd is in this node, reserve its memory. 305 */ 306 if (node == initrd_node) { 307 reserve_bootmem_node(pgdat, phys_initrd_start, 308 phys_initrd_size); 309 initrd_start = __phys_to_virt(phys_initrd_start); 310 initrd_end = initrd_start + phys_initrd_size; 311 } 312 #endif 313 314 /* 315 * Finally, reserve any node zero regions. 316 */ 317 if (node == 0) 318 reserve_node_zero(pgdat); 319 320 /* 321 * initialise the zones within this node. 322 */ 323 memset(zone_size, 0, sizeof(zone_size)); 324 memset(zhole_size, 0, sizeof(zhole_size)); 325 326 /* 327 * The size of this node has already been determined. If we need 328 * to do anything fancy with the allocation of this memory to the 329 * zones, now is the time to do it. 330 */ 331 zone_size[0] = end_pfn - start_pfn; 332 333 /* 334 * For each bank in this node, calculate the size of the holes. 335 * holes = node_size - sum(bank_sizes_in_node) 336 */ 337 zhole_size[0] = zone_size[0]; 338 for_each_nodebank(i, mi, node) 339 zhole_size[0] -= mi->bank[i].size >> PAGE_SHIFT; 340 341 /* 342 * Adjust the sizes according to any special requirements for 343 * this machine type. 344 */ 345 arch_adjust_zones(node, zone_size, zhole_size); 346 347 free_area_init_node(node, pgdat, zone_size, start_pfn, zhole_size); 348 349 return end_pfn; 350 } 351 352 static void __init bootmem_init(struct meminfo *mi) 353 { 354 unsigned long addr, memend_pfn = 0; 355 int node, initrd_node, i; 356 357 /* 358 * Invalidate the node number for empty or invalid memory banks 359 */ 360 for (i = 0; i < mi->nr_banks; i++) 361 if (mi->bank[i].size == 0 || mi->bank[i].node >= MAX_NUMNODES) 362 mi->bank[i].node = -1; 363 364 memcpy(&meminfo, mi, sizeof(meminfo)); 365 366 #ifdef CONFIG_XIP_KERNEL 367 #error needs fixing 368 p->pfn = __phys_to_pfn(CONFIG_XIP_PHYS_ADDR & PMD_MASK); 369 p->virtual = (unsigned long)&_stext & PMD_MASK; 370 p->length = ((unsigned long)&_etext - p->virtual + ~PMD_MASK) & PMD_MASK; 371 p->type = MT_ROM; 372 p ++; 373 #endif 374 375 /* 376 * Clear out all the mappings below the kernel image. 377 * FIXME: what about XIP? 378 */ 379 for (addr = 0; addr < PAGE_OFFSET; addr += PGDIR_SIZE) 380 pmd_clear(pmd_off_k(addr)); 381 382 /* 383 * Clear out all the kernel space mappings, except for the first 384 * memory bank, up to the end of the vmalloc region. 385 */ 386 for (addr = __phys_to_virt(mi->bank[0].start + mi->bank[0].size); 387 addr < VMALLOC_END; addr += PGDIR_SIZE) 388 pmd_clear(pmd_off_k(addr)); 389 390 /* 391 * Locate which node contains the ramdisk image, if any. 392 */ 393 initrd_node = check_initrd(mi); 394 395 /* 396 * Run through each node initialising the bootmem allocator. 397 */ 398 for_each_node(node) { 399 unsigned long end_pfn; 400 401 end_pfn = bootmem_init_node(node, initrd_node, mi); 402 403 /* 404 * Remember the highest memory PFN. 405 */ 406 if (end_pfn > memend_pfn) 407 memend_pfn = end_pfn; 408 } 409 410 high_memory = __va(memend_pfn << PAGE_SHIFT); 411 412 /* 413 * This doesn't seem to be used by the Linux memory manager any 414 * more, but is used by ll_rw_block. If we can get rid of it, we 415 * also get rid of some of the stuff above as well. 416 * 417 * Note: max_low_pfn and max_pfn reflect the number of _pages_ in 418 * the system, not the maximum PFN. 419 */ 420 max_pfn = max_low_pfn = memend_pfn - PHYS_PFN_OFFSET; 421 } 422 423 /* 424 * Set up device the mappings. Since we clear out the page tables for all 425 * mappings above VMALLOC_END, we will remove any debug device mappings. 426 * This means you have to be careful how you debug this function, or any 427 * called function. (Do it by code inspection!) 428 */ 429 static void __init devicemaps_init(struct machine_desc *mdesc) 430 { 431 struct map_desc map; 432 unsigned long addr; 433 void *vectors; 434 435 for (addr = VMALLOC_END; addr; addr += PGDIR_SIZE) 436 pmd_clear(pmd_off_k(addr)); 437 438 /* 439 * Map the cache flushing regions. 440 */ 441 #ifdef FLUSH_BASE 442 map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS); 443 map.virtual = FLUSH_BASE; 444 map.length = PGDIR_SIZE; 445 map.type = MT_CACHECLEAN; 446 create_mapping(&map); 447 #endif 448 #ifdef FLUSH_BASE_MINICACHE 449 map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS + PGDIR_SIZE); 450 map.virtual = FLUSH_BASE_MINICACHE; 451 map.length = PGDIR_SIZE; 452 map.type = MT_MINICLEAN; 453 create_mapping(&map); 454 #endif 455 456 flush_cache_all(); 457 local_flush_tlb_all(); 458 459 vectors = alloc_bootmem_low_pages(PAGE_SIZE); 460 BUG_ON(!vectors); 461 462 /* 463 * Create a mapping for the machine vectors at the high-vectors 464 * location (0xffff0000). If we aren't using high-vectors, also 465 * create a mapping at the low-vectors virtual address. 466 */ 467 map.pfn = __phys_to_pfn(virt_to_phys(vectors)); 468 map.virtual = 0xffff0000; 469 map.length = PAGE_SIZE; 470 map.type = MT_HIGH_VECTORS; 471 create_mapping(&map); 472 473 if (!vectors_high()) { 474 map.virtual = 0; 475 map.type = MT_LOW_VECTORS; 476 create_mapping(&map); 477 } 478 479 /* 480 * Ask the machine support to map in the statically mapped devices. 481 * After this point, we can start to touch devices again. 482 */ 483 if (mdesc->map_io) 484 mdesc->map_io(); 485 } 486 487 /* 488 * paging_init() sets up the page tables, initialises the zone memory 489 * maps, and sets up the zero page, bad page and bad page tables. 490 */ 491 void __init paging_init(struct meminfo *mi, struct machine_desc *mdesc) 492 { 493 void *zero_page; 494 495 build_mem_type_table(); 496 bootmem_init(mi); 497 devicemaps_init(mdesc); 498 499 top_pmd = pmd_off_k(0xffff0000); 500 501 /* 502 * allocate the zero page. Note that we count on this going ok. 503 */ 504 zero_page = alloc_bootmem_low_pages(PAGE_SIZE); 505 memzero(zero_page, PAGE_SIZE); 506 empty_zero_page = virt_to_page(zero_page); 507 flush_dcache_page(empty_zero_page); 508 } 509 510 static inline void free_area(unsigned long addr, unsigned long end, char *s) 511 { 512 unsigned int size = (end - addr) >> 10; 513 514 for (; addr < end; addr += PAGE_SIZE) { 515 struct page *page = virt_to_page(addr); 516 ClearPageReserved(page); 517 set_page_count(page, 1); 518 free_page(addr); 519 totalram_pages++; 520 } 521 522 if (size && s) 523 printk(KERN_INFO "Freeing %s memory: %dK\n", s, size); 524 } 525 526 static inline void 527 free_memmap(int node, unsigned long start_pfn, unsigned long end_pfn) 528 { 529 struct page *start_pg, *end_pg; 530 unsigned long pg, pgend; 531 532 /* 533 * Convert start_pfn/end_pfn to a struct page pointer. 534 */ 535 start_pg = pfn_to_page(start_pfn); 536 end_pg = pfn_to_page(end_pfn); 537 538 /* 539 * Convert to physical addresses, and 540 * round start upwards and end downwards. 541 */ 542 pg = PAGE_ALIGN(__pa(start_pg)); 543 pgend = __pa(end_pg) & PAGE_MASK; 544 545 /* 546 * If there are free pages between these, 547 * free the section of the memmap array. 548 */ 549 if (pg < pgend) 550 free_bootmem_node(NODE_DATA(node), pg, pgend - pg); 551 } 552 553 /* 554 * The mem_map array can get very big. Free the unused area of the memory map. 555 */ 556 static void __init free_unused_memmap_node(int node, struct meminfo *mi) 557 { 558 unsigned long bank_start, prev_bank_end = 0; 559 unsigned int i; 560 561 /* 562 * [FIXME] This relies on each bank being in address order. This 563 * may not be the case, especially if the user has provided the 564 * information on the command line. 565 */ 566 for_each_nodebank(i, mi, node) { 567 bank_start = mi->bank[i].start >> PAGE_SHIFT; 568 if (bank_start < prev_bank_end) { 569 printk(KERN_ERR "MEM: unordered memory banks. " 570 "Not freeing memmap.\n"); 571 break; 572 } 573 574 /* 575 * If we had a previous bank, and there is a space 576 * between the current bank and the previous, free it. 577 */ 578 if (prev_bank_end && prev_bank_end != bank_start) 579 free_memmap(node, prev_bank_end, bank_start); 580 581 prev_bank_end = (mi->bank[i].start + 582 mi->bank[i].size) >> PAGE_SHIFT; 583 } 584 } 585 586 /* 587 * mem_init() marks the free areas in the mem_map and tells us how much 588 * memory is free. This is done after various parts of the system have 589 * claimed their memory after the kernel image. 590 */ 591 void __init mem_init(void) 592 { 593 unsigned int codepages, datapages, initpages; 594 int i, node; 595 596 codepages = &_etext - &_text; 597 datapages = &_end - &__data_start; 598 initpages = &__init_end - &__init_begin; 599 600 #ifndef CONFIG_DISCONTIGMEM 601 max_mapnr = virt_to_page(high_memory) - mem_map; 602 #endif 603 604 /* this will put all unused low memory onto the freelists */ 605 for_each_online_node(node) { 606 pg_data_t *pgdat = NODE_DATA(node); 607 608 free_unused_memmap_node(node, &meminfo); 609 610 if (pgdat->node_spanned_pages != 0) 611 totalram_pages += free_all_bootmem_node(pgdat); 612 } 613 614 #ifdef CONFIG_SA1111 615 /* now that our DMA memory is actually so designated, we can free it */ 616 free_area(PAGE_OFFSET, (unsigned long)swapper_pg_dir, NULL); 617 #endif 618 619 /* 620 * Since our memory may not be contiguous, calculate the 621 * real number of pages we have in this system 622 */ 623 printk(KERN_INFO "Memory:"); 624 625 num_physpages = 0; 626 for (i = 0; i < meminfo.nr_banks; i++) { 627 num_physpages += meminfo.bank[i].size >> PAGE_SHIFT; 628 printk(" %ldMB", meminfo.bank[i].size >> 20); 629 } 630 631 printk(" = %luMB total\n", num_physpages >> (20 - PAGE_SHIFT)); 632 printk(KERN_NOTICE "Memory: %luKB available (%dK code, " 633 "%dK data, %dK init)\n", 634 (unsigned long) nr_free_pages() << (PAGE_SHIFT-10), 635 codepages >> 10, datapages >> 10, initpages >> 10); 636 637 if (PAGE_SIZE >= 16384 && num_physpages <= 128) { 638 extern int sysctl_overcommit_memory; 639 /* 640 * On a machine this small we won't get 641 * anywhere without overcommit, so turn 642 * it on by default. 643 */ 644 sysctl_overcommit_memory = OVERCOMMIT_ALWAYS; 645 } 646 } 647 648 void free_initmem(void) 649 { 650 if (!machine_is_integrator() && !machine_is_cintegrator()) { 651 free_area((unsigned long)(&__init_begin), 652 (unsigned long)(&__init_end), 653 "init"); 654 } 655 } 656 657 #ifdef CONFIG_BLK_DEV_INITRD 658 659 static int keep_initrd; 660 661 void free_initrd_mem(unsigned long start, unsigned long end) 662 { 663 if (!keep_initrd) 664 free_area(start, end, "initrd"); 665 } 666 667 static int __init keepinitrd_setup(char *__unused) 668 { 669 keep_initrd = 1; 670 return 1; 671 } 672 673 __setup("keepinitrd", keepinitrd_setup); 674 #endif 675