1 /* 2 * linux/arch/arm/mm/mmu.c 3 * 4 * Copyright (C) 1995-2005 Russell King 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 #include <linux/module.h> 11 #include <linux/kernel.h> 12 #include <linux/errno.h> 13 #include <linux/init.h> 14 #include <linux/bootmem.h> 15 #include <linux/mman.h> 16 #include <linux/nodemask.h> 17 18 #include <asm/mach-types.h> 19 #include <asm/setup.h> 20 #include <asm/sizes.h> 21 #include <asm/tlb.h> 22 23 #include <asm/mach/arch.h> 24 #include <asm/mach/map.h> 25 26 #include "mm.h" 27 28 DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); 29 30 extern void _stext, _etext, __data_start, _end; 31 extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; 32 33 /* 34 * empty_zero_page is a special page that is used for 35 * zero-initialized data and COW. 36 */ 37 struct page *empty_zero_page; 38 EXPORT_SYMBOL(empty_zero_page); 39 40 /* 41 * The pmd table for the upper-most set of pages. 42 */ 43 pmd_t *top_pmd; 44 45 #define CPOLICY_UNCACHED 0 46 #define CPOLICY_BUFFERED 1 47 #define CPOLICY_WRITETHROUGH 2 48 #define CPOLICY_WRITEBACK 3 49 #define CPOLICY_WRITEALLOC 4 50 51 static unsigned int cachepolicy __initdata = CPOLICY_WRITEBACK; 52 static unsigned int ecc_mask __initdata = 0; 53 pgprot_t pgprot_user; 54 pgprot_t pgprot_kernel; 55 56 EXPORT_SYMBOL(pgprot_user); 57 EXPORT_SYMBOL(pgprot_kernel); 58 59 struct cachepolicy { 60 const char policy[16]; 61 unsigned int cr_mask; 62 unsigned int pmd; 63 unsigned int pte; 64 }; 65 66 static struct cachepolicy cache_policies[] __initdata = { 67 { 68 .policy = "uncached", 69 .cr_mask = CR_W|CR_C, 70 .pmd = PMD_SECT_UNCACHED, 71 .pte = L_PTE_MT_UNCACHED, 72 }, { 73 .policy = "buffered", 74 .cr_mask = CR_C, 75 .pmd = PMD_SECT_BUFFERED, 76 .pte = L_PTE_MT_BUFFERABLE, 77 }, { 78 .policy = "writethrough", 79 .cr_mask = 0, 80 .pmd = PMD_SECT_WT, 81 .pte = L_PTE_MT_WRITETHROUGH, 82 }, { 83 .policy = "writeback", 84 .cr_mask = 0, 85 .pmd = PMD_SECT_WB, 86 .pte = L_PTE_MT_WRITEBACK, 87 }, { 88 .policy = "writealloc", 89 .cr_mask = 0, 90 .pmd = PMD_SECT_WBWA, 91 .pte = L_PTE_MT_WRITEALLOC, 92 } 93 }; 94 95 /* 96 * These are useful for identifying cache coherency 97 * problems by allowing the cache or the cache and 98 * writebuffer to be turned off. (Note: the write 99 * buffer should not be on and the cache off). 100 */ 101 static void __init early_cachepolicy(char **p) 102 { 103 int i; 104 105 for (i = 0; i < ARRAY_SIZE(cache_policies); i++) { 106 int len = strlen(cache_policies[i].policy); 107 108 if (memcmp(*p, cache_policies[i].policy, len) == 0) { 109 cachepolicy = i; 110 cr_alignment &= ~cache_policies[i].cr_mask; 111 cr_no_alignment &= ~cache_policies[i].cr_mask; 112 *p += len; 113 break; 114 } 115 } 116 if (i == ARRAY_SIZE(cache_policies)) 117 printk(KERN_ERR "ERROR: unknown or unsupported cache policy\n"); 118 if (cpu_architecture() >= CPU_ARCH_ARMv6) { 119 printk(KERN_WARNING "Only cachepolicy=writeback supported on ARMv6 and later\n"); 120 cachepolicy = CPOLICY_WRITEBACK; 121 } 122 flush_cache_all(); 123 set_cr(cr_alignment); 124 } 125 __early_param("cachepolicy=", early_cachepolicy); 126 127 static void __init early_nocache(char **__unused) 128 { 129 char *p = "buffered"; 130 printk(KERN_WARNING "nocache is deprecated; use cachepolicy=%s\n", p); 131 early_cachepolicy(&p); 132 } 133 __early_param("nocache", early_nocache); 134 135 static void __init early_nowrite(char **__unused) 136 { 137 char *p = "uncached"; 138 printk(KERN_WARNING "nowb is deprecated; use cachepolicy=%s\n", p); 139 early_cachepolicy(&p); 140 } 141 __early_param("nowb", early_nowrite); 142 143 static void __init early_ecc(char **p) 144 { 145 if (memcmp(*p, "on", 2) == 0) { 146 ecc_mask = PMD_PROTECTION; 147 *p += 2; 148 } else if (memcmp(*p, "off", 3) == 0) { 149 ecc_mask = 0; 150 *p += 3; 151 } 152 } 153 __early_param("ecc=", early_ecc); 154 155 static int __init noalign_setup(char *__unused) 156 { 157 cr_alignment &= ~CR_A; 158 cr_no_alignment &= ~CR_A; 159 set_cr(cr_alignment); 160 return 1; 161 } 162 __setup("noalign", noalign_setup); 163 164 #ifndef CONFIG_SMP 165 void adjust_cr(unsigned long mask, unsigned long set) 166 { 167 unsigned long flags; 168 169 mask &= ~CR_A; 170 171 set &= mask; 172 173 local_irq_save(flags); 174 175 cr_no_alignment = (cr_no_alignment & ~mask) | set; 176 cr_alignment = (cr_alignment & ~mask) | set; 177 178 set_cr((get_cr() & ~mask) | set); 179 180 local_irq_restore(flags); 181 } 182 #endif 183 184 #define PROT_PTE_DEVICE L_PTE_PRESENT|L_PTE_YOUNG|L_PTE_DIRTY|L_PTE_WRITE 185 #define PROT_SECT_DEVICE PMD_TYPE_SECT|PMD_SECT_XN|PMD_SECT_AP_WRITE 186 187 static struct mem_type mem_types[] = { 188 [MT_DEVICE] = { /* Strongly ordered / ARMv6 shared device */ 189 .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_SHARED | 190 L_PTE_SHARED, 191 .prot_l1 = PMD_TYPE_TABLE, 192 .prot_sect = PROT_SECT_DEVICE | PMD_SECT_UNCACHED, 193 .domain = DOMAIN_IO, 194 }, 195 [MT_DEVICE_NONSHARED] = { /* ARMv6 non-shared device */ 196 .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_NONSHARED, 197 .prot_l1 = PMD_TYPE_TABLE, 198 .prot_sect = PROT_SECT_DEVICE | PMD_SECT_TEX(2), 199 .domain = DOMAIN_IO, 200 }, 201 [MT_DEVICE_CACHED] = { /* ioremap_cached */ 202 .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_CACHED, 203 .prot_l1 = PMD_TYPE_TABLE, 204 .prot_sect = PROT_SECT_DEVICE | PMD_SECT_WB, 205 .domain = DOMAIN_IO, 206 }, 207 [MT_DEVICE_IXP2000] = { /* IXP2400 requires XCB=101 for on-chip I/O */ 208 .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_IXP2000, 209 .prot_l1 = PMD_TYPE_TABLE, 210 .prot_sect = PROT_SECT_DEVICE | PMD_SECT_BUFFERABLE | 211 PMD_SECT_TEX(1), 212 .domain = DOMAIN_IO, 213 }, 214 [MT_DEVICE_WC] = { /* ioremap_wc */ 215 .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_WC, 216 .prot_l1 = PMD_TYPE_TABLE, 217 .prot_sect = PROT_SECT_DEVICE | PMD_SECT_BUFFERABLE, 218 .domain = DOMAIN_IO, 219 }, 220 [MT_CACHECLEAN] = { 221 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN, 222 .domain = DOMAIN_KERNEL, 223 }, 224 [MT_MINICLEAN] = { 225 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN | PMD_SECT_MINICACHE, 226 .domain = DOMAIN_KERNEL, 227 }, 228 [MT_LOW_VECTORS] = { 229 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | 230 L_PTE_EXEC, 231 .prot_l1 = PMD_TYPE_TABLE, 232 .domain = DOMAIN_USER, 233 }, 234 [MT_HIGH_VECTORS] = { 235 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | 236 L_PTE_USER | L_PTE_EXEC, 237 .prot_l1 = PMD_TYPE_TABLE, 238 .domain = DOMAIN_USER, 239 }, 240 [MT_MEMORY] = { 241 .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE, 242 .domain = DOMAIN_KERNEL, 243 }, 244 [MT_ROM] = { 245 .prot_sect = PMD_TYPE_SECT, 246 .domain = DOMAIN_KERNEL, 247 }, 248 }; 249 250 const struct mem_type *get_mem_type(unsigned int type) 251 { 252 return type < ARRAY_SIZE(mem_types) ? &mem_types[type] : NULL; 253 } 254 255 /* 256 * Adjust the PMD section entries according to the CPU in use. 257 */ 258 static void __init build_mem_type_table(void) 259 { 260 struct cachepolicy *cp; 261 unsigned int cr = get_cr(); 262 unsigned int user_pgprot, kern_pgprot, vecs_pgprot; 263 int cpu_arch = cpu_architecture(); 264 int i; 265 266 if (cpu_arch < CPU_ARCH_ARMv6) { 267 #if defined(CONFIG_CPU_DCACHE_DISABLE) 268 if (cachepolicy > CPOLICY_BUFFERED) 269 cachepolicy = CPOLICY_BUFFERED; 270 #elif defined(CONFIG_CPU_DCACHE_WRITETHROUGH) 271 if (cachepolicy > CPOLICY_WRITETHROUGH) 272 cachepolicy = CPOLICY_WRITETHROUGH; 273 #endif 274 } 275 if (cpu_arch < CPU_ARCH_ARMv5) { 276 if (cachepolicy >= CPOLICY_WRITEALLOC) 277 cachepolicy = CPOLICY_WRITEBACK; 278 ecc_mask = 0; 279 } 280 #ifdef CONFIG_SMP 281 cachepolicy = CPOLICY_WRITEALLOC; 282 #endif 283 284 /* 285 * On non-Xscale3 ARMv5-and-older systems, use CB=01 286 * (Uncached/Buffered) for ioremap_wc() mappings. On XScale3 287 * and ARMv6+, use TEXCB=00100 mappings (Inner/Outer Uncacheable 288 * in xsc3 parlance, Uncached Normal in ARMv6 parlance). 289 */ 290 if (cpu_is_xsc3() || cpu_arch >= CPU_ARCH_ARMv6) { 291 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_TEX(1); 292 mem_types[MT_DEVICE_WC].prot_sect &= ~PMD_SECT_BUFFERABLE; 293 } 294 295 /* 296 * ARMv5 and lower, bit 4 must be set for page tables. 297 * (was: cache "update-able on write" bit on ARM610) 298 * However, Xscale cores require this bit to be cleared. 299 */ 300 if (cpu_is_xscale()) { 301 for (i = 0; i < ARRAY_SIZE(mem_types); i++) { 302 mem_types[i].prot_sect &= ~PMD_BIT4; 303 mem_types[i].prot_l1 &= ~PMD_BIT4; 304 } 305 } else if (cpu_arch < CPU_ARCH_ARMv6) { 306 for (i = 0; i < ARRAY_SIZE(mem_types); i++) { 307 if (mem_types[i].prot_l1) 308 mem_types[i].prot_l1 |= PMD_BIT4; 309 if (mem_types[i].prot_sect) 310 mem_types[i].prot_sect |= PMD_BIT4; 311 } 312 } 313 314 cp = &cache_policies[cachepolicy]; 315 vecs_pgprot = kern_pgprot = user_pgprot = cp->pte; 316 317 #ifndef CONFIG_SMP 318 /* 319 * Only use write-through for non-SMP systems 320 */ 321 if (cpu_arch >= CPU_ARCH_ARMv5 && cachepolicy > CPOLICY_WRITETHROUGH) 322 vecs_pgprot = cache_policies[CPOLICY_WRITETHROUGH].pte; 323 #endif 324 325 /* 326 * Enable CPU-specific coherency if supported. 327 * (Only available on XSC3 at the moment.) 328 */ 329 if (arch_is_coherent()) { 330 if (cpu_is_xsc3()) { 331 mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S; 332 mem_types[MT_MEMORY].prot_pte |= L_PTE_SHARED; 333 } 334 } 335 336 /* 337 * ARMv6 and above have extended page tables. 338 */ 339 if (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP)) { 340 /* 341 * Mark cache clean areas and XIP ROM read only 342 * from SVC mode and no access from userspace. 343 */ 344 mem_types[MT_ROM].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; 345 mem_types[MT_MINICLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; 346 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; 347 348 /* 349 * Mark the device area as "shared device" 350 */ 351 mem_types[MT_DEVICE].prot_sect |= PMD_SECT_BUFFERED; 352 353 #ifdef CONFIG_SMP 354 /* 355 * Mark memory with the "shared" attribute for SMP systems 356 */ 357 user_pgprot |= L_PTE_SHARED; 358 kern_pgprot |= L_PTE_SHARED; 359 vecs_pgprot |= L_PTE_SHARED; 360 mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S; 361 #endif 362 } 363 364 for (i = 0; i < 16; i++) { 365 unsigned long v = pgprot_val(protection_map[i]); 366 protection_map[i] = __pgprot(v | user_pgprot); 367 } 368 369 mem_types[MT_LOW_VECTORS].prot_pte |= vecs_pgprot; 370 mem_types[MT_HIGH_VECTORS].prot_pte |= vecs_pgprot; 371 372 if (cpu_arch < CPU_ARCH_ARMv5) 373 mem_types[MT_MINICLEAN].prot_sect &= ~PMD_SECT_TEX(1); 374 375 pgprot_user = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | user_pgprot); 376 pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | 377 L_PTE_DIRTY | L_PTE_WRITE | 378 L_PTE_EXEC | kern_pgprot); 379 380 mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask; 381 mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask; 382 mem_types[MT_MEMORY].prot_sect |= ecc_mask | cp->pmd; 383 mem_types[MT_ROM].prot_sect |= cp->pmd; 384 385 switch (cp->pmd) { 386 case PMD_SECT_WT: 387 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WT; 388 break; 389 case PMD_SECT_WB: 390 case PMD_SECT_WBWA: 391 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WB; 392 break; 393 } 394 printk("Memory policy: ECC %sabled, Data cache %s\n", 395 ecc_mask ? "en" : "dis", cp->policy); 396 397 for (i = 0; i < ARRAY_SIZE(mem_types); i++) { 398 struct mem_type *t = &mem_types[i]; 399 if (t->prot_l1) 400 t->prot_l1 |= PMD_DOMAIN(t->domain); 401 if (t->prot_sect) 402 t->prot_sect |= PMD_DOMAIN(t->domain); 403 } 404 } 405 406 #define vectors_base() (vectors_high() ? 0xffff0000 : 0) 407 408 static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr, 409 unsigned long end, unsigned long pfn, 410 const struct mem_type *type) 411 { 412 pte_t *pte; 413 414 if (pmd_none(*pmd)) { 415 pte = alloc_bootmem_low_pages(2 * PTRS_PER_PTE * sizeof(pte_t)); 416 __pmd_populate(pmd, __pa(pte) | type->prot_l1); 417 } 418 419 pte = pte_offset_kernel(pmd, addr); 420 do { 421 set_pte_ext(pte, pfn_pte(pfn, __pgprot(type->prot_pte)), 0); 422 pfn++; 423 } while (pte++, addr += PAGE_SIZE, addr != end); 424 } 425 426 static void __init alloc_init_section(pgd_t *pgd, unsigned long addr, 427 unsigned long end, unsigned long phys, 428 const struct mem_type *type) 429 { 430 pmd_t *pmd = pmd_offset(pgd, addr); 431 432 /* 433 * Try a section mapping - end, addr and phys must all be aligned 434 * to a section boundary. Note that PMDs refer to the individual 435 * L1 entries, whereas PGDs refer to a group of L1 entries making 436 * up one logical pointer to an L2 table. 437 */ 438 if (((addr | end | phys) & ~SECTION_MASK) == 0) { 439 pmd_t *p = pmd; 440 441 if (addr & SECTION_SIZE) 442 pmd++; 443 444 do { 445 *pmd = __pmd(phys | type->prot_sect); 446 phys += SECTION_SIZE; 447 } while (pmd++, addr += SECTION_SIZE, addr != end); 448 449 flush_pmd_entry(p); 450 } else { 451 /* 452 * No need to loop; pte's aren't interested in the 453 * individual L1 entries. 454 */ 455 alloc_init_pte(pmd, addr, end, __phys_to_pfn(phys), type); 456 } 457 } 458 459 static void __init create_36bit_mapping(struct map_desc *md, 460 const struct mem_type *type) 461 { 462 unsigned long phys, addr, length, end; 463 pgd_t *pgd; 464 465 addr = md->virtual; 466 phys = (unsigned long)__pfn_to_phys(md->pfn); 467 length = PAGE_ALIGN(md->length); 468 469 if (!(cpu_architecture() >= CPU_ARCH_ARMv6 || cpu_is_xsc3())) { 470 printk(KERN_ERR "MM: CPU does not support supersection " 471 "mapping for 0x%08llx at 0x%08lx\n", 472 __pfn_to_phys((u64)md->pfn), addr); 473 return; 474 } 475 476 /* N.B. ARMv6 supersections are only defined to work with domain 0. 477 * Since domain assignments can in fact be arbitrary, the 478 * 'domain == 0' check below is required to insure that ARMv6 479 * supersections are only allocated for domain 0 regardless 480 * of the actual domain assignments in use. 481 */ 482 if (type->domain) { 483 printk(KERN_ERR "MM: invalid domain in supersection " 484 "mapping for 0x%08llx at 0x%08lx\n", 485 __pfn_to_phys((u64)md->pfn), addr); 486 return; 487 } 488 489 if ((addr | length | __pfn_to_phys(md->pfn)) & ~SUPERSECTION_MASK) { 490 printk(KERN_ERR "MM: cannot create mapping for " 491 "0x%08llx at 0x%08lx invalid alignment\n", 492 __pfn_to_phys((u64)md->pfn), addr); 493 return; 494 } 495 496 /* 497 * Shift bits [35:32] of address into bits [23:20] of PMD 498 * (See ARMv6 spec). 499 */ 500 phys |= (((md->pfn >> (32 - PAGE_SHIFT)) & 0xF) << 20); 501 502 pgd = pgd_offset_k(addr); 503 end = addr + length; 504 do { 505 pmd_t *pmd = pmd_offset(pgd, addr); 506 int i; 507 508 for (i = 0; i < 16; i++) 509 *pmd++ = __pmd(phys | type->prot_sect | PMD_SECT_SUPER); 510 511 addr += SUPERSECTION_SIZE; 512 phys += SUPERSECTION_SIZE; 513 pgd += SUPERSECTION_SIZE >> PGDIR_SHIFT; 514 } while (addr != end); 515 } 516 517 /* 518 * Create the page directory entries and any necessary 519 * page tables for the mapping specified by `md'. We 520 * are able to cope here with varying sizes and address 521 * offsets, and we take full advantage of sections and 522 * supersections. 523 */ 524 void __init create_mapping(struct map_desc *md) 525 { 526 unsigned long phys, addr, length, end; 527 const struct mem_type *type; 528 pgd_t *pgd; 529 530 if (md->virtual != vectors_base() && md->virtual < TASK_SIZE) { 531 printk(KERN_WARNING "BUG: not creating mapping for " 532 "0x%08llx at 0x%08lx in user region\n", 533 __pfn_to_phys((u64)md->pfn), md->virtual); 534 return; 535 } 536 537 if ((md->type == MT_DEVICE || md->type == MT_ROM) && 538 md->virtual >= PAGE_OFFSET && md->virtual < VMALLOC_END) { 539 printk(KERN_WARNING "BUG: mapping for 0x%08llx at 0x%08lx " 540 "overlaps vmalloc space\n", 541 __pfn_to_phys((u64)md->pfn), md->virtual); 542 } 543 544 type = &mem_types[md->type]; 545 546 /* 547 * Catch 36-bit addresses 548 */ 549 if (md->pfn >= 0x100000) { 550 create_36bit_mapping(md, type); 551 return; 552 } 553 554 addr = md->virtual & PAGE_MASK; 555 phys = (unsigned long)__pfn_to_phys(md->pfn); 556 length = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK)); 557 558 if (type->prot_l1 == 0 && ((addr | phys | length) & ~SECTION_MASK)) { 559 printk(KERN_WARNING "BUG: map for 0x%08lx at 0x%08lx can not " 560 "be mapped using pages, ignoring.\n", 561 __pfn_to_phys(md->pfn), addr); 562 return; 563 } 564 565 pgd = pgd_offset_k(addr); 566 end = addr + length; 567 do { 568 unsigned long next = pgd_addr_end(addr, end); 569 570 alloc_init_section(pgd, addr, next, phys, type); 571 572 phys += next - addr; 573 addr = next; 574 } while (pgd++, addr != end); 575 } 576 577 /* 578 * Create the architecture specific mappings 579 */ 580 void __init iotable_init(struct map_desc *io_desc, int nr) 581 { 582 int i; 583 584 for (i = 0; i < nr; i++) 585 create_mapping(io_desc + i); 586 } 587 588 static int __init check_membank_valid(struct membank *mb) 589 { 590 /* 591 * Check whether this memory region has non-zero size. 592 */ 593 if (mb->size == 0) 594 return 0; 595 596 /* 597 * Check whether this memory region would entirely overlap 598 * the vmalloc area. 599 */ 600 if (phys_to_virt(mb->start) >= VMALLOC_MIN) { 601 printk(KERN_NOTICE "Ignoring RAM at %.8lx-%.8lx " 602 "(vmalloc region overlap).\n", 603 mb->start, mb->start + mb->size - 1); 604 return 0; 605 } 606 607 /* 608 * Check whether this memory region would partially overlap 609 * the vmalloc area. 610 */ 611 if (phys_to_virt(mb->start + mb->size) < phys_to_virt(mb->start) || 612 phys_to_virt(mb->start + mb->size) > VMALLOC_MIN) { 613 unsigned long newsize = VMALLOC_MIN - phys_to_virt(mb->start); 614 615 printk(KERN_NOTICE "Truncating RAM at %.8lx-%.8lx " 616 "to -%.8lx (vmalloc region overlap).\n", 617 mb->start, mb->start + mb->size - 1, 618 mb->start + newsize - 1); 619 mb->size = newsize; 620 } 621 622 return 1; 623 } 624 625 static void __init sanity_check_meminfo(struct meminfo *mi) 626 { 627 int i; 628 int j; 629 630 for (i = 0, j = 0; i < mi->nr_banks; i++) { 631 if (check_membank_valid(&mi->bank[i])) 632 mi->bank[j++] = mi->bank[i]; 633 } 634 mi->nr_banks = j; 635 } 636 637 static inline void prepare_page_table(struct meminfo *mi) 638 { 639 unsigned long addr; 640 641 /* 642 * Clear out all the mappings below the kernel image. 643 */ 644 for (addr = 0; addr < MODULE_START; addr += PGDIR_SIZE) 645 pmd_clear(pmd_off_k(addr)); 646 647 #ifdef CONFIG_XIP_KERNEL 648 /* The XIP kernel is mapped in the module area -- skip over it */ 649 addr = ((unsigned long)&_etext + PGDIR_SIZE - 1) & PGDIR_MASK; 650 #endif 651 for ( ; addr < PAGE_OFFSET; addr += PGDIR_SIZE) 652 pmd_clear(pmd_off_k(addr)); 653 654 /* 655 * Clear out all the kernel space mappings, except for the first 656 * memory bank, up to the end of the vmalloc region. 657 */ 658 for (addr = __phys_to_virt(mi->bank[0].start + mi->bank[0].size); 659 addr < VMALLOC_END; addr += PGDIR_SIZE) 660 pmd_clear(pmd_off_k(addr)); 661 } 662 663 /* 664 * Reserve the various regions of node 0 665 */ 666 void __init reserve_node_zero(pg_data_t *pgdat) 667 { 668 unsigned long res_size = 0; 669 670 /* 671 * Register the kernel text and data with bootmem. 672 * Note that this can only be in node 0. 673 */ 674 #ifdef CONFIG_XIP_KERNEL 675 reserve_bootmem_node(pgdat, __pa(&__data_start), &_end - &__data_start, 676 BOOTMEM_DEFAULT); 677 #else 678 reserve_bootmem_node(pgdat, __pa(&_stext), &_end - &_stext, 679 BOOTMEM_DEFAULT); 680 #endif 681 682 /* 683 * Reserve the page tables. These are already in use, 684 * and can only be in node 0. 685 */ 686 reserve_bootmem_node(pgdat, __pa(swapper_pg_dir), 687 PTRS_PER_PGD * sizeof(pgd_t), BOOTMEM_DEFAULT); 688 689 /* 690 * Hmm... This should go elsewhere, but we really really need to 691 * stop things allocating the low memory; ideally we need a better 692 * implementation of GFP_DMA which does not assume that DMA-able 693 * memory starts at zero. 694 */ 695 if (machine_is_integrator() || machine_is_cintegrator()) 696 res_size = __pa(swapper_pg_dir) - PHYS_OFFSET; 697 698 /* 699 * These should likewise go elsewhere. They pre-reserve the 700 * screen memory region at the start of main system memory. 701 */ 702 if (machine_is_edb7211()) 703 res_size = 0x00020000; 704 if (machine_is_p720t()) 705 res_size = 0x00014000; 706 707 /* H1940 and RX3715 need to reserve this for suspend */ 708 709 if (machine_is_h1940() || machine_is_rx3715()) { 710 reserve_bootmem_node(pgdat, 0x30003000, 0x1000, 711 BOOTMEM_DEFAULT); 712 reserve_bootmem_node(pgdat, 0x30081000, 0x1000, 713 BOOTMEM_DEFAULT); 714 } 715 716 #ifdef CONFIG_SA1111 717 /* 718 * Because of the SA1111 DMA bug, we want to preserve our 719 * precious DMA-able memory... 720 */ 721 res_size = __pa(swapper_pg_dir) - PHYS_OFFSET; 722 #endif 723 if (res_size) 724 reserve_bootmem_node(pgdat, PHYS_OFFSET, res_size, 725 BOOTMEM_DEFAULT); 726 } 727 728 /* 729 * Set up device the mappings. Since we clear out the page tables for all 730 * mappings above VMALLOC_END, we will remove any debug device mappings. 731 * This means you have to be careful how you debug this function, or any 732 * called function. This means you can't use any function or debugging 733 * method which may touch any device, otherwise the kernel _will_ crash. 734 */ 735 static void __init devicemaps_init(struct machine_desc *mdesc) 736 { 737 struct map_desc map; 738 unsigned long addr; 739 void *vectors; 740 741 /* 742 * Allocate the vector page early. 743 */ 744 vectors = alloc_bootmem_low_pages(PAGE_SIZE); 745 BUG_ON(!vectors); 746 747 for (addr = VMALLOC_END; addr; addr += PGDIR_SIZE) 748 pmd_clear(pmd_off_k(addr)); 749 750 /* 751 * Map the kernel if it is XIP. 752 * It is always first in the modulearea. 753 */ 754 #ifdef CONFIG_XIP_KERNEL 755 map.pfn = __phys_to_pfn(CONFIG_XIP_PHYS_ADDR & SECTION_MASK); 756 map.virtual = MODULE_START; 757 map.length = ((unsigned long)&_etext - map.virtual + ~SECTION_MASK) & SECTION_MASK; 758 map.type = MT_ROM; 759 create_mapping(&map); 760 #endif 761 762 /* 763 * Map the cache flushing regions. 764 */ 765 #ifdef FLUSH_BASE 766 map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS); 767 map.virtual = FLUSH_BASE; 768 map.length = SZ_1M; 769 map.type = MT_CACHECLEAN; 770 create_mapping(&map); 771 #endif 772 #ifdef FLUSH_BASE_MINICACHE 773 map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS + SZ_1M); 774 map.virtual = FLUSH_BASE_MINICACHE; 775 map.length = SZ_1M; 776 map.type = MT_MINICLEAN; 777 create_mapping(&map); 778 #endif 779 780 /* 781 * Create a mapping for the machine vectors at the high-vectors 782 * location (0xffff0000). If we aren't using high-vectors, also 783 * create a mapping at the low-vectors virtual address. 784 */ 785 map.pfn = __phys_to_pfn(virt_to_phys(vectors)); 786 map.virtual = 0xffff0000; 787 map.length = PAGE_SIZE; 788 map.type = MT_HIGH_VECTORS; 789 create_mapping(&map); 790 791 if (!vectors_high()) { 792 map.virtual = 0; 793 map.type = MT_LOW_VECTORS; 794 create_mapping(&map); 795 } 796 797 /* 798 * Ask the machine support to map in the statically mapped devices. 799 */ 800 if (mdesc->map_io) 801 mdesc->map_io(); 802 803 /* 804 * Finally flush the caches and tlb to ensure that we're in a 805 * consistent state wrt the writebuffer. This also ensures that 806 * any write-allocated cache lines in the vector page are written 807 * back. After this point, we can start to touch devices again. 808 */ 809 local_flush_tlb_all(); 810 flush_cache_all(); 811 } 812 813 /* 814 * paging_init() sets up the page tables, initialises the zone memory 815 * maps, and sets up the zero page, bad page and bad page tables. 816 */ 817 void __init paging_init(struct meminfo *mi, struct machine_desc *mdesc) 818 { 819 void *zero_page; 820 821 build_mem_type_table(); 822 sanity_check_meminfo(mi); 823 prepare_page_table(mi); 824 bootmem_init(mi); 825 devicemaps_init(mdesc); 826 827 top_pmd = pmd_off_k(0xffff0000); 828 829 /* 830 * allocate the zero page. Note that we count on this going ok. 831 */ 832 zero_page = alloc_bootmem_low_pages(PAGE_SIZE); 833 memzero(zero_page, PAGE_SIZE); 834 empty_zero_page = virt_to_page(zero_page); 835 flush_dcache_page(empty_zero_page); 836 } 837 838 /* 839 * In order to soft-boot, we need to insert a 1:1 mapping in place of 840 * the user-mode pages. This will then ensure that we have predictable 841 * results when turning the mmu off 842 */ 843 void setup_mm_for_reboot(char mode) 844 { 845 unsigned long base_pmdval; 846 pgd_t *pgd; 847 int i; 848 849 if (current->mm && current->mm->pgd) 850 pgd = current->mm->pgd; 851 else 852 pgd = init_mm.pgd; 853 854 base_pmdval = PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | PMD_TYPE_SECT; 855 if (cpu_architecture() <= CPU_ARCH_ARMv5TEJ && !cpu_is_xscale()) 856 base_pmdval |= PMD_BIT4; 857 858 for (i = 0; i < FIRST_USER_PGD_NR + USER_PTRS_PER_PGD; i++, pgd++) { 859 unsigned long pmdval = (i << PGDIR_SHIFT) | base_pmdval; 860 pmd_t *pmd; 861 862 pmd = pmd_off(pgd, i << PGDIR_SHIFT); 863 pmd[0] = __pmd(pmdval); 864 pmd[1] = __pmd(pmdval + (1 << (PGDIR_SHIFT - 1))); 865 flush_pmd_entry(pmd); 866 } 867 } 868