1c1cc1552SCatalin Marinas /* 2c1cc1552SCatalin Marinas * Based on arch/arm/mm/mmu.c 3c1cc1552SCatalin Marinas * 4c1cc1552SCatalin Marinas * Copyright (C) 1995-2005 Russell King 5c1cc1552SCatalin Marinas * Copyright (C) 2012 ARM Ltd. 6c1cc1552SCatalin Marinas * 7c1cc1552SCatalin Marinas * This program is free software; you can redistribute it and/or modify 8c1cc1552SCatalin Marinas * it under the terms of the GNU General Public License version 2 as 9c1cc1552SCatalin Marinas * published by the Free Software Foundation. 10c1cc1552SCatalin Marinas * 11c1cc1552SCatalin Marinas * This program is distributed in the hope that it will be useful, 12c1cc1552SCatalin Marinas * but WITHOUT ANY WARRANTY; without even the implied warranty of 13c1cc1552SCatalin Marinas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14c1cc1552SCatalin Marinas * GNU General Public License for more details. 15c1cc1552SCatalin Marinas * 16c1cc1552SCatalin Marinas * You should have received a copy of the GNU General Public License 17c1cc1552SCatalin Marinas * along with this program. If not, see <http://www.gnu.org/licenses/>. 18c1cc1552SCatalin Marinas */ 19c1cc1552SCatalin Marinas 20c1cc1552SCatalin Marinas #include <linux/export.h> 21c1cc1552SCatalin Marinas #include <linux/kernel.h> 22c1cc1552SCatalin Marinas #include <linux/errno.h> 23c1cc1552SCatalin Marinas #include <linux/init.h> 2461bd93ceSArd Biesheuvel #include <linux/libfdt.h> 25c1cc1552SCatalin Marinas #include <linux/mman.h> 26c1cc1552SCatalin Marinas #include <linux/nodemask.h> 27c1cc1552SCatalin Marinas #include <linux/memblock.h> 28c1cc1552SCatalin Marinas #include <linux/fs.h> 292475ff9dSCatalin Marinas #include <linux/io.h> 3041089357SCatalin Marinas #include <linux/slab.h> 31da141706SLaura Abbott #include <linux/stop_machine.h> 32c1cc1552SCatalin Marinas 33c1cc1552SCatalin Marinas #include <asm/cputype.h> 34af86e597SLaura Abbott #include <asm/fixmap.h> 35b433dce0SSuzuki K. Poulose #include <asm/kernel-pgtable.h> 36c1cc1552SCatalin Marinas #include <asm/sections.h> 37c1cc1552SCatalin Marinas #include <asm/setup.h> 38c1cc1552SCatalin Marinas #include <asm/sizes.h> 39c1cc1552SCatalin Marinas #include <asm/tlb.h> 40c79b954bSJungseok Lee #include <asm/memblock.h> 41c1cc1552SCatalin Marinas #include <asm/mmu_context.h> 42c1cc1552SCatalin Marinas 43c1cc1552SCatalin Marinas #include "mm.h" 44c1cc1552SCatalin Marinas 45dd006da2SArd Biesheuvel u64 idmap_t0sz = TCR_T0SZ(VA_BITS); 46dd006da2SArd Biesheuvel 47c1cc1552SCatalin Marinas /* 48c1cc1552SCatalin Marinas * Empty_zero_page is a special page that is used for zero-initialized data 49c1cc1552SCatalin Marinas * and COW. 50c1cc1552SCatalin Marinas */ 51c1cc1552SCatalin Marinas struct page *empty_zero_page; 52c1cc1552SCatalin Marinas EXPORT_SYMBOL(empty_zero_page); 53c1cc1552SCatalin Marinas 54c1cc1552SCatalin Marinas pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, 55c1cc1552SCatalin Marinas unsigned long size, pgprot_t vma_prot) 56c1cc1552SCatalin Marinas { 57c1cc1552SCatalin Marinas if (!pfn_valid(pfn)) 58c1cc1552SCatalin Marinas return pgprot_noncached(vma_prot); 59c1cc1552SCatalin Marinas else if (file->f_flags & O_SYNC) 60c1cc1552SCatalin Marinas return pgprot_writecombine(vma_prot); 61c1cc1552SCatalin Marinas return vma_prot; 62c1cc1552SCatalin Marinas } 63c1cc1552SCatalin Marinas EXPORT_SYMBOL(phys_mem_access_prot); 64c1cc1552SCatalin Marinas 65c1cc1552SCatalin Marinas static void __init *early_alloc(unsigned long sz) 66c1cc1552SCatalin Marinas { 677142392dSSuzuki K. Poulose phys_addr_t phys; 687142392dSSuzuki K. Poulose void *ptr; 697142392dSSuzuki K. Poulose 707142392dSSuzuki K. Poulose phys = memblock_alloc(sz, sz); 717142392dSSuzuki K. Poulose BUG_ON(!phys); 727142392dSSuzuki K. Poulose ptr = __va(phys); 73c1cc1552SCatalin Marinas memset(ptr, 0, sz); 74c1cc1552SCatalin Marinas return ptr; 75c1cc1552SCatalin Marinas } 76c1cc1552SCatalin Marinas 77da141706SLaura Abbott /* 78da141706SLaura Abbott * remap a PMD into pages 79da141706SLaura Abbott */ 80da141706SLaura Abbott static void split_pmd(pmd_t *pmd, pte_t *pte) 81da141706SLaura Abbott { 82da141706SLaura Abbott unsigned long pfn = pmd_pfn(*pmd); 83da141706SLaura Abbott int i = 0; 84da141706SLaura Abbott 85da141706SLaura Abbott do { 86da141706SLaura Abbott /* 87da141706SLaura Abbott * Need to have the least restrictive permissions available 88667c2759SCatalin Marinas * permissions will be fixed up later 89da141706SLaura Abbott */ 90667c2759SCatalin Marinas set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC)); 91da141706SLaura Abbott pfn++; 92da141706SLaura Abbott } while (pte++, i++, i < PTRS_PER_PTE); 93da141706SLaura Abbott } 94da141706SLaura Abbott 95da141706SLaura Abbott static void alloc_init_pte(pmd_t *pmd, unsigned long addr, 96667c2759SCatalin Marinas unsigned long end, unsigned long pfn, 97da141706SLaura Abbott pgprot_t prot, 98da141706SLaura Abbott void *(*alloc)(unsigned long size)) 99c1cc1552SCatalin Marinas { 100c1cc1552SCatalin Marinas pte_t *pte; 101c1cc1552SCatalin Marinas 102a1c76574SMark Rutland if (pmd_none(*pmd) || pmd_sect(*pmd)) { 103da141706SLaura Abbott pte = alloc(PTRS_PER_PTE * sizeof(pte_t)); 104da141706SLaura Abbott if (pmd_sect(*pmd)) 105da141706SLaura Abbott split_pmd(pmd, pte); 106c1cc1552SCatalin Marinas __pmd_populate(pmd, __pa(pte), PMD_TYPE_TABLE); 107da141706SLaura Abbott flush_tlb_all(); 108c1cc1552SCatalin Marinas } 109a1c76574SMark Rutland BUG_ON(pmd_bad(*pmd)); 110c1cc1552SCatalin Marinas 111c1cc1552SCatalin Marinas pte = pte_offset_kernel(pmd, addr); 112c1cc1552SCatalin Marinas do { 113667c2759SCatalin Marinas set_pte(pte, pfn_pte(pfn, prot)); 114667c2759SCatalin Marinas pfn++; 115667c2759SCatalin Marinas } while (pte++, addr += PAGE_SIZE, addr != end); 116c1cc1552SCatalin Marinas } 117c1cc1552SCatalin Marinas 1189a17a213SJisheng Zhang static void split_pud(pud_t *old_pud, pmd_t *pmd) 119da141706SLaura Abbott { 120da141706SLaura Abbott unsigned long addr = pud_pfn(*old_pud) << PAGE_SHIFT; 121da141706SLaura Abbott pgprot_t prot = __pgprot(pud_val(*old_pud) ^ addr); 122da141706SLaura Abbott int i = 0; 123da141706SLaura Abbott 124da141706SLaura Abbott do { 1251e43ba9cSArd Biesheuvel set_pmd(pmd, __pmd(addr | pgprot_val(prot))); 126da141706SLaura Abbott addr += PMD_SIZE; 127da141706SLaura Abbott } while (pmd++, i++, i < PTRS_PER_PMD); 128da141706SLaura Abbott } 129da141706SLaura Abbott 130da141706SLaura Abbott static void alloc_init_pmd(struct mm_struct *mm, pud_t *pud, 131e1e1fddaSArd Biesheuvel unsigned long addr, unsigned long end, 132da141706SLaura Abbott phys_addr_t phys, pgprot_t prot, 133da141706SLaura Abbott void *(*alloc)(unsigned long size)) 134c1cc1552SCatalin Marinas { 135c1cc1552SCatalin Marinas pmd_t *pmd; 136c1cc1552SCatalin Marinas unsigned long next; 137c1cc1552SCatalin Marinas 138c1cc1552SCatalin Marinas /* 139c1cc1552SCatalin Marinas * Check for initial section mappings in the pgd/pud and remove them. 140c1cc1552SCatalin Marinas */ 141a1c76574SMark Rutland if (pud_none(*pud) || pud_sect(*pud)) { 142da141706SLaura Abbott pmd = alloc(PTRS_PER_PMD * sizeof(pmd_t)); 143da141706SLaura Abbott if (pud_sect(*pud)) { 144da141706SLaura Abbott /* 145da141706SLaura Abbott * need to have the 1G of mappings continue to be 146da141706SLaura Abbott * present 147da141706SLaura Abbott */ 148da141706SLaura Abbott split_pud(pud, pmd); 149da141706SLaura Abbott } 150e1e1fddaSArd Biesheuvel pud_populate(mm, pud, pmd); 151da141706SLaura Abbott flush_tlb_all(); 152c1cc1552SCatalin Marinas } 153a1c76574SMark Rutland BUG_ON(pud_bad(*pud)); 154c1cc1552SCatalin Marinas 155c1cc1552SCatalin Marinas pmd = pmd_offset(pud, addr); 156c1cc1552SCatalin Marinas do { 157c1cc1552SCatalin Marinas next = pmd_addr_end(addr, end); 158c1cc1552SCatalin Marinas /* try section mapping first */ 159a55f9929SCatalin Marinas if (((addr | next | phys) & ~SECTION_MASK) == 0) { 160a55f9929SCatalin Marinas pmd_t old_pmd =*pmd; 1618ce837ceSArd Biesheuvel set_pmd(pmd, __pmd(phys | 1628ce837ceSArd Biesheuvel pgprot_val(mk_sect_prot(prot)))); 163a55f9929SCatalin Marinas /* 164a55f9929SCatalin Marinas * Check for previous table entries created during 165a55f9929SCatalin Marinas * boot (__create_page_tables) and flush them. 166a55f9929SCatalin Marinas */ 167523d6e9fSzhichang.yuan if (!pmd_none(old_pmd)) { 168a55f9929SCatalin Marinas flush_tlb_all(); 169523d6e9fSzhichang.yuan if (pmd_table(old_pmd)) { 170523d6e9fSzhichang.yuan phys_addr_t table = __pa(pte_offset_map(&old_pmd, 0)); 17141089357SCatalin Marinas if (!WARN_ON_ONCE(slab_is_available())) 172523d6e9fSzhichang.yuan memblock_free(table, PAGE_SIZE); 173523d6e9fSzhichang.yuan } 174523d6e9fSzhichang.yuan } 175a55f9929SCatalin Marinas } else { 176667c2759SCatalin Marinas alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys), 177667c2759SCatalin Marinas prot, alloc); 178a55f9929SCatalin Marinas } 179c1cc1552SCatalin Marinas phys += next - addr; 180c1cc1552SCatalin Marinas } while (pmd++, addr = next, addr != end); 181c1cc1552SCatalin Marinas } 182c1cc1552SCatalin Marinas 183da141706SLaura Abbott static inline bool use_1G_block(unsigned long addr, unsigned long next, 184da141706SLaura Abbott unsigned long phys) 185da141706SLaura Abbott { 186da141706SLaura Abbott if (PAGE_SHIFT != 12) 187da141706SLaura Abbott return false; 188da141706SLaura Abbott 189da141706SLaura Abbott if (((addr | next | phys) & ~PUD_MASK) != 0) 190da141706SLaura Abbott return false; 191da141706SLaura Abbott 192da141706SLaura Abbott return true; 193da141706SLaura Abbott } 194da141706SLaura Abbott 195da141706SLaura Abbott static void alloc_init_pud(struct mm_struct *mm, pgd_t *pgd, 196e1e1fddaSArd Biesheuvel unsigned long addr, unsigned long end, 197da141706SLaura Abbott phys_addr_t phys, pgprot_t prot, 198da141706SLaura Abbott void *(*alloc)(unsigned long size)) 199c1cc1552SCatalin Marinas { 200c79b954bSJungseok Lee pud_t *pud; 201c1cc1552SCatalin Marinas unsigned long next; 202c1cc1552SCatalin Marinas 203c79b954bSJungseok Lee if (pgd_none(*pgd)) { 204da141706SLaura Abbott pud = alloc(PTRS_PER_PUD * sizeof(pud_t)); 205e1e1fddaSArd Biesheuvel pgd_populate(mm, pgd, pud); 206c79b954bSJungseok Lee } 207c79b954bSJungseok Lee BUG_ON(pgd_bad(*pgd)); 208c79b954bSJungseok Lee 209c79b954bSJungseok Lee pud = pud_offset(pgd, addr); 210c1cc1552SCatalin Marinas do { 211c1cc1552SCatalin Marinas next = pud_addr_end(addr, end); 212206a2a73SSteve Capper 213206a2a73SSteve Capper /* 214206a2a73SSteve Capper * For 4K granule only, attempt to put down a 1GB block 215206a2a73SSteve Capper */ 216da141706SLaura Abbott if (use_1G_block(addr, next, phys)) { 217206a2a73SSteve Capper pud_t old_pud = *pud; 2188ce837ceSArd Biesheuvel set_pud(pud, __pud(phys | 2198ce837ceSArd Biesheuvel pgprot_val(mk_sect_prot(prot)))); 220206a2a73SSteve Capper 221206a2a73SSteve Capper /* 222206a2a73SSteve Capper * If we have an old value for a pud, it will 223206a2a73SSteve Capper * be pointing to a pmd table that we no longer 224206a2a73SSteve Capper * need (from swapper_pg_dir). 225206a2a73SSteve Capper * 226206a2a73SSteve Capper * Look up the old pmd table and free it. 227206a2a73SSteve Capper */ 228206a2a73SSteve Capper if (!pud_none(old_pud)) { 229206a2a73SSteve Capper flush_tlb_all(); 230523d6e9fSzhichang.yuan if (pud_table(old_pud)) { 231523d6e9fSzhichang.yuan phys_addr_t table = __pa(pmd_offset(&old_pud, 0)); 23241089357SCatalin Marinas if (!WARN_ON_ONCE(slab_is_available())) 233523d6e9fSzhichang.yuan memblock_free(table, PAGE_SIZE); 234523d6e9fSzhichang.yuan } 235206a2a73SSteve Capper } 236206a2a73SSteve Capper } else { 237da141706SLaura Abbott alloc_init_pmd(mm, pud, addr, next, phys, prot, alloc); 238206a2a73SSteve Capper } 239c1cc1552SCatalin Marinas phys += next - addr; 240c1cc1552SCatalin Marinas } while (pud++, addr = next, addr != end); 241c1cc1552SCatalin Marinas } 242c1cc1552SCatalin Marinas 243c1cc1552SCatalin Marinas /* 244c1cc1552SCatalin Marinas * Create the page directory entries and any necessary page tables for the 245c1cc1552SCatalin Marinas * mapping specified by 'md'. 246c1cc1552SCatalin Marinas */ 247da141706SLaura Abbott static void __create_mapping(struct mm_struct *mm, pgd_t *pgd, 248e1e1fddaSArd Biesheuvel phys_addr_t phys, unsigned long virt, 249da141706SLaura Abbott phys_addr_t size, pgprot_t prot, 250da141706SLaura Abbott void *(*alloc)(unsigned long size)) 251c1cc1552SCatalin Marinas { 252c1cc1552SCatalin Marinas unsigned long addr, length, end, next; 253c1cc1552SCatalin Marinas 254cc5d2b3bSMark Rutland /* 255cc5d2b3bSMark Rutland * If the virtual and physical address don't have the same offset 256cc5d2b3bSMark Rutland * within a page, we cannot map the region as the caller expects. 257cc5d2b3bSMark Rutland */ 258cc5d2b3bSMark Rutland if (WARN_ON((phys ^ virt) & ~PAGE_MASK)) 259cc5d2b3bSMark Rutland return; 260cc5d2b3bSMark Rutland 2619c4e08a3SMark Rutland phys &= PAGE_MASK; 262c1cc1552SCatalin Marinas addr = virt & PAGE_MASK; 263c1cc1552SCatalin Marinas length = PAGE_ALIGN(size + (virt & ~PAGE_MASK)); 264c1cc1552SCatalin Marinas 265c1cc1552SCatalin Marinas end = addr + length; 266c1cc1552SCatalin Marinas do { 267c1cc1552SCatalin Marinas next = pgd_addr_end(addr, end); 268da141706SLaura Abbott alloc_init_pud(mm, pgd, addr, next, phys, prot, alloc); 269c1cc1552SCatalin Marinas phys += next - addr; 270c1cc1552SCatalin Marinas } while (pgd++, addr = next, addr != end); 271c1cc1552SCatalin Marinas } 272c1cc1552SCatalin Marinas 273da141706SLaura Abbott static void *late_alloc(unsigned long size) 274da141706SLaura Abbott { 275da141706SLaura Abbott void *ptr; 276da141706SLaura Abbott 277da141706SLaura Abbott BUG_ON(size > PAGE_SIZE); 278da141706SLaura Abbott ptr = (void *)__get_free_page(PGALLOC_GFP); 279da141706SLaura Abbott BUG_ON(!ptr); 280da141706SLaura Abbott return ptr; 281da141706SLaura Abbott } 282da141706SLaura Abbott 283c53e0baaSMark Rutland static void __init create_mapping(phys_addr_t phys, unsigned long virt, 284da141706SLaura Abbott phys_addr_t size, pgprot_t prot) 285d7ecbddfSMark Salter { 286d7ecbddfSMark Salter if (virt < VMALLOC_START) { 287d7ecbddfSMark Salter pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", 288d7ecbddfSMark Salter &phys, virt); 289d7ecbddfSMark Salter return; 290d7ecbddfSMark Salter } 291e2c30ee3SMark Rutland __create_mapping(&init_mm, pgd_offset_k(virt), phys, virt, 292da141706SLaura Abbott size, prot, early_alloc); 293d7ecbddfSMark Salter } 294d7ecbddfSMark Salter 2958ce837ceSArd Biesheuvel void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, 2968ce837ceSArd Biesheuvel unsigned long virt, phys_addr_t size, 2978ce837ceSArd Biesheuvel pgprot_t prot) 2988ce837ceSArd Biesheuvel { 299da141706SLaura Abbott __create_mapping(mm, pgd_offset(mm, virt), phys, virt, size, prot, 30060305db9SArd Biesheuvel late_alloc); 301d7ecbddfSMark Salter } 302d7ecbddfSMark Salter 303da141706SLaura Abbott static void create_mapping_late(phys_addr_t phys, unsigned long virt, 304da141706SLaura Abbott phys_addr_t size, pgprot_t prot) 305da141706SLaura Abbott { 306da141706SLaura Abbott if (virt < VMALLOC_START) { 307da141706SLaura Abbott pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", 308da141706SLaura Abbott &phys, virt); 309da141706SLaura Abbott return; 310da141706SLaura Abbott } 311da141706SLaura Abbott 312e2c30ee3SMark Rutland return __create_mapping(&init_mm, pgd_offset_k(virt), 313da141706SLaura Abbott phys, virt, size, prot, late_alloc); 314da141706SLaura Abbott } 315da141706SLaura Abbott 316da141706SLaura Abbott #ifdef CONFIG_DEBUG_RODATA 317da141706SLaura Abbott static void __init __map_memblock(phys_addr_t start, phys_addr_t end) 318da141706SLaura Abbott { 319da141706SLaura Abbott /* 320da141706SLaura Abbott * Set up the executable regions using the existing section mappings 321da141706SLaura Abbott * for now. This will get more fine grained later once all memory 322da141706SLaura Abbott * is mapped 323da141706SLaura Abbott */ 3244fee9f36SArd Biesheuvel unsigned long kernel_x_start = round_down(__pa(_stext), SWAPPER_BLOCK_SIZE); 3254fee9f36SArd Biesheuvel unsigned long kernel_x_end = round_up(__pa(__init_end), SWAPPER_BLOCK_SIZE); 326da141706SLaura Abbott 327da141706SLaura Abbott if (end < kernel_x_start) { 328da141706SLaura Abbott create_mapping(start, __phys_to_virt(start), 329da141706SLaura Abbott end - start, PAGE_KERNEL); 330da141706SLaura Abbott } else if (start >= kernel_x_end) { 331da141706SLaura Abbott create_mapping(start, __phys_to_virt(start), 332da141706SLaura Abbott end - start, PAGE_KERNEL); 333da141706SLaura Abbott } else { 334da141706SLaura Abbott if (start < kernel_x_start) 335da141706SLaura Abbott create_mapping(start, __phys_to_virt(start), 336da141706SLaura Abbott kernel_x_start - start, 337da141706SLaura Abbott PAGE_KERNEL); 338da141706SLaura Abbott create_mapping(kernel_x_start, 339da141706SLaura Abbott __phys_to_virt(kernel_x_start), 340da141706SLaura Abbott kernel_x_end - kernel_x_start, 341da141706SLaura Abbott PAGE_KERNEL_EXEC); 342da141706SLaura Abbott if (kernel_x_end < end) 343da141706SLaura Abbott create_mapping(kernel_x_end, 344da141706SLaura Abbott __phys_to_virt(kernel_x_end), 345da141706SLaura Abbott end - kernel_x_end, 346da141706SLaura Abbott PAGE_KERNEL); 347da141706SLaura Abbott } 348da141706SLaura Abbott 349da141706SLaura Abbott } 350da141706SLaura Abbott #else 351da141706SLaura Abbott static void __init __map_memblock(phys_addr_t start, phys_addr_t end) 352da141706SLaura Abbott { 353da141706SLaura Abbott create_mapping(start, __phys_to_virt(start), end - start, 354da141706SLaura Abbott PAGE_KERNEL_EXEC); 355da141706SLaura Abbott } 356da141706SLaura Abbott #endif 357da141706SLaura Abbott 358c1cc1552SCatalin Marinas static void __init map_mem(void) 359c1cc1552SCatalin Marinas { 360c1cc1552SCatalin Marinas struct memblock_region *reg; 361e25208f7SCatalin Marinas phys_addr_t limit; 362c1cc1552SCatalin Marinas 363f6bc87c3SSteve Capper /* 364f6bc87c3SSteve Capper * Temporarily limit the memblock range. We need to do this as 365f6bc87c3SSteve Capper * create_mapping requires puds, pmds and ptes to be allocated from 366f6bc87c3SSteve Capper * memory addressable from the initial direct kernel mapping. 367f6bc87c3SSteve Capper * 3683dec0fe4SCatalin Marinas * The initial direct kernel mapping, located at swapper_pg_dir, gives 369b433dce0SSuzuki K. Poulose * us PUD_SIZE (with SECTION maps) or PMD_SIZE (without SECTION maps, 370b433dce0SSuzuki K. Poulose * memory starting from PHYS_OFFSET (which must be aligned to 2MB as 371b433dce0SSuzuki K. Poulose * per Documentation/arm64/booting.txt). 372f6bc87c3SSteve Capper */ 373b433dce0SSuzuki K. Poulose limit = PHYS_OFFSET + SWAPPER_INIT_MAP_SIZE; 374e25208f7SCatalin Marinas memblock_set_current_limit(limit); 375f6bc87c3SSteve Capper 376c1cc1552SCatalin Marinas /* map all the memory banks */ 377c1cc1552SCatalin Marinas for_each_memblock(memory, reg) { 378c1cc1552SCatalin Marinas phys_addr_t start = reg->base; 379c1cc1552SCatalin Marinas phys_addr_t end = start + reg->size; 380c1cc1552SCatalin Marinas 381c1cc1552SCatalin Marinas if (start >= end) 382c1cc1552SCatalin Marinas break; 383c1cc1552SCatalin Marinas 384b433dce0SSuzuki K. Poulose if (ARM64_SWAPPER_USES_SECTION_MAPS) { 385e25208f7SCatalin Marinas /* 386e25208f7SCatalin Marinas * For the first memory bank align the start address and 387e25208f7SCatalin Marinas * current memblock limit to prevent create_mapping() from 388b433dce0SSuzuki K. Poulose * allocating pte page tables from unmapped memory. With 389b433dce0SSuzuki K. Poulose * the section maps, if the first block doesn't end on section 390b433dce0SSuzuki K. Poulose * size boundary, create_mapping() will try to allocate a pte 391b433dce0SSuzuki K. Poulose * page, which may be returned from an unmapped area. 392b433dce0SSuzuki K. Poulose * When section maps are not used, the pte page table for the 393b433dce0SSuzuki K. Poulose * current limit is already present in swapper_pg_dir. 394e25208f7SCatalin Marinas */ 395e25208f7SCatalin Marinas if (start < limit) 396b433dce0SSuzuki K. Poulose start = ALIGN(start, SECTION_SIZE); 397e25208f7SCatalin Marinas if (end < limit) { 398b433dce0SSuzuki K. Poulose limit = end & SECTION_MASK; 399e25208f7SCatalin Marinas memblock_set_current_limit(limit); 400e25208f7SCatalin Marinas } 401b433dce0SSuzuki K. Poulose } 402da141706SLaura Abbott __map_memblock(start, end); 403c1cc1552SCatalin Marinas } 404f6bc87c3SSteve Capper 405f6bc87c3SSteve Capper /* Limit no longer required. */ 406f6bc87c3SSteve Capper memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE); 407c1cc1552SCatalin Marinas } 408c1cc1552SCatalin Marinas 4099a17a213SJisheng Zhang static void __init fixup_executable(void) 410da141706SLaura Abbott { 411da141706SLaura Abbott #ifdef CONFIG_DEBUG_RODATA 412da141706SLaura Abbott /* now that we are actually fully mapped, make the start/end more fine grained */ 4134fee9f36SArd Biesheuvel if (!IS_ALIGNED((unsigned long)_stext, SWAPPER_BLOCK_SIZE)) { 414da141706SLaura Abbott unsigned long aligned_start = round_down(__pa(_stext), 4154fee9f36SArd Biesheuvel SWAPPER_BLOCK_SIZE); 416da141706SLaura Abbott 417da141706SLaura Abbott create_mapping(aligned_start, __phys_to_virt(aligned_start), 418da141706SLaura Abbott __pa(_stext) - aligned_start, 419da141706SLaura Abbott PAGE_KERNEL); 420da141706SLaura Abbott } 421da141706SLaura Abbott 4224fee9f36SArd Biesheuvel if (!IS_ALIGNED((unsigned long)__init_end, SWAPPER_BLOCK_SIZE)) { 423da141706SLaura Abbott unsigned long aligned_end = round_up(__pa(__init_end), 4244fee9f36SArd Biesheuvel SWAPPER_BLOCK_SIZE); 425da141706SLaura Abbott create_mapping(__pa(__init_end), (unsigned long)__init_end, 426da141706SLaura Abbott aligned_end - __pa(__init_end), 427da141706SLaura Abbott PAGE_KERNEL); 428da141706SLaura Abbott } 429da141706SLaura Abbott #endif 430da141706SLaura Abbott } 431da141706SLaura Abbott 432da141706SLaura Abbott #ifdef CONFIG_DEBUG_RODATA 433da141706SLaura Abbott void mark_rodata_ro(void) 434da141706SLaura Abbott { 435da141706SLaura Abbott create_mapping_late(__pa(_stext), (unsigned long)_stext, 436da141706SLaura Abbott (unsigned long)_etext - (unsigned long)_stext, 4370b2aa5b8SLaura Abbott PAGE_KERNEL_ROX); 438da141706SLaura Abbott 439da141706SLaura Abbott } 440da141706SLaura Abbott #endif 441da141706SLaura Abbott 442da141706SLaura Abbott void fixup_init(void) 443da141706SLaura Abbott { 444da141706SLaura Abbott create_mapping_late(__pa(__init_begin), (unsigned long)__init_begin, 445da141706SLaura Abbott (unsigned long)__init_end - (unsigned long)__init_begin, 446da141706SLaura Abbott PAGE_KERNEL); 447da141706SLaura Abbott } 448da141706SLaura Abbott 449c1cc1552SCatalin Marinas /* 450c1cc1552SCatalin Marinas * paging_init() sets up the page tables, initialises the zone memory 451c1cc1552SCatalin Marinas * maps and sets up the zero page. 452c1cc1552SCatalin Marinas */ 453c1cc1552SCatalin Marinas void __init paging_init(void) 454c1cc1552SCatalin Marinas { 455c1cc1552SCatalin Marinas void *zero_page; 456c1cc1552SCatalin Marinas 457c1cc1552SCatalin Marinas map_mem(); 458da141706SLaura Abbott fixup_executable(); 459c1cc1552SCatalin Marinas 460c1cc1552SCatalin Marinas /* allocate the zero page. */ 461c1cc1552SCatalin Marinas zero_page = early_alloc(PAGE_SIZE); 462c1cc1552SCatalin Marinas 463c1cc1552SCatalin Marinas bootmem_init(); 464c1cc1552SCatalin Marinas 465c1cc1552SCatalin Marinas empty_zero_page = virt_to_page(zero_page); 466c1cc1552SCatalin Marinas 467c1cc1552SCatalin Marinas /* 468c1cc1552SCatalin Marinas * TTBR0 is only used for the identity mapping at this stage. Make it 469c1cc1552SCatalin Marinas * point to zero page to avoid speculatively fetching new entries. 470c1cc1552SCatalin Marinas */ 471c1cc1552SCatalin Marinas cpu_set_reserved_ttbr0(); 4728e63d388SWill Deacon local_flush_tlb_all(); 473dd006da2SArd Biesheuvel cpu_set_default_tcr_t0sz(); 474c1cc1552SCatalin Marinas } 475c1cc1552SCatalin Marinas 476c1cc1552SCatalin Marinas /* 477c1cc1552SCatalin Marinas * Check whether a kernel address is valid (derived from arch/x86/). 478c1cc1552SCatalin Marinas */ 479c1cc1552SCatalin Marinas int kern_addr_valid(unsigned long addr) 480c1cc1552SCatalin Marinas { 481c1cc1552SCatalin Marinas pgd_t *pgd; 482c1cc1552SCatalin Marinas pud_t *pud; 483c1cc1552SCatalin Marinas pmd_t *pmd; 484c1cc1552SCatalin Marinas pte_t *pte; 485c1cc1552SCatalin Marinas 486c1cc1552SCatalin Marinas if ((((long)addr) >> VA_BITS) != -1UL) 487c1cc1552SCatalin Marinas return 0; 488c1cc1552SCatalin Marinas 489c1cc1552SCatalin Marinas pgd = pgd_offset_k(addr); 490c1cc1552SCatalin Marinas if (pgd_none(*pgd)) 491c1cc1552SCatalin Marinas return 0; 492c1cc1552SCatalin Marinas 493c1cc1552SCatalin Marinas pud = pud_offset(pgd, addr); 494c1cc1552SCatalin Marinas if (pud_none(*pud)) 495c1cc1552SCatalin Marinas return 0; 496c1cc1552SCatalin Marinas 497206a2a73SSteve Capper if (pud_sect(*pud)) 498206a2a73SSteve Capper return pfn_valid(pud_pfn(*pud)); 499206a2a73SSteve Capper 500c1cc1552SCatalin Marinas pmd = pmd_offset(pud, addr); 501c1cc1552SCatalin Marinas if (pmd_none(*pmd)) 502c1cc1552SCatalin Marinas return 0; 503c1cc1552SCatalin Marinas 504da6e4cb6SDave Anderson if (pmd_sect(*pmd)) 505da6e4cb6SDave Anderson return pfn_valid(pmd_pfn(*pmd)); 506da6e4cb6SDave Anderson 507c1cc1552SCatalin Marinas pte = pte_offset_kernel(pmd, addr); 508c1cc1552SCatalin Marinas if (pte_none(*pte)) 509c1cc1552SCatalin Marinas return 0; 510c1cc1552SCatalin Marinas 511c1cc1552SCatalin Marinas return pfn_valid(pte_pfn(*pte)); 512c1cc1552SCatalin Marinas } 513c1cc1552SCatalin Marinas #ifdef CONFIG_SPARSEMEM_VMEMMAP 514b433dce0SSuzuki K. Poulose #if !ARM64_SWAPPER_USES_SECTION_MAPS 5150aad818bSJohannes Weiner int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) 516c1cc1552SCatalin Marinas { 5170aad818bSJohannes Weiner return vmemmap_populate_basepages(start, end, node); 518c1cc1552SCatalin Marinas } 519b433dce0SSuzuki K. Poulose #else /* !ARM64_SWAPPER_USES_SECTION_MAPS */ 5200aad818bSJohannes Weiner int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) 521c1cc1552SCatalin Marinas { 5220aad818bSJohannes Weiner unsigned long addr = start; 523c1cc1552SCatalin Marinas unsigned long next; 524c1cc1552SCatalin Marinas pgd_t *pgd; 525c1cc1552SCatalin Marinas pud_t *pud; 526c1cc1552SCatalin Marinas pmd_t *pmd; 527c1cc1552SCatalin Marinas 528c1cc1552SCatalin Marinas do { 529c1cc1552SCatalin Marinas next = pmd_addr_end(addr, end); 530c1cc1552SCatalin Marinas 531c1cc1552SCatalin Marinas pgd = vmemmap_pgd_populate(addr, node); 532c1cc1552SCatalin Marinas if (!pgd) 533c1cc1552SCatalin Marinas return -ENOMEM; 534c1cc1552SCatalin Marinas 535c1cc1552SCatalin Marinas pud = vmemmap_pud_populate(pgd, addr, node); 536c1cc1552SCatalin Marinas if (!pud) 537c1cc1552SCatalin Marinas return -ENOMEM; 538c1cc1552SCatalin Marinas 539c1cc1552SCatalin Marinas pmd = pmd_offset(pud, addr); 540c1cc1552SCatalin Marinas if (pmd_none(*pmd)) { 541c1cc1552SCatalin Marinas void *p = NULL; 542c1cc1552SCatalin Marinas 543c1cc1552SCatalin Marinas p = vmemmap_alloc_block_buf(PMD_SIZE, node); 544c1cc1552SCatalin Marinas if (!p) 545c1cc1552SCatalin Marinas return -ENOMEM; 546c1cc1552SCatalin Marinas 547a501e324SCatalin Marinas set_pmd(pmd, __pmd(__pa(p) | PROT_SECT_NORMAL)); 548c1cc1552SCatalin Marinas } else 549c1cc1552SCatalin Marinas vmemmap_verify((pte_t *)pmd, node, addr, next); 550c1cc1552SCatalin Marinas } while (addr = next, addr != end); 551c1cc1552SCatalin Marinas 552c1cc1552SCatalin Marinas return 0; 553c1cc1552SCatalin Marinas } 554c1cc1552SCatalin Marinas #endif /* CONFIG_ARM64_64K_PAGES */ 5550aad818bSJohannes Weiner void vmemmap_free(unsigned long start, unsigned long end) 5560197518cSTang Chen { 5570197518cSTang Chen } 558c1cc1552SCatalin Marinas #endif /* CONFIG_SPARSEMEM_VMEMMAP */ 559af86e597SLaura Abbott 560af86e597SLaura Abbott static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss; 5619f25e6adSKirill A. Shutemov #if CONFIG_PGTABLE_LEVELS > 2 562af86e597SLaura Abbott static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss; 563af86e597SLaura Abbott #endif 5649f25e6adSKirill A. Shutemov #if CONFIG_PGTABLE_LEVELS > 3 565af86e597SLaura Abbott static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss; 566af86e597SLaura Abbott #endif 567af86e597SLaura Abbott 568af86e597SLaura Abbott static inline pud_t * fixmap_pud(unsigned long addr) 569af86e597SLaura Abbott { 570af86e597SLaura Abbott pgd_t *pgd = pgd_offset_k(addr); 571af86e597SLaura Abbott 572af86e597SLaura Abbott BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd)); 573af86e597SLaura Abbott 574af86e597SLaura Abbott return pud_offset(pgd, addr); 575af86e597SLaura Abbott } 576af86e597SLaura Abbott 577af86e597SLaura Abbott static inline pmd_t * fixmap_pmd(unsigned long addr) 578af86e597SLaura Abbott { 579af86e597SLaura Abbott pud_t *pud = fixmap_pud(addr); 580af86e597SLaura Abbott 581af86e597SLaura Abbott BUG_ON(pud_none(*pud) || pud_bad(*pud)); 582af86e597SLaura Abbott 583af86e597SLaura Abbott return pmd_offset(pud, addr); 584af86e597SLaura Abbott } 585af86e597SLaura Abbott 586af86e597SLaura Abbott static inline pte_t * fixmap_pte(unsigned long addr) 587af86e597SLaura Abbott { 588af86e597SLaura Abbott pmd_t *pmd = fixmap_pmd(addr); 589af86e597SLaura Abbott 590af86e597SLaura Abbott BUG_ON(pmd_none(*pmd) || pmd_bad(*pmd)); 591af86e597SLaura Abbott 592af86e597SLaura Abbott return pte_offset_kernel(pmd, addr); 593af86e597SLaura Abbott } 594af86e597SLaura Abbott 595af86e597SLaura Abbott void __init early_fixmap_init(void) 596af86e597SLaura Abbott { 597af86e597SLaura Abbott pgd_t *pgd; 598af86e597SLaura Abbott pud_t *pud; 599af86e597SLaura Abbott pmd_t *pmd; 600af86e597SLaura Abbott unsigned long addr = FIXADDR_START; 601af86e597SLaura Abbott 602af86e597SLaura Abbott pgd = pgd_offset_k(addr); 603af86e597SLaura Abbott pgd_populate(&init_mm, pgd, bm_pud); 604af86e597SLaura Abbott pud = pud_offset(pgd, addr); 605af86e597SLaura Abbott pud_populate(&init_mm, pud, bm_pmd); 606af86e597SLaura Abbott pmd = pmd_offset(pud, addr); 607af86e597SLaura Abbott pmd_populate_kernel(&init_mm, pmd, bm_pte); 608af86e597SLaura Abbott 609af86e597SLaura Abbott /* 610af86e597SLaura Abbott * The boot-ioremap range spans multiple pmds, for which 611af86e597SLaura Abbott * we are not preparted: 612af86e597SLaura Abbott */ 613af86e597SLaura Abbott BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) 614af86e597SLaura Abbott != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); 615af86e597SLaura Abbott 616af86e597SLaura Abbott if ((pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN))) 617af86e597SLaura Abbott || pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) { 618af86e597SLaura Abbott WARN_ON(1); 619af86e597SLaura Abbott pr_warn("pmd %p != %p, %p\n", 620af86e597SLaura Abbott pmd, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)), 621af86e597SLaura Abbott fixmap_pmd(fix_to_virt(FIX_BTMAP_END))); 622af86e597SLaura Abbott pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", 623af86e597SLaura Abbott fix_to_virt(FIX_BTMAP_BEGIN)); 624af86e597SLaura Abbott pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n", 625af86e597SLaura Abbott fix_to_virt(FIX_BTMAP_END)); 626af86e597SLaura Abbott 627af86e597SLaura Abbott pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END); 628af86e597SLaura Abbott pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN); 629af86e597SLaura Abbott } 630af86e597SLaura Abbott } 631af86e597SLaura Abbott 632af86e597SLaura Abbott void __set_fixmap(enum fixed_addresses idx, 633af86e597SLaura Abbott phys_addr_t phys, pgprot_t flags) 634af86e597SLaura Abbott { 635af86e597SLaura Abbott unsigned long addr = __fix_to_virt(idx); 636af86e597SLaura Abbott pte_t *pte; 637af86e597SLaura Abbott 638b63dbef9SMark Rutland BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); 639af86e597SLaura Abbott 640af86e597SLaura Abbott pte = fixmap_pte(addr); 641af86e597SLaura Abbott 642af86e597SLaura Abbott if (pgprot_val(flags)) { 643af86e597SLaura Abbott set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags)); 644af86e597SLaura Abbott } else { 645af86e597SLaura Abbott pte_clear(&init_mm, addr, pte); 646af86e597SLaura Abbott flush_tlb_kernel_range(addr, addr+PAGE_SIZE); 647af86e597SLaura Abbott } 648af86e597SLaura Abbott } 64961bd93ceSArd Biesheuvel 65061bd93ceSArd Biesheuvel void *__init fixmap_remap_fdt(phys_addr_t dt_phys) 65161bd93ceSArd Biesheuvel { 65261bd93ceSArd Biesheuvel const u64 dt_virt_base = __fix_to_virt(FIX_FDT); 653fb226c3dSArd Biesheuvel pgprot_t prot = PAGE_KERNEL_RO; 654b433dce0SSuzuki K. Poulose int size, offset; 65561bd93ceSArd Biesheuvel void *dt_virt; 65661bd93ceSArd Biesheuvel 65761bd93ceSArd Biesheuvel /* 65861bd93ceSArd Biesheuvel * Check whether the physical FDT address is set and meets the minimum 65961bd93ceSArd Biesheuvel * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be 66061bd93ceSArd Biesheuvel * at least 8 bytes so that we can always access the size field of the 66161bd93ceSArd Biesheuvel * FDT header after mapping the first chunk, double check here if that 66261bd93ceSArd Biesheuvel * is indeed the case. 66361bd93ceSArd Biesheuvel */ 66461bd93ceSArd Biesheuvel BUILD_BUG_ON(MIN_FDT_ALIGN < 8); 66561bd93ceSArd Biesheuvel if (!dt_phys || dt_phys % MIN_FDT_ALIGN) 66661bd93ceSArd Biesheuvel return NULL; 66761bd93ceSArd Biesheuvel 66861bd93ceSArd Biesheuvel /* 66961bd93ceSArd Biesheuvel * Make sure that the FDT region can be mapped without the need to 67061bd93ceSArd Biesheuvel * allocate additional translation table pages, so that it is safe 67161bd93ceSArd Biesheuvel * to call create_mapping() this early. 67261bd93ceSArd Biesheuvel * 67361bd93ceSArd Biesheuvel * On 64k pages, the FDT will be mapped using PTEs, so we need to 67461bd93ceSArd Biesheuvel * be in the same PMD as the rest of the fixmap. 67561bd93ceSArd Biesheuvel * On 4k pages, we'll use section mappings for the FDT so we only 67661bd93ceSArd Biesheuvel * have to be in the same PUD. 67761bd93ceSArd Biesheuvel */ 67861bd93ceSArd Biesheuvel BUILD_BUG_ON(dt_virt_base % SZ_2M); 67961bd93ceSArd Biesheuvel 680b433dce0SSuzuki K. Poulose BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> SWAPPER_TABLE_SHIFT != 681b433dce0SSuzuki K. Poulose __fix_to_virt(FIX_BTMAP_BEGIN) >> SWAPPER_TABLE_SHIFT); 68261bd93ceSArd Biesheuvel 683b433dce0SSuzuki K. Poulose offset = dt_phys % SWAPPER_BLOCK_SIZE; 68461bd93ceSArd Biesheuvel dt_virt = (void *)dt_virt_base + offset; 68561bd93ceSArd Biesheuvel 68661bd93ceSArd Biesheuvel /* map the first chunk so we can read the size from the header */ 687b433dce0SSuzuki K. Poulose create_mapping(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base, 688b433dce0SSuzuki K. Poulose SWAPPER_BLOCK_SIZE, prot); 68961bd93ceSArd Biesheuvel 69061bd93ceSArd Biesheuvel if (fdt_check_header(dt_virt) != 0) 69161bd93ceSArd Biesheuvel return NULL; 69261bd93ceSArd Biesheuvel 69361bd93ceSArd Biesheuvel size = fdt_totalsize(dt_virt); 69461bd93ceSArd Biesheuvel if (size > MAX_FDT_SIZE) 69561bd93ceSArd Biesheuvel return NULL; 69661bd93ceSArd Biesheuvel 697b433dce0SSuzuki K. Poulose if (offset + size > SWAPPER_BLOCK_SIZE) 698b433dce0SSuzuki K. Poulose create_mapping(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base, 699b433dce0SSuzuki K. Poulose round_up(offset + size, SWAPPER_BLOCK_SIZE), prot); 70061bd93ceSArd Biesheuvel 70161bd93ceSArd Biesheuvel memblock_reserve(dt_phys, size); 70261bd93ceSArd Biesheuvel 70361bd93ceSArd Biesheuvel return dt_virt; 70461bd93ceSArd Biesheuvel } 705