1c1cc1552SCatalin Marinas /* 2c1cc1552SCatalin Marinas * Based on arch/arm/mm/mmu.c 3c1cc1552SCatalin Marinas * 4c1cc1552SCatalin Marinas * Copyright (C) 1995-2005 Russell King 5c1cc1552SCatalin Marinas * Copyright (C) 2012 ARM Ltd. 6c1cc1552SCatalin Marinas * 7c1cc1552SCatalin Marinas * This program is free software; you can redistribute it and/or modify 8c1cc1552SCatalin Marinas * it under the terms of the GNU General Public License version 2 as 9c1cc1552SCatalin Marinas * published by the Free Software Foundation. 10c1cc1552SCatalin Marinas * 11c1cc1552SCatalin Marinas * This program is distributed in the hope that it will be useful, 12c1cc1552SCatalin Marinas * but WITHOUT ANY WARRANTY; without even the implied warranty of 13c1cc1552SCatalin Marinas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14c1cc1552SCatalin Marinas * GNU General Public License for more details. 15c1cc1552SCatalin Marinas * 16c1cc1552SCatalin Marinas * You should have received a copy of the GNU General Public License 17c1cc1552SCatalin Marinas * along with this program. If not, see <http://www.gnu.org/licenses/>. 18c1cc1552SCatalin Marinas */ 19c1cc1552SCatalin Marinas 20c1cc1552SCatalin Marinas #include <linux/export.h> 21c1cc1552SCatalin Marinas #include <linux/kernel.h> 22c1cc1552SCatalin Marinas #include <linux/errno.h> 23c1cc1552SCatalin Marinas #include <linux/init.h> 2461bd93ceSArd Biesheuvel #include <linux/libfdt.h> 25c1cc1552SCatalin Marinas #include <linux/mman.h> 26c1cc1552SCatalin Marinas #include <linux/nodemask.h> 27c1cc1552SCatalin Marinas #include <linux/memblock.h> 28c1cc1552SCatalin Marinas #include <linux/fs.h> 292475ff9dSCatalin Marinas #include <linux/io.h> 3041089357SCatalin Marinas #include <linux/slab.h> 31da141706SLaura Abbott #include <linux/stop_machine.h> 32c1cc1552SCatalin Marinas 3321ab99c2SMark Rutland #include <asm/barrier.h> 34c1cc1552SCatalin Marinas #include <asm/cputype.h> 35af86e597SLaura Abbott #include <asm/fixmap.h> 36068a17a5SMark Rutland #include <asm/kasan.h> 37b433dce0SSuzuki K. Poulose #include <asm/kernel-pgtable.h> 38c1cc1552SCatalin Marinas #include <asm/sections.h> 39c1cc1552SCatalin Marinas #include <asm/setup.h> 40c1cc1552SCatalin Marinas #include <asm/sizes.h> 41c1cc1552SCatalin Marinas #include <asm/tlb.h> 42c79b954bSJungseok Lee #include <asm/memblock.h> 43c1cc1552SCatalin Marinas #include <asm/mmu_context.h> 44c1cc1552SCatalin Marinas 45c1cc1552SCatalin Marinas #include "mm.h" 46c1cc1552SCatalin Marinas 47dd006da2SArd Biesheuvel u64 idmap_t0sz = TCR_T0SZ(VA_BITS); 48dd006da2SArd Biesheuvel 49a7f8de16SArd Biesheuvel u64 kimage_voffset __read_mostly; 50a7f8de16SArd Biesheuvel EXPORT_SYMBOL(kimage_voffset); 51a7f8de16SArd Biesheuvel 52c1cc1552SCatalin Marinas /* 53c1cc1552SCatalin Marinas * Empty_zero_page is a special page that is used for zero-initialized data 54c1cc1552SCatalin Marinas * and COW. 55c1cc1552SCatalin Marinas */ 565227cfa7SMark Rutland unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss; 57c1cc1552SCatalin Marinas EXPORT_SYMBOL(empty_zero_page); 58c1cc1552SCatalin Marinas 59f9040773SArd Biesheuvel static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss; 60f9040773SArd Biesheuvel static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; 61f9040773SArd Biesheuvel static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; 62f9040773SArd Biesheuvel 63c1cc1552SCatalin Marinas pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, 64c1cc1552SCatalin Marinas unsigned long size, pgprot_t vma_prot) 65c1cc1552SCatalin Marinas { 66c1cc1552SCatalin Marinas if (!pfn_valid(pfn)) 67c1cc1552SCatalin Marinas return pgprot_noncached(vma_prot); 68c1cc1552SCatalin Marinas else if (file->f_flags & O_SYNC) 69c1cc1552SCatalin Marinas return pgprot_writecombine(vma_prot); 70c1cc1552SCatalin Marinas return vma_prot; 71c1cc1552SCatalin Marinas } 72c1cc1552SCatalin Marinas EXPORT_SYMBOL(phys_mem_access_prot); 73c1cc1552SCatalin Marinas 74f4710445SMark Rutland static phys_addr_t __init early_pgtable_alloc(void) 75c1cc1552SCatalin Marinas { 767142392dSSuzuki K. Poulose phys_addr_t phys; 777142392dSSuzuki K. Poulose void *ptr; 787142392dSSuzuki K. Poulose 7921ab99c2SMark Rutland phys = memblock_alloc(PAGE_SIZE, PAGE_SIZE); 80f4710445SMark Rutland 81f4710445SMark Rutland /* 82f4710445SMark Rutland * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE 83f4710445SMark Rutland * slot will be free, so we can (ab)use the FIX_PTE slot to initialise 84f4710445SMark Rutland * any level of table. 85f4710445SMark Rutland */ 86f4710445SMark Rutland ptr = pte_set_fixmap(phys); 87f4710445SMark Rutland 8821ab99c2SMark Rutland memset(ptr, 0, PAGE_SIZE); 8921ab99c2SMark Rutland 90f4710445SMark Rutland /* 91f4710445SMark Rutland * Implicit barriers also ensure the zeroed page is visible to the page 92f4710445SMark Rutland * table walker 93f4710445SMark Rutland */ 94f4710445SMark Rutland pte_clear_fixmap(); 95f4710445SMark Rutland 96f4710445SMark Rutland return phys; 97c1cc1552SCatalin Marinas } 98c1cc1552SCatalin Marinas 99da141706SLaura Abbott static void alloc_init_pte(pmd_t *pmd, unsigned long addr, 100667c2759SCatalin Marinas unsigned long end, unsigned long pfn, 101da141706SLaura Abbott pgprot_t prot, 102f4710445SMark Rutland phys_addr_t (*pgtable_alloc)(void)) 103c1cc1552SCatalin Marinas { 104c1cc1552SCatalin Marinas pte_t *pte; 105c1cc1552SCatalin Marinas 1064133af6cSCatalin Marinas BUG_ON(pmd_sect(*pmd)); 1074133af6cSCatalin Marinas if (pmd_none(*pmd)) { 108132233a7SLaura Abbott phys_addr_t pte_phys; 109132233a7SLaura Abbott BUG_ON(!pgtable_alloc); 110132233a7SLaura Abbott pte_phys = pgtable_alloc(); 111f4710445SMark Rutland pte = pte_set_fixmap(pte_phys); 112f4710445SMark Rutland __pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE); 113f4710445SMark Rutland pte_clear_fixmap(); 114c1cc1552SCatalin Marinas } 115a1c76574SMark Rutland BUG_ON(pmd_bad(*pmd)); 116c1cc1552SCatalin Marinas 117f4710445SMark Rutland pte = pte_set_fixmap_offset(pmd, addr); 118c1cc1552SCatalin Marinas do { 119667c2759SCatalin Marinas set_pte(pte, pfn_pte(pfn, prot)); 120667c2759SCatalin Marinas pfn++; 121667c2759SCatalin Marinas } while (pte++, addr += PAGE_SIZE, addr != end); 122f4710445SMark Rutland 123f4710445SMark Rutland pte_clear_fixmap(); 124c1cc1552SCatalin Marinas } 125c1cc1552SCatalin Marinas 12611509a30SMark Rutland static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end, 127da141706SLaura Abbott phys_addr_t phys, pgprot_t prot, 12853e1b329SArd Biesheuvel phys_addr_t (*pgtable_alloc)(void), 12953e1b329SArd Biesheuvel bool allow_block_mappings) 130c1cc1552SCatalin Marinas { 131c1cc1552SCatalin Marinas pmd_t *pmd; 132c1cc1552SCatalin Marinas unsigned long next; 133c1cc1552SCatalin Marinas 134c1cc1552SCatalin Marinas /* 135c1cc1552SCatalin Marinas * Check for initial section mappings in the pgd/pud and remove them. 136c1cc1552SCatalin Marinas */ 1374133af6cSCatalin Marinas BUG_ON(pud_sect(*pud)); 1384133af6cSCatalin Marinas if (pud_none(*pud)) { 139132233a7SLaura Abbott phys_addr_t pmd_phys; 140132233a7SLaura Abbott BUG_ON(!pgtable_alloc); 141132233a7SLaura Abbott pmd_phys = pgtable_alloc(); 142f4710445SMark Rutland pmd = pmd_set_fixmap(pmd_phys); 143f4710445SMark Rutland __pud_populate(pud, pmd_phys, PUD_TYPE_TABLE); 144f4710445SMark Rutland pmd_clear_fixmap(); 145c1cc1552SCatalin Marinas } 146a1c76574SMark Rutland BUG_ON(pud_bad(*pud)); 147c1cc1552SCatalin Marinas 148f4710445SMark Rutland pmd = pmd_set_fixmap_offset(pud, addr); 149c1cc1552SCatalin Marinas do { 150c1cc1552SCatalin Marinas next = pmd_addr_end(addr, end); 151c1cc1552SCatalin Marinas /* try section mapping first */ 15283863f25SLaura Abbott if (((addr | next | phys) & ~SECTION_MASK) == 0 && 15353e1b329SArd Biesheuvel allow_block_mappings) { 154a55f9929SCatalin Marinas pmd_t old_pmd =*pmd; 155c661cb1cSMark Rutland pmd_set_huge(pmd, phys, prot); 156a55f9929SCatalin Marinas /* 157a55f9929SCatalin Marinas * Check for previous table entries created during 158a55f9929SCatalin Marinas * boot (__create_page_tables) and flush them. 159a55f9929SCatalin Marinas */ 160523d6e9fSzhichang.yuan if (!pmd_none(old_pmd)) { 161a55f9929SCatalin Marinas flush_tlb_all(); 162523d6e9fSzhichang.yuan if (pmd_table(old_pmd)) { 163316b39dbSMark Rutland phys_addr_t table = pmd_page_paddr(old_pmd); 16441089357SCatalin Marinas if (!WARN_ON_ONCE(slab_is_available())) 165523d6e9fSzhichang.yuan memblock_free(table, PAGE_SIZE); 166523d6e9fSzhichang.yuan } 167523d6e9fSzhichang.yuan } 168a55f9929SCatalin Marinas } else { 169667c2759SCatalin Marinas alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys), 17021ab99c2SMark Rutland prot, pgtable_alloc); 171a55f9929SCatalin Marinas } 172c1cc1552SCatalin Marinas phys += next - addr; 173c1cc1552SCatalin Marinas } while (pmd++, addr = next, addr != end); 174f4710445SMark Rutland 175f4710445SMark Rutland pmd_clear_fixmap(); 176c1cc1552SCatalin Marinas } 177c1cc1552SCatalin Marinas 178da141706SLaura Abbott static inline bool use_1G_block(unsigned long addr, unsigned long next, 179da141706SLaura Abbott unsigned long phys) 180da141706SLaura Abbott { 181da141706SLaura Abbott if (PAGE_SHIFT != 12) 182da141706SLaura Abbott return false; 183da141706SLaura Abbott 184da141706SLaura Abbott if (((addr | next | phys) & ~PUD_MASK) != 0) 185da141706SLaura Abbott return false; 186da141706SLaura Abbott 187da141706SLaura Abbott return true; 188da141706SLaura Abbott } 189da141706SLaura Abbott 19011509a30SMark Rutland static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end, 191da141706SLaura Abbott phys_addr_t phys, pgprot_t prot, 19253e1b329SArd Biesheuvel phys_addr_t (*pgtable_alloc)(void), 19353e1b329SArd Biesheuvel bool allow_block_mappings) 194c1cc1552SCatalin Marinas { 195c79b954bSJungseok Lee pud_t *pud; 196c1cc1552SCatalin Marinas unsigned long next; 197c1cc1552SCatalin Marinas 198c79b954bSJungseok Lee if (pgd_none(*pgd)) { 199132233a7SLaura Abbott phys_addr_t pud_phys; 200132233a7SLaura Abbott BUG_ON(!pgtable_alloc); 201132233a7SLaura Abbott pud_phys = pgtable_alloc(); 202f4710445SMark Rutland __pgd_populate(pgd, pud_phys, PUD_TYPE_TABLE); 203c79b954bSJungseok Lee } 204c79b954bSJungseok Lee BUG_ON(pgd_bad(*pgd)); 205c79b954bSJungseok Lee 206f4710445SMark Rutland pud = pud_set_fixmap_offset(pgd, addr); 207c1cc1552SCatalin Marinas do { 208c1cc1552SCatalin Marinas next = pud_addr_end(addr, end); 209206a2a73SSteve Capper 210206a2a73SSteve Capper /* 211206a2a73SSteve Capper * For 4K granule only, attempt to put down a 1GB block 212206a2a73SSteve Capper */ 21353e1b329SArd Biesheuvel if (use_1G_block(addr, next, phys) && allow_block_mappings) { 214206a2a73SSteve Capper pud_t old_pud = *pud; 215c661cb1cSMark Rutland pud_set_huge(pud, phys, prot); 216206a2a73SSteve Capper 217206a2a73SSteve Capper /* 218206a2a73SSteve Capper * If we have an old value for a pud, it will 219206a2a73SSteve Capper * be pointing to a pmd table that we no longer 220206a2a73SSteve Capper * need (from swapper_pg_dir). 221206a2a73SSteve Capper * 222206a2a73SSteve Capper * Look up the old pmd table and free it. 223206a2a73SSteve Capper */ 224206a2a73SSteve Capper if (!pud_none(old_pud)) { 225206a2a73SSteve Capper flush_tlb_all(); 226523d6e9fSzhichang.yuan if (pud_table(old_pud)) { 227316b39dbSMark Rutland phys_addr_t table = pud_page_paddr(old_pud); 22841089357SCatalin Marinas if (!WARN_ON_ONCE(slab_is_available())) 229523d6e9fSzhichang.yuan memblock_free(table, PAGE_SIZE); 230523d6e9fSzhichang.yuan } 231206a2a73SSteve Capper } 232206a2a73SSteve Capper } else { 23311509a30SMark Rutland alloc_init_pmd(pud, addr, next, phys, prot, 23453e1b329SArd Biesheuvel pgtable_alloc, allow_block_mappings); 235206a2a73SSteve Capper } 236c1cc1552SCatalin Marinas phys += next - addr; 237c1cc1552SCatalin Marinas } while (pud++, addr = next, addr != end); 238f4710445SMark Rutland 239f4710445SMark Rutland pud_clear_fixmap(); 240c1cc1552SCatalin Marinas } 241c1cc1552SCatalin Marinas 242c1cc1552SCatalin Marinas /* 243c1cc1552SCatalin Marinas * Create the page directory entries and any necessary page tables for the 244c1cc1552SCatalin Marinas * mapping specified by 'md'. 245c1cc1552SCatalin Marinas */ 24611509a30SMark Rutland static void init_pgd(pgd_t *pgd, phys_addr_t phys, unsigned long virt, 247da141706SLaura Abbott phys_addr_t size, pgprot_t prot, 24853e1b329SArd Biesheuvel phys_addr_t (*pgtable_alloc)(void), 24953e1b329SArd Biesheuvel bool allow_block_mappings) 250c1cc1552SCatalin Marinas { 251c1cc1552SCatalin Marinas unsigned long addr, length, end, next; 252c1cc1552SCatalin Marinas 253cc5d2b3bSMark Rutland /* 254cc5d2b3bSMark Rutland * If the virtual and physical address don't have the same offset 255cc5d2b3bSMark Rutland * within a page, we cannot map the region as the caller expects. 256cc5d2b3bSMark Rutland */ 257cc5d2b3bSMark Rutland if (WARN_ON((phys ^ virt) & ~PAGE_MASK)) 258cc5d2b3bSMark Rutland return; 259cc5d2b3bSMark Rutland 2609c4e08a3SMark Rutland phys &= PAGE_MASK; 261c1cc1552SCatalin Marinas addr = virt & PAGE_MASK; 262c1cc1552SCatalin Marinas length = PAGE_ALIGN(size + (virt & ~PAGE_MASK)); 263c1cc1552SCatalin Marinas 264c1cc1552SCatalin Marinas end = addr + length; 265c1cc1552SCatalin Marinas do { 266c1cc1552SCatalin Marinas next = pgd_addr_end(addr, end); 26753e1b329SArd Biesheuvel alloc_init_pud(pgd, addr, next, phys, prot, pgtable_alloc, 26853e1b329SArd Biesheuvel allow_block_mappings); 269c1cc1552SCatalin Marinas phys += next - addr; 270c1cc1552SCatalin Marinas } while (pgd++, addr = next, addr != end); 271c1cc1552SCatalin Marinas } 272c1cc1552SCatalin Marinas 273f4710445SMark Rutland static phys_addr_t late_pgtable_alloc(void) 274da141706SLaura Abbott { 27521ab99c2SMark Rutland void *ptr = (void *)__get_free_page(PGALLOC_GFP); 276da141706SLaura Abbott BUG_ON(!ptr); 27721ab99c2SMark Rutland 27821ab99c2SMark Rutland /* Ensure the zeroed page is visible to the page table walker */ 27921ab99c2SMark Rutland dsb(ishst); 280f4710445SMark Rutland return __pa(ptr); 281da141706SLaura Abbott } 282da141706SLaura Abbott 28311509a30SMark Rutland static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, 28411509a30SMark Rutland unsigned long virt, phys_addr_t size, 28511509a30SMark Rutland pgprot_t prot, 28653e1b329SArd Biesheuvel phys_addr_t (*alloc)(void), 28753e1b329SArd Biesheuvel bool allow_block_mappings) 28811509a30SMark Rutland { 28953e1b329SArd Biesheuvel init_pgd(pgd_offset_raw(pgdir, virt), phys, virt, size, prot, alloc, 29053e1b329SArd Biesheuvel allow_block_mappings); 29111509a30SMark Rutland } 29211509a30SMark Rutland 293132233a7SLaura Abbott /* 294132233a7SLaura Abbott * This function can only be used to modify existing table entries, 295132233a7SLaura Abbott * without allocating new levels of table. Note that this permits the 296132233a7SLaura Abbott * creation of new section or page entries. 297132233a7SLaura Abbott */ 298132233a7SLaura Abbott static void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, 299da141706SLaura Abbott phys_addr_t size, pgprot_t prot) 300d7ecbddfSMark Salter { 301d7ecbddfSMark Salter if (virt < VMALLOC_START) { 302d7ecbddfSMark Salter pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", 303d7ecbddfSMark Salter &phys, virt); 304d7ecbddfSMark Salter return; 305d7ecbddfSMark Salter } 30653e1b329SArd Biesheuvel __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, true); 307d7ecbddfSMark Salter } 308d7ecbddfSMark Salter 3098ce837ceSArd Biesheuvel void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, 3108ce837ceSArd Biesheuvel unsigned long virt, phys_addr_t size, 31153e1b329SArd Biesheuvel pgprot_t prot, bool allow_block_mappings) 3128ce837ceSArd Biesheuvel { 31311509a30SMark Rutland __create_pgd_mapping(mm->pgd, phys, virt, size, prot, 31453e1b329SArd Biesheuvel late_pgtable_alloc, allow_block_mappings); 315d7ecbddfSMark Salter } 316d7ecbddfSMark Salter 317da141706SLaura Abbott static void create_mapping_late(phys_addr_t phys, unsigned long virt, 318da141706SLaura Abbott phys_addr_t size, pgprot_t prot) 319da141706SLaura Abbott { 320da141706SLaura Abbott if (virt < VMALLOC_START) { 321da141706SLaura Abbott pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", 322da141706SLaura Abbott &phys, virt); 323da141706SLaura Abbott return; 324da141706SLaura Abbott } 325da141706SLaura Abbott 32611509a30SMark Rutland __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, 32753e1b329SArd Biesheuvel late_pgtable_alloc, !debug_pagealloc_enabled()); 328da141706SLaura Abbott } 329da141706SLaura Abbott 330068a17a5SMark Rutland static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end) 331da141706SLaura Abbott { 3327eb90f2fSArd Biesheuvel unsigned long kernel_start = __pa(_text); 3339fdc14c5SArd Biesheuvel unsigned long kernel_end = __pa(__init_begin); 334068a17a5SMark Rutland 335da141706SLaura Abbott /* 336f9040773SArd Biesheuvel * Take care not to create a writable alias for the 337f9040773SArd Biesheuvel * read-only text and rodata sections of the kernel image. 338da141706SLaura Abbott */ 339da141706SLaura Abbott 3409fdc14c5SArd Biesheuvel /* No overlap with the kernel text/rodata */ 341068a17a5SMark Rutland if (end < kernel_start || start >= kernel_end) { 342068a17a5SMark Rutland __create_pgd_mapping(pgd, start, __phys_to_virt(start), 343068a17a5SMark Rutland end - start, PAGE_KERNEL, 34453e1b329SArd Biesheuvel early_pgtable_alloc, 34553e1b329SArd Biesheuvel !debug_pagealloc_enabled()); 346068a17a5SMark Rutland return; 347da141706SLaura Abbott } 348da141706SLaura Abbott 349068a17a5SMark Rutland /* 3509fdc14c5SArd Biesheuvel * This block overlaps the kernel text/rodata mappings. 351f9040773SArd Biesheuvel * Map the portion(s) which don't overlap. 352068a17a5SMark Rutland */ 353068a17a5SMark Rutland if (start < kernel_start) 354068a17a5SMark Rutland __create_pgd_mapping(pgd, start, 355068a17a5SMark Rutland __phys_to_virt(start), 356068a17a5SMark Rutland kernel_start - start, PAGE_KERNEL, 35753e1b329SArd Biesheuvel early_pgtable_alloc, 35853e1b329SArd Biesheuvel !debug_pagealloc_enabled()); 359068a17a5SMark Rutland if (kernel_end < end) 360068a17a5SMark Rutland __create_pgd_mapping(pgd, kernel_end, 361068a17a5SMark Rutland __phys_to_virt(kernel_end), 362068a17a5SMark Rutland end - kernel_end, PAGE_KERNEL, 36353e1b329SArd Biesheuvel early_pgtable_alloc, 36453e1b329SArd Biesheuvel !debug_pagealloc_enabled()); 365f9040773SArd Biesheuvel 366f9040773SArd Biesheuvel /* 3679fdc14c5SArd Biesheuvel * Map the linear alias of the [_text, __init_begin) interval as 368f9040773SArd Biesheuvel * read-only/non-executable. This makes the contents of the 369f9040773SArd Biesheuvel * region accessible to subsystems such as hibernate, but 370f9040773SArd Biesheuvel * protects it from inadvertent modification or execution. 371f9040773SArd Biesheuvel */ 372f9040773SArd Biesheuvel __create_pgd_mapping(pgd, kernel_start, __phys_to_virt(kernel_start), 373f9040773SArd Biesheuvel kernel_end - kernel_start, PAGE_KERNEL_RO, 37453e1b329SArd Biesheuvel early_pgtable_alloc, !debug_pagealloc_enabled()); 375da141706SLaura Abbott } 376da141706SLaura Abbott 377068a17a5SMark Rutland static void __init map_mem(pgd_t *pgd) 378c1cc1552SCatalin Marinas { 379c1cc1552SCatalin Marinas struct memblock_region *reg; 380f6bc87c3SSteve Capper 381c1cc1552SCatalin Marinas /* map all the memory banks */ 382c1cc1552SCatalin Marinas for_each_memblock(memory, reg) { 383c1cc1552SCatalin Marinas phys_addr_t start = reg->base; 384c1cc1552SCatalin Marinas phys_addr_t end = start + reg->size; 385c1cc1552SCatalin Marinas 386c1cc1552SCatalin Marinas if (start >= end) 387c1cc1552SCatalin Marinas break; 38868709f45SArd Biesheuvel if (memblock_is_nomap(reg)) 38968709f45SArd Biesheuvel continue; 390c1cc1552SCatalin Marinas 391068a17a5SMark Rutland __map_memblock(pgd, start, end); 392c1cc1552SCatalin Marinas } 393c1cc1552SCatalin Marinas } 394c1cc1552SCatalin Marinas 395da141706SLaura Abbott void mark_rodata_ro(void) 396da141706SLaura Abbott { 3972f39b5f9SJeremy Linton unsigned long section_size; 398f9040773SArd Biesheuvel 3999fdc14c5SArd Biesheuvel section_size = (unsigned long)_etext - (unsigned long)_text; 4007eb90f2fSArd Biesheuvel create_mapping_late(__pa(_text), (unsigned long)_text, 4012f39b5f9SJeremy Linton section_size, PAGE_KERNEL_ROX); 4022f39b5f9SJeremy Linton /* 4039fdc14c5SArd Biesheuvel * mark .rodata as read only. Use __init_begin rather than __end_rodata 4049fdc14c5SArd Biesheuvel * to cover NOTES and EXCEPTION_TABLE. 4052f39b5f9SJeremy Linton */ 4069fdc14c5SArd Biesheuvel section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata; 4072f39b5f9SJeremy Linton create_mapping_late(__pa(__start_rodata), (unsigned long)__start_rodata, 4082f39b5f9SJeremy Linton section_size, PAGE_KERNEL_RO); 409da141706SLaura Abbott } 410da141706SLaura Abbott 411da141706SLaura Abbott void fixup_init(void) 412da141706SLaura Abbott { 413f9040773SArd Biesheuvel /* 414f9040773SArd Biesheuvel * Unmap the __init region but leave the VM area in place. This 415f9040773SArd Biesheuvel * prevents the region from being reused for kernel modules, which 416f9040773SArd Biesheuvel * is not supported by kallsyms. 417f9040773SArd Biesheuvel */ 418f9040773SArd Biesheuvel unmap_kernel_range((u64)__init_begin, (u64)(__init_end - __init_begin)); 419da141706SLaura Abbott } 420da141706SLaura Abbott 4212c09ec06SArd Biesheuvel static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end, 422f9040773SArd Biesheuvel pgprot_t prot, struct vm_struct *vma) 423068a17a5SMark Rutland { 424068a17a5SMark Rutland phys_addr_t pa_start = __pa(va_start); 425068a17a5SMark Rutland unsigned long size = va_end - va_start; 426068a17a5SMark Rutland 427068a17a5SMark Rutland BUG_ON(!PAGE_ALIGNED(pa_start)); 428068a17a5SMark Rutland BUG_ON(!PAGE_ALIGNED(size)); 429068a17a5SMark Rutland 430068a17a5SMark Rutland __create_pgd_mapping(pgd, pa_start, (unsigned long)va_start, size, prot, 43153e1b329SArd Biesheuvel early_pgtable_alloc, !debug_pagealloc_enabled()); 432f9040773SArd Biesheuvel 433f9040773SArd Biesheuvel vma->addr = va_start; 434f9040773SArd Biesheuvel vma->phys_addr = pa_start; 435f9040773SArd Biesheuvel vma->size = size; 436f9040773SArd Biesheuvel vma->flags = VM_MAP; 437f9040773SArd Biesheuvel vma->caller = __builtin_return_address(0); 438f9040773SArd Biesheuvel 439f9040773SArd Biesheuvel vm_area_add_early(vma); 440068a17a5SMark Rutland } 441068a17a5SMark Rutland 442068a17a5SMark Rutland /* 443068a17a5SMark Rutland * Create fine-grained mappings for the kernel. 444068a17a5SMark Rutland */ 445068a17a5SMark Rutland static void __init map_kernel(pgd_t *pgd) 446068a17a5SMark Rutland { 4472f39b5f9SJeremy Linton static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_init, vmlinux_data; 448068a17a5SMark Rutland 4499fdc14c5SArd Biesheuvel map_kernel_segment(pgd, _text, _etext, PAGE_KERNEL_EXEC, &vmlinux_text); 4509fdc14c5SArd Biesheuvel map_kernel_segment(pgd, __start_rodata, __init_begin, PAGE_KERNEL, &vmlinux_rodata); 4512c09ec06SArd Biesheuvel map_kernel_segment(pgd, __init_begin, __init_end, PAGE_KERNEL_EXEC, 452f9040773SArd Biesheuvel &vmlinux_init); 4532c09ec06SArd Biesheuvel map_kernel_segment(pgd, _data, _end, PAGE_KERNEL, &vmlinux_data); 454068a17a5SMark Rutland 455f9040773SArd Biesheuvel if (!pgd_val(*pgd_offset_raw(pgd, FIXADDR_START))) { 456068a17a5SMark Rutland /* 457f9040773SArd Biesheuvel * The fixmap falls in a separate pgd to the kernel, and doesn't 458f9040773SArd Biesheuvel * live in the carveout for the swapper_pg_dir. We can simply 459f9040773SArd Biesheuvel * re-use the existing dir for the fixmap. 460068a17a5SMark Rutland */ 461f9040773SArd Biesheuvel set_pgd(pgd_offset_raw(pgd, FIXADDR_START), 462f9040773SArd Biesheuvel *pgd_offset_k(FIXADDR_START)); 463f9040773SArd Biesheuvel } else if (CONFIG_PGTABLE_LEVELS > 3) { 464f9040773SArd Biesheuvel /* 465f9040773SArd Biesheuvel * The fixmap shares its top level pgd entry with the kernel 466f9040773SArd Biesheuvel * mapping. This can really only occur when we are running 467f9040773SArd Biesheuvel * with 16k/4 levels, so we can simply reuse the pud level 468f9040773SArd Biesheuvel * entry instead. 469f9040773SArd Biesheuvel */ 470f9040773SArd Biesheuvel BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); 471f9040773SArd Biesheuvel set_pud(pud_set_fixmap_offset(pgd, FIXADDR_START), 472f9040773SArd Biesheuvel __pud(__pa(bm_pmd) | PUD_TYPE_TABLE)); 473f9040773SArd Biesheuvel pud_clear_fixmap(); 474f9040773SArd Biesheuvel } else { 475f9040773SArd Biesheuvel BUG(); 476f9040773SArd Biesheuvel } 477068a17a5SMark Rutland 478068a17a5SMark Rutland kasan_copy_shadow(pgd); 479068a17a5SMark Rutland } 480068a17a5SMark Rutland 481c1cc1552SCatalin Marinas /* 482c1cc1552SCatalin Marinas * paging_init() sets up the page tables, initialises the zone memory 483c1cc1552SCatalin Marinas * maps and sets up the zero page. 484c1cc1552SCatalin Marinas */ 485c1cc1552SCatalin Marinas void __init paging_init(void) 486c1cc1552SCatalin Marinas { 487068a17a5SMark Rutland phys_addr_t pgd_phys = early_pgtable_alloc(); 488068a17a5SMark Rutland pgd_t *pgd = pgd_set_fixmap(pgd_phys); 489068a17a5SMark Rutland 490068a17a5SMark Rutland map_kernel(pgd); 491068a17a5SMark Rutland map_mem(pgd); 492068a17a5SMark Rutland 493068a17a5SMark Rutland /* 494068a17a5SMark Rutland * We want to reuse the original swapper_pg_dir so we don't have to 495068a17a5SMark Rutland * communicate the new address to non-coherent secondaries in 496068a17a5SMark Rutland * secondary_entry, and so cpu_switch_mm can generate the address with 497068a17a5SMark Rutland * adrp+add rather than a load from some global variable. 498068a17a5SMark Rutland * 499068a17a5SMark Rutland * To do this we need to go via a temporary pgd. 500068a17a5SMark Rutland */ 501068a17a5SMark Rutland cpu_replace_ttbr1(__va(pgd_phys)); 502068a17a5SMark Rutland memcpy(swapper_pg_dir, pgd, PAGE_SIZE); 503068a17a5SMark Rutland cpu_replace_ttbr1(swapper_pg_dir); 504068a17a5SMark Rutland 505068a17a5SMark Rutland pgd_clear_fixmap(); 506068a17a5SMark Rutland memblock_free(pgd_phys, PAGE_SIZE); 507068a17a5SMark Rutland 508068a17a5SMark Rutland /* 509068a17a5SMark Rutland * We only reuse the PGD from the swapper_pg_dir, not the pud + pmd 510068a17a5SMark Rutland * allocated with it. 511068a17a5SMark Rutland */ 512068a17a5SMark Rutland memblock_free(__pa(swapper_pg_dir) + PAGE_SIZE, 513068a17a5SMark Rutland SWAPPER_DIR_SIZE - PAGE_SIZE); 514c1cc1552SCatalin Marinas } 515c1cc1552SCatalin Marinas 516c1cc1552SCatalin Marinas /* 517c1cc1552SCatalin Marinas * Check whether a kernel address is valid (derived from arch/x86/). 518c1cc1552SCatalin Marinas */ 519c1cc1552SCatalin Marinas int kern_addr_valid(unsigned long addr) 520c1cc1552SCatalin Marinas { 521c1cc1552SCatalin Marinas pgd_t *pgd; 522c1cc1552SCatalin Marinas pud_t *pud; 523c1cc1552SCatalin Marinas pmd_t *pmd; 524c1cc1552SCatalin Marinas pte_t *pte; 525c1cc1552SCatalin Marinas 526c1cc1552SCatalin Marinas if ((((long)addr) >> VA_BITS) != -1UL) 527c1cc1552SCatalin Marinas return 0; 528c1cc1552SCatalin Marinas 529c1cc1552SCatalin Marinas pgd = pgd_offset_k(addr); 530c1cc1552SCatalin Marinas if (pgd_none(*pgd)) 531c1cc1552SCatalin Marinas return 0; 532c1cc1552SCatalin Marinas 533c1cc1552SCatalin Marinas pud = pud_offset(pgd, addr); 534c1cc1552SCatalin Marinas if (pud_none(*pud)) 535c1cc1552SCatalin Marinas return 0; 536c1cc1552SCatalin Marinas 537206a2a73SSteve Capper if (pud_sect(*pud)) 538206a2a73SSteve Capper return pfn_valid(pud_pfn(*pud)); 539206a2a73SSteve Capper 540c1cc1552SCatalin Marinas pmd = pmd_offset(pud, addr); 541c1cc1552SCatalin Marinas if (pmd_none(*pmd)) 542c1cc1552SCatalin Marinas return 0; 543c1cc1552SCatalin Marinas 544da6e4cb6SDave Anderson if (pmd_sect(*pmd)) 545da6e4cb6SDave Anderson return pfn_valid(pmd_pfn(*pmd)); 546da6e4cb6SDave Anderson 547c1cc1552SCatalin Marinas pte = pte_offset_kernel(pmd, addr); 548c1cc1552SCatalin Marinas if (pte_none(*pte)) 549c1cc1552SCatalin Marinas return 0; 550c1cc1552SCatalin Marinas 551c1cc1552SCatalin Marinas return pfn_valid(pte_pfn(*pte)); 552c1cc1552SCatalin Marinas } 553c1cc1552SCatalin Marinas #ifdef CONFIG_SPARSEMEM_VMEMMAP 554b433dce0SSuzuki K. Poulose #if !ARM64_SWAPPER_USES_SECTION_MAPS 5550aad818bSJohannes Weiner int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) 556c1cc1552SCatalin Marinas { 5570aad818bSJohannes Weiner return vmemmap_populate_basepages(start, end, node); 558c1cc1552SCatalin Marinas } 559b433dce0SSuzuki K. Poulose #else /* !ARM64_SWAPPER_USES_SECTION_MAPS */ 5600aad818bSJohannes Weiner int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) 561c1cc1552SCatalin Marinas { 5620aad818bSJohannes Weiner unsigned long addr = start; 563c1cc1552SCatalin Marinas unsigned long next; 564c1cc1552SCatalin Marinas pgd_t *pgd; 565c1cc1552SCatalin Marinas pud_t *pud; 566c1cc1552SCatalin Marinas pmd_t *pmd; 567c1cc1552SCatalin Marinas 568c1cc1552SCatalin Marinas do { 569c1cc1552SCatalin Marinas next = pmd_addr_end(addr, end); 570c1cc1552SCatalin Marinas 571c1cc1552SCatalin Marinas pgd = vmemmap_pgd_populate(addr, node); 572c1cc1552SCatalin Marinas if (!pgd) 573c1cc1552SCatalin Marinas return -ENOMEM; 574c1cc1552SCatalin Marinas 575c1cc1552SCatalin Marinas pud = vmemmap_pud_populate(pgd, addr, node); 576c1cc1552SCatalin Marinas if (!pud) 577c1cc1552SCatalin Marinas return -ENOMEM; 578c1cc1552SCatalin Marinas 579c1cc1552SCatalin Marinas pmd = pmd_offset(pud, addr); 580c1cc1552SCatalin Marinas if (pmd_none(*pmd)) { 581c1cc1552SCatalin Marinas void *p = NULL; 582c1cc1552SCatalin Marinas 583c1cc1552SCatalin Marinas p = vmemmap_alloc_block_buf(PMD_SIZE, node); 584c1cc1552SCatalin Marinas if (!p) 585c1cc1552SCatalin Marinas return -ENOMEM; 586c1cc1552SCatalin Marinas 587a501e324SCatalin Marinas set_pmd(pmd, __pmd(__pa(p) | PROT_SECT_NORMAL)); 588c1cc1552SCatalin Marinas } else 589c1cc1552SCatalin Marinas vmemmap_verify((pte_t *)pmd, node, addr, next); 590c1cc1552SCatalin Marinas } while (addr = next, addr != end); 591c1cc1552SCatalin Marinas 592c1cc1552SCatalin Marinas return 0; 593c1cc1552SCatalin Marinas } 594c1cc1552SCatalin Marinas #endif /* CONFIG_ARM64_64K_PAGES */ 5950aad818bSJohannes Weiner void vmemmap_free(unsigned long start, unsigned long end) 5960197518cSTang Chen { 5970197518cSTang Chen } 598c1cc1552SCatalin Marinas #endif /* CONFIG_SPARSEMEM_VMEMMAP */ 599af86e597SLaura Abbott 600af86e597SLaura Abbott static inline pud_t * fixmap_pud(unsigned long addr) 601af86e597SLaura Abbott { 602af86e597SLaura Abbott pgd_t *pgd = pgd_offset_k(addr); 603af86e597SLaura Abbott 604af86e597SLaura Abbott BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd)); 605af86e597SLaura Abbott 606157962f5SArd Biesheuvel return pud_offset_kimg(pgd, addr); 607af86e597SLaura Abbott } 608af86e597SLaura Abbott 609af86e597SLaura Abbott static inline pmd_t * fixmap_pmd(unsigned long addr) 610af86e597SLaura Abbott { 611af86e597SLaura Abbott pud_t *pud = fixmap_pud(addr); 612af86e597SLaura Abbott 613af86e597SLaura Abbott BUG_ON(pud_none(*pud) || pud_bad(*pud)); 614af86e597SLaura Abbott 615157962f5SArd Biesheuvel return pmd_offset_kimg(pud, addr); 616af86e597SLaura Abbott } 617af86e597SLaura Abbott 618af86e597SLaura Abbott static inline pte_t * fixmap_pte(unsigned long addr) 619af86e597SLaura Abbott { 620157962f5SArd Biesheuvel return &bm_pte[pte_index(addr)]; 621af86e597SLaura Abbott } 622af86e597SLaura Abbott 623af86e597SLaura Abbott void __init early_fixmap_init(void) 624af86e597SLaura Abbott { 625af86e597SLaura Abbott pgd_t *pgd; 626af86e597SLaura Abbott pud_t *pud; 627af86e597SLaura Abbott pmd_t *pmd; 628af86e597SLaura Abbott unsigned long addr = FIXADDR_START; 629af86e597SLaura Abbott 630af86e597SLaura Abbott pgd = pgd_offset_k(addr); 631f80fb3a3SArd Biesheuvel if (CONFIG_PGTABLE_LEVELS > 3 && 632f80fb3a3SArd Biesheuvel !(pgd_none(*pgd) || pgd_page_paddr(*pgd) == __pa(bm_pud))) { 633f9040773SArd Biesheuvel /* 634f9040773SArd Biesheuvel * We only end up here if the kernel mapping and the fixmap 635f9040773SArd Biesheuvel * share the top level pgd entry, which should only happen on 636f9040773SArd Biesheuvel * 16k/4 levels configurations. 637f9040773SArd Biesheuvel */ 638f9040773SArd Biesheuvel BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); 639f9040773SArd Biesheuvel pud = pud_offset_kimg(pgd, addr); 640f9040773SArd Biesheuvel } else { 641af86e597SLaura Abbott pgd_populate(&init_mm, pgd, bm_pud); 642157962f5SArd Biesheuvel pud = fixmap_pud(addr); 643f9040773SArd Biesheuvel } 644af86e597SLaura Abbott pud_populate(&init_mm, pud, bm_pmd); 645157962f5SArd Biesheuvel pmd = fixmap_pmd(addr); 646af86e597SLaura Abbott pmd_populate_kernel(&init_mm, pmd, bm_pte); 647af86e597SLaura Abbott 648af86e597SLaura Abbott /* 649af86e597SLaura Abbott * The boot-ioremap range spans multiple pmds, for which 650157962f5SArd Biesheuvel * we are not prepared: 651af86e597SLaura Abbott */ 652af86e597SLaura Abbott BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) 653af86e597SLaura Abbott != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); 654af86e597SLaura Abbott 655af86e597SLaura Abbott if ((pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN))) 656af86e597SLaura Abbott || pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) { 657af86e597SLaura Abbott WARN_ON(1); 658af86e597SLaura Abbott pr_warn("pmd %p != %p, %p\n", 659af86e597SLaura Abbott pmd, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)), 660af86e597SLaura Abbott fixmap_pmd(fix_to_virt(FIX_BTMAP_END))); 661af86e597SLaura Abbott pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", 662af86e597SLaura Abbott fix_to_virt(FIX_BTMAP_BEGIN)); 663af86e597SLaura Abbott pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n", 664af86e597SLaura Abbott fix_to_virt(FIX_BTMAP_END)); 665af86e597SLaura Abbott 666af86e597SLaura Abbott pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END); 667af86e597SLaura Abbott pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN); 668af86e597SLaura Abbott } 669af86e597SLaura Abbott } 670af86e597SLaura Abbott 671af86e597SLaura Abbott void __set_fixmap(enum fixed_addresses idx, 672af86e597SLaura Abbott phys_addr_t phys, pgprot_t flags) 673af86e597SLaura Abbott { 674af86e597SLaura Abbott unsigned long addr = __fix_to_virt(idx); 675af86e597SLaura Abbott pte_t *pte; 676af86e597SLaura Abbott 677b63dbef9SMark Rutland BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); 678af86e597SLaura Abbott 679af86e597SLaura Abbott pte = fixmap_pte(addr); 680af86e597SLaura Abbott 681af86e597SLaura Abbott if (pgprot_val(flags)) { 682af86e597SLaura Abbott set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags)); 683af86e597SLaura Abbott } else { 684af86e597SLaura Abbott pte_clear(&init_mm, addr, pte); 685af86e597SLaura Abbott flush_tlb_kernel_range(addr, addr+PAGE_SIZE); 686af86e597SLaura Abbott } 687af86e597SLaura Abbott } 68861bd93ceSArd Biesheuvel 689f80fb3a3SArd Biesheuvel void *__init __fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot) 69061bd93ceSArd Biesheuvel { 69161bd93ceSArd Biesheuvel const u64 dt_virt_base = __fix_to_virt(FIX_FDT); 692f80fb3a3SArd Biesheuvel int offset; 69361bd93ceSArd Biesheuvel void *dt_virt; 69461bd93ceSArd Biesheuvel 69561bd93ceSArd Biesheuvel /* 69661bd93ceSArd Biesheuvel * Check whether the physical FDT address is set and meets the minimum 69761bd93ceSArd Biesheuvel * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be 69861bd93ceSArd Biesheuvel * at least 8 bytes so that we can always access the size field of the 69961bd93ceSArd Biesheuvel * FDT header after mapping the first chunk, double check here if that 70061bd93ceSArd Biesheuvel * is indeed the case. 70161bd93ceSArd Biesheuvel */ 70261bd93ceSArd Biesheuvel BUILD_BUG_ON(MIN_FDT_ALIGN < 8); 70361bd93ceSArd Biesheuvel if (!dt_phys || dt_phys % MIN_FDT_ALIGN) 70461bd93ceSArd Biesheuvel return NULL; 70561bd93ceSArd Biesheuvel 70661bd93ceSArd Biesheuvel /* 70761bd93ceSArd Biesheuvel * Make sure that the FDT region can be mapped without the need to 70861bd93ceSArd Biesheuvel * allocate additional translation table pages, so that it is safe 709132233a7SLaura Abbott * to call create_mapping_noalloc() this early. 71061bd93ceSArd Biesheuvel * 71161bd93ceSArd Biesheuvel * On 64k pages, the FDT will be mapped using PTEs, so we need to 71261bd93ceSArd Biesheuvel * be in the same PMD as the rest of the fixmap. 71361bd93ceSArd Biesheuvel * On 4k pages, we'll use section mappings for the FDT so we only 71461bd93ceSArd Biesheuvel * have to be in the same PUD. 71561bd93ceSArd Biesheuvel */ 71661bd93ceSArd Biesheuvel BUILD_BUG_ON(dt_virt_base % SZ_2M); 71761bd93ceSArd Biesheuvel 718b433dce0SSuzuki K. Poulose BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> SWAPPER_TABLE_SHIFT != 719b433dce0SSuzuki K. Poulose __fix_to_virt(FIX_BTMAP_BEGIN) >> SWAPPER_TABLE_SHIFT); 72061bd93ceSArd Biesheuvel 721b433dce0SSuzuki K. Poulose offset = dt_phys % SWAPPER_BLOCK_SIZE; 72261bd93ceSArd Biesheuvel dt_virt = (void *)dt_virt_base + offset; 72361bd93ceSArd Biesheuvel 72461bd93ceSArd Biesheuvel /* map the first chunk so we can read the size from the header */ 725132233a7SLaura Abbott create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), 726132233a7SLaura Abbott dt_virt_base, SWAPPER_BLOCK_SIZE, prot); 72761bd93ceSArd Biesheuvel 72861bd93ceSArd Biesheuvel if (fdt_check_header(dt_virt) != 0) 72961bd93ceSArd Biesheuvel return NULL; 73061bd93ceSArd Biesheuvel 731f80fb3a3SArd Biesheuvel *size = fdt_totalsize(dt_virt); 732f80fb3a3SArd Biesheuvel if (*size > MAX_FDT_SIZE) 73361bd93ceSArd Biesheuvel return NULL; 73461bd93ceSArd Biesheuvel 735f80fb3a3SArd Biesheuvel if (offset + *size > SWAPPER_BLOCK_SIZE) 736132233a7SLaura Abbott create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base, 737f80fb3a3SArd Biesheuvel round_up(offset + *size, SWAPPER_BLOCK_SIZE), prot); 738f80fb3a3SArd Biesheuvel 739f80fb3a3SArd Biesheuvel return dt_virt; 740f80fb3a3SArd Biesheuvel } 741f80fb3a3SArd Biesheuvel 742f80fb3a3SArd Biesheuvel void *__init fixmap_remap_fdt(phys_addr_t dt_phys) 743f80fb3a3SArd Biesheuvel { 744f80fb3a3SArd Biesheuvel void *dt_virt; 745f80fb3a3SArd Biesheuvel int size; 746f80fb3a3SArd Biesheuvel 747f80fb3a3SArd Biesheuvel dt_virt = __fixmap_remap_fdt(dt_phys, &size, PAGE_KERNEL_RO); 748f80fb3a3SArd Biesheuvel if (!dt_virt) 749f80fb3a3SArd Biesheuvel return NULL; 75061bd93ceSArd Biesheuvel 75161bd93ceSArd Biesheuvel memblock_reserve(dt_phys, size); 75261bd93ceSArd Biesheuvel return dt_virt; 75361bd93ceSArd Biesheuvel } 754324420bfSArd Biesheuvel 755324420bfSArd Biesheuvel int __init arch_ioremap_pud_supported(void) 756324420bfSArd Biesheuvel { 757324420bfSArd Biesheuvel /* only 4k granule supports level 1 block mappings */ 758324420bfSArd Biesheuvel return IS_ENABLED(CONFIG_ARM64_4K_PAGES); 759324420bfSArd Biesheuvel } 760324420bfSArd Biesheuvel 761324420bfSArd Biesheuvel int __init arch_ioremap_pmd_supported(void) 762324420bfSArd Biesheuvel { 763324420bfSArd Biesheuvel return 1; 764324420bfSArd Biesheuvel } 765324420bfSArd Biesheuvel 766324420bfSArd Biesheuvel int pud_set_huge(pud_t *pud, phys_addr_t phys, pgprot_t prot) 767324420bfSArd Biesheuvel { 768324420bfSArd Biesheuvel BUG_ON(phys & ~PUD_MASK); 769324420bfSArd Biesheuvel set_pud(pud, __pud(phys | PUD_TYPE_SECT | pgprot_val(mk_sect_prot(prot)))); 770324420bfSArd Biesheuvel return 1; 771324420bfSArd Biesheuvel } 772324420bfSArd Biesheuvel 773324420bfSArd Biesheuvel int pmd_set_huge(pmd_t *pmd, phys_addr_t phys, pgprot_t prot) 774324420bfSArd Biesheuvel { 775324420bfSArd Biesheuvel BUG_ON(phys & ~PMD_MASK); 776324420bfSArd Biesheuvel set_pmd(pmd, __pmd(phys | PMD_TYPE_SECT | pgprot_val(mk_sect_prot(prot)))); 777324420bfSArd Biesheuvel return 1; 778324420bfSArd Biesheuvel } 779324420bfSArd Biesheuvel 780324420bfSArd Biesheuvel int pud_clear_huge(pud_t *pud) 781324420bfSArd Biesheuvel { 782324420bfSArd Biesheuvel if (!pud_sect(*pud)) 783324420bfSArd Biesheuvel return 0; 784324420bfSArd Biesheuvel pud_clear(pud); 785324420bfSArd Biesheuvel return 1; 786324420bfSArd Biesheuvel } 787324420bfSArd Biesheuvel 788324420bfSArd Biesheuvel int pmd_clear_huge(pmd_t *pmd) 789324420bfSArd Biesheuvel { 790324420bfSArd Biesheuvel if (!pmd_sect(*pmd)) 791324420bfSArd Biesheuvel return 0; 792324420bfSArd Biesheuvel pmd_clear(pmd); 793324420bfSArd Biesheuvel return 1; 794324420bfSArd Biesheuvel } 795