1c1cc1552SCatalin Marinas /* 2c1cc1552SCatalin Marinas * Based on arch/arm/mm/mmu.c 3c1cc1552SCatalin Marinas * 4c1cc1552SCatalin Marinas * Copyright (C) 1995-2005 Russell King 5c1cc1552SCatalin Marinas * Copyright (C) 2012 ARM Ltd. 6c1cc1552SCatalin Marinas * 7c1cc1552SCatalin Marinas * This program is free software; you can redistribute it and/or modify 8c1cc1552SCatalin Marinas * it under the terms of the GNU General Public License version 2 as 9c1cc1552SCatalin Marinas * published by the Free Software Foundation. 10c1cc1552SCatalin Marinas * 11c1cc1552SCatalin Marinas * This program is distributed in the hope that it will be useful, 12c1cc1552SCatalin Marinas * but WITHOUT ANY WARRANTY; without even the implied warranty of 13c1cc1552SCatalin Marinas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14c1cc1552SCatalin Marinas * GNU General Public License for more details. 15c1cc1552SCatalin Marinas * 16c1cc1552SCatalin Marinas * You should have received a copy of the GNU General Public License 17c1cc1552SCatalin Marinas * along with this program. If not, see <http://www.gnu.org/licenses/>. 18c1cc1552SCatalin Marinas */ 19c1cc1552SCatalin Marinas 20c1cc1552SCatalin Marinas #include <linux/export.h> 21c1cc1552SCatalin Marinas #include <linux/kernel.h> 22c1cc1552SCatalin Marinas #include <linux/errno.h> 23c1cc1552SCatalin Marinas #include <linux/init.h> 2461bd93ceSArd Biesheuvel #include <linux/libfdt.h> 25c1cc1552SCatalin Marinas #include <linux/mman.h> 26c1cc1552SCatalin Marinas #include <linux/nodemask.h> 27c1cc1552SCatalin Marinas #include <linux/memblock.h> 28c1cc1552SCatalin Marinas #include <linux/fs.h> 292475ff9dSCatalin Marinas #include <linux/io.h> 3041089357SCatalin Marinas #include <linux/slab.h> 31da141706SLaura Abbott #include <linux/stop_machine.h> 32c1cc1552SCatalin Marinas 3321ab99c2SMark Rutland #include <asm/barrier.h> 34c1cc1552SCatalin Marinas #include <asm/cputype.h> 35af86e597SLaura Abbott #include <asm/fixmap.h> 36068a17a5SMark Rutland #include <asm/kasan.h> 37b433dce0SSuzuki K. Poulose #include <asm/kernel-pgtable.h> 38c1cc1552SCatalin Marinas #include <asm/sections.h> 39c1cc1552SCatalin Marinas #include <asm/setup.h> 40c1cc1552SCatalin Marinas #include <asm/sizes.h> 41c1cc1552SCatalin Marinas #include <asm/tlb.h> 42c79b954bSJungseok Lee #include <asm/memblock.h> 43c1cc1552SCatalin Marinas #include <asm/mmu_context.h> 44c1cc1552SCatalin Marinas 45c1cc1552SCatalin Marinas #include "mm.h" 46c1cc1552SCatalin Marinas 47dd006da2SArd Biesheuvel u64 idmap_t0sz = TCR_T0SZ(VA_BITS); 48dd006da2SArd Biesheuvel 49a7f8de16SArd Biesheuvel u64 kimage_voffset __read_mostly; 50a7f8de16SArd Biesheuvel EXPORT_SYMBOL(kimage_voffset); 51a7f8de16SArd Biesheuvel 52c1cc1552SCatalin Marinas /* 53c1cc1552SCatalin Marinas * Empty_zero_page is a special page that is used for zero-initialized data 54c1cc1552SCatalin Marinas * and COW. 55c1cc1552SCatalin Marinas */ 565227cfa7SMark Rutland unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss; 57c1cc1552SCatalin Marinas EXPORT_SYMBOL(empty_zero_page); 58c1cc1552SCatalin Marinas 59f9040773SArd Biesheuvel static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss; 60f9040773SArd Biesheuvel static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; 61f9040773SArd Biesheuvel static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; 62f9040773SArd Biesheuvel 63c1cc1552SCatalin Marinas pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, 64c1cc1552SCatalin Marinas unsigned long size, pgprot_t vma_prot) 65c1cc1552SCatalin Marinas { 66c1cc1552SCatalin Marinas if (!pfn_valid(pfn)) 67c1cc1552SCatalin Marinas return pgprot_noncached(vma_prot); 68c1cc1552SCatalin Marinas else if (file->f_flags & O_SYNC) 69c1cc1552SCatalin Marinas return pgprot_writecombine(vma_prot); 70c1cc1552SCatalin Marinas return vma_prot; 71c1cc1552SCatalin Marinas } 72c1cc1552SCatalin Marinas EXPORT_SYMBOL(phys_mem_access_prot); 73c1cc1552SCatalin Marinas 74f4710445SMark Rutland static phys_addr_t __init early_pgtable_alloc(void) 75c1cc1552SCatalin Marinas { 767142392dSSuzuki K. Poulose phys_addr_t phys; 777142392dSSuzuki K. Poulose void *ptr; 787142392dSSuzuki K. Poulose 7921ab99c2SMark Rutland phys = memblock_alloc(PAGE_SIZE, PAGE_SIZE); 807142392dSSuzuki K. Poulose BUG_ON(!phys); 81f4710445SMark Rutland 82f4710445SMark Rutland /* 83f4710445SMark Rutland * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE 84f4710445SMark Rutland * slot will be free, so we can (ab)use the FIX_PTE slot to initialise 85f4710445SMark Rutland * any level of table. 86f4710445SMark Rutland */ 87f4710445SMark Rutland ptr = pte_set_fixmap(phys); 88f4710445SMark Rutland 8921ab99c2SMark Rutland memset(ptr, 0, PAGE_SIZE); 9021ab99c2SMark Rutland 91f4710445SMark Rutland /* 92f4710445SMark Rutland * Implicit barriers also ensure the zeroed page is visible to the page 93f4710445SMark Rutland * table walker 94f4710445SMark Rutland */ 95f4710445SMark Rutland pte_clear_fixmap(); 96f4710445SMark Rutland 97f4710445SMark Rutland return phys; 98c1cc1552SCatalin Marinas } 99c1cc1552SCatalin Marinas 100da141706SLaura Abbott /* 101da141706SLaura Abbott * remap a PMD into pages 102da141706SLaura Abbott */ 103da141706SLaura Abbott static void split_pmd(pmd_t *pmd, pte_t *pte) 104da141706SLaura Abbott { 105da141706SLaura Abbott unsigned long pfn = pmd_pfn(*pmd); 106da141706SLaura Abbott int i = 0; 107da141706SLaura Abbott 108da141706SLaura Abbott do { 109da141706SLaura Abbott /* 110da141706SLaura Abbott * Need to have the least restrictive permissions available 111667c2759SCatalin Marinas * permissions will be fixed up later 112da141706SLaura Abbott */ 113667c2759SCatalin Marinas set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC)); 114da141706SLaura Abbott pfn++; 115da141706SLaura Abbott } while (pte++, i++, i < PTRS_PER_PTE); 116da141706SLaura Abbott } 117da141706SLaura Abbott 118da141706SLaura Abbott static void alloc_init_pte(pmd_t *pmd, unsigned long addr, 119667c2759SCatalin Marinas unsigned long end, unsigned long pfn, 120da141706SLaura Abbott pgprot_t prot, 121f4710445SMark Rutland phys_addr_t (*pgtable_alloc)(void)) 122c1cc1552SCatalin Marinas { 123c1cc1552SCatalin Marinas pte_t *pte; 124c1cc1552SCatalin Marinas 125a1c76574SMark Rutland if (pmd_none(*pmd) || pmd_sect(*pmd)) { 126132233a7SLaura Abbott phys_addr_t pte_phys; 127132233a7SLaura Abbott BUG_ON(!pgtable_alloc); 128132233a7SLaura Abbott pte_phys = pgtable_alloc(); 129f4710445SMark Rutland pte = pte_set_fixmap(pte_phys); 130da141706SLaura Abbott if (pmd_sect(*pmd)) 131da141706SLaura Abbott split_pmd(pmd, pte); 132f4710445SMark Rutland __pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE); 133da141706SLaura Abbott flush_tlb_all(); 134f4710445SMark Rutland pte_clear_fixmap(); 135c1cc1552SCatalin Marinas } 136a1c76574SMark Rutland BUG_ON(pmd_bad(*pmd)); 137c1cc1552SCatalin Marinas 138f4710445SMark Rutland pte = pte_set_fixmap_offset(pmd, addr); 139c1cc1552SCatalin Marinas do { 140667c2759SCatalin Marinas set_pte(pte, pfn_pte(pfn, prot)); 141667c2759SCatalin Marinas pfn++; 142667c2759SCatalin Marinas } while (pte++, addr += PAGE_SIZE, addr != end); 143f4710445SMark Rutland 144f4710445SMark Rutland pte_clear_fixmap(); 145c1cc1552SCatalin Marinas } 146c1cc1552SCatalin Marinas 1479a17a213SJisheng Zhang static void split_pud(pud_t *old_pud, pmd_t *pmd) 148da141706SLaura Abbott { 149da141706SLaura Abbott unsigned long addr = pud_pfn(*old_pud) << PAGE_SHIFT; 150da141706SLaura Abbott pgprot_t prot = __pgprot(pud_val(*old_pud) ^ addr); 151da141706SLaura Abbott int i = 0; 152da141706SLaura Abbott 153da141706SLaura Abbott do { 1541e43ba9cSArd Biesheuvel set_pmd(pmd, __pmd(addr | pgprot_val(prot))); 155da141706SLaura Abbott addr += PMD_SIZE; 156da141706SLaura Abbott } while (pmd++, i++, i < PTRS_PER_PMD); 157da141706SLaura Abbott } 158da141706SLaura Abbott 15983863f25SLaura Abbott #ifdef CONFIG_DEBUG_PAGEALLOC 16083863f25SLaura Abbott static bool block_mappings_allowed(phys_addr_t (*pgtable_alloc)(void)) 16183863f25SLaura Abbott { 16283863f25SLaura Abbott 16383863f25SLaura Abbott /* 16483863f25SLaura Abbott * If debug_page_alloc is enabled we must map the linear map 16583863f25SLaura Abbott * using pages. However, other mappings created by 16683863f25SLaura Abbott * create_mapping_noalloc must use sections in some cases. Allow 16783863f25SLaura Abbott * sections to be used in those cases, where no pgtable_alloc 16883863f25SLaura Abbott * function is provided. 16983863f25SLaura Abbott */ 17083863f25SLaura Abbott return !pgtable_alloc || !debug_pagealloc_enabled(); 17183863f25SLaura Abbott } 17283863f25SLaura Abbott #else 17383863f25SLaura Abbott static bool block_mappings_allowed(phys_addr_t (*pgtable_alloc)(void)) 17483863f25SLaura Abbott { 17583863f25SLaura Abbott return true; 17683863f25SLaura Abbott } 17783863f25SLaura Abbott #endif 17883863f25SLaura Abbott 17911509a30SMark Rutland static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end, 180da141706SLaura Abbott phys_addr_t phys, pgprot_t prot, 181f4710445SMark Rutland phys_addr_t (*pgtable_alloc)(void)) 182c1cc1552SCatalin Marinas { 183c1cc1552SCatalin Marinas pmd_t *pmd; 184c1cc1552SCatalin Marinas unsigned long next; 185c1cc1552SCatalin Marinas 186c1cc1552SCatalin Marinas /* 187c1cc1552SCatalin Marinas * Check for initial section mappings in the pgd/pud and remove them. 188c1cc1552SCatalin Marinas */ 189a1c76574SMark Rutland if (pud_none(*pud) || pud_sect(*pud)) { 190132233a7SLaura Abbott phys_addr_t pmd_phys; 191132233a7SLaura Abbott BUG_ON(!pgtable_alloc); 192132233a7SLaura Abbott pmd_phys = pgtable_alloc(); 193f4710445SMark Rutland pmd = pmd_set_fixmap(pmd_phys); 194da141706SLaura Abbott if (pud_sect(*pud)) { 195da141706SLaura Abbott /* 196da141706SLaura Abbott * need to have the 1G of mappings continue to be 197da141706SLaura Abbott * present 198da141706SLaura Abbott */ 199da141706SLaura Abbott split_pud(pud, pmd); 200da141706SLaura Abbott } 201f4710445SMark Rutland __pud_populate(pud, pmd_phys, PUD_TYPE_TABLE); 202da141706SLaura Abbott flush_tlb_all(); 203f4710445SMark Rutland pmd_clear_fixmap(); 204c1cc1552SCatalin Marinas } 205a1c76574SMark Rutland BUG_ON(pud_bad(*pud)); 206c1cc1552SCatalin Marinas 207f4710445SMark Rutland pmd = pmd_set_fixmap_offset(pud, addr); 208c1cc1552SCatalin Marinas do { 209c1cc1552SCatalin Marinas next = pmd_addr_end(addr, end); 210c1cc1552SCatalin Marinas /* try section mapping first */ 21183863f25SLaura Abbott if (((addr | next | phys) & ~SECTION_MASK) == 0 && 21283863f25SLaura Abbott block_mappings_allowed(pgtable_alloc)) { 213a55f9929SCatalin Marinas pmd_t old_pmd =*pmd; 214c661cb1cSMark Rutland pmd_set_huge(pmd, phys, prot); 215a55f9929SCatalin Marinas /* 216a55f9929SCatalin Marinas * Check for previous table entries created during 217a55f9929SCatalin Marinas * boot (__create_page_tables) and flush them. 218a55f9929SCatalin Marinas */ 219523d6e9fSzhichang.yuan if (!pmd_none(old_pmd)) { 220a55f9929SCatalin Marinas flush_tlb_all(); 221523d6e9fSzhichang.yuan if (pmd_table(old_pmd)) { 222316b39dbSMark Rutland phys_addr_t table = pmd_page_paddr(old_pmd); 22341089357SCatalin Marinas if (!WARN_ON_ONCE(slab_is_available())) 224523d6e9fSzhichang.yuan memblock_free(table, PAGE_SIZE); 225523d6e9fSzhichang.yuan } 226523d6e9fSzhichang.yuan } 227a55f9929SCatalin Marinas } else { 228667c2759SCatalin Marinas alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys), 22921ab99c2SMark Rutland prot, pgtable_alloc); 230a55f9929SCatalin Marinas } 231c1cc1552SCatalin Marinas phys += next - addr; 232c1cc1552SCatalin Marinas } while (pmd++, addr = next, addr != end); 233f4710445SMark Rutland 234f4710445SMark Rutland pmd_clear_fixmap(); 235c1cc1552SCatalin Marinas } 236c1cc1552SCatalin Marinas 237da141706SLaura Abbott static inline bool use_1G_block(unsigned long addr, unsigned long next, 238da141706SLaura Abbott unsigned long phys) 239da141706SLaura Abbott { 240da141706SLaura Abbott if (PAGE_SHIFT != 12) 241da141706SLaura Abbott return false; 242da141706SLaura Abbott 243da141706SLaura Abbott if (((addr | next | phys) & ~PUD_MASK) != 0) 244da141706SLaura Abbott return false; 245da141706SLaura Abbott 246da141706SLaura Abbott return true; 247da141706SLaura Abbott } 248da141706SLaura Abbott 24911509a30SMark Rutland static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end, 250da141706SLaura Abbott phys_addr_t phys, pgprot_t prot, 251f4710445SMark Rutland phys_addr_t (*pgtable_alloc)(void)) 252c1cc1552SCatalin Marinas { 253c79b954bSJungseok Lee pud_t *pud; 254c1cc1552SCatalin Marinas unsigned long next; 255c1cc1552SCatalin Marinas 256c79b954bSJungseok Lee if (pgd_none(*pgd)) { 257132233a7SLaura Abbott phys_addr_t pud_phys; 258132233a7SLaura Abbott BUG_ON(!pgtable_alloc); 259132233a7SLaura Abbott pud_phys = pgtable_alloc(); 260f4710445SMark Rutland __pgd_populate(pgd, pud_phys, PUD_TYPE_TABLE); 261c79b954bSJungseok Lee } 262c79b954bSJungseok Lee BUG_ON(pgd_bad(*pgd)); 263c79b954bSJungseok Lee 264f4710445SMark Rutland pud = pud_set_fixmap_offset(pgd, addr); 265c1cc1552SCatalin Marinas do { 266c1cc1552SCatalin Marinas next = pud_addr_end(addr, end); 267206a2a73SSteve Capper 268206a2a73SSteve Capper /* 269206a2a73SSteve Capper * For 4K granule only, attempt to put down a 1GB block 270206a2a73SSteve Capper */ 27183863f25SLaura Abbott if (use_1G_block(addr, next, phys) && 27283863f25SLaura Abbott block_mappings_allowed(pgtable_alloc)) { 273206a2a73SSteve Capper pud_t old_pud = *pud; 274c661cb1cSMark Rutland pud_set_huge(pud, phys, prot); 275206a2a73SSteve Capper 276206a2a73SSteve Capper /* 277206a2a73SSteve Capper * If we have an old value for a pud, it will 278206a2a73SSteve Capper * be pointing to a pmd table that we no longer 279206a2a73SSteve Capper * need (from swapper_pg_dir). 280206a2a73SSteve Capper * 281206a2a73SSteve Capper * Look up the old pmd table and free it. 282206a2a73SSteve Capper */ 283206a2a73SSteve Capper if (!pud_none(old_pud)) { 284206a2a73SSteve Capper flush_tlb_all(); 285523d6e9fSzhichang.yuan if (pud_table(old_pud)) { 286316b39dbSMark Rutland phys_addr_t table = pud_page_paddr(old_pud); 28741089357SCatalin Marinas if (!WARN_ON_ONCE(slab_is_available())) 288523d6e9fSzhichang.yuan memblock_free(table, PAGE_SIZE); 289523d6e9fSzhichang.yuan } 290206a2a73SSteve Capper } 291206a2a73SSteve Capper } else { 29211509a30SMark Rutland alloc_init_pmd(pud, addr, next, phys, prot, 29321ab99c2SMark Rutland pgtable_alloc); 294206a2a73SSteve Capper } 295c1cc1552SCatalin Marinas phys += next - addr; 296c1cc1552SCatalin Marinas } while (pud++, addr = next, addr != end); 297f4710445SMark Rutland 298f4710445SMark Rutland pud_clear_fixmap(); 299c1cc1552SCatalin Marinas } 300c1cc1552SCatalin Marinas 301c1cc1552SCatalin Marinas /* 302c1cc1552SCatalin Marinas * Create the page directory entries and any necessary page tables for the 303c1cc1552SCatalin Marinas * mapping specified by 'md'. 304c1cc1552SCatalin Marinas */ 30511509a30SMark Rutland static void init_pgd(pgd_t *pgd, phys_addr_t phys, unsigned long virt, 306da141706SLaura Abbott phys_addr_t size, pgprot_t prot, 307f4710445SMark Rutland phys_addr_t (*pgtable_alloc)(void)) 308c1cc1552SCatalin Marinas { 309c1cc1552SCatalin Marinas unsigned long addr, length, end, next; 310c1cc1552SCatalin Marinas 311cc5d2b3bSMark Rutland /* 312cc5d2b3bSMark Rutland * If the virtual and physical address don't have the same offset 313cc5d2b3bSMark Rutland * within a page, we cannot map the region as the caller expects. 314cc5d2b3bSMark Rutland */ 315cc5d2b3bSMark Rutland if (WARN_ON((phys ^ virt) & ~PAGE_MASK)) 316cc5d2b3bSMark Rutland return; 317cc5d2b3bSMark Rutland 3189c4e08a3SMark Rutland phys &= PAGE_MASK; 319c1cc1552SCatalin Marinas addr = virt & PAGE_MASK; 320c1cc1552SCatalin Marinas length = PAGE_ALIGN(size + (virt & ~PAGE_MASK)); 321c1cc1552SCatalin Marinas 322c1cc1552SCatalin Marinas end = addr + length; 323c1cc1552SCatalin Marinas do { 324c1cc1552SCatalin Marinas next = pgd_addr_end(addr, end); 32511509a30SMark Rutland alloc_init_pud(pgd, addr, next, phys, prot, pgtable_alloc); 326c1cc1552SCatalin Marinas phys += next - addr; 327c1cc1552SCatalin Marinas } while (pgd++, addr = next, addr != end); 328c1cc1552SCatalin Marinas } 329c1cc1552SCatalin Marinas 330f4710445SMark Rutland static phys_addr_t late_pgtable_alloc(void) 331da141706SLaura Abbott { 33221ab99c2SMark Rutland void *ptr = (void *)__get_free_page(PGALLOC_GFP); 333da141706SLaura Abbott BUG_ON(!ptr); 33421ab99c2SMark Rutland 33521ab99c2SMark Rutland /* Ensure the zeroed page is visible to the page table walker */ 33621ab99c2SMark Rutland dsb(ishst); 337f4710445SMark Rutland return __pa(ptr); 338da141706SLaura Abbott } 339da141706SLaura Abbott 34011509a30SMark Rutland static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, 34111509a30SMark Rutland unsigned long virt, phys_addr_t size, 34211509a30SMark Rutland pgprot_t prot, 34311509a30SMark Rutland phys_addr_t (*alloc)(void)) 34411509a30SMark Rutland { 34511509a30SMark Rutland init_pgd(pgd_offset_raw(pgdir, virt), phys, virt, size, prot, alloc); 34611509a30SMark Rutland } 34711509a30SMark Rutland 348132233a7SLaura Abbott /* 349132233a7SLaura Abbott * This function can only be used to modify existing table entries, 350132233a7SLaura Abbott * without allocating new levels of table. Note that this permits the 351132233a7SLaura Abbott * creation of new section or page entries. 352132233a7SLaura Abbott */ 353132233a7SLaura Abbott static void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, 354da141706SLaura Abbott phys_addr_t size, pgprot_t prot) 355d7ecbddfSMark Salter { 356d7ecbddfSMark Salter if (virt < VMALLOC_START) { 357d7ecbddfSMark Salter pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", 358d7ecbddfSMark Salter &phys, virt); 359d7ecbddfSMark Salter return; 360d7ecbddfSMark Salter } 36111509a30SMark Rutland __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, 362132233a7SLaura Abbott NULL); 363d7ecbddfSMark Salter } 364d7ecbddfSMark Salter 3658ce837ceSArd Biesheuvel void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, 3668ce837ceSArd Biesheuvel unsigned long virt, phys_addr_t size, 3678ce837ceSArd Biesheuvel pgprot_t prot) 3688ce837ceSArd Biesheuvel { 36911509a30SMark Rutland __create_pgd_mapping(mm->pgd, phys, virt, size, prot, 37021ab99c2SMark Rutland late_pgtable_alloc); 371d7ecbddfSMark Salter } 372d7ecbddfSMark Salter 373da141706SLaura Abbott static void create_mapping_late(phys_addr_t phys, unsigned long virt, 374da141706SLaura Abbott phys_addr_t size, pgprot_t prot) 375da141706SLaura Abbott { 376da141706SLaura Abbott if (virt < VMALLOC_START) { 377da141706SLaura Abbott pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", 378da141706SLaura Abbott &phys, virt); 379da141706SLaura Abbott return; 380da141706SLaura Abbott } 381da141706SLaura Abbott 38211509a30SMark Rutland __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, 38311509a30SMark Rutland late_pgtable_alloc); 384da141706SLaura Abbott } 385da141706SLaura Abbott 386068a17a5SMark Rutland static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end) 387da141706SLaura Abbott { 3887eb90f2fSArd Biesheuvel unsigned long kernel_start = __pa(_text); 389*9fdc14c5SArd Biesheuvel unsigned long kernel_end = __pa(__init_begin); 390068a17a5SMark Rutland 391da141706SLaura Abbott /* 392f9040773SArd Biesheuvel * Take care not to create a writable alias for the 393f9040773SArd Biesheuvel * read-only text and rodata sections of the kernel image. 394da141706SLaura Abbott */ 395da141706SLaura Abbott 396*9fdc14c5SArd Biesheuvel /* No overlap with the kernel text/rodata */ 397068a17a5SMark Rutland if (end < kernel_start || start >= kernel_end) { 398068a17a5SMark Rutland __create_pgd_mapping(pgd, start, __phys_to_virt(start), 399068a17a5SMark Rutland end - start, PAGE_KERNEL, 400068a17a5SMark Rutland early_pgtable_alloc); 401068a17a5SMark Rutland return; 402da141706SLaura Abbott } 403da141706SLaura Abbott 404068a17a5SMark Rutland /* 405*9fdc14c5SArd Biesheuvel * This block overlaps the kernel text/rodata mappings. 406f9040773SArd Biesheuvel * Map the portion(s) which don't overlap. 407068a17a5SMark Rutland */ 408068a17a5SMark Rutland if (start < kernel_start) 409068a17a5SMark Rutland __create_pgd_mapping(pgd, start, 410068a17a5SMark Rutland __phys_to_virt(start), 411068a17a5SMark Rutland kernel_start - start, PAGE_KERNEL, 412068a17a5SMark Rutland early_pgtable_alloc); 413068a17a5SMark Rutland if (kernel_end < end) 414068a17a5SMark Rutland __create_pgd_mapping(pgd, kernel_end, 415068a17a5SMark Rutland __phys_to_virt(kernel_end), 416068a17a5SMark Rutland end - kernel_end, PAGE_KERNEL, 417068a17a5SMark Rutland early_pgtable_alloc); 418f9040773SArd Biesheuvel 419f9040773SArd Biesheuvel /* 420*9fdc14c5SArd Biesheuvel * Map the linear alias of the [_text, __init_begin) interval as 421f9040773SArd Biesheuvel * read-only/non-executable. This makes the contents of the 422f9040773SArd Biesheuvel * region accessible to subsystems such as hibernate, but 423f9040773SArd Biesheuvel * protects it from inadvertent modification or execution. 424f9040773SArd Biesheuvel */ 425f9040773SArd Biesheuvel __create_pgd_mapping(pgd, kernel_start, __phys_to_virt(kernel_start), 426f9040773SArd Biesheuvel kernel_end - kernel_start, PAGE_KERNEL_RO, 427f9040773SArd Biesheuvel early_pgtable_alloc); 428da141706SLaura Abbott } 429da141706SLaura Abbott 430068a17a5SMark Rutland static void __init map_mem(pgd_t *pgd) 431c1cc1552SCatalin Marinas { 432c1cc1552SCatalin Marinas struct memblock_region *reg; 433f6bc87c3SSteve Capper 434c1cc1552SCatalin Marinas /* map all the memory banks */ 435c1cc1552SCatalin Marinas for_each_memblock(memory, reg) { 436c1cc1552SCatalin Marinas phys_addr_t start = reg->base; 437c1cc1552SCatalin Marinas phys_addr_t end = start + reg->size; 438c1cc1552SCatalin Marinas 439c1cc1552SCatalin Marinas if (start >= end) 440c1cc1552SCatalin Marinas break; 44168709f45SArd Biesheuvel if (memblock_is_nomap(reg)) 44268709f45SArd Biesheuvel continue; 443c1cc1552SCatalin Marinas 444068a17a5SMark Rutland __map_memblock(pgd, start, end); 445c1cc1552SCatalin Marinas } 446c1cc1552SCatalin Marinas } 447c1cc1552SCatalin Marinas 448da141706SLaura Abbott void mark_rodata_ro(void) 449da141706SLaura Abbott { 4502f39b5f9SJeremy Linton unsigned long section_size; 451f9040773SArd Biesheuvel 452*9fdc14c5SArd Biesheuvel section_size = (unsigned long)_etext - (unsigned long)_text; 4537eb90f2fSArd Biesheuvel create_mapping_late(__pa(_text), (unsigned long)_text, 4542f39b5f9SJeremy Linton section_size, PAGE_KERNEL_ROX); 4552f39b5f9SJeremy Linton /* 456*9fdc14c5SArd Biesheuvel * mark .rodata as read only. Use __init_begin rather than __end_rodata 457*9fdc14c5SArd Biesheuvel * to cover NOTES and EXCEPTION_TABLE. 4582f39b5f9SJeremy Linton */ 459*9fdc14c5SArd Biesheuvel section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata; 4602f39b5f9SJeremy Linton create_mapping_late(__pa(__start_rodata), (unsigned long)__start_rodata, 4612f39b5f9SJeremy Linton section_size, PAGE_KERNEL_RO); 462da141706SLaura Abbott } 463da141706SLaura Abbott 464da141706SLaura Abbott void fixup_init(void) 465da141706SLaura Abbott { 466f9040773SArd Biesheuvel /* 467f9040773SArd Biesheuvel * Unmap the __init region but leave the VM area in place. This 468f9040773SArd Biesheuvel * prevents the region from being reused for kernel modules, which 469f9040773SArd Biesheuvel * is not supported by kallsyms. 470f9040773SArd Biesheuvel */ 471f9040773SArd Biesheuvel unmap_kernel_range((u64)__init_begin, (u64)(__init_end - __init_begin)); 472da141706SLaura Abbott } 473da141706SLaura Abbott 4742c09ec06SArd Biesheuvel static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end, 475f9040773SArd Biesheuvel pgprot_t prot, struct vm_struct *vma) 476068a17a5SMark Rutland { 477068a17a5SMark Rutland phys_addr_t pa_start = __pa(va_start); 478068a17a5SMark Rutland unsigned long size = va_end - va_start; 479068a17a5SMark Rutland 480068a17a5SMark Rutland BUG_ON(!PAGE_ALIGNED(pa_start)); 481068a17a5SMark Rutland BUG_ON(!PAGE_ALIGNED(size)); 482068a17a5SMark Rutland 483068a17a5SMark Rutland __create_pgd_mapping(pgd, pa_start, (unsigned long)va_start, size, prot, 484068a17a5SMark Rutland early_pgtable_alloc); 485f9040773SArd Biesheuvel 486f9040773SArd Biesheuvel vma->addr = va_start; 487f9040773SArd Biesheuvel vma->phys_addr = pa_start; 488f9040773SArd Biesheuvel vma->size = size; 489f9040773SArd Biesheuvel vma->flags = VM_MAP; 490f9040773SArd Biesheuvel vma->caller = __builtin_return_address(0); 491f9040773SArd Biesheuvel 492f9040773SArd Biesheuvel vm_area_add_early(vma); 493068a17a5SMark Rutland } 494068a17a5SMark Rutland 495068a17a5SMark Rutland /* 496068a17a5SMark Rutland * Create fine-grained mappings for the kernel. 497068a17a5SMark Rutland */ 498068a17a5SMark Rutland static void __init map_kernel(pgd_t *pgd) 499068a17a5SMark Rutland { 5002f39b5f9SJeremy Linton static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_init, vmlinux_data; 501068a17a5SMark Rutland 502*9fdc14c5SArd Biesheuvel map_kernel_segment(pgd, _text, _etext, PAGE_KERNEL_EXEC, &vmlinux_text); 503*9fdc14c5SArd Biesheuvel map_kernel_segment(pgd, __start_rodata, __init_begin, PAGE_KERNEL, &vmlinux_rodata); 5042c09ec06SArd Biesheuvel map_kernel_segment(pgd, __init_begin, __init_end, PAGE_KERNEL_EXEC, 505f9040773SArd Biesheuvel &vmlinux_init); 5062c09ec06SArd Biesheuvel map_kernel_segment(pgd, _data, _end, PAGE_KERNEL, &vmlinux_data); 507068a17a5SMark Rutland 508f9040773SArd Biesheuvel if (!pgd_val(*pgd_offset_raw(pgd, FIXADDR_START))) { 509068a17a5SMark Rutland /* 510f9040773SArd Biesheuvel * The fixmap falls in a separate pgd to the kernel, and doesn't 511f9040773SArd Biesheuvel * live in the carveout for the swapper_pg_dir. We can simply 512f9040773SArd Biesheuvel * re-use the existing dir for the fixmap. 513068a17a5SMark Rutland */ 514f9040773SArd Biesheuvel set_pgd(pgd_offset_raw(pgd, FIXADDR_START), 515f9040773SArd Biesheuvel *pgd_offset_k(FIXADDR_START)); 516f9040773SArd Biesheuvel } else if (CONFIG_PGTABLE_LEVELS > 3) { 517f9040773SArd Biesheuvel /* 518f9040773SArd Biesheuvel * The fixmap shares its top level pgd entry with the kernel 519f9040773SArd Biesheuvel * mapping. This can really only occur when we are running 520f9040773SArd Biesheuvel * with 16k/4 levels, so we can simply reuse the pud level 521f9040773SArd Biesheuvel * entry instead. 522f9040773SArd Biesheuvel */ 523f9040773SArd Biesheuvel BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); 524f9040773SArd Biesheuvel set_pud(pud_set_fixmap_offset(pgd, FIXADDR_START), 525f9040773SArd Biesheuvel __pud(__pa(bm_pmd) | PUD_TYPE_TABLE)); 526f9040773SArd Biesheuvel pud_clear_fixmap(); 527f9040773SArd Biesheuvel } else { 528f9040773SArd Biesheuvel BUG(); 529f9040773SArd Biesheuvel } 530068a17a5SMark Rutland 531068a17a5SMark Rutland kasan_copy_shadow(pgd); 532068a17a5SMark Rutland } 533068a17a5SMark Rutland 534c1cc1552SCatalin Marinas /* 535c1cc1552SCatalin Marinas * paging_init() sets up the page tables, initialises the zone memory 536c1cc1552SCatalin Marinas * maps and sets up the zero page. 537c1cc1552SCatalin Marinas */ 538c1cc1552SCatalin Marinas void __init paging_init(void) 539c1cc1552SCatalin Marinas { 540068a17a5SMark Rutland phys_addr_t pgd_phys = early_pgtable_alloc(); 541068a17a5SMark Rutland pgd_t *pgd = pgd_set_fixmap(pgd_phys); 542068a17a5SMark Rutland 543068a17a5SMark Rutland map_kernel(pgd); 544068a17a5SMark Rutland map_mem(pgd); 545068a17a5SMark Rutland 546068a17a5SMark Rutland /* 547068a17a5SMark Rutland * We want to reuse the original swapper_pg_dir so we don't have to 548068a17a5SMark Rutland * communicate the new address to non-coherent secondaries in 549068a17a5SMark Rutland * secondary_entry, and so cpu_switch_mm can generate the address with 550068a17a5SMark Rutland * adrp+add rather than a load from some global variable. 551068a17a5SMark Rutland * 552068a17a5SMark Rutland * To do this we need to go via a temporary pgd. 553068a17a5SMark Rutland */ 554068a17a5SMark Rutland cpu_replace_ttbr1(__va(pgd_phys)); 555068a17a5SMark Rutland memcpy(swapper_pg_dir, pgd, PAGE_SIZE); 556068a17a5SMark Rutland cpu_replace_ttbr1(swapper_pg_dir); 557068a17a5SMark Rutland 558068a17a5SMark Rutland pgd_clear_fixmap(); 559068a17a5SMark Rutland memblock_free(pgd_phys, PAGE_SIZE); 560068a17a5SMark Rutland 561068a17a5SMark Rutland /* 562068a17a5SMark Rutland * We only reuse the PGD from the swapper_pg_dir, not the pud + pmd 563068a17a5SMark Rutland * allocated with it. 564068a17a5SMark Rutland */ 565068a17a5SMark Rutland memblock_free(__pa(swapper_pg_dir) + PAGE_SIZE, 566068a17a5SMark Rutland SWAPPER_DIR_SIZE - PAGE_SIZE); 567c1cc1552SCatalin Marinas } 568c1cc1552SCatalin Marinas 569c1cc1552SCatalin Marinas /* 570c1cc1552SCatalin Marinas * Check whether a kernel address is valid (derived from arch/x86/). 571c1cc1552SCatalin Marinas */ 572c1cc1552SCatalin Marinas int kern_addr_valid(unsigned long addr) 573c1cc1552SCatalin Marinas { 574c1cc1552SCatalin Marinas pgd_t *pgd; 575c1cc1552SCatalin Marinas pud_t *pud; 576c1cc1552SCatalin Marinas pmd_t *pmd; 577c1cc1552SCatalin Marinas pte_t *pte; 578c1cc1552SCatalin Marinas 579c1cc1552SCatalin Marinas if ((((long)addr) >> VA_BITS) != -1UL) 580c1cc1552SCatalin Marinas return 0; 581c1cc1552SCatalin Marinas 582c1cc1552SCatalin Marinas pgd = pgd_offset_k(addr); 583c1cc1552SCatalin Marinas if (pgd_none(*pgd)) 584c1cc1552SCatalin Marinas return 0; 585c1cc1552SCatalin Marinas 586c1cc1552SCatalin Marinas pud = pud_offset(pgd, addr); 587c1cc1552SCatalin Marinas if (pud_none(*pud)) 588c1cc1552SCatalin Marinas return 0; 589c1cc1552SCatalin Marinas 590206a2a73SSteve Capper if (pud_sect(*pud)) 591206a2a73SSteve Capper return pfn_valid(pud_pfn(*pud)); 592206a2a73SSteve Capper 593c1cc1552SCatalin Marinas pmd = pmd_offset(pud, addr); 594c1cc1552SCatalin Marinas if (pmd_none(*pmd)) 595c1cc1552SCatalin Marinas return 0; 596c1cc1552SCatalin Marinas 597da6e4cb6SDave Anderson if (pmd_sect(*pmd)) 598da6e4cb6SDave Anderson return pfn_valid(pmd_pfn(*pmd)); 599da6e4cb6SDave Anderson 600c1cc1552SCatalin Marinas pte = pte_offset_kernel(pmd, addr); 601c1cc1552SCatalin Marinas if (pte_none(*pte)) 602c1cc1552SCatalin Marinas return 0; 603c1cc1552SCatalin Marinas 604c1cc1552SCatalin Marinas return pfn_valid(pte_pfn(*pte)); 605c1cc1552SCatalin Marinas } 606c1cc1552SCatalin Marinas #ifdef CONFIG_SPARSEMEM_VMEMMAP 607b433dce0SSuzuki K. Poulose #if !ARM64_SWAPPER_USES_SECTION_MAPS 6080aad818bSJohannes Weiner int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) 609c1cc1552SCatalin Marinas { 6100aad818bSJohannes Weiner return vmemmap_populate_basepages(start, end, node); 611c1cc1552SCatalin Marinas } 612b433dce0SSuzuki K. Poulose #else /* !ARM64_SWAPPER_USES_SECTION_MAPS */ 6130aad818bSJohannes Weiner int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) 614c1cc1552SCatalin Marinas { 6150aad818bSJohannes Weiner unsigned long addr = start; 616c1cc1552SCatalin Marinas unsigned long next; 617c1cc1552SCatalin Marinas pgd_t *pgd; 618c1cc1552SCatalin Marinas pud_t *pud; 619c1cc1552SCatalin Marinas pmd_t *pmd; 620c1cc1552SCatalin Marinas 621c1cc1552SCatalin Marinas do { 622c1cc1552SCatalin Marinas next = pmd_addr_end(addr, end); 623c1cc1552SCatalin Marinas 624c1cc1552SCatalin Marinas pgd = vmemmap_pgd_populate(addr, node); 625c1cc1552SCatalin Marinas if (!pgd) 626c1cc1552SCatalin Marinas return -ENOMEM; 627c1cc1552SCatalin Marinas 628c1cc1552SCatalin Marinas pud = vmemmap_pud_populate(pgd, addr, node); 629c1cc1552SCatalin Marinas if (!pud) 630c1cc1552SCatalin Marinas return -ENOMEM; 631c1cc1552SCatalin Marinas 632c1cc1552SCatalin Marinas pmd = pmd_offset(pud, addr); 633c1cc1552SCatalin Marinas if (pmd_none(*pmd)) { 634c1cc1552SCatalin Marinas void *p = NULL; 635c1cc1552SCatalin Marinas 636c1cc1552SCatalin Marinas p = vmemmap_alloc_block_buf(PMD_SIZE, node); 637c1cc1552SCatalin Marinas if (!p) 638c1cc1552SCatalin Marinas return -ENOMEM; 639c1cc1552SCatalin Marinas 640a501e324SCatalin Marinas set_pmd(pmd, __pmd(__pa(p) | PROT_SECT_NORMAL)); 641c1cc1552SCatalin Marinas } else 642c1cc1552SCatalin Marinas vmemmap_verify((pte_t *)pmd, node, addr, next); 643c1cc1552SCatalin Marinas } while (addr = next, addr != end); 644c1cc1552SCatalin Marinas 645c1cc1552SCatalin Marinas return 0; 646c1cc1552SCatalin Marinas } 647c1cc1552SCatalin Marinas #endif /* CONFIG_ARM64_64K_PAGES */ 6480aad818bSJohannes Weiner void vmemmap_free(unsigned long start, unsigned long end) 6490197518cSTang Chen { 6500197518cSTang Chen } 651c1cc1552SCatalin Marinas #endif /* CONFIG_SPARSEMEM_VMEMMAP */ 652af86e597SLaura Abbott 653af86e597SLaura Abbott static inline pud_t * fixmap_pud(unsigned long addr) 654af86e597SLaura Abbott { 655af86e597SLaura Abbott pgd_t *pgd = pgd_offset_k(addr); 656af86e597SLaura Abbott 657af86e597SLaura Abbott BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd)); 658af86e597SLaura Abbott 659157962f5SArd Biesheuvel return pud_offset_kimg(pgd, addr); 660af86e597SLaura Abbott } 661af86e597SLaura Abbott 662af86e597SLaura Abbott static inline pmd_t * fixmap_pmd(unsigned long addr) 663af86e597SLaura Abbott { 664af86e597SLaura Abbott pud_t *pud = fixmap_pud(addr); 665af86e597SLaura Abbott 666af86e597SLaura Abbott BUG_ON(pud_none(*pud) || pud_bad(*pud)); 667af86e597SLaura Abbott 668157962f5SArd Biesheuvel return pmd_offset_kimg(pud, addr); 669af86e597SLaura Abbott } 670af86e597SLaura Abbott 671af86e597SLaura Abbott static inline pte_t * fixmap_pte(unsigned long addr) 672af86e597SLaura Abbott { 673157962f5SArd Biesheuvel return &bm_pte[pte_index(addr)]; 674af86e597SLaura Abbott } 675af86e597SLaura Abbott 676af86e597SLaura Abbott void __init early_fixmap_init(void) 677af86e597SLaura Abbott { 678af86e597SLaura Abbott pgd_t *pgd; 679af86e597SLaura Abbott pud_t *pud; 680af86e597SLaura Abbott pmd_t *pmd; 681af86e597SLaura Abbott unsigned long addr = FIXADDR_START; 682af86e597SLaura Abbott 683af86e597SLaura Abbott pgd = pgd_offset_k(addr); 684f80fb3a3SArd Biesheuvel if (CONFIG_PGTABLE_LEVELS > 3 && 685f80fb3a3SArd Biesheuvel !(pgd_none(*pgd) || pgd_page_paddr(*pgd) == __pa(bm_pud))) { 686f9040773SArd Biesheuvel /* 687f9040773SArd Biesheuvel * We only end up here if the kernel mapping and the fixmap 688f9040773SArd Biesheuvel * share the top level pgd entry, which should only happen on 689f9040773SArd Biesheuvel * 16k/4 levels configurations. 690f9040773SArd Biesheuvel */ 691f9040773SArd Biesheuvel BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); 692f9040773SArd Biesheuvel pud = pud_offset_kimg(pgd, addr); 693f9040773SArd Biesheuvel } else { 694af86e597SLaura Abbott pgd_populate(&init_mm, pgd, bm_pud); 695157962f5SArd Biesheuvel pud = fixmap_pud(addr); 696f9040773SArd Biesheuvel } 697af86e597SLaura Abbott pud_populate(&init_mm, pud, bm_pmd); 698157962f5SArd Biesheuvel pmd = fixmap_pmd(addr); 699af86e597SLaura Abbott pmd_populate_kernel(&init_mm, pmd, bm_pte); 700af86e597SLaura Abbott 701af86e597SLaura Abbott /* 702af86e597SLaura Abbott * The boot-ioremap range spans multiple pmds, for which 703157962f5SArd Biesheuvel * we are not prepared: 704af86e597SLaura Abbott */ 705af86e597SLaura Abbott BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) 706af86e597SLaura Abbott != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); 707af86e597SLaura Abbott 708af86e597SLaura Abbott if ((pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN))) 709af86e597SLaura Abbott || pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) { 710af86e597SLaura Abbott WARN_ON(1); 711af86e597SLaura Abbott pr_warn("pmd %p != %p, %p\n", 712af86e597SLaura Abbott pmd, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)), 713af86e597SLaura Abbott fixmap_pmd(fix_to_virt(FIX_BTMAP_END))); 714af86e597SLaura Abbott pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", 715af86e597SLaura Abbott fix_to_virt(FIX_BTMAP_BEGIN)); 716af86e597SLaura Abbott pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n", 717af86e597SLaura Abbott fix_to_virt(FIX_BTMAP_END)); 718af86e597SLaura Abbott 719af86e597SLaura Abbott pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END); 720af86e597SLaura Abbott pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN); 721af86e597SLaura Abbott } 722af86e597SLaura Abbott } 723af86e597SLaura Abbott 724af86e597SLaura Abbott void __set_fixmap(enum fixed_addresses idx, 725af86e597SLaura Abbott phys_addr_t phys, pgprot_t flags) 726af86e597SLaura Abbott { 727af86e597SLaura Abbott unsigned long addr = __fix_to_virt(idx); 728af86e597SLaura Abbott pte_t *pte; 729af86e597SLaura Abbott 730b63dbef9SMark Rutland BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); 731af86e597SLaura Abbott 732af86e597SLaura Abbott pte = fixmap_pte(addr); 733af86e597SLaura Abbott 734af86e597SLaura Abbott if (pgprot_val(flags)) { 735af86e597SLaura Abbott set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags)); 736af86e597SLaura Abbott } else { 737af86e597SLaura Abbott pte_clear(&init_mm, addr, pte); 738af86e597SLaura Abbott flush_tlb_kernel_range(addr, addr+PAGE_SIZE); 739af86e597SLaura Abbott } 740af86e597SLaura Abbott } 74161bd93ceSArd Biesheuvel 742f80fb3a3SArd Biesheuvel void *__init __fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot) 74361bd93ceSArd Biesheuvel { 74461bd93ceSArd Biesheuvel const u64 dt_virt_base = __fix_to_virt(FIX_FDT); 745f80fb3a3SArd Biesheuvel int offset; 74661bd93ceSArd Biesheuvel void *dt_virt; 74761bd93ceSArd Biesheuvel 74861bd93ceSArd Biesheuvel /* 74961bd93ceSArd Biesheuvel * Check whether the physical FDT address is set and meets the minimum 75061bd93ceSArd Biesheuvel * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be 75161bd93ceSArd Biesheuvel * at least 8 bytes so that we can always access the size field of the 75261bd93ceSArd Biesheuvel * FDT header after mapping the first chunk, double check here if that 75361bd93ceSArd Biesheuvel * is indeed the case. 75461bd93ceSArd Biesheuvel */ 75561bd93ceSArd Biesheuvel BUILD_BUG_ON(MIN_FDT_ALIGN < 8); 75661bd93ceSArd Biesheuvel if (!dt_phys || dt_phys % MIN_FDT_ALIGN) 75761bd93ceSArd Biesheuvel return NULL; 75861bd93ceSArd Biesheuvel 75961bd93ceSArd Biesheuvel /* 76061bd93ceSArd Biesheuvel * Make sure that the FDT region can be mapped without the need to 76161bd93ceSArd Biesheuvel * allocate additional translation table pages, so that it is safe 762132233a7SLaura Abbott * to call create_mapping_noalloc() this early. 76361bd93ceSArd Biesheuvel * 76461bd93ceSArd Biesheuvel * On 64k pages, the FDT will be mapped using PTEs, so we need to 76561bd93ceSArd Biesheuvel * be in the same PMD as the rest of the fixmap. 76661bd93ceSArd Biesheuvel * On 4k pages, we'll use section mappings for the FDT so we only 76761bd93ceSArd Biesheuvel * have to be in the same PUD. 76861bd93ceSArd Biesheuvel */ 76961bd93ceSArd Biesheuvel BUILD_BUG_ON(dt_virt_base % SZ_2M); 77061bd93ceSArd Biesheuvel 771b433dce0SSuzuki K. Poulose BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> SWAPPER_TABLE_SHIFT != 772b433dce0SSuzuki K. Poulose __fix_to_virt(FIX_BTMAP_BEGIN) >> SWAPPER_TABLE_SHIFT); 77361bd93ceSArd Biesheuvel 774b433dce0SSuzuki K. Poulose offset = dt_phys % SWAPPER_BLOCK_SIZE; 77561bd93ceSArd Biesheuvel dt_virt = (void *)dt_virt_base + offset; 77661bd93ceSArd Biesheuvel 77761bd93ceSArd Biesheuvel /* map the first chunk so we can read the size from the header */ 778132233a7SLaura Abbott create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), 779132233a7SLaura Abbott dt_virt_base, SWAPPER_BLOCK_SIZE, prot); 78061bd93ceSArd Biesheuvel 78161bd93ceSArd Biesheuvel if (fdt_check_header(dt_virt) != 0) 78261bd93ceSArd Biesheuvel return NULL; 78361bd93ceSArd Biesheuvel 784f80fb3a3SArd Biesheuvel *size = fdt_totalsize(dt_virt); 785f80fb3a3SArd Biesheuvel if (*size > MAX_FDT_SIZE) 78661bd93ceSArd Biesheuvel return NULL; 78761bd93ceSArd Biesheuvel 788f80fb3a3SArd Biesheuvel if (offset + *size > SWAPPER_BLOCK_SIZE) 789132233a7SLaura Abbott create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base, 790f80fb3a3SArd Biesheuvel round_up(offset + *size, SWAPPER_BLOCK_SIZE), prot); 791f80fb3a3SArd Biesheuvel 792f80fb3a3SArd Biesheuvel return dt_virt; 793f80fb3a3SArd Biesheuvel } 794f80fb3a3SArd Biesheuvel 795f80fb3a3SArd Biesheuvel void *__init fixmap_remap_fdt(phys_addr_t dt_phys) 796f80fb3a3SArd Biesheuvel { 797f80fb3a3SArd Biesheuvel void *dt_virt; 798f80fb3a3SArd Biesheuvel int size; 799f80fb3a3SArd Biesheuvel 800f80fb3a3SArd Biesheuvel dt_virt = __fixmap_remap_fdt(dt_phys, &size, PAGE_KERNEL_RO); 801f80fb3a3SArd Biesheuvel if (!dt_virt) 802f80fb3a3SArd Biesheuvel return NULL; 80361bd93ceSArd Biesheuvel 80461bd93ceSArd Biesheuvel memblock_reserve(dt_phys, size); 80561bd93ceSArd Biesheuvel return dt_virt; 80661bd93ceSArd Biesheuvel } 807324420bfSArd Biesheuvel 808324420bfSArd Biesheuvel int __init arch_ioremap_pud_supported(void) 809324420bfSArd Biesheuvel { 810324420bfSArd Biesheuvel /* only 4k granule supports level 1 block mappings */ 811324420bfSArd Biesheuvel return IS_ENABLED(CONFIG_ARM64_4K_PAGES); 812324420bfSArd Biesheuvel } 813324420bfSArd Biesheuvel 814324420bfSArd Biesheuvel int __init arch_ioremap_pmd_supported(void) 815324420bfSArd Biesheuvel { 816324420bfSArd Biesheuvel return 1; 817324420bfSArd Biesheuvel } 818324420bfSArd Biesheuvel 819324420bfSArd Biesheuvel int pud_set_huge(pud_t *pud, phys_addr_t phys, pgprot_t prot) 820324420bfSArd Biesheuvel { 821324420bfSArd Biesheuvel BUG_ON(phys & ~PUD_MASK); 822324420bfSArd Biesheuvel set_pud(pud, __pud(phys | PUD_TYPE_SECT | pgprot_val(mk_sect_prot(prot)))); 823324420bfSArd Biesheuvel return 1; 824324420bfSArd Biesheuvel } 825324420bfSArd Biesheuvel 826324420bfSArd Biesheuvel int pmd_set_huge(pmd_t *pmd, phys_addr_t phys, pgprot_t prot) 827324420bfSArd Biesheuvel { 828324420bfSArd Biesheuvel BUG_ON(phys & ~PMD_MASK); 829324420bfSArd Biesheuvel set_pmd(pmd, __pmd(phys | PMD_TYPE_SECT | pgprot_val(mk_sect_prot(prot)))); 830324420bfSArd Biesheuvel return 1; 831324420bfSArd Biesheuvel } 832324420bfSArd Biesheuvel 833324420bfSArd Biesheuvel int pud_clear_huge(pud_t *pud) 834324420bfSArd Biesheuvel { 835324420bfSArd Biesheuvel if (!pud_sect(*pud)) 836324420bfSArd Biesheuvel return 0; 837324420bfSArd Biesheuvel pud_clear(pud); 838324420bfSArd Biesheuvel return 1; 839324420bfSArd Biesheuvel } 840324420bfSArd Biesheuvel 841324420bfSArd Biesheuvel int pmd_clear_huge(pmd_t *pmd) 842324420bfSArd Biesheuvel { 843324420bfSArd Biesheuvel if (!pmd_sect(*pmd)) 844324420bfSArd Biesheuvel return 0; 845324420bfSArd Biesheuvel pmd_clear(pmd); 846324420bfSArd Biesheuvel return 1; 847324420bfSArd Biesheuvel } 848