1c1cc1552SCatalin Marinas /* 2c1cc1552SCatalin Marinas * Based on arch/arm/mm/mmu.c 3c1cc1552SCatalin Marinas * 4c1cc1552SCatalin Marinas * Copyright (C) 1995-2005 Russell King 5c1cc1552SCatalin Marinas * Copyright (C) 2012 ARM Ltd. 6c1cc1552SCatalin Marinas * 7c1cc1552SCatalin Marinas * This program is free software; you can redistribute it and/or modify 8c1cc1552SCatalin Marinas * it under the terms of the GNU General Public License version 2 as 9c1cc1552SCatalin Marinas * published by the Free Software Foundation. 10c1cc1552SCatalin Marinas * 11c1cc1552SCatalin Marinas * This program is distributed in the hope that it will be useful, 12c1cc1552SCatalin Marinas * but WITHOUT ANY WARRANTY; without even the implied warranty of 13c1cc1552SCatalin Marinas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14c1cc1552SCatalin Marinas * GNU General Public License for more details. 15c1cc1552SCatalin Marinas * 16c1cc1552SCatalin Marinas * You should have received a copy of the GNU General Public License 17c1cc1552SCatalin Marinas * along with this program. If not, see <http://www.gnu.org/licenses/>. 18c1cc1552SCatalin Marinas */ 19c1cc1552SCatalin Marinas 20c1cc1552SCatalin Marinas #include <linux/export.h> 21c1cc1552SCatalin Marinas #include <linux/kernel.h> 22c1cc1552SCatalin Marinas #include <linux/errno.h> 23c1cc1552SCatalin Marinas #include <linux/init.h> 2461bd93ceSArd Biesheuvel #include <linux/libfdt.h> 25c1cc1552SCatalin Marinas #include <linux/mman.h> 26c1cc1552SCatalin Marinas #include <linux/nodemask.h> 27c1cc1552SCatalin Marinas #include <linux/memblock.h> 28c1cc1552SCatalin Marinas #include <linux/fs.h> 292475ff9dSCatalin Marinas #include <linux/io.h> 3041089357SCatalin Marinas #include <linux/slab.h> 31da141706SLaura Abbott #include <linux/stop_machine.h> 32c1cc1552SCatalin Marinas 3321ab99c2SMark Rutland #include <asm/barrier.h> 34c1cc1552SCatalin Marinas #include <asm/cputype.h> 35af86e597SLaura Abbott #include <asm/fixmap.h> 36068a17a5SMark Rutland #include <asm/kasan.h> 37b433dce0SSuzuki K. Poulose #include <asm/kernel-pgtable.h> 38c1cc1552SCatalin Marinas #include <asm/sections.h> 39c1cc1552SCatalin Marinas #include <asm/setup.h> 40c1cc1552SCatalin Marinas #include <asm/sizes.h> 41c1cc1552SCatalin Marinas #include <asm/tlb.h> 42c79b954bSJungseok Lee #include <asm/memblock.h> 43c1cc1552SCatalin Marinas #include <asm/mmu_context.h> 44c1cc1552SCatalin Marinas 45c1cc1552SCatalin Marinas #include "mm.h" 46c1cc1552SCatalin Marinas 47dd006da2SArd Biesheuvel u64 idmap_t0sz = TCR_T0SZ(VA_BITS); 48dd006da2SArd Biesheuvel 49a7f8de16SArd Biesheuvel u64 kimage_voffset __read_mostly; 50a7f8de16SArd Biesheuvel EXPORT_SYMBOL(kimage_voffset); 51a7f8de16SArd Biesheuvel 52c1cc1552SCatalin Marinas /* 53c1cc1552SCatalin Marinas * Empty_zero_page is a special page that is used for zero-initialized data 54c1cc1552SCatalin Marinas * and COW. 55c1cc1552SCatalin Marinas */ 565227cfa7SMark Rutland unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss; 57c1cc1552SCatalin Marinas EXPORT_SYMBOL(empty_zero_page); 58c1cc1552SCatalin Marinas 59f9040773SArd Biesheuvel static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss; 60f9040773SArd Biesheuvel static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; 61f9040773SArd Biesheuvel static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; 62f9040773SArd Biesheuvel 63c1cc1552SCatalin Marinas pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, 64c1cc1552SCatalin Marinas unsigned long size, pgprot_t vma_prot) 65c1cc1552SCatalin Marinas { 66c1cc1552SCatalin Marinas if (!pfn_valid(pfn)) 67c1cc1552SCatalin Marinas return pgprot_noncached(vma_prot); 68c1cc1552SCatalin Marinas else if (file->f_flags & O_SYNC) 69c1cc1552SCatalin Marinas return pgprot_writecombine(vma_prot); 70c1cc1552SCatalin Marinas return vma_prot; 71c1cc1552SCatalin Marinas } 72c1cc1552SCatalin Marinas EXPORT_SYMBOL(phys_mem_access_prot); 73c1cc1552SCatalin Marinas 74f4710445SMark Rutland static phys_addr_t __init early_pgtable_alloc(void) 75c1cc1552SCatalin Marinas { 767142392dSSuzuki K. Poulose phys_addr_t phys; 777142392dSSuzuki K. Poulose void *ptr; 787142392dSSuzuki K. Poulose 7921ab99c2SMark Rutland phys = memblock_alloc(PAGE_SIZE, PAGE_SIZE); 80f4710445SMark Rutland 81f4710445SMark Rutland /* 82f4710445SMark Rutland * The FIX_{PGD,PUD,PMD} slots may be in active use, but the FIX_PTE 83f4710445SMark Rutland * slot will be free, so we can (ab)use the FIX_PTE slot to initialise 84f4710445SMark Rutland * any level of table. 85f4710445SMark Rutland */ 86f4710445SMark Rutland ptr = pte_set_fixmap(phys); 87f4710445SMark Rutland 8821ab99c2SMark Rutland memset(ptr, 0, PAGE_SIZE); 8921ab99c2SMark Rutland 90f4710445SMark Rutland /* 91f4710445SMark Rutland * Implicit barriers also ensure the zeroed page is visible to the page 92f4710445SMark Rutland * table walker 93f4710445SMark Rutland */ 94f4710445SMark Rutland pte_clear_fixmap(); 95f4710445SMark Rutland 96f4710445SMark Rutland return phys; 97c1cc1552SCatalin Marinas } 98c1cc1552SCatalin Marinas 99da141706SLaura Abbott /* 100da141706SLaura Abbott * remap a PMD into pages 101da141706SLaura Abbott */ 102da141706SLaura Abbott static void split_pmd(pmd_t *pmd, pte_t *pte) 103da141706SLaura Abbott { 104da141706SLaura Abbott unsigned long pfn = pmd_pfn(*pmd); 105da141706SLaura Abbott int i = 0; 106da141706SLaura Abbott 107da141706SLaura Abbott do { 108da141706SLaura Abbott /* 109da141706SLaura Abbott * Need to have the least restrictive permissions available 110667c2759SCatalin Marinas * permissions will be fixed up later 111da141706SLaura Abbott */ 112667c2759SCatalin Marinas set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC)); 113da141706SLaura Abbott pfn++; 114da141706SLaura Abbott } while (pte++, i++, i < PTRS_PER_PTE); 115da141706SLaura Abbott } 116da141706SLaura Abbott 117da141706SLaura Abbott static void alloc_init_pte(pmd_t *pmd, unsigned long addr, 118667c2759SCatalin Marinas unsigned long end, unsigned long pfn, 119da141706SLaura Abbott pgprot_t prot, 120f4710445SMark Rutland phys_addr_t (*pgtable_alloc)(void)) 121c1cc1552SCatalin Marinas { 122c1cc1552SCatalin Marinas pte_t *pte; 123c1cc1552SCatalin Marinas 124a1c76574SMark Rutland if (pmd_none(*pmd) || pmd_sect(*pmd)) { 125132233a7SLaura Abbott phys_addr_t pte_phys; 126132233a7SLaura Abbott BUG_ON(!pgtable_alloc); 127132233a7SLaura Abbott pte_phys = pgtable_alloc(); 128f4710445SMark Rutland pte = pte_set_fixmap(pte_phys); 129da141706SLaura Abbott if (pmd_sect(*pmd)) 130da141706SLaura Abbott split_pmd(pmd, pte); 131f4710445SMark Rutland __pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE); 132da141706SLaura Abbott flush_tlb_all(); 133f4710445SMark Rutland pte_clear_fixmap(); 134c1cc1552SCatalin Marinas } 135a1c76574SMark Rutland BUG_ON(pmd_bad(*pmd)); 136c1cc1552SCatalin Marinas 137f4710445SMark Rutland pte = pte_set_fixmap_offset(pmd, addr); 138c1cc1552SCatalin Marinas do { 139667c2759SCatalin Marinas set_pte(pte, pfn_pte(pfn, prot)); 140667c2759SCatalin Marinas pfn++; 141667c2759SCatalin Marinas } while (pte++, addr += PAGE_SIZE, addr != end); 142f4710445SMark Rutland 143f4710445SMark Rutland pte_clear_fixmap(); 144c1cc1552SCatalin Marinas } 145c1cc1552SCatalin Marinas 1469a17a213SJisheng Zhang static void split_pud(pud_t *old_pud, pmd_t *pmd) 147da141706SLaura Abbott { 148da141706SLaura Abbott unsigned long addr = pud_pfn(*old_pud) << PAGE_SHIFT; 149da141706SLaura Abbott pgprot_t prot = __pgprot(pud_val(*old_pud) ^ addr); 150da141706SLaura Abbott int i = 0; 151da141706SLaura Abbott 152da141706SLaura Abbott do { 1531e43ba9cSArd Biesheuvel set_pmd(pmd, __pmd(addr | pgprot_val(prot))); 154da141706SLaura Abbott addr += PMD_SIZE; 155da141706SLaura Abbott } while (pmd++, i++, i < PTRS_PER_PMD); 156da141706SLaura Abbott } 157da141706SLaura Abbott 15811509a30SMark Rutland static void alloc_init_pmd(pud_t *pud, unsigned long addr, unsigned long end, 159da141706SLaura Abbott phys_addr_t phys, pgprot_t prot, 16053e1b329SArd Biesheuvel phys_addr_t (*pgtable_alloc)(void), 16153e1b329SArd Biesheuvel bool allow_block_mappings) 162c1cc1552SCatalin Marinas { 163c1cc1552SCatalin Marinas pmd_t *pmd; 164c1cc1552SCatalin Marinas unsigned long next; 165c1cc1552SCatalin Marinas 166c1cc1552SCatalin Marinas /* 167c1cc1552SCatalin Marinas * Check for initial section mappings in the pgd/pud and remove them. 168c1cc1552SCatalin Marinas */ 169a1c76574SMark Rutland if (pud_none(*pud) || pud_sect(*pud)) { 170132233a7SLaura Abbott phys_addr_t pmd_phys; 171132233a7SLaura Abbott BUG_ON(!pgtable_alloc); 172132233a7SLaura Abbott pmd_phys = pgtable_alloc(); 173f4710445SMark Rutland pmd = pmd_set_fixmap(pmd_phys); 174da141706SLaura Abbott if (pud_sect(*pud)) { 175da141706SLaura Abbott /* 176da141706SLaura Abbott * need to have the 1G of mappings continue to be 177da141706SLaura Abbott * present 178da141706SLaura Abbott */ 179da141706SLaura Abbott split_pud(pud, pmd); 180da141706SLaura Abbott } 181f4710445SMark Rutland __pud_populate(pud, pmd_phys, PUD_TYPE_TABLE); 182da141706SLaura Abbott flush_tlb_all(); 183f4710445SMark Rutland pmd_clear_fixmap(); 184c1cc1552SCatalin Marinas } 185a1c76574SMark Rutland BUG_ON(pud_bad(*pud)); 186c1cc1552SCatalin Marinas 187f4710445SMark Rutland pmd = pmd_set_fixmap_offset(pud, addr); 188c1cc1552SCatalin Marinas do { 189c1cc1552SCatalin Marinas next = pmd_addr_end(addr, end); 190c1cc1552SCatalin Marinas /* try section mapping first */ 19183863f25SLaura Abbott if (((addr | next | phys) & ~SECTION_MASK) == 0 && 19253e1b329SArd Biesheuvel allow_block_mappings) { 193a55f9929SCatalin Marinas pmd_t old_pmd =*pmd; 194c661cb1cSMark Rutland pmd_set_huge(pmd, phys, prot); 195a55f9929SCatalin Marinas /* 196a55f9929SCatalin Marinas * Check for previous table entries created during 197a55f9929SCatalin Marinas * boot (__create_page_tables) and flush them. 198a55f9929SCatalin Marinas */ 199523d6e9fSzhichang.yuan if (!pmd_none(old_pmd)) { 200a55f9929SCatalin Marinas flush_tlb_all(); 201523d6e9fSzhichang.yuan if (pmd_table(old_pmd)) { 202316b39dbSMark Rutland phys_addr_t table = pmd_page_paddr(old_pmd); 20341089357SCatalin Marinas if (!WARN_ON_ONCE(slab_is_available())) 204523d6e9fSzhichang.yuan memblock_free(table, PAGE_SIZE); 205523d6e9fSzhichang.yuan } 206523d6e9fSzhichang.yuan } 207a55f9929SCatalin Marinas } else { 208667c2759SCatalin Marinas alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys), 20921ab99c2SMark Rutland prot, pgtable_alloc); 210a55f9929SCatalin Marinas } 211c1cc1552SCatalin Marinas phys += next - addr; 212c1cc1552SCatalin Marinas } while (pmd++, addr = next, addr != end); 213f4710445SMark Rutland 214f4710445SMark Rutland pmd_clear_fixmap(); 215c1cc1552SCatalin Marinas } 216c1cc1552SCatalin Marinas 217da141706SLaura Abbott static inline bool use_1G_block(unsigned long addr, unsigned long next, 218da141706SLaura Abbott unsigned long phys) 219da141706SLaura Abbott { 220da141706SLaura Abbott if (PAGE_SHIFT != 12) 221da141706SLaura Abbott return false; 222da141706SLaura Abbott 223da141706SLaura Abbott if (((addr | next | phys) & ~PUD_MASK) != 0) 224da141706SLaura Abbott return false; 225da141706SLaura Abbott 226da141706SLaura Abbott return true; 227da141706SLaura Abbott } 228da141706SLaura Abbott 22911509a30SMark Rutland static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end, 230da141706SLaura Abbott phys_addr_t phys, pgprot_t prot, 23153e1b329SArd Biesheuvel phys_addr_t (*pgtable_alloc)(void), 23253e1b329SArd Biesheuvel bool allow_block_mappings) 233c1cc1552SCatalin Marinas { 234c79b954bSJungseok Lee pud_t *pud; 235c1cc1552SCatalin Marinas unsigned long next; 236c1cc1552SCatalin Marinas 237c79b954bSJungseok Lee if (pgd_none(*pgd)) { 238132233a7SLaura Abbott phys_addr_t pud_phys; 239132233a7SLaura Abbott BUG_ON(!pgtable_alloc); 240132233a7SLaura Abbott pud_phys = pgtable_alloc(); 241f4710445SMark Rutland __pgd_populate(pgd, pud_phys, PUD_TYPE_TABLE); 242c79b954bSJungseok Lee } 243c79b954bSJungseok Lee BUG_ON(pgd_bad(*pgd)); 244c79b954bSJungseok Lee 245f4710445SMark Rutland pud = pud_set_fixmap_offset(pgd, addr); 246c1cc1552SCatalin Marinas do { 247c1cc1552SCatalin Marinas next = pud_addr_end(addr, end); 248206a2a73SSteve Capper 249206a2a73SSteve Capper /* 250206a2a73SSteve Capper * For 4K granule only, attempt to put down a 1GB block 251206a2a73SSteve Capper */ 25253e1b329SArd Biesheuvel if (use_1G_block(addr, next, phys) && allow_block_mappings) { 253206a2a73SSteve Capper pud_t old_pud = *pud; 254c661cb1cSMark Rutland pud_set_huge(pud, phys, prot); 255206a2a73SSteve Capper 256206a2a73SSteve Capper /* 257206a2a73SSteve Capper * If we have an old value for a pud, it will 258206a2a73SSteve Capper * be pointing to a pmd table that we no longer 259206a2a73SSteve Capper * need (from swapper_pg_dir). 260206a2a73SSteve Capper * 261206a2a73SSteve Capper * Look up the old pmd table and free it. 262206a2a73SSteve Capper */ 263206a2a73SSteve Capper if (!pud_none(old_pud)) { 264206a2a73SSteve Capper flush_tlb_all(); 265523d6e9fSzhichang.yuan if (pud_table(old_pud)) { 266316b39dbSMark Rutland phys_addr_t table = pud_page_paddr(old_pud); 26741089357SCatalin Marinas if (!WARN_ON_ONCE(slab_is_available())) 268523d6e9fSzhichang.yuan memblock_free(table, PAGE_SIZE); 269523d6e9fSzhichang.yuan } 270206a2a73SSteve Capper } 271206a2a73SSteve Capper } else { 27211509a30SMark Rutland alloc_init_pmd(pud, addr, next, phys, prot, 27353e1b329SArd Biesheuvel pgtable_alloc, allow_block_mappings); 274206a2a73SSteve Capper } 275c1cc1552SCatalin Marinas phys += next - addr; 276c1cc1552SCatalin Marinas } while (pud++, addr = next, addr != end); 277f4710445SMark Rutland 278f4710445SMark Rutland pud_clear_fixmap(); 279c1cc1552SCatalin Marinas } 280c1cc1552SCatalin Marinas 281c1cc1552SCatalin Marinas /* 282c1cc1552SCatalin Marinas * Create the page directory entries and any necessary page tables for the 283c1cc1552SCatalin Marinas * mapping specified by 'md'. 284c1cc1552SCatalin Marinas */ 28511509a30SMark Rutland static void init_pgd(pgd_t *pgd, phys_addr_t phys, unsigned long virt, 286da141706SLaura Abbott phys_addr_t size, pgprot_t prot, 28753e1b329SArd Biesheuvel phys_addr_t (*pgtable_alloc)(void), 28853e1b329SArd Biesheuvel bool allow_block_mappings) 289c1cc1552SCatalin Marinas { 290c1cc1552SCatalin Marinas unsigned long addr, length, end, next; 291c1cc1552SCatalin Marinas 292cc5d2b3bSMark Rutland /* 293cc5d2b3bSMark Rutland * If the virtual and physical address don't have the same offset 294cc5d2b3bSMark Rutland * within a page, we cannot map the region as the caller expects. 295cc5d2b3bSMark Rutland */ 296cc5d2b3bSMark Rutland if (WARN_ON((phys ^ virt) & ~PAGE_MASK)) 297cc5d2b3bSMark Rutland return; 298cc5d2b3bSMark Rutland 2999c4e08a3SMark Rutland phys &= PAGE_MASK; 300c1cc1552SCatalin Marinas addr = virt & PAGE_MASK; 301c1cc1552SCatalin Marinas length = PAGE_ALIGN(size + (virt & ~PAGE_MASK)); 302c1cc1552SCatalin Marinas 303c1cc1552SCatalin Marinas end = addr + length; 304c1cc1552SCatalin Marinas do { 305c1cc1552SCatalin Marinas next = pgd_addr_end(addr, end); 30653e1b329SArd Biesheuvel alloc_init_pud(pgd, addr, next, phys, prot, pgtable_alloc, 30753e1b329SArd Biesheuvel allow_block_mappings); 308c1cc1552SCatalin Marinas phys += next - addr; 309c1cc1552SCatalin Marinas } while (pgd++, addr = next, addr != end); 310c1cc1552SCatalin Marinas } 311c1cc1552SCatalin Marinas 312f4710445SMark Rutland static phys_addr_t late_pgtable_alloc(void) 313da141706SLaura Abbott { 31421ab99c2SMark Rutland void *ptr = (void *)__get_free_page(PGALLOC_GFP); 315da141706SLaura Abbott BUG_ON(!ptr); 31621ab99c2SMark Rutland 31721ab99c2SMark Rutland /* Ensure the zeroed page is visible to the page table walker */ 31821ab99c2SMark Rutland dsb(ishst); 319f4710445SMark Rutland return __pa(ptr); 320da141706SLaura Abbott } 321da141706SLaura Abbott 32211509a30SMark Rutland static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, 32311509a30SMark Rutland unsigned long virt, phys_addr_t size, 32411509a30SMark Rutland pgprot_t prot, 32553e1b329SArd Biesheuvel phys_addr_t (*alloc)(void), 32653e1b329SArd Biesheuvel bool allow_block_mappings) 32711509a30SMark Rutland { 32853e1b329SArd Biesheuvel init_pgd(pgd_offset_raw(pgdir, virt), phys, virt, size, prot, alloc, 32953e1b329SArd Biesheuvel allow_block_mappings); 33011509a30SMark Rutland } 33111509a30SMark Rutland 332132233a7SLaura Abbott /* 333132233a7SLaura Abbott * This function can only be used to modify existing table entries, 334132233a7SLaura Abbott * without allocating new levels of table. Note that this permits the 335132233a7SLaura Abbott * creation of new section or page entries. 336132233a7SLaura Abbott */ 337132233a7SLaura Abbott static void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, 338da141706SLaura Abbott phys_addr_t size, pgprot_t prot) 339d7ecbddfSMark Salter { 340d7ecbddfSMark Salter if (virt < VMALLOC_START) { 341d7ecbddfSMark Salter pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", 342d7ecbddfSMark Salter &phys, virt); 343d7ecbddfSMark Salter return; 344d7ecbddfSMark Salter } 34553e1b329SArd Biesheuvel __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, true); 346d7ecbddfSMark Salter } 347d7ecbddfSMark Salter 3488ce837ceSArd Biesheuvel void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, 3498ce837ceSArd Biesheuvel unsigned long virt, phys_addr_t size, 35053e1b329SArd Biesheuvel pgprot_t prot, bool allow_block_mappings) 3518ce837ceSArd Biesheuvel { 35211509a30SMark Rutland __create_pgd_mapping(mm->pgd, phys, virt, size, prot, 35353e1b329SArd Biesheuvel late_pgtable_alloc, allow_block_mappings); 354d7ecbddfSMark Salter } 355d7ecbddfSMark Salter 356da141706SLaura Abbott static void create_mapping_late(phys_addr_t phys, unsigned long virt, 357da141706SLaura Abbott phys_addr_t size, pgprot_t prot) 358da141706SLaura Abbott { 359da141706SLaura Abbott if (virt < VMALLOC_START) { 360da141706SLaura Abbott pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", 361da141706SLaura Abbott &phys, virt); 362da141706SLaura Abbott return; 363da141706SLaura Abbott } 364da141706SLaura Abbott 36511509a30SMark Rutland __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, 36653e1b329SArd Biesheuvel late_pgtable_alloc, !debug_pagealloc_enabled()); 367da141706SLaura Abbott } 368da141706SLaura Abbott 369068a17a5SMark Rutland static void __init __map_memblock(pgd_t *pgd, phys_addr_t start, phys_addr_t end) 370da141706SLaura Abbott { 3717eb90f2fSArd Biesheuvel unsigned long kernel_start = __pa(_text); 3729fdc14c5SArd Biesheuvel unsigned long kernel_end = __pa(__init_begin); 373068a17a5SMark Rutland 374da141706SLaura Abbott /* 375f9040773SArd Biesheuvel * Take care not to create a writable alias for the 376f9040773SArd Biesheuvel * read-only text and rodata sections of the kernel image. 377da141706SLaura Abbott */ 378da141706SLaura Abbott 3799fdc14c5SArd Biesheuvel /* No overlap with the kernel text/rodata */ 380068a17a5SMark Rutland if (end < kernel_start || start >= kernel_end) { 381068a17a5SMark Rutland __create_pgd_mapping(pgd, start, __phys_to_virt(start), 382068a17a5SMark Rutland end - start, PAGE_KERNEL, 38353e1b329SArd Biesheuvel early_pgtable_alloc, 38453e1b329SArd Biesheuvel !debug_pagealloc_enabled()); 385068a17a5SMark Rutland return; 386da141706SLaura Abbott } 387da141706SLaura Abbott 388068a17a5SMark Rutland /* 3899fdc14c5SArd Biesheuvel * This block overlaps the kernel text/rodata mappings. 390f9040773SArd Biesheuvel * Map the portion(s) which don't overlap. 391068a17a5SMark Rutland */ 392068a17a5SMark Rutland if (start < kernel_start) 393068a17a5SMark Rutland __create_pgd_mapping(pgd, start, 394068a17a5SMark Rutland __phys_to_virt(start), 395068a17a5SMark Rutland kernel_start - start, PAGE_KERNEL, 39653e1b329SArd Biesheuvel early_pgtable_alloc, 39753e1b329SArd Biesheuvel !debug_pagealloc_enabled()); 398068a17a5SMark Rutland if (kernel_end < end) 399068a17a5SMark Rutland __create_pgd_mapping(pgd, kernel_end, 400068a17a5SMark Rutland __phys_to_virt(kernel_end), 401068a17a5SMark Rutland end - kernel_end, PAGE_KERNEL, 40253e1b329SArd Biesheuvel early_pgtable_alloc, 40353e1b329SArd Biesheuvel !debug_pagealloc_enabled()); 404f9040773SArd Biesheuvel 405f9040773SArd Biesheuvel /* 4069fdc14c5SArd Biesheuvel * Map the linear alias of the [_text, __init_begin) interval as 407f9040773SArd Biesheuvel * read-only/non-executable. This makes the contents of the 408f9040773SArd Biesheuvel * region accessible to subsystems such as hibernate, but 409f9040773SArd Biesheuvel * protects it from inadvertent modification or execution. 410f9040773SArd Biesheuvel */ 411f9040773SArd Biesheuvel __create_pgd_mapping(pgd, kernel_start, __phys_to_virt(kernel_start), 412f9040773SArd Biesheuvel kernel_end - kernel_start, PAGE_KERNEL_RO, 41353e1b329SArd Biesheuvel early_pgtable_alloc, !debug_pagealloc_enabled()); 414da141706SLaura Abbott } 415da141706SLaura Abbott 416068a17a5SMark Rutland static void __init map_mem(pgd_t *pgd) 417c1cc1552SCatalin Marinas { 418c1cc1552SCatalin Marinas struct memblock_region *reg; 419f6bc87c3SSteve Capper 420c1cc1552SCatalin Marinas /* map all the memory banks */ 421c1cc1552SCatalin Marinas for_each_memblock(memory, reg) { 422c1cc1552SCatalin Marinas phys_addr_t start = reg->base; 423c1cc1552SCatalin Marinas phys_addr_t end = start + reg->size; 424c1cc1552SCatalin Marinas 425c1cc1552SCatalin Marinas if (start >= end) 426c1cc1552SCatalin Marinas break; 42768709f45SArd Biesheuvel if (memblock_is_nomap(reg)) 42868709f45SArd Biesheuvel continue; 429c1cc1552SCatalin Marinas 430068a17a5SMark Rutland __map_memblock(pgd, start, end); 431c1cc1552SCatalin Marinas } 432c1cc1552SCatalin Marinas } 433c1cc1552SCatalin Marinas 434da141706SLaura Abbott void mark_rodata_ro(void) 435da141706SLaura Abbott { 4362f39b5f9SJeremy Linton unsigned long section_size; 437f9040773SArd Biesheuvel 4389fdc14c5SArd Biesheuvel section_size = (unsigned long)_etext - (unsigned long)_text; 4397eb90f2fSArd Biesheuvel create_mapping_late(__pa(_text), (unsigned long)_text, 4402f39b5f9SJeremy Linton section_size, PAGE_KERNEL_ROX); 4412f39b5f9SJeremy Linton /* 4429fdc14c5SArd Biesheuvel * mark .rodata as read only. Use __init_begin rather than __end_rodata 4439fdc14c5SArd Biesheuvel * to cover NOTES and EXCEPTION_TABLE. 4442f39b5f9SJeremy Linton */ 4459fdc14c5SArd Biesheuvel section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata; 4462f39b5f9SJeremy Linton create_mapping_late(__pa(__start_rodata), (unsigned long)__start_rodata, 4472f39b5f9SJeremy Linton section_size, PAGE_KERNEL_RO); 448da141706SLaura Abbott } 449da141706SLaura Abbott 450da141706SLaura Abbott void fixup_init(void) 451da141706SLaura Abbott { 452f9040773SArd Biesheuvel /* 453f9040773SArd Biesheuvel * Unmap the __init region but leave the VM area in place. This 454f9040773SArd Biesheuvel * prevents the region from being reused for kernel modules, which 455f9040773SArd Biesheuvel * is not supported by kallsyms. 456f9040773SArd Biesheuvel */ 457f9040773SArd Biesheuvel unmap_kernel_range((u64)__init_begin, (u64)(__init_end - __init_begin)); 458da141706SLaura Abbott } 459da141706SLaura Abbott 4602c09ec06SArd Biesheuvel static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end, 461f9040773SArd Biesheuvel pgprot_t prot, struct vm_struct *vma) 462068a17a5SMark Rutland { 463068a17a5SMark Rutland phys_addr_t pa_start = __pa(va_start); 464068a17a5SMark Rutland unsigned long size = va_end - va_start; 465068a17a5SMark Rutland 466068a17a5SMark Rutland BUG_ON(!PAGE_ALIGNED(pa_start)); 467068a17a5SMark Rutland BUG_ON(!PAGE_ALIGNED(size)); 468068a17a5SMark Rutland 469068a17a5SMark Rutland __create_pgd_mapping(pgd, pa_start, (unsigned long)va_start, size, prot, 47053e1b329SArd Biesheuvel early_pgtable_alloc, !debug_pagealloc_enabled()); 471f9040773SArd Biesheuvel 472f9040773SArd Biesheuvel vma->addr = va_start; 473f9040773SArd Biesheuvel vma->phys_addr = pa_start; 474f9040773SArd Biesheuvel vma->size = size; 475f9040773SArd Biesheuvel vma->flags = VM_MAP; 476f9040773SArd Biesheuvel vma->caller = __builtin_return_address(0); 477f9040773SArd Biesheuvel 478f9040773SArd Biesheuvel vm_area_add_early(vma); 479068a17a5SMark Rutland } 480068a17a5SMark Rutland 481068a17a5SMark Rutland /* 482068a17a5SMark Rutland * Create fine-grained mappings for the kernel. 483068a17a5SMark Rutland */ 484068a17a5SMark Rutland static void __init map_kernel(pgd_t *pgd) 485068a17a5SMark Rutland { 4862f39b5f9SJeremy Linton static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_init, vmlinux_data; 487068a17a5SMark Rutland 4889fdc14c5SArd Biesheuvel map_kernel_segment(pgd, _text, _etext, PAGE_KERNEL_EXEC, &vmlinux_text); 4899fdc14c5SArd Biesheuvel map_kernel_segment(pgd, __start_rodata, __init_begin, PAGE_KERNEL, &vmlinux_rodata); 4902c09ec06SArd Biesheuvel map_kernel_segment(pgd, __init_begin, __init_end, PAGE_KERNEL_EXEC, 491f9040773SArd Biesheuvel &vmlinux_init); 4922c09ec06SArd Biesheuvel map_kernel_segment(pgd, _data, _end, PAGE_KERNEL, &vmlinux_data); 493068a17a5SMark Rutland 494f9040773SArd Biesheuvel if (!pgd_val(*pgd_offset_raw(pgd, FIXADDR_START))) { 495068a17a5SMark Rutland /* 496f9040773SArd Biesheuvel * The fixmap falls in a separate pgd to the kernel, and doesn't 497f9040773SArd Biesheuvel * live in the carveout for the swapper_pg_dir. We can simply 498f9040773SArd Biesheuvel * re-use the existing dir for the fixmap. 499068a17a5SMark Rutland */ 500f9040773SArd Biesheuvel set_pgd(pgd_offset_raw(pgd, FIXADDR_START), 501f9040773SArd Biesheuvel *pgd_offset_k(FIXADDR_START)); 502f9040773SArd Biesheuvel } else if (CONFIG_PGTABLE_LEVELS > 3) { 503f9040773SArd Biesheuvel /* 504f9040773SArd Biesheuvel * The fixmap shares its top level pgd entry with the kernel 505f9040773SArd Biesheuvel * mapping. This can really only occur when we are running 506f9040773SArd Biesheuvel * with 16k/4 levels, so we can simply reuse the pud level 507f9040773SArd Biesheuvel * entry instead. 508f9040773SArd Biesheuvel */ 509f9040773SArd Biesheuvel BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); 510f9040773SArd Biesheuvel set_pud(pud_set_fixmap_offset(pgd, FIXADDR_START), 511f9040773SArd Biesheuvel __pud(__pa(bm_pmd) | PUD_TYPE_TABLE)); 512f9040773SArd Biesheuvel pud_clear_fixmap(); 513f9040773SArd Biesheuvel } else { 514f9040773SArd Biesheuvel BUG(); 515f9040773SArd Biesheuvel } 516068a17a5SMark Rutland 517068a17a5SMark Rutland kasan_copy_shadow(pgd); 518068a17a5SMark Rutland } 519068a17a5SMark Rutland 520c1cc1552SCatalin Marinas /* 521c1cc1552SCatalin Marinas * paging_init() sets up the page tables, initialises the zone memory 522c1cc1552SCatalin Marinas * maps and sets up the zero page. 523c1cc1552SCatalin Marinas */ 524c1cc1552SCatalin Marinas void __init paging_init(void) 525c1cc1552SCatalin Marinas { 526068a17a5SMark Rutland phys_addr_t pgd_phys = early_pgtable_alloc(); 527068a17a5SMark Rutland pgd_t *pgd = pgd_set_fixmap(pgd_phys); 528068a17a5SMark Rutland 529068a17a5SMark Rutland map_kernel(pgd); 530068a17a5SMark Rutland map_mem(pgd); 531068a17a5SMark Rutland 532068a17a5SMark Rutland /* 533068a17a5SMark Rutland * We want to reuse the original swapper_pg_dir so we don't have to 534068a17a5SMark Rutland * communicate the new address to non-coherent secondaries in 535068a17a5SMark Rutland * secondary_entry, and so cpu_switch_mm can generate the address with 536068a17a5SMark Rutland * adrp+add rather than a load from some global variable. 537068a17a5SMark Rutland * 538068a17a5SMark Rutland * To do this we need to go via a temporary pgd. 539068a17a5SMark Rutland */ 540068a17a5SMark Rutland cpu_replace_ttbr1(__va(pgd_phys)); 541068a17a5SMark Rutland memcpy(swapper_pg_dir, pgd, PAGE_SIZE); 542068a17a5SMark Rutland cpu_replace_ttbr1(swapper_pg_dir); 543068a17a5SMark Rutland 544068a17a5SMark Rutland pgd_clear_fixmap(); 545068a17a5SMark Rutland memblock_free(pgd_phys, PAGE_SIZE); 546068a17a5SMark Rutland 547068a17a5SMark Rutland /* 548068a17a5SMark Rutland * We only reuse the PGD from the swapper_pg_dir, not the pud + pmd 549068a17a5SMark Rutland * allocated with it. 550068a17a5SMark Rutland */ 551068a17a5SMark Rutland memblock_free(__pa(swapper_pg_dir) + PAGE_SIZE, 552068a17a5SMark Rutland SWAPPER_DIR_SIZE - PAGE_SIZE); 553c1cc1552SCatalin Marinas } 554c1cc1552SCatalin Marinas 555c1cc1552SCatalin Marinas /* 556c1cc1552SCatalin Marinas * Check whether a kernel address is valid (derived from arch/x86/). 557c1cc1552SCatalin Marinas */ 558c1cc1552SCatalin Marinas int kern_addr_valid(unsigned long addr) 559c1cc1552SCatalin Marinas { 560c1cc1552SCatalin Marinas pgd_t *pgd; 561c1cc1552SCatalin Marinas pud_t *pud; 562c1cc1552SCatalin Marinas pmd_t *pmd; 563c1cc1552SCatalin Marinas pte_t *pte; 564c1cc1552SCatalin Marinas 565c1cc1552SCatalin Marinas if ((((long)addr) >> VA_BITS) != -1UL) 566c1cc1552SCatalin Marinas return 0; 567c1cc1552SCatalin Marinas 568c1cc1552SCatalin Marinas pgd = pgd_offset_k(addr); 569c1cc1552SCatalin Marinas if (pgd_none(*pgd)) 570c1cc1552SCatalin Marinas return 0; 571c1cc1552SCatalin Marinas 572c1cc1552SCatalin Marinas pud = pud_offset(pgd, addr); 573c1cc1552SCatalin Marinas if (pud_none(*pud)) 574c1cc1552SCatalin Marinas return 0; 575c1cc1552SCatalin Marinas 576206a2a73SSteve Capper if (pud_sect(*pud)) 577206a2a73SSteve Capper return pfn_valid(pud_pfn(*pud)); 578206a2a73SSteve Capper 579c1cc1552SCatalin Marinas pmd = pmd_offset(pud, addr); 580c1cc1552SCatalin Marinas if (pmd_none(*pmd)) 581c1cc1552SCatalin Marinas return 0; 582c1cc1552SCatalin Marinas 583da6e4cb6SDave Anderson if (pmd_sect(*pmd)) 584da6e4cb6SDave Anderson return pfn_valid(pmd_pfn(*pmd)); 585da6e4cb6SDave Anderson 586c1cc1552SCatalin Marinas pte = pte_offset_kernel(pmd, addr); 587c1cc1552SCatalin Marinas if (pte_none(*pte)) 588c1cc1552SCatalin Marinas return 0; 589c1cc1552SCatalin Marinas 590c1cc1552SCatalin Marinas return pfn_valid(pte_pfn(*pte)); 591c1cc1552SCatalin Marinas } 592c1cc1552SCatalin Marinas #ifdef CONFIG_SPARSEMEM_VMEMMAP 593b433dce0SSuzuki K. Poulose #if !ARM64_SWAPPER_USES_SECTION_MAPS 5940aad818bSJohannes Weiner int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) 595c1cc1552SCatalin Marinas { 5960aad818bSJohannes Weiner return vmemmap_populate_basepages(start, end, node); 597c1cc1552SCatalin Marinas } 598b433dce0SSuzuki K. Poulose #else /* !ARM64_SWAPPER_USES_SECTION_MAPS */ 5990aad818bSJohannes Weiner int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) 600c1cc1552SCatalin Marinas { 6010aad818bSJohannes Weiner unsigned long addr = start; 602c1cc1552SCatalin Marinas unsigned long next; 603c1cc1552SCatalin Marinas pgd_t *pgd; 604c1cc1552SCatalin Marinas pud_t *pud; 605c1cc1552SCatalin Marinas pmd_t *pmd; 606c1cc1552SCatalin Marinas 607c1cc1552SCatalin Marinas do { 608c1cc1552SCatalin Marinas next = pmd_addr_end(addr, end); 609c1cc1552SCatalin Marinas 610c1cc1552SCatalin Marinas pgd = vmemmap_pgd_populate(addr, node); 611c1cc1552SCatalin Marinas if (!pgd) 612c1cc1552SCatalin Marinas return -ENOMEM; 613c1cc1552SCatalin Marinas 614c1cc1552SCatalin Marinas pud = vmemmap_pud_populate(pgd, addr, node); 615c1cc1552SCatalin Marinas if (!pud) 616c1cc1552SCatalin Marinas return -ENOMEM; 617c1cc1552SCatalin Marinas 618c1cc1552SCatalin Marinas pmd = pmd_offset(pud, addr); 619c1cc1552SCatalin Marinas if (pmd_none(*pmd)) { 620c1cc1552SCatalin Marinas void *p = NULL; 621c1cc1552SCatalin Marinas 622c1cc1552SCatalin Marinas p = vmemmap_alloc_block_buf(PMD_SIZE, node); 623c1cc1552SCatalin Marinas if (!p) 624c1cc1552SCatalin Marinas return -ENOMEM; 625c1cc1552SCatalin Marinas 626a501e324SCatalin Marinas set_pmd(pmd, __pmd(__pa(p) | PROT_SECT_NORMAL)); 627c1cc1552SCatalin Marinas } else 628c1cc1552SCatalin Marinas vmemmap_verify((pte_t *)pmd, node, addr, next); 629c1cc1552SCatalin Marinas } while (addr = next, addr != end); 630c1cc1552SCatalin Marinas 631c1cc1552SCatalin Marinas return 0; 632c1cc1552SCatalin Marinas } 633c1cc1552SCatalin Marinas #endif /* CONFIG_ARM64_64K_PAGES */ 6340aad818bSJohannes Weiner void vmemmap_free(unsigned long start, unsigned long end) 6350197518cSTang Chen { 6360197518cSTang Chen } 637c1cc1552SCatalin Marinas #endif /* CONFIG_SPARSEMEM_VMEMMAP */ 638af86e597SLaura Abbott 639af86e597SLaura Abbott static inline pud_t * fixmap_pud(unsigned long addr) 640af86e597SLaura Abbott { 641af86e597SLaura Abbott pgd_t *pgd = pgd_offset_k(addr); 642af86e597SLaura Abbott 643af86e597SLaura Abbott BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd)); 644af86e597SLaura Abbott 645157962f5SArd Biesheuvel return pud_offset_kimg(pgd, addr); 646af86e597SLaura Abbott } 647af86e597SLaura Abbott 648af86e597SLaura Abbott static inline pmd_t * fixmap_pmd(unsigned long addr) 649af86e597SLaura Abbott { 650af86e597SLaura Abbott pud_t *pud = fixmap_pud(addr); 651af86e597SLaura Abbott 652af86e597SLaura Abbott BUG_ON(pud_none(*pud) || pud_bad(*pud)); 653af86e597SLaura Abbott 654157962f5SArd Biesheuvel return pmd_offset_kimg(pud, addr); 655af86e597SLaura Abbott } 656af86e597SLaura Abbott 657af86e597SLaura Abbott static inline pte_t * fixmap_pte(unsigned long addr) 658af86e597SLaura Abbott { 659157962f5SArd Biesheuvel return &bm_pte[pte_index(addr)]; 660af86e597SLaura Abbott } 661af86e597SLaura Abbott 662af86e597SLaura Abbott void __init early_fixmap_init(void) 663af86e597SLaura Abbott { 664af86e597SLaura Abbott pgd_t *pgd; 665af86e597SLaura Abbott pud_t *pud; 666af86e597SLaura Abbott pmd_t *pmd; 667af86e597SLaura Abbott unsigned long addr = FIXADDR_START; 668af86e597SLaura Abbott 669af86e597SLaura Abbott pgd = pgd_offset_k(addr); 670f80fb3a3SArd Biesheuvel if (CONFIG_PGTABLE_LEVELS > 3 && 671f80fb3a3SArd Biesheuvel !(pgd_none(*pgd) || pgd_page_paddr(*pgd) == __pa(bm_pud))) { 672f9040773SArd Biesheuvel /* 673f9040773SArd Biesheuvel * We only end up here if the kernel mapping and the fixmap 674f9040773SArd Biesheuvel * share the top level pgd entry, which should only happen on 675f9040773SArd Biesheuvel * 16k/4 levels configurations. 676f9040773SArd Biesheuvel */ 677f9040773SArd Biesheuvel BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); 678f9040773SArd Biesheuvel pud = pud_offset_kimg(pgd, addr); 679f9040773SArd Biesheuvel } else { 680af86e597SLaura Abbott pgd_populate(&init_mm, pgd, bm_pud); 681157962f5SArd Biesheuvel pud = fixmap_pud(addr); 682f9040773SArd Biesheuvel } 683af86e597SLaura Abbott pud_populate(&init_mm, pud, bm_pmd); 684157962f5SArd Biesheuvel pmd = fixmap_pmd(addr); 685af86e597SLaura Abbott pmd_populate_kernel(&init_mm, pmd, bm_pte); 686af86e597SLaura Abbott 687af86e597SLaura Abbott /* 688af86e597SLaura Abbott * The boot-ioremap range spans multiple pmds, for which 689157962f5SArd Biesheuvel * we are not prepared: 690af86e597SLaura Abbott */ 691af86e597SLaura Abbott BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) 692af86e597SLaura Abbott != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); 693af86e597SLaura Abbott 694af86e597SLaura Abbott if ((pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN))) 695af86e597SLaura Abbott || pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) { 696af86e597SLaura Abbott WARN_ON(1); 697af86e597SLaura Abbott pr_warn("pmd %p != %p, %p\n", 698af86e597SLaura Abbott pmd, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)), 699af86e597SLaura Abbott fixmap_pmd(fix_to_virt(FIX_BTMAP_END))); 700af86e597SLaura Abbott pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", 701af86e597SLaura Abbott fix_to_virt(FIX_BTMAP_BEGIN)); 702af86e597SLaura Abbott pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n", 703af86e597SLaura Abbott fix_to_virt(FIX_BTMAP_END)); 704af86e597SLaura Abbott 705af86e597SLaura Abbott pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END); 706af86e597SLaura Abbott pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN); 707af86e597SLaura Abbott } 708af86e597SLaura Abbott } 709af86e597SLaura Abbott 710af86e597SLaura Abbott void __set_fixmap(enum fixed_addresses idx, 711af86e597SLaura Abbott phys_addr_t phys, pgprot_t flags) 712af86e597SLaura Abbott { 713af86e597SLaura Abbott unsigned long addr = __fix_to_virt(idx); 714af86e597SLaura Abbott pte_t *pte; 715af86e597SLaura Abbott 716b63dbef9SMark Rutland BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); 717af86e597SLaura Abbott 718af86e597SLaura Abbott pte = fixmap_pte(addr); 719af86e597SLaura Abbott 720af86e597SLaura Abbott if (pgprot_val(flags)) { 721af86e597SLaura Abbott set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags)); 722af86e597SLaura Abbott } else { 723af86e597SLaura Abbott pte_clear(&init_mm, addr, pte); 724af86e597SLaura Abbott flush_tlb_kernel_range(addr, addr+PAGE_SIZE); 725af86e597SLaura Abbott } 726af86e597SLaura Abbott } 72761bd93ceSArd Biesheuvel 728f80fb3a3SArd Biesheuvel void *__init __fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot) 72961bd93ceSArd Biesheuvel { 73061bd93ceSArd Biesheuvel const u64 dt_virt_base = __fix_to_virt(FIX_FDT); 731f80fb3a3SArd Biesheuvel int offset; 73261bd93ceSArd Biesheuvel void *dt_virt; 73361bd93ceSArd Biesheuvel 73461bd93ceSArd Biesheuvel /* 73561bd93ceSArd Biesheuvel * Check whether the physical FDT address is set and meets the minimum 73661bd93ceSArd Biesheuvel * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be 73761bd93ceSArd Biesheuvel * at least 8 bytes so that we can always access the size field of the 73861bd93ceSArd Biesheuvel * FDT header after mapping the first chunk, double check here if that 73961bd93ceSArd Biesheuvel * is indeed the case. 74061bd93ceSArd Biesheuvel */ 74161bd93ceSArd Biesheuvel BUILD_BUG_ON(MIN_FDT_ALIGN < 8); 74261bd93ceSArd Biesheuvel if (!dt_phys || dt_phys % MIN_FDT_ALIGN) 74361bd93ceSArd Biesheuvel return NULL; 74461bd93ceSArd Biesheuvel 74561bd93ceSArd Biesheuvel /* 74661bd93ceSArd Biesheuvel * Make sure that the FDT region can be mapped without the need to 74761bd93ceSArd Biesheuvel * allocate additional translation table pages, so that it is safe 748132233a7SLaura Abbott * to call create_mapping_noalloc() this early. 74961bd93ceSArd Biesheuvel * 75061bd93ceSArd Biesheuvel * On 64k pages, the FDT will be mapped using PTEs, so we need to 75161bd93ceSArd Biesheuvel * be in the same PMD as the rest of the fixmap. 75261bd93ceSArd Biesheuvel * On 4k pages, we'll use section mappings for the FDT so we only 75361bd93ceSArd Biesheuvel * have to be in the same PUD. 75461bd93ceSArd Biesheuvel */ 75561bd93ceSArd Biesheuvel BUILD_BUG_ON(dt_virt_base % SZ_2M); 75661bd93ceSArd Biesheuvel 757b433dce0SSuzuki K. Poulose BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> SWAPPER_TABLE_SHIFT != 758b433dce0SSuzuki K. Poulose __fix_to_virt(FIX_BTMAP_BEGIN) >> SWAPPER_TABLE_SHIFT); 75961bd93ceSArd Biesheuvel 760b433dce0SSuzuki K. Poulose offset = dt_phys % SWAPPER_BLOCK_SIZE; 76161bd93ceSArd Biesheuvel dt_virt = (void *)dt_virt_base + offset; 76261bd93ceSArd Biesheuvel 76361bd93ceSArd Biesheuvel /* map the first chunk so we can read the size from the header */ 764132233a7SLaura Abbott create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), 765132233a7SLaura Abbott dt_virt_base, SWAPPER_BLOCK_SIZE, prot); 76661bd93ceSArd Biesheuvel 76761bd93ceSArd Biesheuvel if (fdt_check_header(dt_virt) != 0) 76861bd93ceSArd Biesheuvel return NULL; 76961bd93ceSArd Biesheuvel 770f80fb3a3SArd Biesheuvel *size = fdt_totalsize(dt_virt); 771f80fb3a3SArd Biesheuvel if (*size > MAX_FDT_SIZE) 77261bd93ceSArd Biesheuvel return NULL; 77361bd93ceSArd Biesheuvel 774f80fb3a3SArd Biesheuvel if (offset + *size > SWAPPER_BLOCK_SIZE) 775132233a7SLaura Abbott create_mapping_noalloc(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base, 776f80fb3a3SArd Biesheuvel round_up(offset + *size, SWAPPER_BLOCK_SIZE), prot); 777f80fb3a3SArd Biesheuvel 778f80fb3a3SArd Biesheuvel return dt_virt; 779f80fb3a3SArd Biesheuvel } 780f80fb3a3SArd Biesheuvel 781f80fb3a3SArd Biesheuvel void *__init fixmap_remap_fdt(phys_addr_t dt_phys) 782f80fb3a3SArd Biesheuvel { 783f80fb3a3SArd Biesheuvel void *dt_virt; 784f80fb3a3SArd Biesheuvel int size; 785f80fb3a3SArd Biesheuvel 786f80fb3a3SArd Biesheuvel dt_virt = __fixmap_remap_fdt(dt_phys, &size, PAGE_KERNEL_RO); 787f80fb3a3SArd Biesheuvel if (!dt_virt) 788f80fb3a3SArd Biesheuvel return NULL; 78961bd93ceSArd Biesheuvel 79061bd93ceSArd Biesheuvel memblock_reserve(dt_phys, size); 79161bd93ceSArd Biesheuvel return dt_virt; 79261bd93ceSArd Biesheuvel } 793324420bfSArd Biesheuvel 794324420bfSArd Biesheuvel int __init arch_ioremap_pud_supported(void) 795324420bfSArd Biesheuvel { 796324420bfSArd Biesheuvel /* only 4k granule supports level 1 block mappings */ 797324420bfSArd Biesheuvel return IS_ENABLED(CONFIG_ARM64_4K_PAGES); 798324420bfSArd Biesheuvel } 799324420bfSArd Biesheuvel 800324420bfSArd Biesheuvel int __init arch_ioremap_pmd_supported(void) 801324420bfSArd Biesheuvel { 802324420bfSArd Biesheuvel return 1; 803324420bfSArd Biesheuvel } 804324420bfSArd Biesheuvel 805324420bfSArd Biesheuvel int pud_set_huge(pud_t *pud, phys_addr_t phys, pgprot_t prot) 806324420bfSArd Biesheuvel { 807324420bfSArd Biesheuvel BUG_ON(phys & ~PUD_MASK); 808324420bfSArd Biesheuvel set_pud(pud, __pud(phys | PUD_TYPE_SECT | pgprot_val(mk_sect_prot(prot)))); 809324420bfSArd Biesheuvel return 1; 810324420bfSArd Biesheuvel } 811324420bfSArd Biesheuvel 812324420bfSArd Biesheuvel int pmd_set_huge(pmd_t *pmd, phys_addr_t phys, pgprot_t prot) 813324420bfSArd Biesheuvel { 814324420bfSArd Biesheuvel BUG_ON(phys & ~PMD_MASK); 815324420bfSArd Biesheuvel set_pmd(pmd, __pmd(phys | PMD_TYPE_SECT | pgprot_val(mk_sect_prot(prot)))); 816324420bfSArd Biesheuvel return 1; 817324420bfSArd Biesheuvel } 818324420bfSArd Biesheuvel 819324420bfSArd Biesheuvel int pud_clear_huge(pud_t *pud) 820324420bfSArd Biesheuvel { 821324420bfSArd Biesheuvel if (!pud_sect(*pud)) 822324420bfSArd Biesheuvel return 0; 823324420bfSArd Biesheuvel pud_clear(pud); 824324420bfSArd Biesheuvel return 1; 825324420bfSArd Biesheuvel } 826324420bfSArd Biesheuvel 827324420bfSArd Biesheuvel int pmd_clear_huge(pmd_t *pmd) 828324420bfSArd Biesheuvel { 829324420bfSArd Biesheuvel if (!pmd_sect(*pmd)) 830324420bfSArd Biesheuvel return 0; 831324420bfSArd Biesheuvel pmd_clear(pmd); 832324420bfSArd Biesheuvel return 1; 833324420bfSArd Biesheuvel } 834