1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 */ 5 #include <linux/init.h> 6 #include <linux/export.h> 7 #include <linux/mm.h> 8 #include <asm/pgalloc.h> 9 #include <asm/pgtable.h> 10 #include <asm/tlbflush.h> 11 12 pgd_t *pgd_alloc(struct mm_struct *mm) 13 { 14 pgd_t *ret, *init; 15 16 ret = (pgd_t *) __get_free_pages(GFP_KERNEL, PGD_ORDER); 17 if (ret) { 18 init = pgd_offset(&init_mm, 0UL); 19 pgd_init((unsigned long)ret); 20 memcpy(ret + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD, 21 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); 22 } 23 24 return ret; 25 } 26 EXPORT_SYMBOL_GPL(pgd_alloc); 27 28 void pgd_init(unsigned long page) 29 { 30 unsigned long *p, *end; 31 unsigned long entry; 32 33 #if !defined(__PAGETABLE_PUD_FOLDED) 34 entry = (unsigned long)invalid_pud_table; 35 #elif !defined(__PAGETABLE_PMD_FOLDED) 36 entry = (unsigned long)invalid_pmd_table; 37 #else 38 entry = (unsigned long)invalid_pte_table; 39 #endif 40 41 p = (unsigned long *) page; 42 end = p + PTRS_PER_PGD; 43 44 do { 45 p[0] = entry; 46 p[1] = entry; 47 p[2] = entry; 48 p[3] = entry; 49 p[4] = entry; 50 p += 8; 51 p[-3] = entry; 52 p[-2] = entry; 53 p[-1] = entry; 54 } while (p != end); 55 } 56 EXPORT_SYMBOL_GPL(pgd_init); 57 58 #ifndef __PAGETABLE_PMD_FOLDED 59 void pmd_init(unsigned long addr, unsigned long pagetable) 60 { 61 unsigned long *p, *end; 62 63 p = (unsigned long *) addr; 64 end = p + PTRS_PER_PMD; 65 66 do { 67 p[0] = pagetable; 68 p[1] = pagetable; 69 p[2] = pagetable; 70 p[3] = pagetable; 71 p[4] = pagetable; 72 p += 8; 73 p[-3] = pagetable; 74 p[-2] = pagetable; 75 p[-1] = pagetable; 76 } while (p != end); 77 } 78 EXPORT_SYMBOL_GPL(pmd_init); 79 #endif 80 81 #ifndef __PAGETABLE_PUD_FOLDED 82 void pud_init(unsigned long addr, unsigned long pagetable) 83 { 84 unsigned long *p, *end; 85 86 p = (unsigned long *)addr; 87 end = p + PTRS_PER_PUD; 88 89 do { 90 p[0] = pagetable; 91 p[1] = pagetable; 92 p[2] = pagetable; 93 p[3] = pagetable; 94 p[4] = pagetable; 95 p += 8; 96 p[-3] = pagetable; 97 p[-2] = pagetable; 98 p[-1] = pagetable; 99 } while (p != end); 100 } 101 #endif 102 103 pmd_t mk_pmd(struct page *page, pgprot_t prot) 104 { 105 pmd_t pmd; 106 107 pmd_val(pmd) = (page_to_pfn(page) << _PFN_SHIFT) | pgprot_val(prot); 108 109 return pmd; 110 } 111 112 void set_pmd_at(struct mm_struct *mm, unsigned long addr, 113 pmd_t *pmdp, pmd_t pmd) 114 { 115 *pmdp = pmd; 116 flush_tlb_all(); 117 } 118 119 void __init pagetable_init(void) 120 { 121 /* Initialize the entire pgd. */ 122 pgd_init((unsigned long)swapper_pg_dir); 123 pgd_init((unsigned long)invalid_pg_dir); 124 #ifndef __PAGETABLE_PUD_FOLDED 125 pud_init((unsigned long)invalid_pud_table, (unsigned long)invalid_pmd_table); 126 #endif 127 #ifndef __PAGETABLE_PMD_FOLDED 128 pmd_init((unsigned long)invalid_pmd_table, (unsigned long)invalid_pte_table); 129 #endif 130 } 131