1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 */ 5 #include <linux/init.h> 6 #include <linux/export.h> 7 #include <linux/mm.h> 8 #include <asm/pgalloc.h> 9 #include <asm/pgtable.h> 10 #include <asm/tlbflush.h> 11 12 pgd_t *pgd_alloc(struct mm_struct *mm) 13 { 14 pgd_t *ret, *init; 15 16 ret = (pgd_t *) __get_free_page(GFP_KERNEL); 17 if (ret) { 18 init = pgd_offset(&init_mm, 0UL); 19 pgd_init(ret); 20 memcpy(ret + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD, 21 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); 22 } 23 24 return ret; 25 } 26 EXPORT_SYMBOL_GPL(pgd_alloc); 27 28 void pgd_init(void *addr) 29 { 30 unsigned long *p, *end; 31 unsigned long entry; 32 33 #if !defined(__PAGETABLE_PUD_FOLDED) 34 entry = (unsigned long)invalid_pud_table; 35 #elif !defined(__PAGETABLE_PMD_FOLDED) 36 entry = (unsigned long)invalid_pmd_table; 37 #else 38 entry = (unsigned long)invalid_pte_table; 39 #endif 40 41 p = (unsigned long *)addr; 42 end = p + PTRS_PER_PGD; 43 44 do { 45 p[0] = entry; 46 p[1] = entry; 47 p[2] = entry; 48 p[3] = entry; 49 p[4] = entry; 50 p += 8; 51 p[-3] = entry; 52 p[-2] = entry; 53 p[-1] = entry; 54 } while (p != end); 55 } 56 EXPORT_SYMBOL_GPL(pgd_init); 57 58 #ifndef __PAGETABLE_PMD_FOLDED 59 void pmd_init(void *addr) 60 { 61 unsigned long *p, *end; 62 unsigned long pagetable = (unsigned long)invalid_pte_table; 63 64 p = (unsigned long *)addr; 65 end = p + PTRS_PER_PMD; 66 67 do { 68 p[0] = pagetable; 69 p[1] = pagetable; 70 p[2] = pagetable; 71 p[3] = pagetable; 72 p[4] = pagetable; 73 p += 8; 74 p[-3] = pagetable; 75 p[-2] = pagetable; 76 p[-1] = pagetable; 77 } while (p != end); 78 } 79 EXPORT_SYMBOL_GPL(pmd_init); 80 #endif 81 82 #ifndef __PAGETABLE_PUD_FOLDED 83 void pud_init(void *addr) 84 { 85 unsigned long *p, *end; 86 unsigned long pagetable = (unsigned long)invalid_pmd_table; 87 88 p = (unsigned long *)addr; 89 end = p + PTRS_PER_PUD; 90 91 do { 92 p[0] = pagetable; 93 p[1] = pagetable; 94 p[2] = pagetable; 95 p[3] = pagetable; 96 p[4] = pagetable; 97 p += 8; 98 p[-3] = pagetable; 99 p[-2] = pagetable; 100 p[-1] = pagetable; 101 } while (p != end); 102 } 103 EXPORT_SYMBOL_GPL(pud_init); 104 #endif 105 106 pmd_t mk_pmd(struct page *page, pgprot_t prot) 107 { 108 pmd_t pmd; 109 110 pmd_val(pmd) = (page_to_pfn(page) << _PFN_SHIFT) | pgprot_val(prot); 111 112 return pmd; 113 } 114 115 void set_pmd_at(struct mm_struct *mm, unsigned long addr, 116 pmd_t *pmdp, pmd_t pmd) 117 { 118 *pmdp = pmd; 119 flush_tlb_all(); 120 } 121 122 void __init pagetable_init(void) 123 { 124 /* Initialize the entire pgd. */ 125 pgd_init(swapper_pg_dir); 126 pgd_init(invalid_pg_dir); 127 #ifndef __PAGETABLE_PUD_FOLDED 128 pud_init(invalid_pud_table); 129 #endif 130 #ifndef __PAGETABLE_PMD_FOLDED 131 pmd_init(invalid_pmd_table); 132 #endif 133 } 134