1 // SPDX-License-Identifier: GPL-2.0 2 3 /* 4 * Copyright (c) 2021, Google LLC. 5 * Pasha Tatashin <pasha.tatashin@soleen.com> 6 */ 7 #include <linux/kstrtox.h> 8 #include <linux/mm.h> 9 #include <linux/page_table_check.h> 10 11 #undef pr_fmt 12 #define pr_fmt(fmt) "page_table_check: " fmt 13 14 struct page_table_check { 15 atomic_t anon_map_count; 16 atomic_t file_map_count; 17 }; 18 19 static bool __page_table_check_enabled __initdata = 20 IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED); 21 22 DEFINE_STATIC_KEY_TRUE(page_table_check_disabled); 23 EXPORT_SYMBOL(page_table_check_disabled); 24 25 static int __init early_page_table_check_param(char *buf) 26 { 27 return kstrtobool(buf, &__page_table_check_enabled); 28 } 29 30 early_param("page_table_check", early_page_table_check_param); 31 32 static bool __init need_page_table_check(void) 33 { 34 return __page_table_check_enabled; 35 } 36 37 static void __init init_page_table_check(void) 38 { 39 if (!__page_table_check_enabled) 40 return; 41 static_branch_disable(&page_table_check_disabled); 42 } 43 44 struct page_ext_operations page_table_check_ops = { 45 .size = sizeof(struct page_table_check), 46 .need = need_page_table_check, 47 .init = init_page_table_check, 48 }; 49 50 static struct page_table_check *get_page_table_check(struct page_ext *page_ext) 51 { 52 BUG_ON(!page_ext); 53 return (void *)(page_ext) + page_table_check_ops.offset; 54 } 55 56 /* 57 * An entry is removed from the page table, decrement the counters for that page 58 * verify that it is of correct type and counters do not become negative. 59 */ 60 static void page_table_check_clear(struct mm_struct *mm, unsigned long addr, 61 unsigned long pfn, unsigned long pgcnt) 62 { 63 struct page_ext *page_ext; 64 struct page *page; 65 unsigned long i; 66 bool anon; 67 68 if (!pfn_valid(pfn)) 69 return; 70 71 page = pfn_to_page(pfn); 72 page_ext = page_ext_get(page); 73 anon = PageAnon(page); 74 75 for (i = 0; i < pgcnt; i++) { 76 struct page_table_check *ptc = get_page_table_check(page_ext); 77 78 if (anon) { 79 BUG_ON(atomic_read(&ptc->file_map_count)); 80 BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0); 81 } else { 82 BUG_ON(atomic_read(&ptc->anon_map_count)); 83 BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0); 84 } 85 page_ext = page_ext_next(page_ext); 86 } 87 page_ext_put(page_ext); 88 } 89 90 /* 91 * A new entry is added to the page table, increment the counters for that page 92 * verify that it is of correct type and is not being mapped with a different 93 * type to a different process. 94 */ 95 static void page_table_check_set(struct mm_struct *mm, unsigned long addr, 96 unsigned long pfn, unsigned long pgcnt, 97 bool rw) 98 { 99 struct page_ext *page_ext; 100 struct page *page; 101 unsigned long i; 102 bool anon; 103 104 if (!pfn_valid(pfn)) 105 return; 106 107 page = pfn_to_page(pfn); 108 page_ext = page_ext_get(page); 109 anon = PageAnon(page); 110 111 for (i = 0; i < pgcnt; i++) { 112 struct page_table_check *ptc = get_page_table_check(page_ext); 113 114 if (anon) { 115 BUG_ON(atomic_read(&ptc->file_map_count)); 116 BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw); 117 } else { 118 BUG_ON(atomic_read(&ptc->anon_map_count)); 119 BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0); 120 } 121 page_ext = page_ext_next(page_ext); 122 } 123 page_ext_put(page_ext); 124 } 125 126 /* 127 * page is on free list, or is being allocated, verify that counters are zeroes 128 * crash if they are not. 129 */ 130 void __page_table_check_zero(struct page *page, unsigned int order) 131 { 132 struct page_ext *page_ext; 133 unsigned long i; 134 135 page_ext = page_ext_get(page); 136 BUG_ON(!page_ext); 137 for (i = 0; i < (1ul << order); i++) { 138 struct page_table_check *ptc = get_page_table_check(page_ext); 139 140 BUG_ON(atomic_read(&ptc->anon_map_count)); 141 BUG_ON(atomic_read(&ptc->file_map_count)); 142 page_ext = page_ext_next(page_ext); 143 } 144 page_ext_put(page_ext); 145 } 146 147 void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr, 148 pte_t pte) 149 { 150 if (&init_mm == mm) 151 return; 152 153 if (pte_user_accessible_page(pte)) { 154 page_table_check_clear(mm, addr, pte_pfn(pte), 155 PAGE_SIZE >> PAGE_SHIFT); 156 } 157 } 158 EXPORT_SYMBOL(__page_table_check_pte_clear); 159 160 void __page_table_check_pmd_clear(struct mm_struct *mm, unsigned long addr, 161 pmd_t pmd) 162 { 163 if (&init_mm == mm) 164 return; 165 166 if (pmd_user_accessible_page(pmd)) { 167 page_table_check_clear(mm, addr, pmd_pfn(pmd), 168 PMD_SIZE >> PAGE_SHIFT); 169 } 170 } 171 EXPORT_SYMBOL(__page_table_check_pmd_clear); 172 173 void __page_table_check_pud_clear(struct mm_struct *mm, unsigned long addr, 174 pud_t pud) 175 { 176 if (&init_mm == mm) 177 return; 178 179 if (pud_user_accessible_page(pud)) { 180 page_table_check_clear(mm, addr, pud_pfn(pud), 181 PUD_SIZE >> PAGE_SHIFT); 182 } 183 } 184 EXPORT_SYMBOL(__page_table_check_pud_clear); 185 186 void __page_table_check_pte_set(struct mm_struct *mm, unsigned long addr, 187 pte_t *ptep, pte_t pte) 188 { 189 if (&init_mm == mm) 190 return; 191 192 __page_table_check_pte_clear(mm, addr, *ptep); 193 if (pte_user_accessible_page(pte)) { 194 page_table_check_set(mm, addr, pte_pfn(pte), 195 PAGE_SIZE >> PAGE_SHIFT, 196 pte_write(pte)); 197 } 198 } 199 EXPORT_SYMBOL(__page_table_check_pte_set); 200 201 void __page_table_check_pmd_set(struct mm_struct *mm, unsigned long addr, 202 pmd_t *pmdp, pmd_t pmd) 203 { 204 if (&init_mm == mm) 205 return; 206 207 __page_table_check_pmd_clear(mm, addr, *pmdp); 208 if (pmd_user_accessible_page(pmd)) { 209 page_table_check_set(mm, addr, pmd_pfn(pmd), 210 PMD_SIZE >> PAGE_SHIFT, 211 pmd_write(pmd)); 212 } 213 } 214 EXPORT_SYMBOL(__page_table_check_pmd_set); 215 216 void __page_table_check_pud_set(struct mm_struct *mm, unsigned long addr, 217 pud_t *pudp, pud_t pud) 218 { 219 if (&init_mm == mm) 220 return; 221 222 __page_table_check_pud_clear(mm, addr, *pudp); 223 if (pud_user_accessible_page(pud)) { 224 page_table_check_set(mm, addr, pud_pfn(pud), 225 PUD_SIZE >> PAGE_SHIFT, 226 pud_write(pud)); 227 } 228 } 229 EXPORT_SYMBOL(__page_table_check_pud_set); 230 231 void __page_table_check_pte_clear_range(struct mm_struct *mm, 232 unsigned long addr, 233 pmd_t pmd) 234 { 235 if (&init_mm == mm) 236 return; 237 238 if (!pmd_bad(pmd) && !pmd_leaf(pmd)) { 239 pte_t *ptep = pte_offset_map(&pmd, addr); 240 unsigned long i; 241 242 for (i = 0; i < PTRS_PER_PTE; i++) { 243 __page_table_check_pte_clear(mm, addr, *ptep); 244 addr += PAGE_SIZE; 245 ptep++; 246 } 247 pte_unmap(ptep - PTRS_PER_PTE); 248 } 249 } 250