xref: /openbmc/linux/mm/page_table_check.c (revision 8d81cd1a) 
1 // SPDX-License-Identifier: GPL-2.0
2 
3 /*
4  * Copyright (c) 2021, Google LLC.
5  * Pasha Tatashin <pasha.tatashin@soleen.com>
6  */
7 #include <linux/kstrtox.h>
8 #include <linux/mm.h>
9 #include <linux/page_table_check.h>
10 #include <linux/swap.h>
11 #include <linux/swapops.h>
12 
13 #undef pr_fmt
14 #define pr_fmt(fmt)	"page_table_check: " fmt
15 
16 struct page_table_check {
17 	atomic_t anon_map_count;
18 	atomic_t file_map_count;
19 };
20 
21 static bool __page_table_check_enabled __initdata =
22 				IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED);
23 
24 DEFINE_STATIC_KEY_TRUE(page_table_check_disabled);
25 EXPORT_SYMBOL(page_table_check_disabled);
26 
27 static int __init early_page_table_check_param(char *buf)
28 {
29 	return kstrtobool(buf, &__page_table_check_enabled);
30 }
31 
32 early_param("page_table_check", early_page_table_check_param);
33 
34 static bool __init need_page_table_check(void)
35 {
36 	return __page_table_check_enabled;
37 }
38 
39 static void __init init_page_table_check(void)
40 {
41 	if (!__page_table_check_enabled)
42 		return;
43 	static_branch_disable(&page_table_check_disabled);
44 }
45 
46 struct page_ext_operations page_table_check_ops = {
47 	.size = sizeof(struct page_table_check),
48 	.need = need_page_table_check,
49 	.init = init_page_table_check,
50 	.need_shared_flags = false,
51 };
52 
53 static struct page_table_check *get_page_table_check(struct page_ext *page_ext)
54 {
55 	BUG_ON(!page_ext);
56 	return page_ext_data(page_ext, &page_table_check_ops);
57 }
58 
59 /*
60  * An entry is removed from the page table, decrement the counters for that page
61  * verify that it is of correct type and counters do not become negative.
62  */
63 static void page_table_check_clear(unsigned long pfn, unsigned long pgcnt)
64 {
65 	struct page_ext *page_ext;
66 	struct page *page;
67 	unsigned long i;
68 	bool anon;
69 
70 	if (!pfn_valid(pfn))
71 		return;
72 
73 	page = pfn_to_page(pfn);
74 	page_ext = page_ext_get(page);
75 
76 	if (!page_ext)
77 		return;
78 
79 	BUG_ON(PageSlab(page));
80 	anon = PageAnon(page);
81 
82 	for (i = 0; i < pgcnt; i++) {
83 		struct page_table_check *ptc = get_page_table_check(page_ext);
84 
85 		if (anon) {
86 			BUG_ON(atomic_read(&ptc->file_map_count));
87 			BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0);
88 		} else {
89 			BUG_ON(atomic_read(&ptc->anon_map_count));
90 			BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0);
91 		}
92 		page_ext = page_ext_next(page_ext);
93 	}
94 	page_ext_put(page_ext);
95 }
96 
97 /*
98  * A new entry is added to the page table, increment the counters for that page
99  * verify that it is of correct type and is not being mapped with a different
100  * type to a different process.
101  */
102 static void page_table_check_set(unsigned long pfn, unsigned long pgcnt,
103 				 bool rw)
104 {
105 	struct page_ext *page_ext;
106 	struct page *page;
107 	unsigned long i;
108 	bool anon;
109 
110 	if (!pfn_valid(pfn))
111 		return;
112 
113 	page = pfn_to_page(pfn);
114 	page_ext = page_ext_get(page);
115 
116 	if (!page_ext)
117 		return;
118 
119 	BUG_ON(PageSlab(page));
120 	anon = PageAnon(page);
121 
122 	for (i = 0; i < pgcnt; i++) {
123 		struct page_table_check *ptc = get_page_table_check(page_ext);
124 
125 		if (anon) {
126 			BUG_ON(atomic_read(&ptc->file_map_count));
127 			BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw);
128 		} else {
129 			BUG_ON(atomic_read(&ptc->anon_map_count));
130 			BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0);
131 		}
132 		page_ext = page_ext_next(page_ext);
133 	}
134 	page_ext_put(page_ext);
135 }
136 
137 /*
138  * page is on free list, or is being allocated, verify that counters are zeroes
139  * crash if they are not.
140  */
141 void __page_table_check_zero(struct page *page, unsigned int order)
142 {
143 	struct page_ext *page_ext;
144 	unsigned long i;
145 
146 	BUG_ON(PageSlab(page));
147 
148 	page_ext = page_ext_get(page);
149 
150 	if (!page_ext)
151 		return;
152 
153 	for (i = 0; i < (1ul << order); i++) {
154 		struct page_table_check *ptc = get_page_table_check(page_ext);
155 
156 		BUG_ON(atomic_read(&ptc->anon_map_count));
157 		BUG_ON(atomic_read(&ptc->file_map_count));
158 		page_ext = page_ext_next(page_ext);
159 	}
160 	page_ext_put(page_ext);
161 }
162 
163 void __page_table_check_pte_clear(struct mm_struct *mm, pte_t pte)
164 {
165 	if (&init_mm == mm)
166 		return;
167 
168 	if (pte_user_accessible_page(pte)) {
169 		page_table_check_clear(pte_pfn(pte), PAGE_SIZE >> PAGE_SHIFT);
170 	}
171 }
172 EXPORT_SYMBOL(__page_table_check_pte_clear);
173 
174 void __page_table_check_pmd_clear(struct mm_struct *mm, pmd_t pmd)
175 {
176 	if (&init_mm == mm)
177 		return;
178 
179 	if (pmd_user_accessible_page(pmd)) {
180 		page_table_check_clear(pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT);
181 	}
182 }
183 EXPORT_SYMBOL(__page_table_check_pmd_clear);
184 
185 void __page_table_check_pud_clear(struct mm_struct *mm, pud_t pud)
186 {
187 	if (&init_mm == mm)
188 		return;
189 
190 	if (pud_user_accessible_page(pud)) {
191 		page_table_check_clear(pud_pfn(pud), PUD_SIZE >> PAGE_SHIFT);
192 	}
193 }
194 EXPORT_SYMBOL(__page_table_check_pud_clear);
195 
196 /* Whether the swap entry cached writable information */
197 static inline bool swap_cached_writable(swp_entry_t entry)
198 {
199 	return is_writable_device_exclusive_entry(entry) ||
200 	    is_writable_device_private_entry(entry) ||
201 	    is_writable_migration_entry(entry);
202 }
203 
204 static inline void page_table_check_pte_flags(pte_t pte)
205 {
206 	if (pte_present(pte) && pte_uffd_wp(pte))
207 		WARN_ON_ONCE(pte_write(pte));
208 	else if (is_swap_pte(pte) && pte_swp_uffd_wp(pte))
209 		WARN_ON_ONCE(swap_cached_writable(pte_to_swp_entry(pte)));
210 }
211 
212 void __page_table_check_ptes_set(struct mm_struct *mm, pte_t *ptep, pte_t pte,
213 		unsigned int nr)
214 {
215 	unsigned int i;
216 
217 	if (&init_mm == mm)
218 		return;
219 
220 	page_table_check_pte_flags(pte);
221 
222 	for (i = 0; i < nr; i++)
223 		__page_table_check_pte_clear(mm, ptep_get(ptep + i));
224 	if (pte_user_accessible_page(pte))
225 		page_table_check_set(pte_pfn(pte), nr, pte_write(pte));
226 }
227 EXPORT_SYMBOL(__page_table_check_ptes_set);
228 
229 static inline void page_table_check_pmd_flags(pmd_t pmd)
230 {
231 	if (pmd_present(pmd) && pmd_uffd_wp(pmd))
232 		WARN_ON_ONCE(pmd_write(pmd));
233 	else if (is_swap_pmd(pmd) && pmd_swp_uffd_wp(pmd))
234 		WARN_ON_ONCE(swap_cached_writable(pmd_to_swp_entry(pmd)));
235 }
236 
237 void __page_table_check_pmd_set(struct mm_struct *mm, pmd_t *pmdp, pmd_t pmd)
238 {
239 	if (&init_mm == mm)
240 		return;
241 
242 	page_table_check_pmd_flags(pmd);
243 
244 	__page_table_check_pmd_clear(mm, *pmdp);
245 	if (pmd_user_accessible_page(pmd)) {
246 		page_table_check_set(pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT,
247 				     pmd_write(pmd));
248 	}
249 }
250 EXPORT_SYMBOL(__page_table_check_pmd_set);
251 
252 void __page_table_check_pud_set(struct mm_struct *mm, pud_t *pudp, pud_t pud)
253 {
254 	if (&init_mm == mm)
255 		return;
256 
257 	__page_table_check_pud_clear(mm, *pudp);
258 	if (pud_user_accessible_page(pud)) {
259 		page_table_check_set(pud_pfn(pud), PUD_SIZE >> PAGE_SHIFT,
260 				     pud_write(pud));
261 	}
262 }
263 EXPORT_SYMBOL(__page_table_check_pud_set);
264 
265 void __page_table_check_pte_clear_range(struct mm_struct *mm,
266 					unsigned long addr,
267 					pmd_t pmd)
268 {
269 	if (&init_mm == mm)
270 		return;
271 
272 	if (!pmd_bad(pmd) && !pmd_leaf(pmd)) {
273 		pte_t *ptep = pte_offset_map(&pmd, addr);
274 		unsigned long i;
275 
276 		if (WARN_ON(!ptep))
277 			return;
278 		for (i = 0; i < PTRS_PER_PTE; i++) {
279 			__page_table_check_pte_clear(mm, ptep_get(ptep));
280 			addr += PAGE_SIZE;
281 			ptep++;
282 		}
283 		pte_unmap(ptep - PTRS_PER_PTE);
284 	}
285 }
286