xref: /openbmc/linux/arch/x86/mm/dump_pagetables.c (revision 8ee90c5c)
1 /*
2  * Debug helper to dump the current kernel pagetables of the system
3  * so that we can see what the various memory ranges are set to.
4  *
5  * (C) Copyright 2008 Intel Corporation
6  *
7  * Author: Arjan van de Ven <arjan@linux.intel.com>
8  *
9  * This program is free software; you can redistribute it and/or
10  * modify it under the terms of the GNU General Public License
11  * as published by the Free Software Foundation; version 2
12  * of the License.
13  */
14 
15 #include <linux/debugfs.h>
16 #include <linux/kasan.h>
17 #include <linux/mm.h>
18 #include <linux/init.h>
19 #include <linux/sched.h>
20 #include <linux/seq_file.h>
21 
22 #include <asm/pgtable.h>
23 
24 /*
25  * The dumper groups pagetable entries of the same type into one, and for
26  * that it needs to keep some state when walking, and flush this state
27  * when a "break" in the continuity is found.
28  */
29 struct pg_state {
30 	int level;
31 	pgprot_t current_prot;
32 	unsigned long start_address;
33 	unsigned long current_address;
34 	const struct addr_marker *marker;
35 	unsigned long lines;
36 	bool to_dmesg;
37 	bool check_wx;
38 	unsigned long wx_pages;
39 };
40 
41 struct addr_marker {
42 	unsigned long start_address;
43 	const char *name;
44 	unsigned long max_lines;
45 };
46 
47 /* indices for address_markers; keep sync'd w/ address_markers below */
48 enum address_markers_idx {
49 	USER_SPACE_NR = 0,
50 #ifdef CONFIG_X86_64
51 	KERNEL_SPACE_NR,
52 	LOW_KERNEL_NR,
53 	VMALLOC_START_NR,
54 	VMEMMAP_START_NR,
55 #ifdef CONFIG_KASAN
56 	KASAN_SHADOW_START_NR,
57 	KASAN_SHADOW_END_NR,
58 #endif
59 # ifdef CONFIG_X86_ESPFIX64
60 	ESPFIX_START_NR,
61 # endif
62 	HIGH_KERNEL_NR,
63 	MODULES_VADDR_NR,
64 	MODULES_END_NR,
65 #else
66 	KERNEL_SPACE_NR,
67 	VMALLOC_START_NR,
68 	VMALLOC_END_NR,
69 # ifdef CONFIG_HIGHMEM
70 	PKMAP_BASE_NR,
71 # endif
72 	FIXADDR_START_NR,
73 #endif
74 };
75 
76 /* Address space markers hints */
77 static struct addr_marker address_markers[] = {
78 	{ 0, "User Space" },
79 #ifdef CONFIG_X86_64
80 	{ 0x8000000000000000UL, "Kernel Space" },
81 	{ 0/* PAGE_OFFSET */,   "Low Kernel Mapping" },
82 	{ 0/* VMALLOC_START */, "vmalloc() Area" },
83 	{ 0/* VMEMMAP_START */, "Vmemmap" },
84 #ifdef CONFIG_KASAN
85 	{ KASAN_SHADOW_START,	"KASAN shadow" },
86 	{ KASAN_SHADOW_END,	"KASAN shadow end" },
87 #endif
88 # ifdef CONFIG_X86_ESPFIX64
89 	{ ESPFIX_BASE_ADDR,	"ESPfix Area", 16 },
90 # endif
91 # ifdef CONFIG_EFI
92 	{ EFI_VA_END,		"EFI Runtime Services" },
93 # endif
94 	{ __START_KERNEL_map,   "High Kernel Mapping" },
95 	{ MODULES_VADDR,        "Modules" },
96 	{ MODULES_END,          "End Modules" },
97 #else
98 	{ PAGE_OFFSET,          "Kernel Mapping" },
99 	{ 0/* VMALLOC_START */, "vmalloc() Area" },
100 	{ 0/*VMALLOC_END*/,     "vmalloc() End" },
101 # ifdef CONFIG_HIGHMEM
102 	{ 0/*PKMAP_BASE*/,      "Persistent kmap() Area" },
103 # endif
104 	{ 0/*FIXADDR_START*/,   "Fixmap Area" },
105 #endif
106 	{ -1, NULL }		/* End of list */
107 };
108 
109 /* Multipliers for offsets within the PTEs */
110 #define PTE_LEVEL_MULT (PAGE_SIZE)
111 #define PMD_LEVEL_MULT (PTRS_PER_PTE * PTE_LEVEL_MULT)
112 #define PUD_LEVEL_MULT (PTRS_PER_PMD * PMD_LEVEL_MULT)
113 #define P4D_LEVEL_MULT (PTRS_PER_PUD * PUD_LEVEL_MULT)
114 #define PGD_LEVEL_MULT (PTRS_PER_P4D * P4D_LEVEL_MULT)
115 
116 #define pt_dump_seq_printf(m, to_dmesg, fmt, args...)		\
117 ({								\
118 	if (to_dmesg)					\
119 		printk(KERN_INFO fmt, ##args);			\
120 	else							\
121 		if (m)						\
122 			seq_printf(m, fmt, ##args);		\
123 })
124 
125 #define pt_dump_cont_printf(m, to_dmesg, fmt, args...)		\
126 ({								\
127 	if (to_dmesg)					\
128 		printk(KERN_CONT fmt, ##args);			\
129 	else							\
130 		if (m)						\
131 			seq_printf(m, fmt, ##args);		\
132 })
133 
134 /*
135  * Print a readable form of a pgprot_t to the seq_file
136  */
137 static void printk_prot(struct seq_file *m, pgprot_t prot, int level, bool dmsg)
138 {
139 	pgprotval_t pr = pgprot_val(prot);
140 	static const char * const level_name[] =
141 		{ "cr3", "pgd", "p4d", "pud", "pmd", "pte" };
142 
143 	if (!pgprot_val(prot)) {
144 		/* Not present */
145 		pt_dump_cont_printf(m, dmsg, "                              ");
146 	} else {
147 		if (pr & _PAGE_USER)
148 			pt_dump_cont_printf(m, dmsg, "USR ");
149 		else
150 			pt_dump_cont_printf(m, dmsg, "    ");
151 		if (pr & _PAGE_RW)
152 			pt_dump_cont_printf(m, dmsg, "RW ");
153 		else
154 			pt_dump_cont_printf(m, dmsg, "ro ");
155 		if (pr & _PAGE_PWT)
156 			pt_dump_cont_printf(m, dmsg, "PWT ");
157 		else
158 			pt_dump_cont_printf(m, dmsg, "    ");
159 		if (pr & _PAGE_PCD)
160 			pt_dump_cont_printf(m, dmsg, "PCD ");
161 		else
162 			pt_dump_cont_printf(m, dmsg, "    ");
163 
164 		/* Bit 7 has a different meaning on level 3 vs 4 */
165 		if (level <= 4 && pr & _PAGE_PSE)
166 			pt_dump_cont_printf(m, dmsg, "PSE ");
167 		else
168 			pt_dump_cont_printf(m, dmsg, "    ");
169 		if ((level == 5 && pr & _PAGE_PAT) ||
170 		    ((level == 4 || level == 3) && pr & _PAGE_PAT_LARGE))
171 			pt_dump_cont_printf(m, dmsg, "PAT ");
172 		else
173 			pt_dump_cont_printf(m, dmsg, "    ");
174 		if (pr & _PAGE_GLOBAL)
175 			pt_dump_cont_printf(m, dmsg, "GLB ");
176 		else
177 			pt_dump_cont_printf(m, dmsg, "    ");
178 		if (pr & _PAGE_NX)
179 			pt_dump_cont_printf(m, dmsg, "NX ");
180 		else
181 			pt_dump_cont_printf(m, dmsg, "x  ");
182 	}
183 	pt_dump_cont_printf(m, dmsg, "%s\n", level_name[level]);
184 }
185 
186 /*
187  * On 64 bits, sign-extend the 48 bit address to 64 bit
188  */
189 static unsigned long normalize_addr(unsigned long u)
190 {
191 	int shift;
192 	if (!IS_ENABLED(CONFIG_X86_64))
193 		return u;
194 
195 	shift = 64 - (__VIRTUAL_MASK_SHIFT + 1);
196 	return (signed long)(u << shift) >> shift;
197 }
198 
199 /*
200  * This function gets called on a break in a continuous series
201  * of PTE entries; the next one is different so we need to
202  * print what we collected so far.
203  */
204 static void note_page(struct seq_file *m, struct pg_state *st,
205 		      pgprot_t new_prot, int level)
206 {
207 	pgprotval_t prot, cur;
208 	static const char units[] = "BKMGTPE";
209 
210 	/*
211 	 * If we have a "break" in the series, we need to flush the state that
212 	 * we have now. "break" is either changing perms, levels or
213 	 * address space marker.
214 	 */
215 	prot = pgprot_val(new_prot);
216 	cur = pgprot_val(st->current_prot);
217 
218 	if (!st->level) {
219 		/* First entry */
220 		st->current_prot = new_prot;
221 		st->level = level;
222 		st->marker = address_markers;
223 		st->lines = 0;
224 		pt_dump_seq_printf(m, st->to_dmesg, "---[ %s ]---\n",
225 				   st->marker->name);
226 	} else if (prot != cur || level != st->level ||
227 		   st->current_address >= st->marker[1].start_address) {
228 		const char *unit = units;
229 		unsigned long delta;
230 		int width = sizeof(unsigned long) * 2;
231 		pgprotval_t pr = pgprot_val(st->current_prot);
232 
233 		if (st->check_wx && (pr & _PAGE_RW) && !(pr & _PAGE_NX)) {
234 			WARN_ONCE(1,
235 				  "x86/mm: Found insecure W+X mapping at address %p/%pS\n",
236 				  (void *)st->start_address,
237 				  (void *)st->start_address);
238 			st->wx_pages += (st->current_address -
239 					 st->start_address) / PAGE_SIZE;
240 		}
241 
242 		/*
243 		 * Now print the actual finished series
244 		 */
245 		if (!st->marker->max_lines ||
246 		    st->lines < st->marker->max_lines) {
247 			pt_dump_seq_printf(m, st->to_dmesg,
248 					   "0x%0*lx-0x%0*lx   ",
249 					   width, st->start_address,
250 					   width, st->current_address);
251 
252 			delta = st->current_address - st->start_address;
253 			while (!(delta & 1023) && unit[1]) {
254 				delta >>= 10;
255 				unit++;
256 			}
257 			pt_dump_cont_printf(m, st->to_dmesg, "%9lu%c ",
258 					    delta, *unit);
259 			printk_prot(m, st->current_prot, st->level,
260 				    st->to_dmesg);
261 		}
262 		st->lines++;
263 
264 		/*
265 		 * We print markers for special areas of address space,
266 		 * such as the start of vmalloc space etc.
267 		 * This helps in the interpretation.
268 		 */
269 		if (st->current_address >= st->marker[1].start_address) {
270 			if (st->marker->max_lines &&
271 			    st->lines > st->marker->max_lines) {
272 				unsigned long nskip =
273 					st->lines - st->marker->max_lines;
274 				pt_dump_seq_printf(m, st->to_dmesg,
275 						   "... %lu entr%s skipped ... \n",
276 						   nskip,
277 						   nskip == 1 ? "y" : "ies");
278 			}
279 			st->marker++;
280 			st->lines = 0;
281 			pt_dump_seq_printf(m, st->to_dmesg, "---[ %s ]---\n",
282 					   st->marker->name);
283 		}
284 
285 		st->start_address = st->current_address;
286 		st->current_prot = new_prot;
287 		st->level = level;
288 	}
289 }
290 
291 static void walk_pte_level(struct seq_file *m, struct pg_state *st, pmd_t addr, unsigned long P)
292 {
293 	int i;
294 	pte_t *start;
295 	pgprotval_t prot;
296 
297 	start = (pte_t *)pmd_page_vaddr(addr);
298 	for (i = 0; i < PTRS_PER_PTE; i++) {
299 		prot = pte_flags(*start);
300 		st->current_address = normalize_addr(P + i * PTE_LEVEL_MULT);
301 		note_page(m, st, __pgprot(prot), 5);
302 		start++;
303 	}
304 }
305 #ifdef CONFIG_KASAN
306 
307 /*
308  * This is an optimization for KASAN=y case. Since all kasan page tables
309  * eventually point to the kasan_zero_page we could call note_page()
310  * right away without walking through lower level page tables. This saves
311  * us dozens of seconds (minutes for 5-level config) while checking for
312  * W+X mapping or reading kernel_page_tables debugfs file.
313  */
314 static inline bool kasan_page_table(struct seq_file *m, struct pg_state *st,
315 				void *pt)
316 {
317 	if (__pa(pt) == __pa(kasan_zero_pmd) ||
318 #ifdef CONFIG_X86_5LEVEL
319 	    __pa(pt) == __pa(kasan_zero_p4d) ||
320 #endif
321 	    __pa(pt) == __pa(kasan_zero_pud)) {
322 		pgprotval_t prot = pte_flags(kasan_zero_pte[0]);
323 		note_page(m, st, __pgprot(prot), 5);
324 		return true;
325 	}
326 	return false;
327 }
328 #else
329 static inline bool kasan_page_table(struct seq_file *m, struct pg_state *st,
330 				void *pt)
331 {
332 	return false;
333 }
334 #endif
335 
336 #if PTRS_PER_PMD > 1
337 
338 static void walk_pmd_level(struct seq_file *m, struct pg_state *st, pud_t addr, unsigned long P)
339 {
340 	int i;
341 	pmd_t *start, *pmd_start;
342 	pgprotval_t prot;
343 
344 	pmd_start = start = (pmd_t *)pud_page_vaddr(addr);
345 	for (i = 0; i < PTRS_PER_PMD; i++) {
346 		st->current_address = normalize_addr(P + i * PMD_LEVEL_MULT);
347 		if (!pmd_none(*start)) {
348 			if (pmd_large(*start) || !pmd_present(*start)) {
349 				prot = pmd_flags(*start);
350 				note_page(m, st, __pgprot(prot), 4);
351 			} else if (!kasan_page_table(m, st, pmd_start)) {
352 				walk_pte_level(m, st, *start,
353 					       P + i * PMD_LEVEL_MULT);
354 			}
355 		} else
356 			note_page(m, st, __pgprot(0), 4);
357 		start++;
358 	}
359 }
360 
361 #else
362 #define walk_pmd_level(m,s,a,p) walk_pte_level(m,s,__pmd(pud_val(a)),p)
363 #define pud_large(a) pmd_large(__pmd(pud_val(a)))
364 #define pud_none(a)  pmd_none(__pmd(pud_val(a)))
365 #endif
366 
367 #if PTRS_PER_PUD > 1
368 
369 static void walk_pud_level(struct seq_file *m, struct pg_state *st, p4d_t addr, unsigned long P)
370 {
371 	int i;
372 	pud_t *start, *pud_start;
373 	pgprotval_t prot;
374 	pud_t *prev_pud = NULL;
375 
376 	pud_start = start = (pud_t *)p4d_page_vaddr(addr);
377 
378 	for (i = 0; i < PTRS_PER_PUD; i++) {
379 		st->current_address = normalize_addr(P + i * PUD_LEVEL_MULT);
380 		if (!pud_none(*start)) {
381 			if (pud_large(*start) || !pud_present(*start)) {
382 				prot = pud_flags(*start);
383 				note_page(m, st, __pgprot(prot), 3);
384 			} else if (!kasan_page_table(m, st, pud_start)) {
385 				walk_pmd_level(m, st, *start,
386 					       P + i * PUD_LEVEL_MULT);
387 			}
388 		} else
389 			note_page(m, st, __pgprot(0), 3);
390 
391 		prev_pud = start;
392 		start++;
393 	}
394 }
395 
396 #else
397 #define walk_pud_level(m,s,a,p) walk_pmd_level(m,s,__pud(p4d_val(a)),p)
398 #define p4d_large(a) pud_large(__pud(p4d_val(a)))
399 #define p4d_none(a)  pud_none(__pud(p4d_val(a)))
400 #endif
401 
402 #if PTRS_PER_P4D > 1
403 
404 static void walk_p4d_level(struct seq_file *m, struct pg_state *st, pgd_t addr, unsigned long P)
405 {
406 	int i;
407 	p4d_t *start, *p4d_start;
408 	pgprotval_t prot;
409 
410 	p4d_start = start = (p4d_t *)pgd_page_vaddr(addr);
411 
412 	for (i = 0; i < PTRS_PER_P4D; i++) {
413 		st->current_address = normalize_addr(P + i * P4D_LEVEL_MULT);
414 		if (!p4d_none(*start)) {
415 			if (p4d_large(*start) || !p4d_present(*start)) {
416 				prot = p4d_flags(*start);
417 				note_page(m, st, __pgprot(prot), 2);
418 			} else if (!kasan_page_table(m, st, p4d_start)) {
419 				walk_pud_level(m, st, *start,
420 					       P + i * P4D_LEVEL_MULT);
421 			}
422 		} else
423 			note_page(m, st, __pgprot(0), 2);
424 
425 		start++;
426 	}
427 }
428 
429 #else
430 #define walk_p4d_level(m,s,a,p) walk_pud_level(m,s,__p4d(pgd_val(a)),p)
431 #define pgd_large(a) p4d_large(__p4d(pgd_val(a)))
432 #define pgd_none(a)  p4d_none(__p4d(pgd_val(a)))
433 #endif
434 
435 static inline bool is_hypervisor_range(int idx)
436 {
437 #ifdef CONFIG_X86_64
438 	/*
439 	 * ffff800000000000 - ffff87ffffffffff is reserved for
440 	 * the hypervisor.
441 	 */
442 	return	(idx >= pgd_index(__PAGE_OFFSET) - 16) &&
443 		(idx <  pgd_index(__PAGE_OFFSET));
444 #else
445 	return false;
446 #endif
447 }
448 
449 static void ptdump_walk_pgd_level_core(struct seq_file *m, pgd_t *pgd,
450 				       bool checkwx)
451 {
452 #ifdef CONFIG_X86_64
453 	pgd_t *start = (pgd_t *) &init_top_pgt;
454 #else
455 	pgd_t *start = swapper_pg_dir;
456 #endif
457 	pgprotval_t prot;
458 	int i;
459 	struct pg_state st = {};
460 
461 	if (pgd) {
462 		start = pgd;
463 		st.to_dmesg = true;
464 	}
465 
466 	st.check_wx = checkwx;
467 	if (checkwx)
468 		st.wx_pages = 0;
469 
470 	for (i = 0; i < PTRS_PER_PGD; i++) {
471 		st.current_address = normalize_addr(i * PGD_LEVEL_MULT);
472 		if (!pgd_none(*start) && !is_hypervisor_range(i)) {
473 			if (pgd_large(*start) || !pgd_present(*start)) {
474 				prot = pgd_flags(*start);
475 				note_page(m, &st, __pgprot(prot), 1);
476 			} else {
477 				walk_p4d_level(m, &st, *start,
478 					       i * PGD_LEVEL_MULT);
479 			}
480 		} else
481 			note_page(m, &st, __pgprot(0), 1);
482 
483 		cond_resched();
484 		start++;
485 	}
486 
487 	/* Flush out the last page */
488 	st.current_address = normalize_addr(PTRS_PER_PGD*PGD_LEVEL_MULT);
489 	note_page(m, &st, __pgprot(0), 0);
490 	if (!checkwx)
491 		return;
492 	if (st.wx_pages)
493 		pr_info("x86/mm: Checked W+X mappings: FAILED, %lu W+X pages found.\n",
494 			st.wx_pages);
495 	else
496 		pr_info("x86/mm: Checked W+X mappings: passed, no W+X pages found.\n");
497 }
498 
499 void ptdump_walk_pgd_level(struct seq_file *m, pgd_t *pgd)
500 {
501 	ptdump_walk_pgd_level_core(m, pgd, false);
502 }
503 EXPORT_SYMBOL_GPL(ptdump_walk_pgd_level);
504 
505 void ptdump_walk_pgd_level_checkwx(void)
506 {
507 	ptdump_walk_pgd_level_core(NULL, NULL, true);
508 }
509 
510 static int __init pt_dump_init(void)
511 {
512 	/*
513 	 * Various markers are not compile-time constants, so assign them
514 	 * here.
515 	 */
516 #ifdef CONFIG_X86_64
517 	address_markers[LOW_KERNEL_NR].start_address = PAGE_OFFSET;
518 	address_markers[VMALLOC_START_NR].start_address = VMALLOC_START;
519 	address_markers[VMEMMAP_START_NR].start_address = VMEMMAP_START;
520 #endif
521 #ifdef CONFIG_X86_32
522 	address_markers[VMALLOC_START_NR].start_address = VMALLOC_START;
523 	address_markers[VMALLOC_END_NR].start_address = VMALLOC_END;
524 # ifdef CONFIG_HIGHMEM
525 	address_markers[PKMAP_BASE_NR].start_address = PKMAP_BASE;
526 # endif
527 	address_markers[FIXADDR_START_NR].start_address = FIXADDR_START;
528 #endif
529 
530 	return 0;
531 }
532 __initcall(pt_dump_init);
533