xref: /openbmc/linux/fs/proc/task_mmu.c (revision da2ef666)
1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/mm.h>
3 #include <linux/vmacache.h>
4 #include <linux/hugetlb.h>
5 #include <linux/huge_mm.h>
6 #include <linux/mount.h>
7 #include <linux/seq_file.h>
8 #include <linux/highmem.h>
9 #include <linux/ptrace.h>
10 #include <linux/slab.h>
11 #include <linux/pagemap.h>
12 #include <linux/mempolicy.h>
13 #include <linux/rmap.h>
14 #include <linux/swap.h>
15 #include <linux/sched/mm.h>
16 #include <linux/swapops.h>
17 #include <linux/mmu_notifier.h>
18 #include <linux/page_idle.h>
19 #include <linux/shmem_fs.h>
20 #include <linux/uaccess.h>
21 #include <linux/pkeys.h>
22 
23 #include <asm/elf.h>
24 #include <asm/tlb.h>
25 #include <asm/tlbflush.h>
26 #include "internal.h"
27 
28 #define SEQ_PUT_DEC(str, val) \
29 		seq_put_decimal_ull_width(m, str, (val) << (PAGE_SHIFT-10), 8)
30 void task_mem(struct seq_file *m, struct mm_struct *mm)
31 {
32 	unsigned long text, lib, swap, anon, file, shmem;
33 	unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss;
34 
35 	anon = get_mm_counter(mm, MM_ANONPAGES);
36 	file = get_mm_counter(mm, MM_FILEPAGES);
37 	shmem = get_mm_counter(mm, MM_SHMEMPAGES);
38 
39 	/*
40 	 * Note: to minimize their overhead, mm maintains hiwater_vm and
41 	 * hiwater_rss only when about to *lower* total_vm or rss.  Any
42 	 * collector of these hiwater stats must therefore get total_vm
43 	 * and rss too, which will usually be the higher.  Barriers? not
44 	 * worth the effort, such snapshots can always be inconsistent.
45 	 */
46 	hiwater_vm = total_vm = mm->total_vm;
47 	if (hiwater_vm < mm->hiwater_vm)
48 		hiwater_vm = mm->hiwater_vm;
49 	hiwater_rss = total_rss = anon + file + shmem;
50 	if (hiwater_rss < mm->hiwater_rss)
51 		hiwater_rss = mm->hiwater_rss;
52 
53 	/* split executable areas between text and lib */
54 	text = PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK);
55 	text = min(text, mm->exec_vm << PAGE_SHIFT);
56 	lib = (mm->exec_vm << PAGE_SHIFT) - text;
57 
58 	swap = get_mm_counter(mm, MM_SWAPENTS);
59 	SEQ_PUT_DEC("VmPeak:\t", hiwater_vm);
60 	SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm);
61 	SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm);
62 	SEQ_PUT_DEC(" kB\nVmPin:\t", mm->pinned_vm);
63 	SEQ_PUT_DEC(" kB\nVmHWM:\t", hiwater_rss);
64 	SEQ_PUT_DEC(" kB\nVmRSS:\t", total_rss);
65 	SEQ_PUT_DEC(" kB\nRssAnon:\t", anon);
66 	SEQ_PUT_DEC(" kB\nRssFile:\t", file);
67 	SEQ_PUT_DEC(" kB\nRssShmem:\t", shmem);
68 	SEQ_PUT_DEC(" kB\nVmData:\t", mm->data_vm);
69 	SEQ_PUT_DEC(" kB\nVmStk:\t", mm->stack_vm);
70 	seq_put_decimal_ull_width(m,
71 		    " kB\nVmExe:\t", text >> 10, 8);
72 	seq_put_decimal_ull_width(m,
73 		    " kB\nVmLib:\t", lib >> 10, 8);
74 	seq_put_decimal_ull_width(m,
75 		    " kB\nVmPTE:\t", mm_pgtables_bytes(mm) >> 10, 8);
76 	SEQ_PUT_DEC(" kB\nVmSwap:\t", swap);
77 	seq_puts(m, " kB\n");
78 	hugetlb_report_usage(m, mm);
79 }
80 #undef SEQ_PUT_DEC
81 
82 unsigned long task_vsize(struct mm_struct *mm)
83 {
84 	return PAGE_SIZE * mm->total_vm;
85 }
86 
87 unsigned long task_statm(struct mm_struct *mm,
88 			 unsigned long *shared, unsigned long *text,
89 			 unsigned long *data, unsigned long *resident)
90 {
91 	*shared = get_mm_counter(mm, MM_FILEPAGES) +
92 			get_mm_counter(mm, MM_SHMEMPAGES);
93 	*text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))
94 								>> PAGE_SHIFT;
95 	*data = mm->data_vm + mm->stack_vm;
96 	*resident = *shared + get_mm_counter(mm, MM_ANONPAGES);
97 	return mm->total_vm;
98 }
99 
100 #ifdef CONFIG_NUMA
101 /*
102  * Save get_task_policy() for show_numa_map().
103  */
104 static void hold_task_mempolicy(struct proc_maps_private *priv)
105 {
106 	struct task_struct *task = priv->task;
107 
108 	task_lock(task);
109 	priv->task_mempolicy = get_task_policy(task);
110 	mpol_get(priv->task_mempolicy);
111 	task_unlock(task);
112 }
113 static void release_task_mempolicy(struct proc_maps_private *priv)
114 {
115 	mpol_put(priv->task_mempolicy);
116 }
117 #else
118 static void hold_task_mempolicy(struct proc_maps_private *priv)
119 {
120 }
121 static void release_task_mempolicy(struct proc_maps_private *priv)
122 {
123 }
124 #endif
125 
126 static void vma_stop(struct proc_maps_private *priv)
127 {
128 	struct mm_struct *mm = priv->mm;
129 
130 	release_task_mempolicy(priv);
131 	up_read(&mm->mmap_sem);
132 	mmput(mm);
133 }
134 
135 static struct vm_area_struct *
136 m_next_vma(struct proc_maps_private *priv, struct vm_area_struct *vma)
137 {
138 	if (vma == priv->tail_vma)
139 		return NULL;
140 	return vma->vm_next ?: priv->tail_vma;
141 }
142 
143 static void m_cache_vma(struct seq_file *m, struct vm_area_struct *vma)
144 {
145 	if (m->count < m->size)	/* vma is copied successfully */
146 		m->version = m_next_vma(m->private, vma) ? vma->vm_end : -1UL;
147 }
148 
149 static void *m_start(struct seq_file *m, loff_t *ppos)
150 {
151 	struct proc_maps_private *priv = m->private;
152 	unsigned long last_addr = m->version;
153 	struct mm_struct *mm;
154 	struct vm_area_struct *vma;
155 	unsigned int pos = *ppos;
156 
157 	/* See m_cache_vma(). Zero at the start or after lseek. */
158 	if (last_addr == -1UL)
159 		return NULL;
160 
161 	priv->task = get_proc_task(priv->inode);
162 	if (!priv->task)
163 		return ERR_PTR(-ESRCH);
164 
165 	mm = priv->mm;
166 	if (!mm || !mmget_not_zero(mm))
167 		return NULL;
168 
169 	down_read(&mm->mmap_sem);
170 	hold_task_mempolicy(priv);
171 	priv->tail_vma = get_gate_vma(mm);
172 
173 	if (last_addr) {
174 		vma = find_vma(mm, last_addr - 1);
175 		if (vma && vma->vm_start <= last_addr)
176 			vma = m_next_vma(priv, vma);
177 		if (vma)
178 			return vma;
179 	}
180 
181 	m->version = 0;
182 	if (pos < mm->map_count) {
183 		for (vma = mm->mmap; pos; pos--) {
184 			m->version = vma->vm_start;
185 			vma = vma->vm_next;
186 		}
187 		return vma;
188 	}
189 
190 	/* we do not bother to update m->version in this case */
191 	if (pos == mm->map_count && priv->tail_vma)
192 		return priv->tail_vma;
193 
194 	vma_stop(priv);
195 	return NULL;
196 }
197 
198 static void *m_next(struct seq_file *m, void *v, loff_t *pos)
199 {
200 	struct proc_maps_private *priv = m->private;
201 	struct vm_area_struct *next;
202 
203 	(*pos)++;
204 	next = m_next_vma(priv, v);
205 	if (!next)
206 		vma_stop(priv);
207 	return next;
208 }
209 
210 static void m_stop(struct seq_file *m, void *v)
211 {
212 	struct proc_maps_private *priv = m->private;
213 
214 	if (!IS_ERR_OR_NULL(v))
215 		vma_stop(priv);
216 	if (priv->task) {
217 		put_task_struct(priv->task);
218 		priv->task = NULL;
219 	}
220 }
221 
222 static int proc_maps_open(struct inode *inode, struct file *file,
223 			const struct seq_operations *ops, int psize)
224 {
225 	struct proc_maps_private *priv = __seq_open_private(file, ops, psize);
226 
227 	if (!priv)
228 		return -ENOMEM;
229 
230 	priv->inode = inode;
231 	priv->mm = proc_mem_open(inode, PTRACE_MODE_READ);
232 	if (IS_ERR(priv->mm)) {
233 		int err = PTR_ERR(priv->mm);
234 
235 		seq_release_private(inode, file);
236 		return err;
237 	}
238 
239 	return 0;
240 }
241 
242 static int proc_map_release(struct inode *inode, struct file *file)
243 {
244 	struct seq_file *seq = file->private_data;
245 	struct proc_maps_private *priv = seq->private;
246 
247 	if (priv->mm)
248 		mmdrop(priv->mm);
249 
250 	return seq_release_private(inode, file);
251 }
252 
253 static int do_maps_open(struct inode *inode, struct file *file,
254 			const struct seq_operations *ops)
255 {
256 	return proc_maps_open(inode, file, ops,
257 				sizeof(struct proc_maps_private));
258 }
259 
260 /*
261  * Indicate if the VMA is a stack for the given task; for
262  * /proc/PID/maps that is the stack of the main task.
263  */
264 static int is_stack(struct vm_area_struct *vma)
265 {
266 	/*
267 	 * We make no effort to guess what a given thread considers to be
268 	 * its "stack".  It's not even well-defined for programs written
269 	 * languages like Go.
270 	 */
271 	return vma->vm_start <= vma->vm_mm->start_stack &&
272 		vma->vm_end >= vma->vm_mm->start_stack;
273 }
274 
275 static void show_vma_header_prefix(struct seq_file *m,
276 				   unsigned long start, unsigned long end,
277 				   vm_flags_t flags, unsigned long long pgoff,
278 				   dev_t dev, unsigned long ino)
279 {
280 	seq_setwidth(m, 25 + sizeof(void *) * 6 - 1);
281 	seq_put_hex_ll(m, NULL, start, 8);
282 	seq_put_hex_ll(m, "-", end, 8);
283 	seq_putc(m, ' ');
284 	seq_putc(m, flags & VM_READ ? 'r' : '-');
285 	seq_putc(m, flags & VM_WRITE ? 'w' : '-');
286 	seq_putc(m, flags & VM_EXEC ? 'x' : '-');
287 	seq_putc(m, flags & VM_MAYSHARE ? 's' : 'p');
288 	seq_put_hex_ll(m, " ", pgoff, 8);
289 	seq_put_hex_ll(m, " ", MAJOR(dev), 2);
290 	seq_put_hex_ll(m, ":", MINOR(dev), 2);
291 	seq_put_decimal_ull(m, " ", ino);
292 	seq_putc(m, ' ');
293 }
294 
295 static void
296 show_map_vma(struct seq_file *m, struct vm_area_struct *vma)
297 {
298 	struct mm_struct *mm = vma->vm_mm;
299 	struct file *file = vma->vm_file;
300 	vm_flags_t flags = vma->vm_flags;
301 	unsigned long ino = 0;
302 	unsigned long long pgoff = 0;
303 	unsigned long start, end;
304 	dev_t dev = 0;
305 	const char *name = NULL;
306 
307 	if (file) {
308 		struct inode *inode = file_inode(vma->vm_file);
309 		dev = inode->i_sb->s_dev;
310 		ino = inode->i_ino;
311 		pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT;
312 	}
313 
314 	start = vma->vm_start;
315 	end = vma->vm_end;
316 	show_vma_header_prefix(m, start, end, flags, pgoff, dev, ino);
317 
318 	/*
319 	 * Print the dentry name for named mappings, and a
320 	 * special [heap] marker for the heap:
321 	 */
322 	if (file) {
323 		seq_pad(m, ' ');
324 		seq_file_path(m, file, "\n");
325 		goto done;
326 	}
327 
328 	if (vma->vm_ops && vma->vm_ops->name) {
329 		name = vma->vm_ops->name(vma);
330 		if (name)
331 			goto done;
332 	}
333 
334 	name = arch_vma_name(vma);
335 	if (!name) {
336 		if (!mm) {
337 			name = "[vdso]";
338 			goto done;
339 		}
340 
341 		if (vma->vm_start <= mm->brk &&
342 		    vma->vm_end >= mm->start_brk) {
343 			name = "[heap]";
344 			goto done;
345 		}
346 
347 		if (is_stack(vma))
348 			name = "[stack]";
349 	}
350 
351 done:
352 	if (name) {
353 		seq_pad(m, ' ');
354 		seq_puts(m, name);
355 	}
356 	seq_putc(m, '\n');
357 }
358 
359 static int show_map(struct seq_file *m, void *v)
360 {
361 	show_map_vma(m, v);
362 	m_cache_vma(m, v);
363 	return 0;
364 }
365 
366 static const struct seq_operations proc_pid_maps_op = {
367 	.start	= m_start,
368 	.next	= m_next,
369 	.stop	= m_stop,
370 	.show	= show_map
371 };
372 
373 static int pid_maps_open(struct inode *inode, struct file *file)
374 {
375 	return do_maps_open(inode, file, &proc_pid_maps_op);
376 }
377 
378 const struct file_operations proc_pid_maps_operations = {
379 	.open		= pid_maps_open,
380 	.read		= seq_read,
381 	.llseek		= seq_lseek,
382 	.release	= proc_map_release,
383 };
384 
385 /*
386  * Proportional Set Size(PSS): my share of RSS.
387  *
388  * PSS of a process is the count of pages it has in memory, where each
389  * page is divided by the number of processes sharing it.  So if a
390  * process has 1000 pages all to itself, and 1000 shared with one other
391  * process, its PSS will be 1500.
392  *
393  * To keep (accumulated) division errors low, we adopt a 64bit
394  * fixed-point pss counter to minimize division errors. So (pss >>
395  * PSS_SHIFT) would be the real byte count.
396  *
397  * A shift of 12 before division means (assuming 4K page size):
398  * 	- 1M 3-user-pages add up to 8KB errors;
399  * 	- supports mapcount up to 2^24, or 16M;
400  * 	- supports PSS up to 2^52 bytes, or 4PB.
401  */
402 #define PSS_SHIFT 12
403 
404 #ifdef CONFIG_PROC_PAGE_MONITOR
405 struct mem_size_stats {
406 	unsigned long resident;
407 	unsigned long shared_clean;
408 	unsigned long shared_dirty;
409 	unsigned long private_clean;
410 	unsigned long private_dirty;
411 	unsigned long referenced;
412 	unsigned long anonymous;
413 	unsigned long lazyfree;
414 	unsigned long anonymous_thp;
415 	unsigned long shmem_thp;
416 	unsigned long swap;
417 	unsigned long shared_hugetlb;
418 	unsigned long private_hugetlb;
419 	u64 pss;
420 	u64 pss_locked;
421 	u64 swap_pss;
422 	bool check_shmem_swap;
423 };
424 
425 static void smaps_account(struct mem_size_stats *mss, struct page *page,
426 		bool compound, bool young, bool dirty)
427 {
428 	int i, nr = compound ? 1 << compound_order(page) : 1;
429 	unsigned long size = nr * PAGE_SIZE;
430 
431 	if (PageAnon(page)) {
432 		mss->anonymous += size;
433 		if (!PageSwapBacked(page) && !dirty && !PageDirty(page))
434 			mss->lazyfree += size;
435 	}
436 
437 	mss->resident += size;
438 	/* Accumulate the size in pages that have been accessed. */
439 	if (young || page_is_young(page) || PageReferenced(page))
440 		mss->referenced += size;
441 
442 	/*
443 	 * page_count(page) == 1 guarantees the page is mapped exactly once.
444 	 * If any subpage of the compound page mapped with PTE it would elevate
445 	 * page_count().
446 	 */
447 	if (page_count(page) == 1) {
448 		if (dirty || PageDirty(page))
449 			mss->private_dirty += size;
450 		else
451 			mss->private_clean += size;
452 		mss->pss += (u64)size << PSS_SHIFT;
453 		return;
454 	}
455 
456 	for (i = 0; i < nr; i++, page++) {
457 		int mapcount = page_mapcount(page);
458 
459 		if (mapcount >= 2) {
460 			if (dirty || PageDirty(page))
461 				mss->shared_dirty += PAGE_SIZE;
462 			else
463 				mss->shared_clean += PAGE_SIZE;
464 			mss->pss += (PAGE_SIZE << PSS_SHIFT) / mapcount;
465 		} else {
466 			if (dirty || PageDirty(page))
467 				mss->private_dirty += PAGE_SIZE;
468 			else
469 				mss->private_clean += PAGE_SIZE;
470 			mss->pss += PAGE_SIZE << PSS_SHIFT;
471 		}
472 	}
473 }
474 
475 #ifdef CONFIG_SHMEM
476 static int smaps_pte_hole(unsigned long addr, unsigned long end,
477 		struct mm_walk *walk)
478 {
479 	struct mem_size_stats *mss = walk->private;
480 
481 	mss->swap += shmem_partial_swap_usage(
482 			walk->vma->vm_file->f_mapping, addr, end);
483 
484 	return 0;
485 }
486 #endif
487 
488 static void smaps_pte_entry(pte_t *pte, unsigned long addr,
489 		struct mm_walk *walk)
490 {
491 	struct mem_size_stats *mss = walk->private;
492 	struct vm_area_struct *vma = walk->vma;
493 	struct page *page = NULL;
494 
495 	if (pte_present(*pte)) {
496 		page = vm_normal_page(vma, addr, *pte);
497 	} else if (is_swap_pte(*pte)) {
498 		swp_entry_t swpent = pte_to_swp_entry(*pte);
499 
500 		if (!non_swap_entry(swpent)) {
501 			int mapcount;
502 
503 			mss->swap += PAGE_SIZE;
504 			mapcount = swp_swapcount(swpent);
505 			if (mapcount >= 2) {
506 				u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT;
507 
508 				do_div(pss_delta, mapcount);
509 				mss->swap_pss += pss_delta;
510 			} else {
511 				mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT;
512 			}
513 		} else if (is_migration_entry(swpent))
514 			page = migration_entry_to_page(swpent);
515 		else if (is_device_private_entry(swpent))
516 			page = device_private_entry_to_page(swpent);
517 	} else if (unlikely(IS_ENABLED(CONFIG_SHMEM) && mss->check_shmem_swap
518 							&& pte_none(*pte))) {
519 		page = find_get_entry(vma->vm_file->f_mapping,
520 						linear_page_index(vma, addr));
521 		if (!page)
522 			return;
523 
524 		if (radix_tree_exceptional_entry(page))
525 			mss->swap += PAGE_SIZE;
526 		else
527 			put_page(page);
528 
529 		return;
530 	}
531 
532 	if (!page)
533 		return;
534 
535 	smaps_account(mss, page, false, pte_young(*pte), pte_dirty(*pte));
536 }
537 
538 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
539 static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
540 		struct mm_walk *walk)
541 {
542 	struct mem_size_stats *mss = walk->private;
543 	struct vm_area_struct *vma = walk->vma;
544 	struct page *page;
545 
546 	/* FOLL_DUMP will return -EFAULT on huge zero page */
547 	page = follow_trans_huge_pmd(vma, addr, pmd, FOLL_DUMP);
548 	if (IS_ERR_OR_NULL(page))
549 		return;
550 	if (PageAnon(page))
551 		mss->anonymous_thp += HPAGE_PMD_SIZE;
552 	else if (PageSwapBacked(page))
553 		mss->shmem_thp += HPAGE_PMD_SIZE;
554 	else if (is_zone_device_page(page))
555 		/* pass */;
556 	else
557 		VM_BUG_ON_PAGE(1, page);
558 	smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd));
559 }
560 #else
561 static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
562 		struct mm_walk *walk)
563 {
564 }
565 #endif
566 
567 static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
568 			   struct mm_walk *walk)
569 {
570 	struct vm_area_struct *vma = walk->vma;
571 	pte_t *pte;
572 	spinlock_t *ptl;
573 
574 	ptl = pmd_trans_huge_lock(pmd, vma);
575 	if (ptl) {
576 		if (pmd_present(*pmd))
577 			smaps_pmd_entry(pmd, addr, walk);
578 		spin_unlock(ptl);
579 		goto out;
580 	}
581 
582 	if (pmd_trans_unstable(pmd))
583 		goto out;
584 	/*
585 	 * The mmap_sem held all the way back in m_start() is what
586 	 * keeps khugepaged out of here and from collapsing things
587 	 * in here.
588 	 */
589 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
590 	for (; addr != end; pte++, addr += PAGE_SIZE)
591 		smaps_pte_entry(pte, addr, walk);
592 	pte_unmap_unlock(pte - 1, ptl);
593 out:
594 	cond_resched();
595 	return 0;
596 }
597 
598 static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
599 {
600 	/*
601 	 * Don't forget to update Documentation/ on changes.
602 	 */
603 	static const char mnemonics[BITS_PER_LONG][2] = {
604 		/*
605 		 * In case if we meet a flag we don't know about.
606 		 */
607 		[0 ... (BITS_PER_LONG-1)] = "??",
608 
609 		[ilog2(VM_READ)]	= "rd",
610 		[ilog2(VM_WRITE)]	= "wr",
611 		[ilog2(VM_EXEC)]	= "ex",
612 		[ilog2(VM_SHARED)]	= "sh",
613 		[ilog2(VM_MAYREAD)]	= "mr",
614 		[ilog2(VM_MAYWRITE)]	= "mw",
615 		[ilog2(VM_MAYEXEC)]	= "me",
616 		[ilog2(VM_MAYSHARE)]	= "ms",
617 		[ilog2(VM_GROWSDOWN)]	= "gd",
618 		[ilog2(VM_PFNMAP)]	= "pf",
619 		[ilog2(VM_DENYWRITE)]	= "dw",
620 #ifdef CONFIG_X86_INTEL_MPX
621 		[ilog2(VM_MPX)]		= "mp",
622 #endif
623 		[ilog2(VM_LOCKED)]	= "lo",
624 		[ilog2(VM_IO)]		= "io",
625 		[ilog2(VM_SEQ_READ)]	= "sr",
626 		[ilog2(VM_RAND_READ)]	= "rr",
627 		[ilog2(VM_DONTCOPY)]	= "dc",
628 		[ilog2(VM_DONTEXPAND)]	= "de",
629 		[ilog2(VM_ACCOUNT)]	= "ac",
630 		[ilog2(VM_NORESERVE)]	= "nr",
631 		[ilog2(VM_HUGETLB)]	= "ht",
632 		[ilog2(VM_SYNC)]	= "sf",
633 		[ilog2(VM_ARCH_1)]	= "ar",
634 		[ilog2(VM_WIPEONFORK)]	= "wf",
635 		[ilog2(VM_DONTDUMP)]	= "dd",
636 #ifdef CONFIG_MEM_SOFT_DIRTY
637 		[ilog2(VM_SOFTDIRTY)]	= "sd",
638 #endif
639 		[ilog2(VM_MIXEDMAP)]	= "mm",
640 		[ilog2(VM_HUGEPAGE)]	= "hg",
641 		[ilog2(VM_NOHUGEPAGE)]	= "nh",
642 		[ilog2(VM_MERGEABLE)]	= "mg",
643 		[ilog2(VM_UFFD_MISSING)]= "um",
644 		[ilog2(VM_UFFD_WP)]	= "uw",
645 #ifdef CONFIG_ARCH_HAS_PKEYS
646 		/* These come out via ProtectionKey: */
647 		[ilog2(VM_PKEY_BIT0)]	= "",
648 		[ilog2(VM_PKEY_BIT1)]	= "",
649 		[ilog2(VM_PKEY_BIT2)]	= "",
650 		[ilog2(VM_PKEY_BIT3)]	= "",
651 #if VM_PKEY_BIT4
652 		[ilog2(VM_PKEY_BIT4)]	= "",
653 #endif
654 #endif /* CONFIG_ARCH_HAS_PKEYS */
655 	};
656 	size_t i;
657 
658 	seq_puts(m, "VmFlags: ");
659 	for (i = 0; i < BITS_PER_LONG; i++) {
660 		if (!mnemonics[i][0])
661 			continue;
662 		if (vma->vm_flags & (1UL << i)) {
663 			seq_putc(m, mnemonics[i][0]);
664 			seq_putc(m, mnemonics[i][1]);
665 			seq_putc(m, ' ');
666 		}
667 	}
668 	seq_putc(m, '\n');
669 }
670 
671 #ifdef CONFIG_HUGETLB_PAGE
672 static int smaps_hugetlb_range(pte_t *pte, unsigned long hmask,
673 				 unsigned long addr, unsigned long end,
674 				 struct mm_walk *walk)
675 {
676 	struct mem_size_stats *mss = walk->private;
677 	struct vm_area_struct *vma = walk->vma;
678 	struct page *page = NULL;
679 
680 	if (pte_present(*pte)) {
681 		page = vm_normal_page(vma, addr, *pte);
682 	} else if (is_swap_pte(*pte)) {
683 		swp_entry_t swpent = pte_to_swp_entry(*pte);
684 
685 		if (is_migration_entry(swpent))
686 			page = migration_entry_to_page(swpent);
687 		else if (is_device_private_entry(swpent))
688 			page = device_private_entry_to_page(swpent);
689 	}
690 	if (page) {
691 		int mapcount = page_mapcount(page);
692 
693 		if (mapcount >= 2)
694 			mss->shared_hugetlb += huge_page_size(hstate_vma(vma));
695 		else
696 			mss->private_hugetlb += huge_page_size(hstate_vma(vma));
697 	}
698 	return 0;
699 }
700 #endif /* HUGETLB_PAGE */
701 
702 static void smap_gather_stats(struct vm_area_struct *vma,
703 			     struct mem_size_stats *mss)
704 {
705 	struct mm_walk smaps_walk = {
706 		.pmd_entry = smaps_pte_range,
707 #ifdef CONFIG_HUGETLB_PAGE
708 		.hugetlb_entry = smaps_hugetlb_range,
709 #endif
710 		.mm = vma->vm_mm,
711 	};
712 
713 	smaps_walk.private = mss;
714 
715 #ifdef CONFIG_SHMEM
716 	if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) {
717 		/*
718 		 * For shared or readonly shmem mappings we know that all
719 		 * swapped out pages belong to the shmem object, and we can
720 		 * obtain the swap value much more efficiently. For private
721 		 * writable mappings, we might have COW pages that are
722 		 * not affected by the parent swapped out pages of the shmem
723 		 * object, so we have to distinguish them during the page walk.
724 		 * Unless we know that the shmem object (or the part mapped by
725 		 * our VMA) has no swapped out pages at all.
726 		 */
727 		unsigned long shmem_swapped = shmem_swap_usage(vma);
728 
729 		if (!shmem_swapped || (vma->vm_flags & VM_SHARED) ||
730 					!(vma->vm_flags & VM_WRITE)) {
731 			mss->swap = shmem_swapped;
732 		} else {
733 			mss->check_shmem_swap = true;
734 			smaps_walk.pte_hole = smaps_pte_hole;
735 		}
736 	}
737 #endif
738 
739 	/* mmap_sem is held in m_start */
740 	walk_page_vma(vma, &smaps_walk);
741 	if (vma->vm_flags & VM_LOCKED)
742 		mss->pss_locked += mss->pss;
743 }
744 
745 #define SEQ_PUT_DEC(str, val) \
746 		seq_put_decimal_ull_width(m, str, (val) >> 10, 8)
747 
748 /* Show the contents common for smaps and smaps_rollup */
749 static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss)
750 {
751 	SEQ_PUT_DEC("Rss:            ", mss->resident);
752 	SEQ_PUT_DEC(" kB\nPss:            ", mss->pss >> PSS_SHIFT);
753 	SEQ_PUT_DEC(" kB\nShared_Clean:   ", mss->shared_clean);
754 	SEQ_PUT_DEC(" kB\nShared_Dirty:   ", mss->shared_dirty);
755 	SEQ_PUT_DEC(" kB\nPrivate_Clean:  ", mss->private_clean);
756 	SEQ_PUT_DEC(" kB\nPrivate_Dirty:  ", mss->private_dirty);
757 	SEQ_PUT_DEC(" kB\nReferenced:     ", mss->referenced);
758 	SEQ_PUT_DEC(" kB\nAnonymous:      ", mss->anonymous);
759 	SEQ_PUT_DEC(" kB\nLazyFree:       ", mss->lazyfree);
760 	SEQ_PUT_DEC(" kB\nAnonHugePages:  ", mss->anonymous_thp);
761 	SEQ_PUT_DEC(" kB\nShmemPmdMapped: ", mss->shmem_thp);
762 	SEQ_PUT_DEC(" kB\nShared_Hugetlb: ", mss->shared_hugetlb);
763 	seq_put_decimal_ull_width(m, " kB\nPrivate_Hugetlb: ",
764 				  mss->private_hugetlb >> 10, 7);
765 	SEQ_PUT_DEC(" kB\nSwap:           ", mss->swap);
766 	SEQ_PUT_DEC(" kB\nSwapPss:        ",
767 					mss->swap_pss >> PSS_SHIFT);
768 	SEQ_PUT_DEC(" kB\nLocked:         ",
769 					mss->pss_locked >> PSS_SHIFT);
770 	seq_puts(m, " kB\n");
771 }
772 
773 static int show_smap(struct seq_file *m, void *v)
774 {
775 	struct vm_area_struct *vma = v;
776 	struct mem_size_stats mss;
777 
778 	memset(&mss, 0, sizeof(mss));
779 
780 	smap_gather_stats(vma, &mss);
781 
782 	show_map_vma(m, vma);
783 
784 	SEQ_PUT_DEC("Size:           ", vma->vm_end - vma->vm_start);
785 	SEQ_PUT_DEC(" kB\nKernelPageSize: ", vma_kernel_pagesize(vma));
786 	SEQ_PUT_DEC(" kB\nMMUPageSize:    ", vma_mmu_pagesize(vma));
787 	seq_puts(m, " kB\n");
788 
789 	__show_smap(m, &mss);
790 
791 	if (arch_pkeys_enabled())
792 		seq_printf(m, "ProtectionKey:  %8u\n", vma_pkey(vma));
793 	show_smap_vma_flags(m, vma);
794 
795 	m_cache_vma(m, vma);
796 
797 	return 0;
798 }
799 
800 static int show_smaps_rollup(struct seq_file *m, void *v)
801 {
802 	struct proc_maps_private *priv = m->private;
803 	struct mem_size_stats mss;
804 	struct mm_struct *mm;
805 	struct vm_area_struct *vma;
806 	unsigned long last_vma_end = 0;
807 	int ret = 0;
808 
809 	priv->task = get_proc_task(priv->inode);
810 	if (!priv->task)
811 		return -ESRCH;
812 
813 	mm = priv->mm;
814 	if (!mm || !mmget_not_zero(mm)) {
815 		ret = -ESRCH;
816 		goto out_put_task;
817 	}
818 
819 	memset(&mss, 0, sizeof(mss));
820 
821 	down_read(&mm->mmap_sem);
822 	hold_task_mempolicy(priv);
823 
824 	for (vma = priv->mm->mmap; vma; vma = vma->vm_next) {
825 		smap_gather_stats(vma, &mss);
826 		last_vma_end = vma->vm_end;
827 	}
828 
829 	show_vma_header_prefix(m, priv->mm->mmap->vm_start,
830 			       last_vma_end, 0, 0, 0, 0);
831 	seq_pad(m, ' ');
832 	seq_puts(m, "[rollup]\n");
833 
834 	__show_smap(m, &mss);
835 
836 	release_task_mempolicy(priv);
837 	up_read(&mm->mmap_sem);
838 	mmput(mm);
839 
840 out_put_task:
841 	put_task_struct(priv->task);
842 	priv->task = NULL;
843 
844 	return ret;
845 }
846 #undef SEQ_PUT_DEC
847 
848 static const struct seq_operations proc_pid_smaps_op = {
849 	.start	= m_start,
850 	.next	= m_next,
851 	.stop	= m_stop,
852 	.show	= show_smap
853 };
854 
855 static int pid_smaps_open(struct inode *inode, struct file *file)
856 {
857 	return do_maps_open(inode, file, &proc_pid_smaps_op);
858 }
859 
860 static int smaps_rollup_open(struct inode *inode, struct file *file)
861 {
862 	int ret;
863 	struct proc_maps_private *priv;
864 
865 	priv = kzalloc(sizeof(*priv), GFP_KERNEL_ACCOUNT);
866 	if (!priv)
867 		return -ENOMEM;
868 
869 	ret = single_open(file, show_smaps_rollup, priv);
870 	if (ret)
871 		goto out_free;
872 
873 	priv->inode = inode;
874 	priv->mm = proc_mem_open(inode, PTRACE_MODE_READ);
875 	if (IS_ERR(priv->mm)) {
876 		ret = PTR_ERR(priv->mm);
877 
878 		single_release(inode, file);
879 		goto out_free;
880 	}
881 
882 	return 0;
883 
884 out_free:
885 	kfree(priv);
886 	return ret;
887 }
888 
889 static int smaps_rollup_release(struct inode *inode, struct file *file)
890 {
891 	struct seq_file *seq = file->private_data;
892 	struct proc_maps_private *priv = seq->private;
893 
894 	if (priv->mm)
895 		mmdrop(priv->mm);
896 
897 	kfree(priv);
898 	return single_release(inode, file);
899 }
900 
901 const struct file_operations proc_pid_smaps_operations = {
902 	.open		= pid_smaps_open,
903 	.read		= seq_read,
904 	.llseek		= seq_lseek,
905 	.release	= proc_map_release,
906 };
907 
908 const struct file_operations proc_pid_smaps_rollup_operations = {
909 	.open		= smaps_rollup_open,
910 	.read		= seq_read,
911 	.llseek		= seq_lseek,
912 	.release	= smaps_rollup_release,
913 };
914 
915 enum clear_refs_types {
916 	CLEAR_REFS_ALL = 1,
917 	CLEAR_REFS_ANON,
918 	CLEAR_REFS_MAPPED,
919 	CLEAR_REFS_SOFT_DIRTY,
920 	CLEAR_REFS_MM_HIWATER_RSS,
921 	CLEAR_REFS_LAST,
922 };
923 
924 struct clear_refs_private {
925 	enum clear_refs_types type;
926 };
927 
928 #ifdef CONFIG_MEM_SOFT_DIRTY
929 static inline void clear_soft_dirty(struct vm_area_struct *vma,
930 		unsigned long addr, pte_t *pte)
931 {
932 	/*
933 	 * The soft-dirty tracker uses #PF-s to catch writes
934 	 * to pages, so write-protect the pte as well. See the
935 	 * Documentation/admin-guide/mm/soft-dirty.rst for full description
936 	 * of how soft-dirty works.
937 	 */
938 	pte_t ptent = *pte;
939 
940 	if (pte_present(ptent)) {
941 		ptent = ptep_modify_prot_start(vma->vm_mm, addr, pte);
942 		ptent = pte_wrprotect(ptent);
943 		ptent = pte_clear_soft_dirty(ptent);
944 		ptep_modify_prot_commit(vma->vm_mm, addr, pte, ptent);
945 	} else if (is_swap_pte(ptent)) {
946 		ptent = pte_swp_clear_soft_dirty(ptent);
947 		set_pte_at(vma->vm_mm, addr, pte, ptent);
948 	}
949 }
950 #else
951 static inline void clear_soft_dirty(struct vm_area_struct *vma,
952 		unsigned long addr, pte_t *pte)
953 {
954 }
955 #endif
956 
957 #if defined(CONFIG_MEM_SOFT_DIRTY) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
958 static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
959 		unsigned long addr, pmd_t *pmdp)
960 {
961 	pmd_t old, pmd = *pmdp;
962 
963 	if (pmd_present(pmd)) {
964 		/* See comment in change_huge_pmd() */
965 		old = pmdp_invalidate(vma, addr, pmdp);
966 		if (pmd_dirty(old))
967 			pmd = pmd_mkdirty(pmd);
968 		if (pmd_young(old))
969 			pmd = pmd_mkyoung(pmd);
970 
971 		pmd = pmd_wrprotect(pmd);
972 		pmd = pmd_clear_soft_dirty(pmd);
973 
974 		set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
975 	} else if (is_migration_entry(pmd_to_swp_entry(pmd))) {
976 		pmd = pmd_swp_clear_soft_dirty(pmd);
977 		set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
978 	}
979 }
980 #else
981 static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
982 		unsigned long addr, pmd_t *pmdp)
983 {
984 }
985 #endif
986 
987 static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
988 				unsigned long end, struct mm_walk *walk)
989 {
990 	struct clear_refs_private *cp = walk->private;
991 	struct vm_area_struct *vma = walk->vma;
992 	pte_t *pte, ptent;
993 	spinlock_t *ptl;
994 	struct page *page;
995 
996 	ptl = pmd_trans_huge_lock(pmd, vma);
997 	if (ptl) {
998 		if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
999 			clear_soft_dirty_pmd(vma, addr, pmd);
1000 			goto out;
1001 		}
1002 
1003 		if (!pmd_present(*pmd))
1004 			goto out;
1005 
1006 		page = pmd_page(*pmd);
1007 
1008 		/* Clear accessed and referenced bits. */
1009 		pmdp_test_and_clear_young(vma, addr, pmd);
1010 		test_and_clear_page_young(page);
1011 		ClearPageReferenced(page);
1012 out:
1013 		spin_unlock(ptl);
1014 		return 0;
1015 	}
1016 
1017 	if (pmd_trans_unstable(pmd))
1018 		return 0;
1019 
1020 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1021 	for (; addr != end; pte++, addr += PAGE_SIZE) {
1022 		ptent = *pte;
1023 
1024 		if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
1025 			clear_soft_dirty(vma, addr, pte);
1026 			continue;
1027 		}
1028 
1029 		if (!pte_present(ptent))
1030 			continue;
1031 
1032 		page = vm_normal_page(vma, addr, ptent);
1033 		if (!page)
1034 			continue;
1035 
1036 		/* Clear accessed and referenced bits. */
1037 		ptep_test_and_clear_young(vma, addr, pte);
1038 		test_and_clear_page_young(page);
1039 		ClearPageReferenced(page);
1040 	}
1041 	pte_unmap_unlock(pte - 1, ptl);
1042 	cond_resched();
1043 	return 0;
1044 }
1045 
1046 static int clear_refs_test_walk(unsigned long start, unsigned long end,
1047 				struct mm_walk *walk)
1048 {
1049 	struct clear_refs_private *cp = walk->private;
1050 	struct vm_area_struct *vma = walk->vma;
1051 
1052 	if (vma->vm_flags & VM_PFNMAP)
1053 		return 1;
1054 
1055 	/*
1056 	 * Writing 1 to /proc/pid/clear_refs affects all pages.
1057 	 * Writing 2 to /proc/pid/clear_refs only affects anonymous pages.
1058 	 * Writing 3 to /proc/pid/clear_refs only affects file mapped pages.
1059 	 * Writing 4 to /proc/pid/clear_refs affects all pages.
1060 	 */
1061 	if (cp->type == CLEAR_REFS_ANON && vma->vm_file)
1062 		return 1;
1063 	if (cp->type == CLEAR_REFS_MAPPED && !vma->vm_file)
1064 		return 1;
1065 	return 0;
1066 }
1067 
1068 static ssize_t clear_refs_write(struct file *file, const char __user *buf,
1069 				size_t count, loff_t *ppos)
1070 {
1071 	struct task_struct *task;
1072 	char buffer[PROC_NUMBUF];
1073 	struct mm_struct *mm;
1074 	struct vm_area_struct *vma;
1075 	enum clear_refs_types type;
1076 	struct mmu_gather tlb;
1077 	int itype;
1078 	int rv;
1079 
1080 	memset(buffer, 0, sizeof(buffer));
1081 	if (count > sizeof(buffer) - 1)
1082 		count = sizeof(buffer) - 1;
1083 	if (copy_from_user(buffer, buf, count))
1084 		return -EFAULT;
1085 	rv = kstrtoint(strstrip(buffer), 10, &itype);
1086 	if (rv < 0)
1087 		return rv;
1088 	type = (enum clear_refs_types)itype;
1089 	if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST)
1090 		return -EINVAL;
1091 
1092 	task = get_proc_task(file_inode(file));
1093 	if (!task)
1094 		return -ESRCH;
1095 	mm = get_task_mm(task);
1096 	if (mm) {
1097 		struct clear_refs_private cp = {
1098 			.type = type,
1099 		};
1100 		struct mm_walk clear_refs_walk = {
1101 			.pmd_entry = clear_refs_pte_range,
1102 			.test_walk = clear_refs_test_walk,
1103 			.mm = mm,
1104 			.private = &cp,
1105 		};
1106 
1107 		if (type == CLEAR_REFS_MM_HIWATER_RSS) {
1108 			if (down_write_killable(&mm->mmap_sem)) {
1109 				count = -EINTR;
1110 				goto out_mm;
1111 			}
1112 
1113 			/*
1114 			 * Writing 5 to /proc/pid/clear_refs resets the peak
1115 			 * resident set size to this mm's current rss value.
1116 			 */
1117 			reset_mm_hiwater_rss(mm);
1118 			up_write(&mm->mmap_sem);
1119 			goto out_mm;
1120 		}
1121 
1122 		down_read(&mm->mmap_sem);
1123 		tlb_gather_mmu(&tlb, mm, 0, -1);
1124 		if (type == CLEAR_REFS_SOFT_DIRTY) {
1125 			for (vma = mm->mmap; vma; vma = vma->vm_next) {
1126 				if (!(vma->vm_flags & VM_SOFTDIRTY))
1127 					continue;
1128 				up_read(&mm->mmap_sem);
1129 				if (down_write_killable(&mm->mmap_sem)) {
1130 					count = -EINTR;
1131 					goto out_mm;
1132 				}
1133 				for (vma = mm->mmap; vma; vma = vma->vm_next) {
1134 					vma->vm_flags &= ~VM_SOFTDIRTY;
1135 					vma_set_page_prot(vma);
1136 				}
1137 				downgrade_write(&mm->mmap_sem);
1138 				break;
1139 			}
1140 			mmu_notifier_invalidate_range_start(mm, 0, -1);
1141 		}
1142 		walk_page_range(0, mm->highest_vm_end, &clear_refs_walk);
1143 		if (type == CLEAR_REFS_SOFT_DIRTY)
1144 			mmu_notifier_invalidate_range_end(mm, 0, -1);
1145 		tlb_finish_mmu(&tlb, 0, -1);
1146 		up_read(&mm->mmap_sem);
1147 out_mm:
1148 		mmput(mm);
1149 	}
1150 	put_task_struct(task);
1151 
1152 	return count;
1153 }
1154 
1155 const struct file_operations proc_clear_refs_operations = {
1156 	.write		= clear_refs_write,
1157 	.llseek		= noop_llseek,
1158 };
1159 
1160 typedef struct {
1161 	u64 pme;
1162 } pagemap_entry_t;
1163 
1164 struct pagemapread {
1165 	int pos, len;		/* units: PM_ENTRY_BYTES, not bytes */
1166 	pagemap_entry_t *buffer;
1167 	bool show_pfn;
1168 };
1169 
1170 #define PAGEMAP_WALK_SIZE	(PMD_SIZE)
1171 #define PAGEMAP_WALK_MASK	(PMD_MASK)
1172 
1173 #define PM_ENTRY_BYTES		sizeof(pagemap_entry_t)
1174 #define PM_PFRAME_BITS		55
1175 #define PM_PFRAME_MASK		GENMASK_ULL(PM_PFRAME_BITS - 1, 0)
1176 #define PM_SOFT_DIRTY		BIT_ULL(55)
1177 #define PM_MMAP_EXCLUSIVE	BIT_ULL(56)
1178 #define PM_FILE			BIT_ULL(61)
1179 #define PM_SWAP			BIT_ULL(62)
1180 #define PM_PRESENT		BIT_ULL(63)
1181 
1182 #define PM_END_OF_BUFFER    1
1183 
1184 static inline pagemap_entry_t make_pme(u64 frame, u64 flags)
1185 {
1186 	return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags };
1187 }
1188 
1189 static int add_to_pagemap(unsigned long addr, pagemap_entry_t *pme,
1190 			  struct pagemapread *pm)
1191 {
1192 	pm->buffer[pm->pos++] = *pme;
1193 	if (pm->pos >= pm->len)
1194 		return PM_END_OF_BUFFER;
1195 	return 0;
1196 }
1197 
1198 static int pagemap_pte_hole(unsigned long start, unsigned long end,
1199 				struct mm_walk *walk)
1200 {
1201 	struct pagemapread *pm = walk->private;
1202 	unsigned long addr = start;
1203 	int err = 0;
1204 
1205 	while (addr < end) {
1206 		struct vm_area_struct *vma = find_vma(walk->mm, addr);
1207 		pagemap_entry_t pme = make_pme(0, 0);
1208 		/* End of address space hole, which we mark as non-present. */
1209 		unsigned long hole_end;
1210 
1211 		if (vma)
1212 			hole_end = min(end, vma->vm_start);
1213 		else
1214 			hole_end = end;
1215 
1216 		for (; addr < hole_end; addr += PAGE_SIZE) {
1217 			err = add_to_pagemap(addr, &pme, pm);
1218 			if (err)
1219 				goto out;
1220 		}
1221 
1222 		if (!vma)
1223 			break;
1224 
1225 		/* Addresses in the VMA. */
1226 		if (vma->vm_flags & VM_SOFTDIRTY)
1227 			pme = make_pme(0, PM_SOFT_DIRTY);
1228 		for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) {
1229 			err = add_to_pagemap(addr, &pme, pm);
1230 			if (err)
1231 				goto out;
1232 		}
1233 	}
1234 out:
1235 	return err;
1236 }
1237 
1238 static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm,
1239 		struct vm_area_struct *vma, unsigned long addr, pte_t pte)
1240 {
1241 	u64 frame = 0, flags = 0;
1242 	struct page *page = NULL;
1243 
1244 	if (pte_present(pte)) {
1245 		if (pm->show_pfn)
1246 			frame = pte_pfn(pte);
1247 		flags |= PM_PRESENT;
1248 		page = _vm_normal_page(vma, addr, pte, true);
1249 		if (pte_soft_dirty(pte))
1250 			flags |= PM_SOFT_DIRTY;
1251 	} else if (is_swap_pte(pte)) {
1252 		swp_entry_t entry;
1253 		if (pte_swp_soft_dirty(pte))
1254 			flags |= PM_SOFT_DIRTY;
1255 		entry = pte_to_swp_entry(pte);
1256 		if (pm->show_pfn)
1257 			frame = swp_type(entry) |
1258 				(swp_offset(entry) << MAX_SWAPFILES_SHIFT);
1259 		flags |= PM_SWAP;
1260 		if (is_migration_entry(entry))
1261 			page = migration_entry_to_page(entry);
1262 
1263 		if (is_device_private_entry(entry))
1264 			page = device_private_entry_to_page(entry);
1265 	}
1266 
1267 	if (page && !PageAnon(page))
1268 		flags |= PM_FILE;
1269 	if (page && page_mapcount(page) == 1)
1270 		flags |= PM_MMAP_EXCLUSIVE;
1271 	if (vma->vm_flags & VM_SOFTDIRTY)
1272 		flags |= PM_SOFT_DIRTY;
1273 
1274 	return make_pme(frame, flags);
1275 }
1276 
1277 static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end,
1278 			     struct mm_walk *walk)
1279 {
1280 	struct vm_area_struct *vma = walk->vma;
1281 	struct pagemapread *pm = walk->private;
1282 	spinlock_t *ptl;
1283 	pte_t *pte, *orig_pte;
1284 	int err = 0;
1285 
1286 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1287 	ptl = pmd_trans_huge_lock(pmdp, vma);
1288 	if (ptl) {
1289 		u64 flags = 0, frame = 0;
1290 		pmd_t pmd = *pmdp;
1291 		struct page *page = NULL;
1292 
1293 		if (vma->vm_flags & VM_SOFTDIRTY)
1294 			flags |= PM_SOFT_DIRTY;
1295 
1296 		if (pmd_present(pmd)) {
1297 			page = pmd_page(pmd);
1298 
1299 			flags |= PM_PRESENT;
1300 			if (pmd_soft_dirty(pmd))
1301 				flags |= PM_SOFT_DIRTY;
1302 			if (pm->show_pfn)
1303 				frame = pmd_pfn(pmd) +
1304 					((addr & ~PMD_MASK) >> PAGE_SHIFT);
1305 		}
1306 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1307 		else if (is_swap_pmd(pmd)) {
1308 			swp_entry_t entry = pmd_to_swp_entry(pmd);
1309 			unsigned long offset;
1310 
1311 			if (pm->show_pfn) {
1312 				offset = swp_offset(entry) +
1313 					((addr & ~PMD_MASK) >> PAGE_SHIFT);
1314 				frame = swp_type(entry) |
1315 					(offset << MAX_SWAPFILES_SHIFT);
1316 			}
1317 			flags |= PM_SWAP;
1318 			if (pmd_swp_soft_dirty(pmd))
1319 				flags |= PM_SOFT_DIRTY;
1320 			VM_BUG_ON(!is_pmd_migration_entry(pmd));
1321 			page = migration_entry_to_page(entry);
1322 		}
1323 #endif
1324 
1325 		if (page && page_mapcount(page) == 1)
1326 			flags |= PM_MMAP_EXCLUSIVE;
1327 
1328 		for (; addr != end; addr += PAGE_SIZE) {
1329 			pagemap_entry_t pme = make_pme(frame, flags);
1330 
1331 			err = add_to_pagemap(addr, &pme, pm);
1332 			if (err)
1333 				break;
1334 			if (pm->show_pfn) {
1335 				if (flags & PM_PRESENT)
1336 					frame++;
1337 				else if (flags & PM_SWAP)
1338 					frame += (1 << MAX_SWAPFILES_SHIFT);
1339 			}
1340 		}
1341 		spin_unlock(ptl);
1342 		return err;
1343 	}
1344 
1345 	if (pmd_trans_unstable(pmdp))
1346 		return 0;
1347 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1348 
1349 	/*
1350 	 * We can assume that @vma always points to a valid one and @end never
1351 	 * goes beyond vma->vm_end.
1352 	 */
1353 	orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl);
1354 	for (; addr < end; pte++, addr += PAGE_SIZE) {
1355 		pagemap_entry_t pme;
1356 
1357 		pme = pte_to_pagemap_entry(pm, vma, addr, *pte);
1358 		err = add_to_pagemap(addr, &pme, pm);
1359 		if (err)
1360 			break;
1361 	}
1362 	pte_unmap_unlock(orig_pte, ptl);
1363 
1364 	cond_resched();
1365 
1366 	return err;
1367 }
1368 
1369 #ifdef CONFIG_HUGETLB_PAGE
1370 /* This function walks within one hugetlb entry in the single call */
1371 static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask,
1372 				 unsigned long addr, unsigned long end,
1373 				 struct mm_walk *walk)
1374 {
1375 	struct pagemapread *pm = walk->private;
1376 	struct vm_area_struct *vma = walk->vma;
1377 	u64 flags = 0, frame = 0;
1378 	int err = 0;
1379 	pte_t pte;
1380 
1381 	if (vma->vm_flags & VM_SOFTDIRTY)
1382 		flags |= PM_SOFT_DIRTY;
1383 
1384 	pte = huge_ptep_get(ptep);
1385 	if (pte_present(pte)) {
1386 		struct page *page = pte_page(pte);
1387 
1388 		if (!PageAnon(page))
1389 			flags |= PM_FILE;
1390 
1391 		if (page_mapcount(page) == 1)
1392 			flags |= PM_MMAP_EXCLUSIVE;
1393 
1394 		flags |= PM_PRESENT;
1395 		if (pm->show_pfn)
1396 			frame = pte_pfn(pte) +
1397 				((addr & ~hmask) >> PAGE_SHIFT);
1398 	}
1399 
1400 	for (; addr != end; addr += PAGE_SIZE) {
1401 		pagemap_entry_t pme = make_pme(frame, flags);
1402 
1403 		err = add_to_pagemap(addr, &pme, pm);
1404 		if (err)
1405 			return err;
1406 		if (pm->show_pfn && (flags & PM_PRESENT))
1407 			frame++;
1408 	}
1409 
1410 	cond_resched();
1411 
1412 	return err;
1413 }
1414 #endif /* HUGETLB_PAGE */
1415 
1416 /*
1417  * /proc/pid/pagemap - an array mapping virtual pages to pfns
1418  *
1419  * For each page in the address space, this file contains one 64-bit entry
1420  * consisting of the following:
1421  *
1422  * Bits 0-54  page frame number (PFN) if present
1423  * Bits 0-4   swap type if swapped
1424  * Bits 5-54  swap offset if swapped
1425  * Bit  55    pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst)
1426  * Bit  56    page exclusively mapped
1427  * Bits 57-60 zero
1428  * Bit  61    page is file-page or shared-anon
1429  * Bit  62    page swapped
1430  * Bit  63    page present
1431  *
1432  * If the page is not present but in swap, then the PFN contains an
1433  * encoding of the swap file number and the page's offset into the
1434  * swap. Unmapped pages return a null PFN. This allows determining
1435  * precisely which pages are mapped (or in swap) and comparing mapped
1436  * pages between processes.
1437  *
1438  * Efficient users of this interface will use /proc/pid/maps to
1439  * determine which areas of memory are actually mapped and llseek to
1440  * skip over unmapped regions.
1441  */
1442 static ssize_t pagemap_read(struct file *file, char __user *buf,
1443 			    size_t count, loff_t *ppos)
1444 {
1445 	struct mm_struct *mm = file->private_data;
1446 	struct pagemapread pm;
1447 	struct mm_walk pagemap_walk = {};
1448 	unsigned long src;
1449 	unsigned long svpfn;
1450 	unsigned long start_vaddr;
1451 	unsigned long end_vaddr;
1452 	int ret = 0, copied = 0;
1453 
1454 	if (!mm || !mmget_not_zero(mm))
1455 		goto out;
1456 
1457 	ret = -EINVAL;
1458 	/* file position must be aligned */
1459 	if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES))
1460 		goto out_mm;
1461 
1462 	ret = 0;
1463 	if (!count)
1464 		goto out_mm;
1465 
1466 	/* do not disclose physical addresses: attack vector */
1467 	pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN);
1468 
1469 	pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT);
1470 	pm.buffer = kmalloc_array(pm.len, PM_ENTRY_BYTES, GFP_KERNEL);
1471 	ret = -ENOMEM;
1472 	if (!pm.buffer)
1473 		goto out_mm;
1474 
1475 	pagemap_walk.pmd_entry = pagemap_pmd_range;
1476 	pagemap_walk.pte_hole = pagemap_pte_hole;
1477 #ifdef CONFIG_HUGETLB_PAGE
1478 	pagemap_walk.hugetlb_entry = pagemap_hugetlb_range;
1479 #endif
1480 	pagemap_walk.mm = mm;
1481 	pagemap_walk.private = &pm;
1482 
1483 	src = *ppos;
1484 	svpfn = src / PM_ENTRY_BYTES;
1485 	start_vaddr = svpfn << PAGE_SHIFT;
1486 	end_vaddr = mm->task_size;
1487 
1488 	/* watch out for wraparound */
1489 	if (svpfn > mm->task_size >> PAGE_SHIFT)
1490 		start_vaddr = end_vaddr;
1491 
1492 	/*
1493 	 * The odds are that this will stop walking way
1494 	 * before end_vaddr, because the length of the
1495 	 * user buffer is tracked in "pm", and the walk
1496 	 * will stop when we hit the end of the buffer.
1497 	 */
1498 	ret = 0;
1499 	while (count && (start_vaddr < end_vaddr)) {
1500 		int len;
1501 		unsigned long end;
1502 
1503 		pm.pos = 0;
1504 		end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK;
1505 		/* overflow ? */
1506 		if (end < start_vaddr || end > end_vaddr)
1507 			end = end_vaddr;
1508 		down_read(&mm->mmap_sem);
1509 		ret = walk_page_range(start_vaddr, end, &pagemap_walk);
1510 		up_read(&mm->mmap_sem);
1511 		start_vaddr = end;
1512 
1513 		len = min(count, PM_ENTRY_BYTES * pm.pos);
1514 		if (copy_to_user(buf, pm.buffer, len)) {
1515 			ret = -EFAULT;
1516 			goto out_free;
1517 		}
1518 		copied += len;
1519 		buf += len;
1520 		count -= len;
1521 	}
1522 	*ppos += copied;
1523 	if (!ret || ret == PM_END_OF_BUFFER)
1524 		ret = copied;
1525 
1526 out_free:
1527 	kfree(pm.buffer);
1528 out_mm:
1529 	mmput(mm);
1530 out:
1531 	return ret;
1532 }
1533 
1534 static int pagemap_open(struct inode *inode, struct file *file)
1535 {
1536 	struct mm_struct *mm;
1537 
1538 	mm = proc_mem_open(inode, PTRACE_MODE_READ);
1539 	if (IS_ERR(mm))
1540 		return PTR_ERR(mm);
1541 	file->private_data = mm;
1542 	return 0;
1543 }
1544 
1545 static int pagemap_release(struct inode *inode, struct file *file)
1546 {
1547 	struct mm_struct *mm = file->private_data;
1548 
1549 	if (mm)
1550 		mmdrop(mm);
1551 	return 0;
1552 }
1553 
1554 const struct file_operations proc_pagemap_operations = {
1555 	.llseek		= mem_lseek, /* borrow this */
1556 	.read		= pagemap_read,
1557 	.open		= pagemap_open,
1558 	.release	= pagemap_release,
1559 };
1560 #endif /* CONFIG_PROC_PAGE_MONITOR */
1561 
1562 #ifdef CONFIG_NUMA
1563 
1564 struct numa_maps {
1565 	unsigned long pages;
1566 	unsigned long anon;
1567 	unsigned long active;
1568 	unsigned long writeback;
1569 	unsigned long mapcount_max;
1570 	unsigned long dirty;
1571 	unsigned long swapcache;
1572 	unsigned long node[MAX_NUMNODES];
1573 };
1574 
1575 struct numa_maps_private {
1576 	struct proc_maps_private proc_maps;
1577 	struct numa_maps md;
1578 };
1579 
1580 static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty,
1581 			unsigned long nr_pages)
1582 {
1583 	int count = page_mapcount(page);
1584 
1585 	md->pages += nr_pages;
1586 	if (pte_dirty || PageDirty(page))
1587 		md->dirty += nr_pages;
1588 
1589 	if (PageSwapCache(page))
1590 		md->swapcache += nr_pages;
1591 
1592 	if (PageActive(page) || PageUnevictable(page))
1593 		md->active += nr_pages;
1594 
1595 	if (PageWriteback(page))
1596 		md->writeback += nr_pages;
1597 
1598 	if (PageAnon(page))
1599 		md->anon += nr_pages;
1600 
1601 	if (count > md->mapcount_max)
1602 		md->mapcount_max = count;
1603 
1604 	md->node[page_to_nid(page)] += nr_pages;
1605 }
1606 
1607 static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma,
1608 		unsigned long addr)
1609 {
1610 	struct page *page;
1611 	int nid;
1612 
1613 	if (!pte_present(pte))
1614 		return NULL;
1615 
1616 	page = vm_normal_page(vma, addr, pte);
1617 	if (!page)
1618 		return NULL;
1619 
1620 	if (PageReserved(page))
1621 		return NULL;
1622 
1623 	nid = page_to_nid(page);
1624 	if (!node_isset(nid, node_states[N_MEMORY]))
1625 		return NULL;
1626 
1627 	return page;
1628 }
1629 
1630 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1631 static struct page *can_gather_numa_stats_pmd(pmd_t pmd,
1632 					      struct vm_area_struct *vma,
1633 					      unsigned long addr)
1634 {
1635 	struct page *page;
1636 	int nid;
1637 
1638 	if (!pmd_present(pmd))
1639 		return NULL;
1640 
1641 	page = vm_normal_page_pmd(vma, addr, pmd);
1642 	if (!page)
1643 		return NULL;
1644 
1645 	if (PageReserved(page))
1646 		return NULL;
1647 
1648 	nid = page_to_nid(page);
1649 	if (!node_isset(nid, node_states[N_MEMORY]))
1650 		return NULL;
1651 
1652 	return page;
1653 }
1654 #endif
1655 
1656 static int gather_pte_stats(pmd_t *pmd, unsigned long addr,
1657 		unsigned long end, struct mm_walk *walk)
1658 {
1659 	struct numa_maps *md = walk->private;
1660 	struct vm_area_struct *vma = walk->vma;
1661 	spinlock_t *ptl;
1662 	pte_t *orig_pte;
1663 	pte_t *pte;
1664 
1665 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1666 	ptl = pmd_trans_huge_lock(pmd, vma);
1667 	if (ptl) {
1668 		struct page *page;
1669 
1670 		page = can_gather_numa_stats_pmd(*pmd, vma, addr);
1671 		if (page)
1672 			gather_stats(page, md, pmd_dirty(*pmd),
1673 				     HPAGE_PMD_SIZE/PAGE_SIZE);
1674 		spin_unlock(ptl);
1675 		return 0;
1676 	}
1677 
1678 	if (pmd_trans_unstable(pmd))
1679 		return 0;
1680 #endif
1681 	orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
1682 	do {
1683 		struct page *page = can_gather_numa_stats(*pte, vma, addr);
1684 		if (!page)
1685 			continue;
1686 		gather_stats(page, md, pte_dirty(*pte), 1);
1687 
1688 	} while (pte++, addr += PAGE_SIZE, addr != end);
1689 	pte_unmap_unlock(orig_pte, ptl);
1690 	cond_resched();
1691 	return 0;
1692 }
1693 #ifdef CONFIG_HUGETLB_PAGE
1694 static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
1695 		unsigned long addr, unsigned long end, struct mm_walk *walk)
1696 {
1697 	pte_t huge_pte = huge_ptep_get(pte);
1698 	struct numa_maps *md;
1699 	struct page *page;
1700 
1701 	if (!pte_present(huge_pte))
1702 		return 0;
1703 
1704 	page = pte_page(huge_pte);
1705 	if (!page)
1706 		return 0;
1707 
1708 	md = walk->private;
1709 	gather_stats(page, md, pte_dirty(huge_pte), 1);
1710 	return 0;
1711 }
1712 
1713 #else
1714 static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
1715 		unsigned long addr, unsigned long end, struct mm_walk *walk)
1716 {
1717 	return 0;
1718 }
1719 #endif
1720 
1721 /*
1722  * Display pages allocated per node and memory policy via /proc.
1723  */
1724 static int show_numa_map(struct seq_file *m, void *v)
1725 {
1726 	struct numa_maps_private *numa_priv = m->private;
1727 	struct proc_maps_private *proc_priv = &numa_priv->proc_maps;
1728 	struct vm_area_struct *vma = v;
1729 	struct numa_maps *md = &numa_priv->md;
1730 	struct file *file = vma->vm_file;
1731 	struct mm_struct *mm = vma->vm_mm;
1732 	struct mm_walk walk = {
1733 		.hugetlb_entry = gather_hugetlb_stats,
1734 		.pmd_entry = gather_pte_stats,
1735 		.private = md,
1736 		.mm = mm,
1737 	};
1738 	struct mempolicy *pol;
1739 	char buffer[64];
1740 	int nid;
1741 
1742 	if (!mm)
1743 		return 0;
1744 
1745 	/* Ensure we start with an empty set of numa_maps statistics. */
1746 	memset(md, 0, sizeof(*md));
1747 
1748 	pol = __get_vma_policy(vma, vma->vm_start);
1749 	if (pol) {
1750 		mpol_to_str(buffer, sizeof(buffer), pol);
1751 		mpol_cond_put(pol);
1752 	} else {
1753 		mpol_to_str(buffer, sizeof(buffer), proc_priv->task_mempolicy);
1754 	}
1755 
1756 	seq_printf(m, "%08lx %s", vma->vm_start, buffer);
1757 
1758 	if (file) {
1759 		seq_puts(m, " file=");
1760 		seq_file_path(m, file, "\n\t= ");
1761 	} else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
1762 		seq_puts(m, " heap");
1763 	} else if (is_stack(vma)) {
1764 		seq_puts(m, " stack");
1765 	}
1766 
1767 	if (is_vm_hugetlb_page(vma))
1768 		seq_puts(m, " huge");
1769 
1770 	/* mmap_sem is held by m_start */
1771 	walk_page_vma(vma, &walk);
1772 
1773 	if (!md->pages)
1774 		goto out;
1775 
1776 	if (md->anon)
1777 		seq_printf(m, " anon=%lu", md->anon);
1778 
1779 	if (md->dirty)
1780 		seq_printf(m, " dirty=%lu", md->dirty);
1781 
1782 	if (md->pages != md->anon && md->pages != md->dirty)
1783 		seq_printf(m, " mapped=%lu", md->pages);
1784 
1785 	if (md->mapcount_max > 1)
1786 		seq_printf(m, " mapmax=%lu", md->mapcount_max);
1787 
1788 	if (md->swapcache)
1789 		seq_printf(m, " swapcache=%lu", md->swapcache);
1790 
1791 	if (md->active < md->pages && !is_vm_hugetlb_page(vma))
1792 		seq_printf(m, " active=%lu", md->active);
1793 
1794 	if (md->writeback)
1795 		seq_printf(m, " writeback=%lu", md->writeback);
1796 
1797 	for_each_node_state(nid, N_MEMORY)
1798 		if (md->node[nid])
1799 			seq_printf(m, " N%d=%lu", nid, md->node[nid]);
1800 
1801 	seq_printf(m, " kernelpagesize_kB=%lu", vma_kernel_pagesize(vma) >> 10);
1802 out:
1803 	seq_putc(m, '\n');
1804 	m_cache_vma(m, vma);
1805 	return 0;
1806 }
1807 
1808 static const struct seq_operations proc_pid_numa_maps_op = {
1809 	.start  = m_start,
1810 	.next   = m_next,
1811 	.stop   = m_stop,
1812 	.show   = show_numa_map,
1813 };
1814 
1815 static int pid_numa_maps_open(struct inode *inode, struct file *file)
1816 {
1817 	return proc_maps_open(inode, file, &proc_pid_numa_maps_op,
1818 				sizeof(struct numa_maps_private));
1819 }
1820 
1821 const struct file_operations proc_pid_numa_maps_operations = {
1822 	.open		= pid_numa_maps_open,
1823 	.read		= seq_read,
1824 	.llseek		= seq_lseek,
1825 	.release	= proc_map_release,
1826 };
1827 
1828 #endif /* CONFIG_NUMA */
1829