xref: /openbmc/linux/fs/proc/task_mmu.c (revision ff4a7481c3898ffc3cc271d6aca431d190c37247)
1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/mm.h>
3 #include <linux/vmacache.h>
4 #include <linux/hugetlb.h>
5 #include <linux/huge_mm.h>
6 #include <linux/mount.h>
7 #include <linux/seq_file.h>
8 #include <linux/highmem.h>
9 #include <linux/ptrace.h>
10 #include <linux/slab.h>
11 #include <linux/pagemap.h>
12 #include <linux/mempolicy.h>
13 #include <linux/rmap.h>
14 #include <linux/swap.h>
15 #include <linux/sched/mm.h>
16 #include <linux/swapops.h>
17 #include <linux/mmu_notifier.h>
18 #include <linux/page_idle.h>
19 #include <linux/shmem_fs.h>
20 #include <linux/uaccess.h>
21 #include <linux/pkeys.h>
22 
23 #include <asm/elf.h>
24 #include <asm/tlb.h>
25 #include <asm/tlbflush.h>
26 #include "internal.h"
27 
28 #define SEQ_PUT_DEC(str, val) \
29 		seq_put_decimal_ull_width(m, str, (val) << (PAGE_SHIFT-10), 8)
30 void task_mem(struct seq_file *m, struct mm_struct *mm)
31 {
32 	unsigned long text, lib, swap, anon, file, shmem;
33 	unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss;
34 
35 	anon = get_mm_counter(mm, MM_ANONPAGES);
36 	file = get_mm_counter(mm, MM_FILEPAGES);
37 	shmem = get_mm_counter(mm, MM_SHMEMPAGES);
38 
39 	/*
40 	 * Note: to minimize their overhead, mm maintains hiwater_vm and
41 	 * hiwater_rss only when about to *lower* total_vm or rss.  Any
42 	 * collector of these hiwater stats must therefore get total_vm
43 	 * and rss too, which will usually be the higher.  Barriers? not
44 	 * worth the effort, such snapshots can always be inconsistent.
45 	 */
46 	hiwater_vm = total_vm = mm->total_vm;
47 	if (hiwater_vm < mm->hiwater_vm)
48 		hiwater_vm = mm->hiwater_vm;
49 	hiwater_rss = total_rss = anon + file + shmem;
50 	if (hiwater_rss < mm->hiwater_rss)
51 		hiwater_rss = mm->hiwater_rss;
52 
53 	/* split executable areas between text and lib */
54 	text = PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK);
55 	text = min(text, mm->exec_vm << PAGE_SHIFT);
56 	lib = (mm->exec_vm << PAGE_SHIFT) - text;
57 
58 	swap = get_mm_counter(mm, MM_SWAPENTS);
59 	SEQ_PUT_DEC("VmPeak:\t", hiwater_vm);
60 	SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm);
61 	SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm);
62 	SEQ_PUT_DEC(" kB\nVmPin:\t", mm->pinned_vm);
63 	SEQ_PUT_DEC(" kB\nVmHWM:\t", hiwater_rss);
64 	SEQ_PUT_DEC(" kB\nVmRSS:\t", total_rss);
65 	SEQ_PUT_DEC(" kB\nRssAnon:\t", anon);
66 	SEQ_PUT_DEC(" kB\nRssFile:\t", file);
67 	SEQ_PUT_DEC(" kB\nRssShmem:\t", shmem);
68 	SEQ_PUT_DEC(" kB\nVmData:\t", mm->data_vm);
69 	SEQ_PUT_DEC(" kB\nVmStk:\t", mm->stack_vm);
70 	seq_put_decimal_ull_width(m,
71 		    " kB\nVmExe:\t", text >> 10, 8);
72 	seq_put_decimal_ull_width(m,
73 		    " kB\nVmLib:\t", lib >> 10, 8);
74 	seq_put_decimal_ull_width(m,
75 		    " kB\nVmPTE:\t", mm_pgtables_bytes(mm) >> 10, 8);
76 	SEQ_PUT_DEC(" kB\nVmSwap:\t", swap);
77 	seq_puts(m, " kB\n");
78 	hugetlb_report_usage(m, mm);
79 }
80 #undef SEQ_PUT_DEC
81 
82 unsigned long task_vsize(struct mm_struct *mm)
83 {
84 	return PAGE_SIZE * mm->total_vm;
85 }
86 
87 unsigned long task_statm(struct mm_struct *mm,
88 			 unsigned long *shared, unsigned long *text,
89 			 unsigned long *data, unsigned long *resident)
90 {
91 	*shared = get_mm_counter(mm, MM_FILEPAGES) +
92 			get_mm_counter(mm, MM_SHMEMPAGES);
93 	*text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))
94 								>> PAGE_SHIFT;
95 	*data = mm->data_vm + mm->stack_vm;
96 	*resident = *shared + get_mm_counter(mm, MM_ANONPAGES);
97 	return mm->total_vm;
98 }
99 
100 #ifdef CONFIG_NUMA
101 /*
102  * Save get_task_policy() for show_numa_map().
103  */
104 static void hold_task_mempolicy(struct proc_maps_private *priv)
105 {
106 	struct task_struct *task = priv->task;
107 
108 	task_lock(task);
109 	priv->task_mempolicy = get_task_policy(task);
110 	mpol_get(priv->task_mempolicy);
111 	task_unlock(task);
112 }
113 static void release_task_mempolicy(struct proc_maps_private *priv)
114 {
115 	mpol_put(priv->task_mempolicy);
116 }
117 #else
118 static void hold_task_mempolicy(struct proc_maps_private *priv)
119 {
120 }
121 static void release_task_mempolicy(struct proc_maps_private *priv)
122 {
123 }
124 #endif
125 
126 static void vma_stop(struct proc_maps_private *priv)
127 {
128 	struct mm_struct *mm = priv->mm;
129 
130 	release_task_mempolicy(priv);
131 	up_read(&mm->mmap_sem);
132 	mmput(mm);
133 }
134 
135 static struct vm_area_struct *
136 m_next_vma(struct proc_maps_private *priv, struct vm_area_struct *vma)
137 {
138 	if (vma == priv->tail_vma)
139 		return NULL;
140 	return vma->vm_next ?: priv->tail_vma;
141 }
142 
143 static void m_cache_vma(struct seq_file *m, struct vm_area_struct *vma)
144 {
145 	if (m->count < m->size)	/* vma is copied successfully */
146 		m->version = m_next_vma(m->private, vma) ? vma->vm_end : -1UL;
147 }
148 
149 static void *m_start(struct seq_file *m, loff_t *ppos)
150 {
151 	struct proc_maps_private *priv = m->private;
152 	unsigned long last_addr = m->version;
153 	struct mm_struct *mm;
154 	struct vm_area_struct *vma;
155 	unsigned int pos = *ppos;
156 
157 	/* See m_cache_vma(). Zero at the start or after lseek. */
158 	if (last_addr == -1UL)
159 		return NULL;
160 
161 	priv->task = get_proc_task(priv->inode);
162 	if (!priv->task)
163 		return ERR_PTR(-ESRCH);
164 
165 	mm = priv->mm;
166 	if (!mm || !mmget_not_zero(mm))
167 		return NULL;
168 
169 	down_read(&mm->mmap_sem);
170 	hold_task_mempolicy(priv);
171 	priv->tail_vma = get_gate_vma(mm);
172 
173 	if (last_addr) {
174 		vma = find_vma(mm, last_addr - 1);
175 		if (vma && vma->vm_start <= last_addr)
176 			vma = m_next_vma(priv, vma);
177 		if (vma)
178 			return vma;
179 	}
180 
181 	m->version = 0;
182 	if (pos < mm->map_count) {
183 		for (vma = mm->mmap; pos; pos--) {
184 			m->version = vma->vm_start;
185 			vma = vma->vm_next;
186 		}
187 		return vma;
188 	}
189 
190 	/* we do not bother to update m->version in this case */
191 	if (pos == mm->map_count && priv->tail_vma)
192 		return priv->tail_vma;
193 
194 	vma_stop(priv);
195 	return NULL;
196 }
197 
198 static void *m_next(struct seq_file *m, void *v, loff_t *pos)
199 {
200 	struct proc_maps_private *priv = m->private;
201 	struct vm_area_struct *next;
202 
203 	(*pos)++;
204 	next = m_next_vma(priv, v);
205 	if (!next)
206 		vma_stop(priv);
207 	return next;
208 }
209 
210 static void m_stop(struct seq_file *m, void *v)
211 {
212 	struct proc_maps_private *priv = m->private;
213 
214 	if (!IS_ERR_OR_NULL(v))
215 		vma_stop(priv);
216 	if (priv->task) {
217 		put_task_struct(priv->task);
218 		priv->task = NULL;
219 	}
220 }
221 
222 static int proc_maps_open(struct inode *inode, struct file *file,
223 			const struct seq_operations *ops, int psize)
224 {
225 	struct proc_maps_private *priv = __seq_open_private(file, ops, psize);
226 
227 	if (!priv)
228 		return -ENOMEM;
229 
230 	priv->inode = inode;
231 	priv->mm = proc_mem_open(inode, PTRACE_MODE_READ);
232 	if (IS_ERR(priv->mm)) {
233 		int err = PTR_ERR(priv->mm);
234 
235 		seq_release_private(inode, file);
236 		return err;
237 	}
238 
239 	return 0;
240 }
241 
242 static int proc_map_release(struct inode *inode, struct file *file)
243 {
244 	struct seq_file *seq = file->private_data;
245 	struct proc_maps_private *priv = seq->private;
246 
247 	if (priv->mm)
248 		mmdrop(priv->mm);
249 
250 	return seq_release_private(inode, file);
251 }
252 
253 static int do_maps_open(struct inode *inode, struct file *file,
254 			const struct seq_operations *ops)
255 {
256 	return proc_maps_open(inode, file, ops,
257 				sizeof(struct proc_maps_private));
258 }
259 
260 /*
261  * Indicate if the VMA is a stack for the given task; for
262  * /proc/PID/maps that is the stack of the main task.
263  */
264 static int is_stack(struct vm_area_struct *vma)
265 {
266 	/*
267 	 * We make no effort to guess what a given thread considers to be
268 	 * its "stack".  It's not even well-defined for programs written
269 	 * languages like Go.
270 	 */
271 	return vma->vm_start <= vma->vm_mm->start_stack &&
272 		vma->vm_end >= vma->vm_mm->start_stack;
273 }
274 
275 static void show_vma_header_prefix(struct seq_file *m,
276 				   unsigned long start, unsigned long end,
277 				   vm_flags_t flags, unsigned long long pgoff,
278 				   dev_t dev, unsigned long ino)
279 {
280 	seq_setwidth(m, 25 + sizeof(void *) * 6 - 1);
281 	seq_put_hex_ll(m, NULL, start, 8);
282 	seq_put_hex_ll(m, "-", end, 8);
283 	seq_putc(m, ' ');
284 	seq_putc(m, flags & VM_READ ? 'r' : '-');
285 	seq_putc(m, flags & VM_WRITE ? 'w' : '-');
286 	seq_putc(m, flags & VM_EXEC ? 'x' : '-');
287 	seq_putc(m, flags & VM_MAYSHARE ? 's' : 'p');
288 	seq_put_hex_ll(m, " ", pgoff, 8);
289 	seq_put_hex_ll(m, " ", MAJOR(dev), 2);
290 	seq_put_hex_ll(m, ":", MINOR(dev), 2);
291 	seq_put_decimal_ull(m, " ", ino);
292 	seq_putc(m, ' ');
293 }
294 
295 static void
296 show_map_vma(struct seq_file *m, struct vm_area_struct *vma)
297 {
298 	struct mm_struct *mm = vma->vm_mm;
299 	struct file *file = vma->vm_file;
300 	vm_flags_t flags = vma->vm_flags;
301 	unsigned long ino = 0;
302 	unsigned long long pgoff = 0;
303 	unsigned long start, end;
304 	dev_t dev = 0;
305 	const char *name = NULL;
306 
307 	if (file) {
308 		struct inode *inode = file_inode(vma->vm_file);
309 		dev = inode->i_sb->s_dev;
310 		ino = inode->i_ino;
311 		pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT;
312 	}
313 
314 	start = vma->vm_start;
315 	end = vma->vm_end;
316 	show_vma_header_prefix(m, start, end, flags, pgoff, dev, ino);
317 
318 	/*
319 	 * Print the dentry name for named mappings, and a
320 	 * special [heap] marker for the heap:
321 	 */
322 	if (file) {
323 		seq_pad(m, ' ');
324 		seq_file_path(m, file, "\n");
325 		goto done;
326 	}
327 
328 	if (vma->vm_ops && vma->vm_ops->name) {
329 		name = vma->vm_ops->name(vma);
330 		if (name)
331 			goto done;
332 	}
333 
334 	name = arch_vma_name(vma);
335 	if (!name) {
336 		if (!mm) {
337 			name = "[vdso]";
338 			goto done;
339 		}
340 
341 		if (vma->vm_start <= mm->brk &&
342 		    vma->vm_end >= mm->start_brk) {
343 			name = "[heap]";
344 			goto done;
345 		}
346 
347 		if (is_stack(vma))
348 			name = "[stack]";
349 	}
350 
351 done:
352 	if (name) {
353 		seq_pad(m, ' ');
354 		seq_puts(m, name);
355 	}
356 	seq_putc(m, '\n');
357 }
358 
359 static int show_map(struct seq_file *m, void *v)
360 {
361 	show_map_vma(m, v);
362 	m_cache_vma(m, v);
363 	return 0;
364 }
365 
366 static const struct seq_operations proc_pid_maps_op = {
367 	.start	= m_start,
368 	.next	= m_next,
369 	.stop	= m_stop,
370 	.show	= show_map
371 };
372 
373 static int pid_maps_open(struct inode *inode, struct file *file)
374 {
375 	return do_maps_open(inode, file, &proc_pid_maps_op);
376 }
377 
378 const struct file_operations proc_pid_maps_operations = {
379 	.open		= pid_maps_open,
380 	.read		= seq_read,
381 	.llseek		= seq_lseek,
382 	.release	= proc_map_release,
383 };
384 
385 /*
386  * Proportional Set Size(PSS): my share of RSS.
387  *
388  * PSS of a process is the count of pages it has in memory, where each
389  * page is divided by the number of processes sharing it.  So if a
390  * process has 1000 pages all to itself, and 1000 shared with one other
391  * process, its PSS will be 1500.
392  *
393  * To keep (accumulated) division errors low, we adopt a 64bit
394  * fixed-point pss counter to minimize division errors. So (pss >>
395  * PSS_SHIFT) would be the real byte count.
396  *
397  * A shift of 12 before division means (assuming 4K page size):
398  * 	- 1M 3-user-pages add up to 8KB errors;
399  * 	- supports mapcount up to 2^24, or 16M;
400  * 	- supports PSS up to 2^52 bytes, or 4PB.
401  */
402 #define PSS_SHIFT 12
403 
404 #ifdef CONFIG_PROC_PAGE_MONITOR
405 struct mem_size_stats {
406 	unsigned long resident;
407 	unsigned long shared_clean;
408 	unsigned long shared_dirty;
409 	unsigned long private_clean;
410 	unsigned long private_dirty;
411 	unsigned long referenced;
412 	unsigned long anonymous;
413 	unsigned long lazyfree;
414 	unsigned long anonymous_thp;
415 	unsigned long shmem_thp;
416 	unsigned long swap;
417 	unsigned long shared_hugetlb;
418 	unsigned long private_hugetlb;
419 	u64 pss;
420 	u64 pss_locked;
421 	u64 swap_pss;
422 	bool check_shmem_swap;
423 };
424 
425 static void smaps_account(struct mem_size_stats *mss, struct page *page,
426 		bool compound, bool young, bool dirty)
427 {
428 	int i, nr = compound ? 1 << compound_order(page) : 1;
429 	unsigned long size = nr * PAGE_SIZE;
430 
431 	if (PageAnon(page)) {
432 		mss->anonymous += size;
433 		if (!PageSwapBacked(page) && !dirty && !PageDirty(page))
434 			mss->lazyfree += size;
435 	}
436 
437 	mss->resident += size;
438 	/* Accumulate the size in pages that have been accessed. */
439 	if (young || page_is_young(page) || PageReferenced(page))
440 		mss->referenced += size;
441 
442 	/*
443 	 * page_count(page) == 1 guarantees the page is mapped exactly once.
444 	 * If any subpage of the compound page mapped with PTE it would elevate
445 	 * page_count().
446 	 */
447 	if (page_count(page) == 1) {
448 		if (dirty || PageDirty(page))
449 			mss->private_dirty += size;
450 		else
451 			mss->private_clean += size;
452 		mss->pss += (u64)size << PSS_SHIFT;
453 		return;
454 	}
455 
456 	for (i = 0; i < nr; i++, page++) {
457 		int mapcount = page_mapcount(page);
458 
459 		if (mapcount >= 2) {
460 			if (dirty || PageDirty(page))
461 				mss->shared_dirty += PAGE_SIZE;
462 			else
463 				mss->shared_clean += PAGE_SIZE;
464 			mss->pss += (PAGE_SIZE << PSS_SHIFT) / mapcount;
465 		} else {
466 			if (dirty || PageDirty(page))
467 				mss->private_dirty += PAGE_SIZE;
468 			else
469 				mss->private_clean += PAGE_SIZE;
470 			mss->pss += PAGE_SIZE << PSS_SHIFT;
471 		}
472 	}
473 }
474 
475 #ifdef CONFIG_SHMEM
476 static int smaps_pte_hole(unsigned long addr, unsigned long end,
477 		struct mm_walk *walk)
478 {
479 	struct mem_size_stats *mss = walk->private;
480 
481 	mss->swap += shmem_partial_swap_usage(
482 			walk->vma->vm_file->f_mapping, addr, end);
483 
484 	return 0;
485 }
486 #endif
487 
488 static void smaps_pte_entry(pte_t *pte, unsigned long addr,
489 		struct mm_walk *walk)
490 {
491 	struct mem_size_stats *mss = walk->private;
492 	struct vm_area_struct *vma = walk->vma;
493 	struct page *page = NULL;
494 
495 	if (pte_present(*pte)) {
496 		page = vm_normal_page(vma, addr, *pte);
497 	} else if (is_swap_pte(*pte)) {
498 		swp_entry_t swpent = pte_to_swp_entry(*pte);
499 
500 		if (!non_swap_entry(swpent)) {
501 			int mapcount;
502 
503 			mss->swap += PAGE_SIZE;
504 			mapcount = swp_swapcount(swpent);
505 			if (mapcount >= 2) {
506 				u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT;
507 
508 				do_div(pss_delta, mapcount);
509 				mss->swap_pss += pss_delta;
510 			} else {
511 				mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT;
512 			}
513 		} else if (is_migration_entry(swpent))
514 			page = migration_entry_to_page(swpent);
515 		else if (is_device_private_entry(swpent))
516 			page = device_private_entry_to_page(swpent);
517 	} else if (unlikely(IS_ENABLED(CONFIG_SHMEM) && mss->check_shmem_swap
518 							&& pte_none(*pte))) {
519 		page = find_get_entry(vma->vm_file->f_mapping,
520 						linear_page_index(vma, addr));
521 		if (!page)
522 			return;
523 
524 		if (xa_is_value(page))
525 			mss->swap += PAGE_SIZE;
526 		else
527 			put_page(page);
528 
529 		return;
530 	}
531 
532 	if (!page)
533 		return;
534 
535 	smaps_account(mss, page, false, pte_young(*pte), pte_dirty(*pte));
536 }
537 
538 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
539 static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
540 		struct mm_walk *walk)
541 {
542 	struct mem_size_stats *mss = walk->private;
543 	struct vm_area_struct *vma = walk->vma;
544 	struct page *page;
545 
546 	/* FOLL_DUMP will return -EFAULT on huge zero page */
547 	page = follow_trans_huge_pmd(vma, addr, pmd, FOLL_DUMP);
548 	if (IS_ERR_OR_NULL(page))
549 		return;
550 	if (PageAnon(page))
551 		mss->anonymous_thp += HPAGE_PMD_SIZE;
552 	else if (PageSwapBacked(page))
553 		mss->shmem_thp += HPAGE_PMD_SIZE;
554 	else if (is_zone_device_page(page))
555 		/* pass */;
556 	else
557 		VM_BUG_ON_PAGE(1, page);
558 	smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd));
559 }
560 #else
561 static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
562 		struct mm_walk *walk)
563 {
564 }
565 #endif
566 
567 static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
568 			   struct mm_walk *walk)
569 {
570 	struct vm_area_struct *vma = walk->vma;
571 	pte_t *pte;
572 	spinlock_t *ptl;
573 
574 	ptl = pmd_trans_huge_lock(pmd, vma);
575 	if (ptl) {
576 		if (pmd_present(*pmd))
577 			smaps_pmd_entry(pmd, addr, walk);
578 		spin_unlock(ptl);
579 		goto out;
580 	}
581 
582 	if (pmd_trans_unstable(pmd))
583 		goto out;
584 	/*
585 	 * The mmap_sem held all the way back in m_start() is what
586 	 * keeps khugepaged out of here and from collapsing things
587 	 * in here.
588 	 */
589 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
590 	for (; addr != end; pte++, addr += PAGE_SIZE)
591 		smaps_pte_entry(pte, addr, walk);
592 	pte_unmap_unlock(pte - 1, ptl);
593 out:
594 	cond_resched();
595 	return 0;
596 }
597 
598 static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
599 {
600 	/*
601 	 * Don't forget to update Documentation/ on changes.
602 	 */
603 	static const char mnemonics[BITS_PER_LONG][2] = {
604 		/*
605 		 * In case if we meet a flag we don't know about.
606 		 */
607 		[0 ... (BITS_PER_LONG-1)] = "??",
608 
609 		[ilog2(VM_READ)]	= "rd",
610 		[ilog2(VM_WRITE)]	= "wr",
611 		[ilog2(VM_EXEC)]	= "ex",
612 		[ilog2(VM_SHARED)]	= "sh",
613 		[ilog2(VM_MAYREAD)]	= "mr",
614 		[ilog2(VM_MAYWRITE)]	= "mw",
615 		[ilog2(VM_MAYEXEC)]	= "me",
616 		[ilog2(VM_MAYSHARE)]	= "ms",
617 		[ilog2(VM_GROWSDOWN)]	= "gd",
618 		[ilog2(VM_PFNMAP)]	= "pf",
619 		[ilog2(VM_DENYWRITE)]	= "dw",
620 #ifdef CONFIG_X86_INTEL_MPX
621 		[ilog2(VM_MPX)]		= "mp",
622 #endif
623 		[ilog2(VM_LOCKED)]	= "lo",
624 		[ilog2(VM_IO)]		= "io",
625 		[ilog2(VM_SEQ_READ)]	= "sr",
626 		[ilog2(VM_RAND_READ)]	= "rr",
627 		[ilog2(VM_DONTCOPY)]	= "dc",
628 		[ilog2(VM_DONTEXPAND)]	= "de",
629 		[ilog2(VM_ACCOUNT)]	= "ac",
630 		[ilog2(VM_NORESERVE)]	= "nr",
631 		[ilog2(VM_HUGETLB)]	= "ht",
632 		[ilog2(VM_SYNC)]	= "sf",
633 		[ilog2(VM_ARCH_1)]	= "ar",
634 		[ilog2(VM_WIPEONFORK)]	= "wf",
635 		[ilog2(VM_DONTDUMP)]	= "dd",
636 #ifdef CONFIG_MEM_SOFT_DIRTY
637 		[ilog2(VM_SOFTDIRTY)]	= "sd",
638 #endif
639 		[ilog2(VM_MIXEDMAP)]	= "mm",
640 		[ilog2(VM_HUGEPAGE)]	= "hg",
641 		[ilog2(VM_NOHUGEPAGE)]	= "nh",
642 		[ilog2(VM_MERGEABLE)]	= "mg",
643 		[ilog2(VM_UFFD_MISSING)]= "um",
644 		[ilog2(VM_UFFD_WP)]	= "uw",
645 #ifdef CONFIG_ARCH_HAS_PKEYS
646 		/* These come out via ProtectionKey: */
647 		[ilog2(VM_PKEY_BIT0)]	= "",
648 		[ilog2(VM_PKEY_BIT1)]	= "",
649 		[ilog2(VM_PKEY_BIT2)]	= "",
650 		[ilog2(VM_PKEY_BIT3)]	= "",
651 #if VM_PKEY_BIT4
652 		[ilog2(VM_PKEY_BIT4)]	= "",
653 #endif
654 #endif /* CONFIG_ARCH_HAS_PKEYS */
655 	};
656 	size_t i;
657 
658 	seq_puts(m, "VmFlags: ");
659 	for (i = 0; i < BITS_PER_LONG; i++) {
660 		if (!mnemonics[i][0])
661 			continue;
662 		if (vma->vm_flags & (1UL << i)) {
663 			seq_putc(m, mnemonics[i][0]);
664 			seq_putc(m, mnemonics[i][1]);
665 			seq_putc(m, ' ');
666 		}
667 	}
668 	seq_putc(m, '\n');
669 }
670 
671 #ifdef CONFIG_HUGETLB_PAGE
672 static int smaps_hugetlb_range(pte_t *pte, unsigned long hmask,
673 				 unsigned long addr, unsigned long end,
674 				 struct mm_walk *walk)
675 {
676 	struct mem_size_stats *mss = walk->private;
677 	struct vm_area_struct *vma = walk->vma;
678 	struct page *page = NULL;
679 
680 	if (pte_present(*pte)) {
681 		page = vm_normal_page(vma, addr, *pte);
682 	} else if (is_swap_pte(*pte)) {
683 		swp_entry_t swpent = pte_to_swp_entry(*pte);
684 
685 		if (is_migration_entry(swpent))
686 			page = migration_entry_to_page(swpent);
687 		else if (is_device_private_entry(swpent))
688 			page = device_private_entry_to_page(swpent);
689 	}
690 	if (page) {
691 		int mapcount = page_mapcount(page);
692 
693 		if (mapcount >= 2)
694 			mss->shared_hugetlb += huge_page_size(hstate_vma(vma));
695 		else
696 			mss->private_hugetlb += huge_page_size(hstate_vma(vma));
697 	}
698 	return 0;
699 }
700 #endif /* HUGETLB_PAGE */
701 
702 static void smap_gather_stats(struct vm_area_struct *vma,
703 			     struct mem_size_stats *mss)
704 {
705 	struct mm_walk smaps_walk = {
706 		.pmd_entry = smaps_pte_range,
707 #ifdef CONFIG_HUGETLB_PAGE
708 		.hugetlb_entry = smaps_hugetlb_range,
709 #endif
710 		.mm = vma->vm_mm,
711 	};
712 
713 	smaps_walk.private = mss;
714 
715 #ifdef CONFIG_SHMEM
716 	/* In case of smaps_rollup, reset the value from previous vma */
717 	mss->check_shmem_swap = false;
718 	if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) {
719 		/*
720 		 * For shared or readonly shmem mappings we know that all
721 		 * swapped out pages belong to the shmem object, and we can
722 		 * obtain the swap value much more efficiently. For private
723 		 * writable mappings, we might have COW pages that are
724 		 * not affected by the parent swapped out pages of the shmem
725 		 * object, so we have to distinguish them during the page walk.
726 		 * Unless we know that the shmem object (or the part mapped by
727 		 * our VMA) has no swapped out pages at all.
728 		 */
729 		unsigned long shmem_swapped = shmem_swap_usage(vma);
730 
731 		if (!shmem_swapped || (vma->vm_flags & VM_SHARED) ||
732 					!(vma->vm_flags & VM_WRITE)) {
733 			mss->swap += shmem_swapped;
734 		} else {
735 			mss->check_shmem_swap = true;
736 			smaps_walk.pte_hole = smaps_pte_hole;
737 		}
738 	}
739 #endif
740 
741 	/* mmap_sem is held in m_start */
742 	walk_page_vma(vma, &smaps_walk);
743 	if (vma->vm_flags & VM_LOCKED)
744 		mss->pss_locked += mss->pss;
745 }
746 
747 #define SEQ_PUT_DEC(str, val) \
748 		seq_put_decimal_ull_width(m, str, (val) >> 10, 8)
749 
750 /* Show the contents common for smaps and smaps_rollup */
751 static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss)
752 {
753 	SEQ_PUT_DEC("Rss:            ", mss->resident);
754 	SEQ_PUT_DEC(" kB\nPss:            ", mss->pss >> PSS_SHIFT);
755 	SEQ_PUT_DEC(" kB\nShared_Clean:   ", mss->shared_clean);
756 	SEQ_PUT_DEC(" kB\nShared_Dirty:   ", mss->shared_dirty);
757 	SEQ_PUT_DEC(" kB\nPrivate_Clean:  ", mss->private_clean);
758 	SEQ_PUT_DEC(" kB\nPrivate_Dirty:  ", mss->private_dirty);
759 	SEQ_PUT_DEC(" kB\nReferenced:     ", mss->referenced);
760 	SEQ_PUT_DEC(" kB\nAnonymous:      ", mss->anonymous);
761 	SEQ_PUT_DEC(" kB\nLazyFree:       ", mss->lazyfree);
762 	SEQ_PUT_DEC(" kB\nAnonHugePages:  ", mss->anonymous_thp);
763 	SEQ_PUT_DEC(" kB\nShmemPmdMapped: ", mss->shmem_thp);
764 	SEQ_PUT_DEC(" kB\nShared_Hugetlb: ", mss->shared_hugetlb);
765 	seq_put_decimal_ull_width(m, " kB\nPrivate_Hugetlb: ",
766 				  mss->private_hugetlb >> 10, 7);
767 	SEQ_PUT_DEC(" kB\nSwap:           ", mss->swap);
768 	SEQ_PUT_DEC(" kB\nSwapPss:        ",
769 					mss->swap_pss >> PSS_SHIFT);
770 	SEQ_PUT_DEC(" kB\nLocked:         ",
771 					mss->pss_locked >> PSS_SHIFT);
772 	seq_puts(m, " kB\n");
773 }
774 
775 static int show_smap(struct seq_file *m, void *v)
776 {
777 	struct vm_area_struct *vma = v;
778 	struct mem_size_stats mss;
779 
780 	memset(&mss, 0, sizeof(mss));
781 
782 	smap_gather_stats(vma, &mss);
783 
784 	show_map_vma(m, vma);
785 
786 	SEQ_PUT_DEC("Size:           ", vma->vm_end - vma->vm_start);
787 	SEQ_PUT_DEC(" kB\nKernelPageSize: ", vma_kernel_pagesize(vma));
788 	SEQ_PUT_DEC(" kB\nMMUPageSize:    ", vma_mmu_pagesize(vma));
789 	seq_puts(m, " kB\n");
790 
791 	__show_smap(m, &mss);
792 
793 	if (arch_pkeys_enabled())
794 		seq_printf(m, "ProtectionKey:  %8u\n", vma_pkey(vma));
795 	show_smap_vma_flags(m, vma);
796 
797 	m_cache_vma(m, vma);
798 
799 	return 0;
800 }
801 
802 static int show_smaps_rollup(struct seq_file *m, void *v)
803 {
804 	struct proc_maps_private *priv = m->private;
805 	struct mem_size_stats mss;
806 	struct mm_struct *mm;
807 	struct vm_area_struct *vma;
808 	unsigned long last_vma_end = 0;
809 	int ret = 0;
810 
811 	priv->task = get_proc_task(priv->inode);
812 	if (!priv->task)
813 		return -ESRCH;
814 
815 	mm = priv->mm;
816 	if (!mm || !mmget_not_zero(mm)) {
817 		ret = -ESRCH;
818 		goto out_put_task;
819 	}
820 
821 	memset(&mss, 0, sizeof(mss));
822 
823 	down_read(&mm->mmap_sem);
824 	hold_task_mempolicy(priv);
825 
826 	for (vma = priv->mm->mmap; vma; vma = vma->vm_next) {
827 		smap_gather_stats(vma, &mss);
828 		last_vma_end = vma->vm_end;
829 	}
830 
831 	show_vma_header_prefix(m, priv->mm->mmap->vm_start,
832 			       last_vma_end, 0, 0, 0, 0);
833 	seq_pad(m, ' ');
834 	seq_puts(m, "[rollup]\n");
835 
836 	__show_smap(m, &mss);
837 
838 	release_task_mempolicy(priv);
839 	up_read(&mm->mmap_sem);
840 	mmput(mm);
841 
842 out_put_task:
843 	put_task_struct(priv->task);
844 	priv->task = NULL;
845 
846 	return ret;
847 }
848 #undef SEQ_PUT_DEC
849 
850 static const struct seq_operations proc_pid_smaps_op = {
851 	.start	= m_start,
852 	.next	= m_next,
853 	.stop	= m_stop,
854 	.show	= show_smap
855 };
856 
857 static int pid_smaps_open(struct inode *inode, struct file *file)
858 {
859 	return do_maps_open(inode, file, &proc_pid_smaps_op);
860 }
861 
862 static int smaps_rollup_open(struct inode *inode, struct file *file)
863 {
864 	int ret;
865 	struct proc_maps_private *priv;
866 
867 	priv = kzalloc(sizeof(*priv), GFP_KERNEL_ACCOUNT);
868 	if (!priv)
869 		return -ENOMEM;
870 
871 	ret = single_open(file, show_smaps_rollup, priv);
872 	if (ret)
873 		goto out_free;
874 
875 	priv->inode = inode;
876 	priv->mm = proc_mem_open(inode, PTRACE_MODE_READ);
877 	if (IS_ERR(priv->mm)) {
878 		ret = PTR_ERR(priv->mm);
879 
880 		single_release(inode, file);
881 		goto out_free;
882 	}
883 
884 	return 0;
885 
886 out_free:
887 	kfree(priv);
888 	return ret;
889 }
890 
891 static int smaps_rollup_release(struct inode *inode, struct file *file)
892 {
893 	struct seq_file *seq = file->private_data;
894 	struct proc_maps_private *priv = seq->private;
895 
896 	if (priv->mm)
897 		mmdrop(priv->mm);
898 
899 	kfree(priv);
900 	return single_release(inode, file);
901 }
902 
903 const struct file_operations proc_pid_smaps_operations = {
904 	.open		= pid_smaps_open,
905 	.read		= seq_read,
906 	.llseek		= seq_lseek,
907 	.release	= proc_map_release,
908 };
909 
910 const struct file_operations proc_pid_smaps_rollup_operations = {
911 	.open		= smaps_rollup_open,
912 	.read		= seq_read,
913 	.llseek		= seq_lseek,
914 	.release	= smaps_rollup_release,
915 };
916 
917 enum clear_refs_types {
918 	CLEAR_REFS_ALL = 1,
919 	CLEAR_REFS_ANON,
920 	CLEAR_REFS_MAPPED,
921 	CLEAR_REFS_SOFT_DIRTY,
922 	CLEAR_REFS_MM_HIWATER_RSS,
923 	CLEAR_REFS_LAST,
924 };
925 
926 struct clear_refs_private {
927 	enum clear_refs_types type;
928 };
929 
930 #ifdef CONFIG_MEM_SOFT_DIRTY
931 static inline void clear_soft_dirty(struct vm_area_struct *vma,
932 		unsigned long addr, pte_t *pte)
933 {
934 	/*
935 	 * The soft-dirty tracker uses #PF-s to catch writes
936 	 * to pages, so write-protect the pte as well. See the
937 	 * Documentation/admin-guide/mm/soft-dirty.rst for full description
938 	 * of how soft-dirty works.
939 	 */
940 	pte_t ptent = *pte;
941 
942 	if (pte_present(ptent)) {
943 		ptent = ptep_modify_prot_start(vma->vm_mm, addr, pte);
944 		ptent = pte_wrprotect(ptent);
945 		ptent = pte_clear_soft_dirty(ptent);
946 		ptep_modify_prot_commit(vma->vm_mm, addr, pte, ptent);
947 	} else if (is_swap_pte(ptent)) {
948 		ptent = pte_swp_clear_soft_dirty(ptent);
949 		set_pte_at(vma->vm_mm, addr, pte, ptent);
950 	}
951 }
952 #else
953 static inline void clear_soft_dirty(struct vm_area_struct *vma,
954 		unsigned long addr, pte_t *pte)
955 {
956 }
957 #endif
958 
959 #if defined(CONFIG_MEM_SOFT_DIRTY) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
960 static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
961 		unsigned long addr, pmd_t *pmdp)
962 {
963 	pmd_t old, pmd = *pmdp;
964 
965 	if (pmd_present(pmd)) {
966 		/* See comment in change_huge_pmd() */
967 		old = pmdp_invalidate(vma, addr, pmdp);
968 		if (pmd_dirty(old))
969 			pmd = pmd_mkdirty(pmd);
970 		if (pmd_young(old))
971 			pmd = pmd_mkyoung(pmd);
972 
973 		pmd = pmd_wrprotect(pmd);
974 		pmd = pmd_clear_soft_dirty(pmd);
975 
976 		set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
977 	} else if (is_migration_entry(pmd_to_swp_entry(pmd))) {
978 		pmd = pmd_swp_clear_soft_dirty(pmd);
979 		set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
980 	}
981 }
982 #else
983 static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
984 		unsigned long addr, pmd_t *pmdp)
985 {
986 }
987 #endif
988 
989 static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
990 				unsigned long end, struct mm_walk *walk)
991 {
992 	struct clear_refs_private *cp = walk->private;
993 	struct vm_area_struct *vma = walk->vma;
994 	pte_t *pte, ptent;
995 	spinlock_t *ptl;
996 	struct page *page;
997 
998 	ptl = pmd_trans_huge_lock(pmd, vma);
999 	if (ptl) {
1000 		if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
1001 			clear_soft_dirty_pmd(vma, addr, pmd);
1002 			goto out;
1003 		}
1004 
1005 		if (!pmd_present(*pmd))
1006 			goto out;
1007 
1008 		page = pmd_page(*pmd);
1009 
1010 		/* Clear accessed and referenced bits. */
1011 		pmdp_test_and_clear_young(vma, addr, pmd);
1012 		test_and_clear_page_young(page);
1013 		ClearPageReferenced(page);
1014 out:
1015 		spin_unlock(ptl);
1016 		return 0;
1017 	}
1018 
1019 	if (pmd_trans_unstable(pmd))
1020 		return 0;
1021 
1022 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1023 	for (; addr != end; pte++, addr += PAGE_SIZE) {
1024 		ptent = *pte;
1025 
1026 		if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
1027 			clear_soft_dirty(vma, addr, pte);
1028 			continue;
1029 		}
1030 
1031 		if (!pte_present(ptent))
1032 			continue;
1033 
1034 		page = vm_normal_page(vma, addr, ptent);
1035 		if (!page)
1036 			continue;
1037 
1038 		/* Clear accessed and referenced bits. */
1039 		ptep_test_and_clear_young(vma, addr, pte);
1040 		test_and_clear_page_young(page);
1041 		ClearPageReferenced(page);
1042 	}
1043 	pte_unmap_unlock(pte - 1, ptl);
1044 	cond_resched();
1045 	return 0;
1046 }
1047 
1048 static int clear_refs_test_walk(unsigned long start, unsigned long end,
1049 				struct mm_walk *walk)
1050 {
1051 	struct clear_refs_private *cp = walk->private;
1052 	struct vm_area_struct *vma = walk->vma;
1053 
1054 	if (vma->vm_flags & VM_PFNMAP)
1055 		return 1;
1056 
1057 	/*
1058 	 * Writing 1 to /proc/pid/clear_refs affects all pages.
1059 	 * Writing 2 to /proc/pid/clear_refs only affects anonymous pages.
1060 	 * Writing 3 to /proc/pid/clear_refs only affects file mapped pages.
1061 	 * Writing 4 to /proc/pid/clear_refs affects all pages.
1062 	 */
1063 	if (cp->type == CLEAR_REFS_ANON && vma->vm_file)
1064 		return 1;
1065 	if (cp->type == CLEAR_REFS_MAPPED && !vma->vm_file)
1066 		return 1;
1067 	return 0;
1068 }
1069 
1070 static ssize_t clear_refs_write(struct file *file, const char __user *buf,
1071 				size_t count, loff_t *ppos)
1072 {
1073 	struct task_struct *task;
1074 	char buffer[PROC_NUMBUF];
1075 	struct mm_struct *mm;
1076 	struct vm_area_struct *vma;
1077 	enum clear_refs_types type;
1078 	struct mmu_gather tlb;
1079 	int itype;
1080 	int rv;
1081 
1082 	memset(buffer, 0, sizeof(buffer));
1083 	if (count > sizeof(buffer) - 1)
1084 		count = sizeof(buffer) - 1;
1085 	if (copy_from_user(buffer, buf, count))
1086 		return -EFAULT;
1087 	rv = kstrtoint(strstrip(buffer), 10, &itype);
1088 	if (rv < 0)
1089 		return rv;
1090 	type = (enum clear_refs_types)itype;
1091 	if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST)
1092 		return -EINVAL;
1093 
1094 	task = get_proc_task(file_inode(file));
1095 	if (!task)
1096 		return -ESRCH;
1097 	mm = get_task_mm(task);
1098 	if (mm) {
1099 		struct clear_refs_private cp = {
1100 			.type = type,
1101 		};
1102 		struct mm_walk clear_refs_walk = {
1103 			.pmd_entry = clear_refs_pte_range,
1104 			.test_walk = clear_refs_test_walk,
1105 			.mm = mm,
1106 			.private = &cp,
1107 		};
1108 
1109 		if (type == CLEAR_REFS_MM_HIWATER_RSS) {
1110 			if (down_write_killable(&mm->mmap_sem)) {
1111 				count = -EINTR;
1112 				goto out_mm;
1113 			}
1114 
1115 			/*
1116 			 * Writing 5 to /proc/pid/clear_refs resets the peak
1117 			 * resident set size to this mm's current rss value.
1118 			 */
1119 			reset_mm_hiwater_rss(mm);
1120 			up_write(&mm->mmap_sem);
1121 			goto out_mm;
1122 		}
1123 
1124 		down_read(&mm->mmap_sem);
1125 		tlb_gather_mmu(&tlb, mm, 0, -1);
1126 		if (type == CLEAR_REFS_SOFT_DIRTY) {
1127 			for (vma = mm->mmap; vma; vma = vma->vm_next) {
1128 				if (!(vma->vm_flags & VM_SOFTDIRTY))
1129 					continue;
1130 				up_read(&mm->mmap_sem);
1131 				if (down_write_killable(&mm->mmap_sem)) {
1132 					count = -EINTR;
1133 					goto out_mm;
1134 				}
1135 				for (vma = mm->mmap; vma; vma = vma->vm_next) {
1136 					vma->vm_flags &= ~VM_SOFTDIRTY;
1137 					vma_set_page_prot(vma);
1138 				}
1139 				downgrade_write(&mm->mmap_sem);
1140 				break;
1141 			}
1142 			mmu_notifier_invalidate_range_start(mm, 0, -1);
1143 		}
1144 		walk_page_range(0, mm->highest_vm_end, &clear_refs_walk);
1145 		if (type == CLEAR_REFS_SOFT_DIRTY)
1146 			mmu_notifier_invalidate_range_end(mm, 0, -1);
1147 		tlb_finish_mmu(&tlb, 0, -1);
1148 		up_read(&mm->mmap_sem);
1149 out_mm:
1150 		mmput(mm);
1151 	}
1152 	put_task_struct(task);
1153 
1154 	return count;
1155 }
1156 
1157 const struct file_operations proc_clear_refs_operations = {
1158 	.write		= clear_refs_write,
1159 	.llseek		= noop_llseek,
1160 };
1161 
1162 typedef struct {
1163 	u64 pme;
1164 } pagemap_entry_t;
1165 
1166 struct pagemapread {
1167 	int pos, len;		/* units: PM_ENTRY_BYTES, not bytes */
1168 	pagemap_entry_t *buffer;
1169 	bool show_pfn;
1170 };
1171 
1172 #define PAGEMAP_WALK_SIZE	(PMD_SIZE)
1173 #define PAGEMAP_WALK_MASK	(PMD_MASK)
1174 
1175 #define PM_ENTRY_BYTES		sizeof(pagemap_entry_t)
1176 #define PM_PFRAME_BITS		55
1177 #define PM_PFRAME_MASK		GENMASK_ULL(PM_PFRAME_BITS - 1, 0)
1178 #define PM_SOFT_DIRTY		BIT_ULL(55)
1179 #define PM_MMAP_EXCLUSIVE	BIT_ULL(56)
1180 #define PM_FILE			BIT_ULL(61)
1181 #define PM_SWAP			BIT_ULL(62)
1182 #define PM_PRESENT		BIT_ULL(63)
1183 
1184 #define PM_END_OF_BUFFER    1
1185 
1186 static inline pagemap_entry_t make_pme(u64 frame, u64 flags)
1187 {
1188 	return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags };
1189 }
1190 
1191 static int add_to_pagemap(unsigned long addr, pagemap_entry_t *pme,
1192 			  struct pagemapread *pm)
1193 {
1194 	pm->buffer[pm->pos++] = *pme;
1195 	if (pm->pos >= pm->len)
1196 		return PM_END_OF_BUFFER;
1197 	return 0;
1198 }
1199 
1200 static int pagemap_pte_hole(unsigned long start, unsigned long end,
1201 				struct mm_walk *walk)
1202 {
1203 	struct pagemapread *pm = walk->private;
1204 	unsigned long addr = start;
1205 	int err = 0;
1206 
1207 	while (addr < end) {
1208 		struct vm_area_struct *vma = find_vma(walk->mm, addr);
1209 		pagemap_entry_t pme = make_pme(0, 0);
1210 		/* End of address space hole, which we mark as non-present. */
1211 		unsigned long hole_end;
1212 
1213 		if (vma)
1214 			hole_end = min(end, vma->vm_start);
1215 		else
1216 			hole_end = end;
1217 
1218 		for (; addr < hole_end; addr += PAGE_SIZE) {
1219 			err = add_to_pagemap(addr, &pme, pm);
1220 			if (err)
1221 				goto out;
1222 		}
1223 
1224 		if (!vma)
1225 			break;
1226 
1227 		/* Addresses in the VMA. */
1228 		if (vma->vm_flags & VM_SOFTDIRTY)
1229 			pme = make_pme(0, PM_SOFT_DIRTY);
1230 		for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) {
1231 			err = add_to_pagemap(addr, &pme, pm);
1232 			if (err)
1233 				goto out;
1234 		}
1235 	}
1236 out:
1237 	return err;
1238 }
1239 
1240 static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm,
1241 		struct vm_area_struct *vma, unsigned long addr, pte_t pte)
1242 {
1243 	u64 frame = 0, flags = 0;
1244 	struct page *page = NULL;
1245 
1246 	if (pte_present(pte)) {
1247 		if (pm->show_pfn)
1248 			frame = pte_pfn(pte);
1249 		flags |= PM_PRESENT;
1250 		page = _vm_normal_page(vma, addr, pte, true);
1251 		if (pte_soft_dirty(pte))
1252 			flags |= PM_SOFT_DIRTY;
1253 	} else if (is_swap_pte(pte)) {
1254 		swp_entry_t entry;
1255 		if (pte_swp_soft_dirty(pte))
1256 			flags |= PM_SOFT_DIRTY;
1257 		entry = pte_to_swp_entry(pte);
1258 		if (pm->show_pfn)
1259 			frame = swp_type(entry) |
1260 				(swp_offset(entry) << MAX_SWAPFILES_SHIFT);
1261 		flags |= PM_SWAP;
1262 		if (is_migration_entry(entry))
1263 			page = migration_entry_to_page(entry);
1264 
1265 		if (is_device_private_entry(entry))
1266 			page = device_private_entry_to_page(entry);
1267 	}
1268 
1269 	if (page && !PageAnon(page))
1270 		flags |= PM_FILE;
1271 	if (page && page_mapcount(page) == 1)
1272 		flags |= PM_MMAP_EXCLUSIVE;
1273 	if (vma->vm_flags & VM_SOFTDIRTY)
1274 		flags |= PM_SOFT_DIRTY;
1275 
1276 	return make_pme(frame, flags);
1277 }
1278 
1279 static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end,
1280 			     struct mm_walk *walk)
1281 {
1282 	struct vm_area_struct *vma = walk->vma;
1283 	struct pagemapread *pm = walk->private;
1284 	spinlock_t *ptl;
1285 	pte_t *pte, *orig_pte;
1286 	int err = 0;
1287 
1288 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1289 	ptl = pmd_trans_huge_lock(pmdp, vma);
1290 	if (ptl) {
1291 		u64 flags = 0, frame = 0;
1292 		pmd_t pmd = *pmdp;
1293 		struct page *page = NULL;
1294 
1295 		if (vma->vm_flags & VM_SOFTDIRTY)
1296 			flags |= PM_SOFT_DIRTY;
1297 
1298 		if (pmd_present(pmd)) {
1299 			page = pmd_page(pmd);
1300 
1301 			flags |= PM_PRESENT;
1302 			if (pmd_soft_dirty(pmd))
1303 				flags |= PM_SOFT_DIRTY;
1304 			if (pm->show_pfn)
1305 				frame = pmd_pfn(pmd) +
1306 					((addr & ~PMD_MASK) >> PAGE_SHIFT);
1307 		}
1308 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1309 		else if (is_swap_pmd(pmd)) {
1310 			swp_entry_t entry = pmd_to_swp_entry(pmd);
1311 			unsigned long offset;
1312 
1313 			if (pm->show_pfn) {
1314 				offset = swp_offset(entry) +
1315 					((addr & ~PMD_MASK) >> PAGE_SHIFT);
1316 				frame = swp_type(entry) |
1317 					(offset << MAX_SWAPFILES_SHIFT);
1318 			}
1319 			flags |= PM_SWAP;
1320 			if (pmd_swp_soft_dirty(pmd))
1321 				flags |= PM_SOFT_DIRTY;
1322 			VM_BUG_ON(!is_pmd_migration_entry(pmd));
1323 			page = migration_entry_to_page(entry);
1324 		}
1325 #endif
1326 
1327 		if (page && page_mapcount(page) == 1)
1328 			flags |= PM_MMAP_EXCLUSIVE;
1329 
1330 		for (; addr != end; addr += PAGE_SIZE) {
1331 			pagemap_entry_t pme = make_pme(frame, flags);
1332 
1333 			err = add_to_pagemap(addr, &pme, pm);
1334 			if (err)
1335 				break;
1336 			if (pm->show_pfn) {
1337 				if (flags & PM_PRESENT)
1338 					frame++;
1339 				else if (flags & PM_SWAP)
1340 					frame += (1 << MAX_SWAPFILES_SHIFT);
1341 			}
1342 		}
1343 		spin_unlock(ptl);
1344 		return err;
1345 	}
1346 
1347 	if (pmd_trans_unstable(pmdp))
1348 		return 0;
1349 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1350 
1351 	/*
1352 	 * We can assume that @vma always points to a valid one and @end never
1353 	 * goes beyond vma->vm_end.
1354 	 */
1355 	orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl);
1356 	for (; addr < end; pte++, addr += PAGE_SIZE) {
1357 		pagemap_entry_t pme;
1358 
1359 		pme = pte_to_pagemap_entry(pm, vma, addr, *pte);
1360 		err = add_to_pagemap(addr, &pme, pm);
1361 		if (err)
1362 			break;
1363 	}
1364 	pte_unmap_unlock(orig_pte, ptl);
1365 
1366 	cond_resched();
1367 
1368 	return err;
1369 }
1370 
1371 #ifdef CONFIG_HUGETLB_PAGE
1372 /* This function walks within one hugetlb entry in the single call */
1373 static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask,
1374 				 unsigned long addr, unsigned long end,
1375 				 struct mm_walk *walk)
1376 {
1377 	struct pagemapread *pm = walk->private;
1378 	struct vm_area_struct *vma = walk->vma;
1379 	u64 flags = 0, frame = 0;
1380 	int err = 0;
1381 	pte_t pte;
1382 
1383 	if (vma->vm_flags & VM_SOFTDIRTY)
1384 		flags |= PM_SOFT_DIRTY;
1385 
1386 	pte = huge_ptep_get(ptep);
1387 	if (pte_present(pte)) {
1388 		struct page *page = pte_page(pte);
1389 
1390 		if (!PageAnon(page))
1391 			flags |= PM_FILE;
1392 
1393 		if (page_mapcount(page) == 1)
1394 			flags |= PM_MMAP_EXCLUSIVE;
1395 
1396 		flags |= PM_PRESENT;
1397 		if (pm->show_pfn)
1398 			frame = pte_pfn(pte) +
1399 				((addr & ~hmask) >> PAGE_SHIFT);
1400 	}
1401 
1402 	for (; addr != end; addr += PAGE_SIZE) {
1403 		pagemap_entry_t pme = make_pme(frame, flags);
1404 
1405 		err = add_to_pagemap(addr, &pme, pm);
1406 		if (err)
1407 			return err;
1408 		if (pm->show_pfn && (flags & PM_PRESENT))
1409 			frame++;
1410 	}
1411 
1412 	cond_resched();
1413 
1414 	return err;
1415 }
1416 #endif /* HUGETLB_PAGE */
1417 
1418 /*
1419  * /proc/pid/pagemap - an array mapping virtual pages to pfns
1420  *
1421  * For each page in the address space, this file contains one 64-bit entry
1422  * consisting of the following:
1423  *
1424  * Bits 0-54  page frame number (PFN) if present
1425  * Bits 0-4   swap type if swapped
1426  * Bits 5-54  swap offset if swapped
1427  * Bit  55    pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst)
1428  * Bit  56    page exclusively mapped
1429  * Bits 57-60 zero
1430  * Bit  61    page is file-page or shared-anon
1431  * Bit  62    page swapped
1432  * Bit  63    page present
1433  *
1434  * If the page is not present but in swap, then the PFN contains an
1435  * encoding of the swap file number and the page's offset into the
1436  * swap. Unmapped pages return a null PFN. This allows determining
1437  * precisely which pages are mapped (or in swap) and comparing mapped
1438  * pages between processes.
1439  *
1440  * Efficient users of this interface will use /proc/pid/maps to
1441  * determine which areas of memory are actually mapped and llseek to
1442  * skip over unmapped regions.
1443  */
1444 static ssize_t pagemap_read(struct file *file, char __user *buf,
1445 			    size_t count, loff_t *ppos)
1446 {
1447 	struct mm_struct *mm = file->private_data;
1448 	struct pagemapread pm;
1449 	struct mm_walk pagemap_walk = {};
1450 	unsigned long src;
1451 	unsigned long svpfn;
1452 	unsigned long start_vaddr;
1453 	unsigned long end_vaddr;
1454 	int ret = 0, copied = 0;
1455 
1456 	if (!mm || !mmget_not_zero(mm))
1457 		goto out;
1458 
1459 	ret = -EINVAL;
1460 	/* file position must be aligned */
1461 	if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES))
1462 		goto out_mm;
1463 
1464 	ret = 0;
1465 	if (!count)
1466 		goto out_mm;
1467 
1468 	/* do not disclose physical addresses: attack vector */
1469 	pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN);
1470 
1471 	pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT);
1472 	pm.buffer = kmalloc_array(pm.len, PM_ENTRY_BYTES, GFP_KERNEL);
1473 	ret = -ENOMEM;
1474 	if (!pm.buffer)
1475 		goto out_mm;
1476 
1477 	pagemap_walk.pmd_entry = pagemap_pmd_range;
1478 	pagemap_walk.pte_hole = pagemap_pte_hole;
1479 #ifdef CONFIG_HUGETLB_PAGE
1480 	pagemap_walk.hugetlb_entry = pagemap_hugetlb_range;
1481 #endif
1482 	pagemap_walk.mm = mm;
1483 	pagemap_walk.private = &pm;
1484 
1485 	src = *ppos;
1486 	svpfn = src / PM_ENTRY_BYTES;
1487 	start_vaddr = svpfn << PAGE_SHIFT;
1488 	end_vaddr = mm->task_size;
1489 
1490 	/* watch out for wraparound */
1491 	if (svpfn > mm->task_size >> PAGE_SHIFT)
1492 		start_vaddr = end_vaddr;
1493 
1494 	/*
1495 	 * The odds are that this will stop walking way
1496 	 * before end_vaddr, because the length of the
1497 	 * user buffer is tracked in "pm", and the walk
1498 	 * will stop when we hit the end of the buffer.
1499 	 */
1500 	ret = 0;
1501 	while (count && (start_vaddr < end_vaddr)) {
1502 		int len;
1503 		unsigned long end;
1504 
1505 		pm.pos = 0;
1506 		end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK;
1507 		/* overflow ? */
1508 		if (end < start_vaddr || end > end_vaddr)
1509 			end = end_vaddr;
1510 		down_read(&mm->mmap_sem);
1511 		ret = walk_page_range(start_vaddr, end, &pagemap_walk);
1512 		up_read(&mm->mmap_sem);
1513 		start_vaddr = end;
1514 
1515 		len = min(count, PM_ENTRY_BYTES * pm.pos);
1516 		if (copy_to_user(buf, pm.buffer, len)) {
1517 			ret = -EFAULT;
1518 			goto out_free;
1519 		}
1520 		copied += len;
1521 		buf += len;
1522 		count -= len;
1523 	}
1524 	*ppos += copied;
1525 	if (!ret || ret == PM_END_OF_BUFFER)
1526 		ret = copied;
1527 
1528 out_free:
1529 	kfree(pm.buffer);
1530 out_mm:
1531 	mmput(mm);
1532 out:
1533 	return ret;
1534 }
1535 
1536 static int pagemap_open(struct inode *inode, struct file *file)
1537 {
1538 	struct mm_struct *mm;
1539 
1540 	mm = proc_mem_open(inode, PTRACE_MODE_READ);
1541 	if (IS_ERR(mm))
1542 		return PTR_ERR(mm);
1543 	file->private_data = mm;
1544 	return 0;
1545 }
1546 
1547 static int pagemap_release(struct inode *inode, struct file *file)
1548 {
1549 	struct mm_struct *mm = file->private_data;
1550 
1551 	if (mm)
1552 		mmdrop(mm);
1553 	return 0;
1554 }
1555 
1556 const struct file_operations proc_pagemap_operations = {
1557 	.llseek		= mem_lseek, /* borrow this */
1558 	.read		= pagemap_read,
1559 	.open		= pagemap_open,
1560 	.release	= pagemap_release,
1561 };
1562 #endif /* CONFIG_PROC_PAGE_MONITOR */
1563 
1564 #ifdef CONFIG_NUMA
1565 
1566 struct numa_maps {
1567 	unsigned long pages;
1568 	unsigned long anon;
1569 	unsigned long active;
1570 	unsigned long writeback;
1571 	unsigned long mapcount_max;
1572 	unsigned long dirty;
1573 	unsigned long swapcache;
1574 	unsigned long node[MAX_NUMNODES];
1575 };
1576 
1577 struct numa_maps_private {
1578 	struct proc_maps_private proc_maps;
1579 	struct numa_maps md;
1580 };
1581 
1582 static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty,
1583 			unsigned long nr_pages)
1584 {
1585 	int count = page_mapcount(page);
1586 
1587 	md->pages += nr_pages;
1588 	if (pte_dirty || PageDirty(page))
1589 		md->dirty += nr_pages;
1590 
1591 	if (PageSwapCache(page))
1592 		md->swapcache += nr_pages;
1593 
1594 	if (PageActive(page) || PageUnevictable(page))
1595 		md->active += nr_pages;
1596 
1597 	if (PageWriteback(page))
1598 		md->writeback += nr_pages;
1599 
1600 	if (PageAnon(page))
1601 		md->anon += nr_pages;
1602 
1603 	if (count > md->mapcount_max)
1604 		md->mapcount_max = count;
1605 
1606 	md->node[page_to_nid(page)] += nr_pages;
1607 }
1608 
1609 static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma,
1610 		unsigned long addr)
1611 {
1612 	struct page *page;
1613 	int nid;
1614 
1615 	if (!pte_present(pte))
1616 		return NULL;
1617 
1618 	page = vm_normal_page(vma, addr, pte);
1619 	if (!page)
1620 		return NULL;
1621 
1622 	if (PageReserved(page))
1623 		return NULL;
1624 
1625 	nid = page_to_nid(page);
1626 	if (!node_isset(nid, node_states[N_MEMORY]))
1627 		return NULL;
1628 
1629 	return page;
1630 }
1631 
1632 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1633 static struct page *can_gather_numa_stats_pmd(pmd_t pmd,
1634 					      struct vm_area_struct *vma,
1635 					      unsigned long addr)
1636 {
1637 	struct page *page;
1638 	int nid;
1639 
1640 	if (!pmd_present(pmd))
1641 		return NULL;
1642 
1643 	page = vm_normal_page_pmd(vma, addr, pmd);
1644 	if (!page)
1645 		return NULL;
1646 
1647 	if (PageReserved(page))
1648 		return NULL;
1649 
1650 	nid = page_to_nid(page);
1651 	if (!node_isset(nid, node_states[N_MEMORY]))
1652 		return NULL;
1653 
1654 	return page;
1655 }
1656 #endif
1657 
1658 static int gather_pte_stats(pmd_t *pmd, unsigned long addr,
1659 		unsigned long end, struct mm_walk *walk)
1660 {
1661 	struct numa_maps *md = walk->private;
1662 	struct vm_area_struct *vma = walk->vma;
1663 	spinlock_t *ptl;
1664 	pte_t *orig_pte;
1665 	pte_t *pte;
1666 
1667 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1668 	ptl = pmd_trans_huge_lock(pmd, vma);
1669 	if (ptl) {
1670 		struct page *page;
1671 
1672 		page = can_gather_numa_stats_pmd(*pmd, vma, addr);
1673 		if (page)
1674 			gather_stats(page, md, pmd_dirty(*pmd),
1675 				     HPAGE_PMD_SIZE/PAGE_SIZE);
1676 		spin_unlock(ptl);
1677 		return 0;
1678 	}
1679 
1680 	if (pmd_trans_unstable(pmd))
1681 		return 0;
1682 #endif
1683 	orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
1684 	do {
1685 		struct page *page = can_gather_numa_stats(*pte, vma, addr);
1686 		if (!page)
1687 			continue;
1688 		gather_stats(page, md, pte_dirty(*pte), 1);
1689 
1690 	} while (pte++, addr += PAGE_SIZE, addr != end);
1691 	pte_unmap_unlock(orig_pte, ptl);
1692 	cond_resched();
1693 	return 0;
1694 }
1695 #ifdef CONFIG_HUGETLB_PAGE
1696 static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
1697 		unsigned long addr, unsigned long end, struct mm_walk *walk)
1698 {
1699 	pte_t huge_pte = huge_ptep_get(pte);
1700 	struct numa_maps *md;
1701 	struct page *page;
1702 
1703 	if (!pte_present(huge_pte))
1704 		return 0;
1705 
1706 	page = pte_page(huge_pte);
1707 	if (!page)
1708 		return 0;
1709 
1710 	md = walk->private;
1711 	gather_stats(page, md, pte_dirty(huge_pte), 1);
1712 	return 0;
1713 }
1714 
1715 #else
1716 static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
1717 		unsigned long addr, unsigned long end, struct mm_walk *walk)
1718 {
1719 	return 0;
1720 }
1721 #endif
1722 
1723 /*
1724  * Display pages allocated per node and memory policy via /proc.
1725  */
1726 static int show_numa_map(struct seq_file *m, void *v)
1727 {
1728 	struct numa_maps_private *numa_priv = m->private;
1729 	struct proc_maps_private *proc_priv = &numa_priv->proc_maps;
1730 	struct vm_area_struct *vma = v;
1731 	struct numa_maps *md = &numa_priv->md;
1732 	struct file *file = vma->vm_file;
1733 	struct mm_struct *mm = vma->vm_mm;
1734 	struct mm_walk walk = {
1735 		.hugetlb_entry = gather_hugetlb_stats,
1736 		.pmd_entry = gather_pte_stats,
1737 		.private = md,
1738 		.mm = mm,
1739 	};
1740 	struct mempolicy *pol;
1741 	char buffer[64];
1742 	int nid;
1743 
1744 	if (!mm)
1745 		return 0;
1746 
1747 	/* Ensure we start with an empty set of numa_maps statistics. */
1748 	memset(md, 0, sizeof(*md));
1749 
1750 	pol = __get_vma_policy(vma, vma->vm_start);
1751 	if (pol) {
1752 		mpol_to_str(buffer, sizeof(buffer), pol);
1753 		mpol_cond_put(pol);
1754 	} else {
1755 		mpol_to_str(buffer, sizeof(buffer), proc_priv->task_mempolicy);
1756 	}
1757 
1758 	seq_printf(m, "%08lx %s", vma->vm_start, buffer);
1759 
1760 	if (file) {
1761 		seq_puts(m, " file=");
1762 		seq_file_path(m, file, "\n\t= ");
1763 	} else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
1764 		seq_puts(m, " heap");
1765 	} else if (is_stack(vma)) {
1766 		seq_puts(m, " stack");
1767 	}
1768 
1769 	if (is_vm_hugetlb_page(vma))
1770 		seq_puts(m, " huge");
1771 
1772 	/* mmap_sem is held by m_start */
1773 	walk_page_vma(vma, &walk);
1774 
1775 	if (!md->pages)
1776 		goto out;
1777 
1778 	if (md->anon)
1779 		seq_printf(m, " anon=%lu", md->anon);
1780 
1781 	if (md->dirty)
1782 		seq_printf(m, " dirty=%lu", md->dirty);
1783 
1784 	if (md->pages != md->anon && md->pages != md->dirty)
1785 		seq_printf(m, " mapped=%lu", md->pages);
1786 
1787 	if (md->mapcount_max > 1)
1788 		seq_printf(m, " mapmax=%lu", md->mapcount_max);
1789 
1790 	if (md->swapcache)
1791 		seq_printf(m, " swapcache=%lu", md->swapcache);
1792 
1793 	if (md->active < md->pages && !is_vm_hugetlb_page(vma))
1794 		seq_printf(m, " active=%lu", md->active);
1795 
1796 	if (md->writeback)
1797 		seq_printf(m, " writeback=%lu", md->writeback);
1798 
1799 	for_each_node_state(nid, N_MEMORY)
1800 		if (md->node[nid])
1801 			seq_printf(m, " N%d=%lu", nid, md->node[nid]);
1802 
1803 	seq_printf(m, " kernelpagesize_kB=%lu", vma_kernel_pagesize(vma) >> 10);
1804 out:
1805 	seq_putc(m, '\n');
1806 	m_cache_vma(m, vma);
1807 	return 0;
1808 }
1809 
1810 static const struct seq_operations proc_pid_numa_maps_op = {
1811 	.start  = m_start,
1812 	.next   = m_next,
1813 	.stop   = m_stop,
1814 	.show   = show_numa_map,
1815 };
1816 
1817 static int pid_numa_maps_open(struct inode *inode, struct file *file)
1818 {
1819 	return proc_maps_open(inode, file, &proc_pid_numa_maps_op,
1820 				sizeof(struct numa_maps_private));
1821 }
1822 
1823 const struct file_operations proc_pid_numa_maps_operations = {
1824 	.open		= pid_numa_maps_open,
1825 	.read		= seq_read,
1826 	.llseek		= seq_lseek,
1827 	.release	= proc_map_release,
1828 };
1829 
1830 #endif /* CONFIG_NUMA */
1831