xref: /openbmc/linux/fs/proc/task_mmu.c (revision 84d517f3)
1 #include <linux/mm.h>
2 #include <linux/vmacache.h>
3 #include <linux/hugetlb.h>
4 #include <linux/huge_mm.h>
5 #include <linux/mount.h>
6 #include <linux/seq_file.h>
7 #include <linux/highmem.h>
8 #include <linux/ptrace.h>
9 #include <linux/slab.h>
10 #include <linux/pagemap.h>
11 #include <linux/mempolicy.h>
12 #include <linux/rmap.h>
13 #include <linux/swap.h>
14 #include <linux/swapops.h>
15 #include <linux/mmu_notifier.h>
16 
17 #include <asm/elf.h>
18 #include <asm/uaccess.h>
19 #include <asm/tlbflush.h>
20 #include "internal.h"
21 
22 void task_mem(struct seq_file *m, struct mm_struct *mm)
23 {
24 	unsigned long data, text, lib, swap;
25 	unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss;
26 
27 	/*
28 	 * Note: to minimize their overhead, mm maintains hiwater_vm and
29 	 * hiwater_rss only when about to *lower* total_vm or rss.  Any
30 	 * collector of these hiwater stats must therefore get total_vm
31 	 * and rss too, which will usually be the higher.  Barriers? not
32 	 * worth the effort, such snapshots can always be inconsistent.
33 	 */
34 	hiwater_vm = total_vm = mm->total_vm;
35 	if (hiwater_vm < mm->hiwater_vm)
36 		hiwater_vm = mm->hiwater_vm;
37 	hiwater_rss = total_rss = get_mm_rss(mm);
38 	if (hiwater_rss < mm->hiwater_rss)
39 		hiwater_rss = mm->hiwater_rss;
40 
41 	data = mm->total_vm - mm->shared_vm - mm->stack_vm;
42 	text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) >> 10;
43 	lib = (mm->exec_vm << (PAGE_SHIFT-10)) - text;
44 	swap = get_mm_counter(mm, MM_SWAPENTS);
45 	seq_printf(m,
46 		"VmPeak:\t%8lu kB\n"
47 		"VmSize:\t%8lu kB\n"
48 		"VmLck:\t%8lu kB\n"
49 		"VmPin:\t%8lu kB\n"
50 		"VmHWM:\t%8lu kB\n"
51 		"VmRSS:\t%8lu kB\n"
52 		"VmData:\t%8lu kB\n"
53 		"VmStk:\t%8lu kB\n"
54 		"VmExe:\t%8lu kB\n"
55 		"VmLib:\t%8lu kB\n"
56 		"VmPTE:\t%8lu kB\n"
57 		"VmSwap:\t%8lu kB\n",
58 		hiwater_vm << (PAGE_SHIFT-10),
59 		total_vm << (PAGE_SHIFT-10),
60 		mm->locked_vm << (PAGE_SHIFT-10),
61 		mm->pinned_vm << (PAGE_SHIFT-10),
62 		hiwater_rss << (PAGE_SHIFT-10),
63 		total_rss << (PAGE_SHIFT-10),
64 		data << (PAGE_SHIFT-10),
65 		mm->stack_vm << (PAGE_SHIFT-10), text, lib,
66 		(PTRS_PER_PTE * sizeof(pte_t) *
67 		 atomic_long_read(&mm->nr_ptes)) >> 10,
68 		swap << (PAGE_SHIFT-10));
69 }
70 
71 unsigned long task_vsize(struct mm_struct *mm)
72 {
73 	return PAGE_SIZE * mm->total_vm;
74 }
75 
76 unsigned long task_statm(struct mm_struct *mm,
77 			 unsigned long *shared, unsigned long *text,
78 			 unsigned long *data, unsigned long *resident)
79 {
80 	*shared = get_mm_counter(mm, MM_FILEPAGES);
81 	*text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))
82 								>> PAGE_SHIFT;
83 	*data = mm->total_vm - mm->shared_vm;
84 	*resident = *shared + get_mm_counter(mm, MM_ANONPAGES);
85 	return mm->total_vm;
86 }
87 
88 #ifdef CONFIG_NUMA
89 /*
90  * These functions are for numa_maps but called in generic **maps seq_file
91  * ->start(), ->stop() ops.
92  *
93  * numa_maps scans all vmas under mmap_sem and checks their mempolicy.
94  * Each mempolicy object is controlled by reference counting. The problem here
95  * is how to avoid accessing dead mempolicy object.
96  *
97  * Because we're holding mmap_sem while reading seq_file, it's safe to access
98  * each vma's mempolicy, no vma objects will never drop refs to mempolicy.
99  *
100  * A task's mempolicy (task->mempolicy) has different behavior. task->mempolicy
101  * is set and replaced under mmap_sem but unrefed and cleared under task_lock().
102  * So, without task_lock(), we cannot trust get_vma_policy() because we cannot
103  * gurantee the task never exits under us. But taking task_lock() around
104  * get_vma_plicy() causes lock order problem.
105  *
106  * To access task->mempolicy without lock, we hold a reference count of an
107  * object pointed by task->mempolicy and remember it. This will guarantee
108  * that task->mempolicy points to an alive object or NULL in numa_maps accesses.
109  */
110 static void hold_task_mempolicy(struct proc_maps_private *priv)
111 {
112 	struct task_struct *task = priv->task;
113 
114 	task_lock(task);
115 	priv->task_mempolicy = task->mempolicy;
116 	mpol_get(priv->task_mempolicy);
117 	task_unlock(task);
118 }
119 static void release_task_mempolicy(struct proc_maps_private *priv)
120 {
121 	mpol_put(priv->task_mempolicy);
122 }
123 #else
124 static void hold_task_mempolicy(struct proc_maps_private *priv)
125 {
126 }
127 static void release_task_mempolicy(struct proc_maps_private *priv)
128 {
129 }
130 #endif
131 
132 static void vma_stop(struct proc_maps_private *priv, struct vm_area_struct *vma)
133 {
134 	if (vma && vma != priv->tail_vma) {
135 		struct mm_struct *mm = vma->vm_mm;
136 		release_task_mempolicy(priv);
137 		up_read(&mm->mmap_sem);
138 		mmput(mm);
139 	}
140 }
141 
142 static void *m_start(struct seq_file *m, loff_t *pos)
143 {
144 	struct proc_maps_private *priv = m->private;
145 	unsigned long last_addr = m->version;
146 	struct mm_struct *mm;
147 	struct vm_area_struct *vma, *tail_vma = NULL;
148 	loff_t l = *pos;
149 
150 	/* Clear the per syscall fields in priv */
151 	priv->task = NULL;
152 	priv->tail_vma = NULL;
153 
154 	/*
155 	 * We remember last_addr rather than next_addr to hit with
156 	 * vmacache most of the time. We have zero last_addr at
157 	 * the beginning and also after lseek. We will have -1 last_addr
158 	 * after the end of the vmas.
159 	 */
160 
161 	if (last_addr == -1UL)
162 		return NULL;
163 
164 	priv->task = get_pid_task(priv->pid, PIDTYPE_PID);
165 	if (!priv->task)
166 		return ERR_PTR(-ESRCH);
167 
168 	mm = mm_access(priv->task, PTRACE_MODE_READ);
169 	if (!mm || IS_ERR(mm))
170 		return mm;
171 	down_read(&mm->mmap_sem);
172 
173 	tail_vma = get_gate_vma(priv->task->mm);
174 	priv->tail_vma = tail_vma;
175 	hold_task_mempolicy(priv);
176 	/* Start with last addr hint */
177 	vma = find_vma(mm, last_addr);
178 	if (last_addr && vma) {
179 		vma = vma->vm_next;
180 		goto out;
181 	}
182 
183 	/*
184 	 * Check the vma index is within the range and do
185 	 * sequential scan until m_index.
186 	 */
187 	vma = NULL;
188 	if ((unsigned long)l < mm->map_count) {
189 		vma = mm->mmap;
190 		while (l-- && vma)
191 			vma = vma->vm_next;
192 		goto out;
193 	}
194 
195 	if (l != mm->map_count)
196 		tail_vma = NULL; /* After gate vma */
197 
198 out:
199 	if (vma)
200 		return vma;
201 
202 	release_task_mempolicy(priv);
203 	/* End of vmas has been reached */
204 	m->version = (tail_vma != NULL)? 0: -1UL;
205 	up_read(&mm->mmap_sem);
206 	mmput(mm);
207 	return tail_vma;
208 }
209 
210 static void *m_next(struct seq_file *m, void *v, loff_t *pos)
211 {
212 	struct proc_maps_private *priv = m->private;
213 	struct vm_area_struct *vma = v;
214 	struct vm_area_struct *tail_vma = priv->tail_vma;
215 
216 	(*pos)++;
217 	if (vma && (vma != tail_vma) && vma->vm_next)
218 		return vma->vm_next;
219 	vma_stop(priv, vma);
220 	return (vma != tail_vma)? tail_vma: NULL;
221 }
222 
223 static void m_stop(struct seq_file *m, void *v)
224 {
225 	struct proc_maps_private *priv = m->private;
226 	struct vm_area_struct *vma = v;
227 
228 	if (!IS_ERR(vma))
229 		vma_stop(priv, vma);
230 	if (priv->task)
231 		put_task_struct(priv->task);
232 }
233 
234 static int do_maps_open(struct inode *inode, struct file *file,
235 			const struct seq_operations *ops)
236 {
237 	struct proc_maps_private *priv;
238 	int ret = -ENOMEM;
239 	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
240 	if (priv) {
241 		priv->pid = proc_pid(inode);
242 		ret = seq_open(file, ops);
243 		if (!ret) {
244 			struct seq_file *m = file->private_data;
245 			m->private = priv;
246 		} else {
247 			kfree(priv);
248 		}
249 	}
250 	return ret;
251 }
252 
253 static void
254 show_map_vma(struct seq_file *m, struct vm_area_struct *vma, int is_pid)
255 {
256 	struct mm_struct *mm = vma->vm_mm;
257 	struct file *file = vma->vm_file;
258 	struct proc_maps_private *priv = m->private;
259 	struct task_struct *task = priv->task;
260 	vm_flags_t flags = vma->vm_flags;
261 	unsigned long ino = 0;
262 	unsigned long long pgoff = 0;
263 	unsigned long start, end;
264 	dev_t dev = 0;
265 	const char *name = NULL;
266 
267 	if (file) {
268 		struct inode *inode = file_inode(vma->vm_file);
269 		dev = inode->i_sb->s_dev;
270 		ino = inode->i_ino;
271 		pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT;
272 	}
273 
274 	/* We don't show the stack guard page in /proc/maps */
275 	start = vma->vm_start;
276 	if (stack_guard_page_start(vma, start))
277 		start += PAGE_SIZE;
278 	end = vma->vm_end;
279 	if (stack_guard_page_end(vma, end))
280 		end -= PAGE_SIZE;
281 
282 	seq_setwidth(m, 25 + sizeof(void *) * 6 - 1);
283 	seq_printf(m, "%08lx-%08lx %c%c%c%c %08llx %02x:%02x %lu ",
284 			start,
285 			end,
286 			flags & VM_READ ? 'r' : '-',
287 			flags & VM_WRITE ? 'w' : '-',
288 			flags & VM_EXEC ? 'x' : '-',
289 			flags & VM_MAYSHARE ? 's' : 'p',
290 			pgoff,
291 			MAJOR(dev), MINOR(dev), ino);
292 
293 	/*
294 	 * Print the dentry name for named mappings, and a
295 	 * special [heap] marker for the heap:
296 	 */
297 	if (file) {
298 		seq_pad(m, ' ');
299 		seq_path(m, &file->f_path, "\n");
300 		goto done;
301 	}
302 
303 	name = arch_vma_name(vma);
304 	if (!name) {
305 		pid_t tid;
306 
307 		if (!mm) {
308 			name = "[vdso]";
309 			goto done;
310 		}
311 
312 		if (vma->vm_start <= mm->brk &&
313 		    vma->vm_end >= mm->start_brk) {
314 			name = "[heap]";
315 			goto done;
316 		}
317 
318 		tid = vm_is_stack(task, vma, is_pid);
319 
320 		if (tid != 0) {
321 			/*
322 			 * Thread stack in /proc/PID/task/TID/maps or
323 			 * the main process stack.
324 			 */
325 			if (!is_pid || (vma->vm_start <= mm->start_stack &&
326 			    vma->vm_end >= mm->start_stack)) {
327 				name = "[stack]";
328 			} else {
329 				/* Thread stack in /proc/PID/maps */
330 				seq_pad(m, ' ');
331 				seq_printf(m, "[stack:%d]", tid);
332 			}
333 		}
334 	}
335 
336 done:
337 	if (name) {
338 		seq_pad(m, ' ');
339 		seq_puts(m, name);
340 	}
341 	seq_putc(m, '\n');
342 }
343 
344 static int show_map(struct seq_file *m, void *v, int is_pid)
345 {
346 	struct vm_area_struct *vma = v;
347 	struct proc_maps_private *priv = m->private;
348 	struct task_struct *task = priv->task;
349 
350 	show_map_vma(m, vma, is_pid);
351 
352 	if (m->count < m->size)  /* vma is copied successfully */
353 		m->version = (vma != get_gate_vma(task->mm))
354 			? vma->vm_start : 0;
355 	return 0;
356 }
357 
358 static int show_pid_map(struct seq_file *m, void *v)
359 {
360 	return show_map(m, v, 1);
361 }
362 
363 static int show_tid_map(struct seq_file *m, void *v)
364 {
365 	return show_map(m, v, 0);
366 }
367 
368 static const struct seq_operations proc_pid_maps_op = {
369 	.start	= m_start,
370 	.next	= m_next,
371 	.stop	= m_stop,
372 	.show	= show_pid_map
373 };
374 
375 static const struct seq_operations proc_tid_maps_op = {
376 	.start	= m_start,
377 	.next	= m_next,
378 	.stop	= m_stop,
379 	.show	= show_tid_map
380 };
381 
382 static int pid_maps_open(struct inode *inode, struct file *file)
383 {
384 	return do_maps_open(inode, file, &proc_pid_maps_op);
385 }
386 
387 static int tid_maps_open(struct inode *inode, struct file *file)
388 {
389 	return do_maps_open(inode, file, &proc_tid_maps_op);
390 }
391 
392 const struct file_operations proc_pid_maps_operations = {
393 	.open		= pid_maps_open,
394 	.read		= seq_read,
395 	.llseek		= seq_lseek,
396 	.release	= seq_release_private,
397 };
398 
399 const struct file_operations proc_tid_maps_operations = {
400 	.open		= tid_maps_open,
401 	.read		= seq_read,
402 	.llseek		= seq_lseek,
403 	.release	= seq_release_private,
404 };
405 
406 /*
407  * Proportional Set Size(PSS): my share of RSS.
408  *
409  * PSS of a process is the count of pages it has in memory, where each
410  * page is divided by the number of processes sharing it.  So if a
411  * process has 1000 pages all to itself, and 1000 shared with one other
412  * process, its PSS will be 1500.
413  *
414  * To keep (accumulated) division errors low, we adopt a 64bit
415  * fixed-point pss counter to minimize division errors. So (pss >>
416  * PSS_SHIFT) would be the real byte count.
417  *
418  * A shift of 12 before division means (assuming 4K page size):
419  * 	- 1M 3-user-pages add up to 8KB errors;
420  * 	- supports mapcount up to 2^24, or 16M;
421  * 	- supports PSS up to 2^52 bytes, or 4PB.
422  */
423 #define PSS_SHIFT 12
424 
425 #ifdef CONFIG_PROC_PAGE_MONITOR
426 struct mem_size_stats {
427 	struct vm_area_struct *vma;
428 	unsigned long resident;
429 	unsigned long shared_clean;
430 	unsigned long shared_dirty;
431 	unsigned long private_clean;
432 	unsigned long private_dirty;
433 	unsigned long referenced;
434 	unsigned long anonymous;
435 	unsigned long anonymous_thp;
436 	unsigned long swap;
437 	unsigned long nonlinear;
438 	u64 pss;
439 };
440 
441 
442 static void smaps_pte_entry(pte_t ptent, unsigned long addr,
443 		unsigned long ptent_size, struct mm_walk *walk)
444 {
445 	struct mem_size_stats *mss = walk->private;
446 	struct vm_area_struct *vma = mss->vma;
447 	pgoff_t pgoff = linear_page_index(vma, addr);
448 	struct page *page = NULL;
449 	int mapcount;
450 
451 	if (pte_present(ptent)) {
452 		page = vm_normal_page(vma, addr, ptent);
453 	} else if (is_swap_pte(ptent)) {
454 		swp_entry_t swpent = pte_to_swp_entry(ptent);
455 
456 		if (!non_swap_entry(swpent))
457 			mss->swap += ptent_size;
458 		else if (is_migration_entry(swpent))
459 			page = migration_entry_to_page(swpent);
460 	} else if (pte_file(ptent)) {
461 		if (pte_to_pgoff(ptent) != pgoff)
462 			mss->nonlinear += ptent_size;
463 	}
464 
465 	if (!page)
466 		return;
467 
468 	if (PageAnon(page))
469 		mss->anonymous += ptent_size;
470 
471 	if (page->index != pgoff)
472 		mss->nonlinear += ptent_size;
473 
474 	mss->resident += ptent_size;
475 	/* Accumulate the size in pages that have been accessed. */
476 	if (pte_young(ptent) || PageReferenced(page))
477 		mss->referenced += ptent_size;
478 	mapcount = page_mapcount(page);
479 	if (mapcount >= 2) {
480 		if (pte_dirty(ptent) || PageDirty(page))
481 			mss->shared_dirty += ptent_size;
482 		else
483 			mss->shared_clean += ptent_size;
484 		mss->pss += (ptent_size << PSS_SHIFT) / mapcount;
485 	} else {
486 		if (pte_dirty(ptent) || PageDirty(page))
487 			mss->private_dirty += ptent_size;
488 		else
489 			mss->private_clean += ptent_size;
490 		mss->pss += (ptent_size << PSS_SHIFT);
491 	}
492 }
493 
494 static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
495 			   struct mm_walk *walk)
496 {
497 	struct mem_size_stats *mss = walk->private;
498 	struct vm_area_struct *vma = mss->vma;
499 	pte_t *pte;
500 	spinlock_t *ptl;
501 
502 	if (pmd_trans_huge_lock(pmd, vma, &ptl) == 1) {
503 		smaps_pte_entry(*(pte_t *)pmd, addr, HPAGE_PMD_SIZE, walk);
504 		spin_unlock(ptl);
505 		mss->anonymous_thp += HPAGE_PMD_SIZE;
506 		return 0;
507 	}
508 
509 	if (pmd_trans_unstable(pmd))
510 		return 0;
511 	/*
512 	 * The mmap_sem held all the way back in m_start() is what
513 	 * keeps khugepaged out of here and from collapsing things
514 	 * in here.
515 	 */
516 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
517 	for (; addr != end; pte++, addr += PAGE_SIZE)
518 		smaps_pte_entry(*pte, addr, PAGE_SIZE, walk);
519 	pte_unmap_unlock(pte - 1, ptl);
520 	cond_resched();
521 	return 0;
522 }
523 
524 static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
525 {
526 	/*
527 	 * Don't forget to update Documentation/ on changes.
528 	 */
529 	static const char mnemonics[BITS_PER_LONG][2] = {
530 		/*
531 		 * In case if we meet a flag we don't know about.
532 		 */
533 		[0 ... (BITS_PER_LONG-1)] = "??",
534 
535 		[ilog2(VM_READ)]	= "rd",
536 		[ilog2(VM_WRITE)]	= "wr",
537 		[ilog2(VM_EXEC)]	= "ex",
538 		[ilog2(VM_SHARED)]	= "sh",
539 		[ilog2(VM_MAYREAD)]	= "mr",
540 		[ilog2(VM_MAYWRITE)]	= "mw",
541 		[ilog2(VM_MAYEXEC)]	= "me",
542 		[ilog2(VM_MAYSHARE)]	= "ms",
543 		[ilog2(VM_GROWSDOWN)]	= "gd",
544 		[ilog2(VM_PFNMAP)]	= "pf",
545 		[ilog2(VM_DENYWRITE)]	= "dw",
546 		[ilog2(VM_LOCKED)]	= "lo",
547 		[ilog2(VM_IO)]		= "io",
548 		[ilog2(VM_SEQ_READ)]	= "sr",
549 		[ilog2(VM_RAND_READ)]	= "rr",
550 		[ilog2(VM_DONTCOPY)]	= "dc",
551 		[ilog2(VM_DONTEXPAND)]	= "de",
552 		[ilog2(VM_ACCOUNT)]	= "ac",
553 		[ilog2(VM_NORESERVE)]	= "nr",
554 		[ilog2(VM_HUGETLB)]	= "ht",
555 		[ilog2(VM_NONLINEAR)]	= "nl",
556 		[ilog2(VM_ARCH_1)]	= "ar",
557 		[ilog2(VM_DONTDUMP)]	= "dd",
558 #ifdef CONFIG_MEM_SOFT_DIRTY
559 		[ilog2(VM_SOFTDIRTY)]	= "sd",
560 #endif
561 		[ilog2(VM_MIXEDMAP)]	= "mm",
562 		[ilog2(VM_HUGEPAGE)]	= "hg",
563 		[ilog2(VM_NOHUGEPAGE)]	= "nh",
564 		[ilog2(VM_MERGEABLE)]	= "mg",
565 	};
566 	size_t i;
567 
568 	seq_puts(m, "VmFlags: ");
569 	for (i = 0; i < BITS_PER_LONG; i++) {
570 		if (vma->vm_flags & (1UL << i)) {
571 			seq_printf(m, "%c%c ",
572 				   mnemonics[i][0], mnemonics[i][1]);
573 		}
574 	}
575 	seq_putc(m, '\n');
576 }
577 
578 static int show_smap(struct seq_file *m, void *v, int is_pid)
579 {
580 	struct proc_maps_private *priv = m->private;
581 	struct task_struct *task = priv->task;
582 	struct vm_area_struct *vma = v;
583 	struct mem_size_stats mss;
584 	struct mm_walk smaps_walk = {
585 		.pmd_entry = smaps_pte_range,
586 		.mm = vma->vm_mm,
587 		.private = &mss,
588 	};
589 
590 	memset(&mss, 0, sizeof mss);
591 	mss.vma = vma;
592 	/* mmap_sem is held in m_start */
593 	if (vma->vm_mm && !is_vm_hugetlb_page(vma))
594 		walk_page_range(vma->vm_start, vma->vm_end, &smaps_walk);
595 
596 	show_map_vma(m, vma, is_pid);
597 
598 	seq_printf(m,
599 		   "Size:           %8lu kB\n"
600 		   "Rss:            %8lu kB\n"
601 		   "Pss:            %8lu kB\n"
602 		   "Shared_Clean:   %8lu kB\n"
603 		   "Shared_Dirty:   %8lu kB\n"
604 		   "Private_Clean:  %8lu kB\n"
605 		   "Private_Dirty:  %8lu kB\n"
606 		   "Referenced:     %8lu kB\n"
607 		   "Anonymous:      %8lu kB\n"
608 		   "AnonHugePages:  %8lu kB\n"
609 		   "Swap:           %8lu kB\n"
610 		   "KernelPageSize: %8lu kB\n"
611 		   "MMUPageSize:    %8lu kB\n"
612 		   "Locked:         %8lu kB\n",
613 		   (vma->vm_end - vma->vm_start) >> 10,
614 		   mss.resident >> 10,
615 		   (unsigned long)(mss.pss >> (10 + PSS_SHIFT)),
616 		   mss.shared_clean  >> 10,
617 		   mss.shared_dirty  >> 10,
618 		   mss.private_clean >> 10,
619 		   mss.private_dirty >> 10,
620 		   mss.referenced >> 10,
621 		   mss.anonymous >> 10,
622 		   mss.anonymous_thp >> 10,
623 		   mss.swap >> 10,
624 		   vma_kernel_pagesize(vma) >> 10,
625 		   vma_mmu_pagesize(vma) >> 10,
626 		   (vma->vm_flags & VM_LOCKED) ?
627 			(unsigned long)(mss.pss >> (10 + PSS_SHIFT)) : 0);
628 
629 	if (vma->vm_flags & VM_NONLINEAR)
630 		seq_printf(m, "Nonlinear:      %8lu kB\n",
631 				mss.nonlinear >> 10);
632 
633 	show_smap_vma_flags(m, vma);
634 
635 	if (m->count < m->size)  /* vma is copied successfully */
636 		m->version = (vma != get_gate_vma(task->mm))
637 			? vma->vm_start : 0;
638 	return 0;
639 }
640 
641 static int show_pid_smap(struct seq_file *m, void *v)
642 {
643 	return show_smap(m, v, 1);
644 }
645 
646 static int show_tid_smap(struct seq_file *m, void *v)
647 {
648 	return show_smap(m, v, 0);
649 }
650 
651 static const struct seq_operations proc_pid_smaps_op = {
652 	.start	= m_start,
653 	.next	= m_next,
654 	.stop	= m_stop,
655 	.show	= show_pid_smap
656 };
657 
658 static const struct seq_operations proc_tid_smaps_op = {
659 	.start	= m_start,
660 	.next	= m_next,
661 	.stop	= m_stop,
662 	.show	= show_tid_smap
663 };
664 
665 static int pid_smaps_open(struct inode *inode, struct file *file)
666 {
667 	return do_maps_open(inode, file, &proc_pid_smaps_op);
668 }
669 
670 static int tid_smaps_open(struct inode *inode, struct file *file)
671 {
672 	return do_maps_open(inode, file, &proc_tid_smaps_op);
673 }
674 
675 const struct file_operations proc_pid_smaps_operations = {
676 	.open		= pid_smaps_open,
677 	.read		= seq_read,
678 	.llseek		= seq_lseek,
679 	.release	= seq_release_private,
680 };
681 
682 const struct file_operations proc_tid_smaps_operations = {
683 	.open		= tid_smaps_open,
684 	.read		= seq_read,
685 	.llseek		= seq_lseek,
686 	.release	= seq_release_private,
687 };
688 
689 /*
690  * We do not want to have constant page-shift bits sitting in
691  * pagemap entries and are about to reuse them some time soon.
692  *
693  * Here's the "migration strategy":
694  * 1. when the system boots these bits remain what they are,
695  *    but a warning about future change is printed in log;
696  * 2. once anyone clears soft-dirty bits via clear_refs file,
697  *    these flag is set to denote, that user is aware of the
698  *    new API and those page-shift bits change their meaning.
699  *    The respective warning is printed in dmesg;
700  * 3. In a couple of releases we will remove all the mentions
701  *    of page-shift in pagemap entries.
702  */
703 
704 static bool soft_dirty_cleared __read_mostly;
705 
706 enum clear_refs_types {
707 	CLEAR_REFS_ALL = 1,
708 	CLEAR_REFS_ANON,
709 	CLEAR_REFS_MAPPED,
710 	CLEAR_REFS_SOFT_DIRTY,
711 	CLEAR_REFS_LAST,
712 };
713 
714 struct clear_refs_private {
715 	struct vm_area_struct *vma;
716 	enum clear_refs_types type;
717 };
718 
719 static inline void clear_soft_dirty(struct vm_area_struct *vma,
720 		unsigned long addr, pte_t *pte)
721 {
722 #ifdef CONFIG_MEM_SOFT_DIRTY
723 	/*
724 	 * The soft-dirty tracker uses #PF-s to catch writes
725 	 * to pages, so write-protect the pte as well. See the
726 	 * Documentation/vm/soft-dirty.txt for full description
727 	 * of how soft-dirty works.
728 	 */
729 	pte_t ptent = *pte;
730 
731 	if (pte_present(ptent)) {
732 		ptent = pte_wrprotect(ptent);
733 		ptent = pte_clear_flags(ptent, _PAGE_SOFT_DIRTY);
734 	} else if (is_swap_pte(ptent)) {
735 		ptent = pte_swp_clear_soft_dirty(ptent);
736 	} else if (pte_file(ptent)) {
737 		ptent = pte_file_clear_soft_dirty(ptent);
738 	}
739 
740 	set_pte_at(vma->vm_mm, addr, pte, ptent);
741 #endif
742 }
743 
744 static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
745 				unsigned long end, struct mm_walk *walk)
746 {
747 	struct clear_refs_private *cp = walk->private;
748 	struct vm_area_struct *vma = cp->vma;
749 	pte_t *pte, ptent;
750 	spinlock_t *ptl;
751 	struct page *page;
752 
753 	split_huge_page_pmd(vma, addr, pmd);
754 	if (pmd_trans_unstable(pmd))
755 		return 0;
756 
757 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
758 	for (; addr != end; pte++, addr += PAGE_SIZE) {
759 		ptent = *pte;
760 
761 		if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
762 			clear_soft_dirty(vma, addr, pte);
763 			continue;
764 		}
765 
766 		if (!pte_present(ptent))
767 			continue;
768 
769 		page = vm_normal_page(vma, addr, ptent);
770 		if (!page)
771 			continue;
772 
773 		/* Clear accessed and referenced bits. */
774 		ptep_test_and_clear_young(vma, addr, pte);
775 		ClearPageReferenced(page);
776 	}
777 	pte_unmap_unlock(pte - 1, ptl);
778 	cond_resched();
779 	return 0;
780 }
781 
782 static ssize_t clear_refs_write(struct file *file, const char __user *buf,
783 				size_t count, loff_t *ppos)
784 {
785 	struct task_struct *task;
786 	char buffer[PROC_NUMBUF];
787 	struct mm_struct *mm;
788 	struct vm_area_struct *vma;
789 	enum clear_refs_types type;
790 	int itype;
791 	int rv;
792 
793 	memset(buffer, 0, sizeof(buffer));
794 	if (count > sizeof(buffer) - 1)
795 		count = sizeof(buffer) - 1;
796 	if (copy_from_user(buffer, buf, count))
797 		return -EFAULT;
798 	rv = kstrtoint(strstrip(buffer), 10, &itype);
799 	if (rv < 0)
800 		return rv;
801 	type = (enum clear_refs_types)itype;
802 	if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST)
803 		return -EINVAL;
804 
805 	if (type == CLEAR_REFS_SOFT_DIRTY) {
806 		soft_dirty_cleared = true;
807 		pr_warn_once("The pagemap bits 55-60 has changed their meaning!"
808 			     " See the linux/Documentation/vm/pagemap.txt for "
809 			     "details.\n");
810 	}
811 
812 	task = get_proc_task(file_inode(file));
813 	if (!task)
814 		return -ESRCH;
815 	mm = get_task_mm(task);
816 	if (mm) {
817 		struct clear_refs_private cp = {
818 			.type = type,
819 		};
820 		struct mm_walk clear_refs_walk = {
821 			.pmd_entry = clear_refs_pte_range,
822 			.mm = mm,
823 			.private = &cp,
824 		};
825 		down_read(&mm->mmap_sem);
826 		if (type == CLEAR_REFS_SOFT_DIRTY)
827 			mmu_notifier_invalidate_range_start(mm, 0, -1);
828 		for (vma = mm->mmap; vma; vma = vma->vm_next) {
829 			cp.vma = vma;
830 			if (is_vm_hugetlb_page(vma))
831 				continue;
832 			/*
833 			 * Writing 1 to /proc/pid/clear_refs affects all pages.
834 			 *
835 			 * Writing 2 to /proc/pid/clear_refs only affects
836 			 * Anonymous pages.
837 			 *
838 			 * Writing 3 to /proc/pid/clear_refs only affects file
839 			 * mapped pages.
840 			 *
841 			 * Writing 4 to /proc/pid/clear_refs affects all pages.
842 			 */
843 			if (type == CLEAR_REFS_ANON && vma->vm_file)
844 				continue;
845 			if (type == CLEAR_REFS_MAPPED && !vma->vm_file)
846 				continue;
847 			if (type == CLEAR_REFS_SOFT_DIRTY) {
848 				if (vma->vm_flags & VM_SOFTDIRTY)
849 					vma->vm_flags &= ~VM_SOFTDIRTY;
850 			}
851 			walk_page_range(vma->vm_start, vma->vm_end,
852 					&clear_refs_walk);
853 		}
854 		if (type == CLEAR_REFS_SOFT_DIRTY)
855 			mmu_notifier_invalidate_range_end(mm, 0, -1);
856 		flush_tlb_mm(mm);
857 		up_read(&mm->mmap_sem);
858 		mmput(mm);
859 	}
860 	put_task_struct(task);
861 
862 	return count;
863 }
864 
865 const struct file_operations proc_clear_refs_operations = {
866 	.write		= clear_refs_write,
867 	.llseek		= noop_llseek,
868 };
869 
870 typedef struct {
871 	u64 pme;
872 } pagemap_entry_t;
873 
874 struct pagemapread {
875 	int pos, len;		/* units: PM_ENTRY_BYTES, not bytes */
876 	pagemap_entry_t *buffer;
877 	bool v2;
878 };
879 
880 #define PAGEMAP_WALK_SIZE	(PMD_SIZE)
881 #define PAGEMAP_WALK_MASK	(PMD_MASK)
882 
883 #define PM_ENTRY_BYTES      sizeof(pagemap_entry_t)
884 #define PM_STATUS_BITS      3
885 #define PM_STATUS_OFFSET    (64 - PM_STATUS_BITS)
886 #define PM_STATUS_MASK      (((1LL << PM_STATUS_BITS) - 1) << PM_STATUS_OFFSET)
887 #define PM_STATUS(nr)       (((nr) << PM_STATUS_OFFSET) & PM_STATUS_MASK)
888 #define PM_PSHIFT_BITS      6
889 #define PM_PSHIFT_OFFSET    (PM_STATUS_OFFSET - PM_PSHIFT_BITS)
890 #define PM_PSHIFT_MASK      (((1LL << PM_PSHIFT_BITS) - 1) << PM_PSHIFT_OFFSET)
891 #define __PM_PSHIFT(x)      (((u64) (x) << PM_PSHIFT_OFFSET) & PM_PSHIFT_MASK)
892 #define PM_PFRAME_MASK      ((1LL << PM_PSHIFT_OFFSET) - 1)
893 #define PM_PFRAME(x)        ((x) & PM_PFRAME_MASK)
894 /* in "new" pagemap pshift bits are occupied with more status bits */
895 #define PM_STATUS2(v2, x)   (__PM_PSHIFT(v2 ? x : PAGE_SHIFT))
896 
897 #define __PM_SOFT_DIRTY      (1LL)
898 #define PM_PRESENT          PM_STATUS(4LL)
899 #define PM_SWAP             PM_STATUS(2LL)
900 #define PM_FILE             PM_STATUS(1LL)
901 #define PM_NOT_PRESENT(v2)  PM_STATUS2(v2, 0)
902 #define PM_END_OF_BUFFER    1
903 
904 static inline pagemap_entry_t make_pme(u64 val)
905 {
906 	return (pagemap_entry_t) { .pme = val };
907 }
908 
909 static int add_to_pagemap(unsigned long addr, pagemap_entry_t *pme,
910 			  struct pagemapread *pm)
911 {
912 	pm->buffer[pm->pos++] = *pme;
913 	if (pm->pos >= pm->len)
914 		return PM_END_OF_BUFFER;
915 	return 0;
916 }
917 
918 static int pagemap_pte_hole(unsigned long start, unsigned long end,
919 				struct mm_walk *walk)
920 {
921 	struct pagemapread *pm = walk->private;
922 	unsigned long addr;
923 	int err = 0;
924 	pagemap_entry_t pme = make_pme(PM_NOT_PRESENT(pm->v2));
925 
926 	for (addr = start; addr < end; addr += PAGE_SIZE) {
927 		err = add_to_pagemap(addr, &pme, pm);
928 		if (err)
929 			break;
930 	}
931 	return err;
932 }
933 
934 static void pte_to_pagemap_entry(pagemap_entry_t *pme, struct pagemapread *pm,
935 		struct vm_area_struct *vma, unsigned long addr, pte_t pte)
936 {
937 	u64 frame, flags;
938 	struct page *page = NULL;
939 	int flags2 = 0;
940 
941 	if (pte_present(pte)) {
942 		frame = pte_pfn(pte);
943 		flags = PM_PRESENT;
944 		page = vm_normal_page(vma, addr, pte);
945 		if (pte_soft_dirty(pte))
946 			flags2 |= __PM_SOFT_DIRTY;
947 	} else if (is_swap_pte(pte)) {
948 		swp_entry_t entry;
949 		if (pte_swp_soft_dirty(pte))
950 			flags2 |= __PM_SOFT_DIRTY;
951 		entry = pte_to_swp_entry(pte);
952 		frame = swp_type(entry) |
953 			(swp_offset(entry) << MAX_SWAPFILES_SHIFT);
954 		flags = PM_SWAP;
955 		if (is_migration_entry(entry))
956 			page = migration_entry_to_page(entry);
957 	} else {
958 		if (vma->vm_flags & VM_SOFTDIRTY)
959 			flags2 |= __PM_SOFT_DIRTY;
960 		*pme = make_pme(PM_NOT_PRESENT(pm->v2) | PM_STATUS2(pm->v2, flags2));
961 		return;
962 	}
963 
964 	if (page && !PageAnon(page))
965 		flags |= PM_FILE;
966 	if ((vma->vm_flags & VM_SOFTDIRTY))
967 		flags2 |= __PM_SOFT_DIRTY;
968 
969 	*pme = make_pme(PM_PFRAME(frame) | PM_STATUS2(pm->v2, flags2) | flags);
970 }
971 
972 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
973 static void thp_pmd_to_pagemap_entry(pagemap_entry_t *pme, struct pagemapread *pm,
974 		pmd_t pmd, int offset, int pmd_flags2)
975 {
976 	/*
977 	 * Currently pmd for thp is always present because thp can not be
978 	 * swapped-out, migrated, or HWPOISONed (split in such cases instead.)
979 	 * This if-check is just to prepare for future implementation.
980 	 */
981 	if (pmd_present(pmd))
982 		*pme = make_pme(PM_PFRAME(pmd_pfn(pmd) + offset)
983 				| PM_STATUS2(pm->v2, pmd_flags2) | PM_PRESENT);
984 	else
985 		*pme = make_pme(PM_NOT_PRESENT(pm->v2) | PM_STATUS2(pm->v2, pmd_flags2));
986 }
987 #else
988 static inline void thp_pmd_to_pagemap_entry(pagemap_entry_t *pme, struct pagemapread *pm,
989 		pmd_t pmd, int offset, int pmd_flags2)
990 {
991 }
992 #endif
993 
994 static int pagemap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
995 			     struct mm_walk *walk)
996 {
997 	struct vm_area_struct *vma;
998 	struct pagemapread *pm = walk->private;
999 	spinlock_t *ptl;
1000 	pte_t *pte;
1001 	int err = 0;
1002 	pagemap_entry_t pme = make_pme(PM_NOT_PRESENT(pm->v2));
1003 
1004 	/* find the first VMA at or above 'addr' */
1005 	vma = find_vma(walk->mm, addr);
1006 	if (vma && pmd_trans_huge_lock(pmd, vma, &ptl) == 1) {
1007 		int pmd_flags2;
1008 
1009 		if ((vma->vm_flags & VM_SOFTDIRTY) || pmd_soft_dirty(*pmd))
1010 			pmd_flags2 = __PM_SOFT_DIRTY;
1011 		else
1012 			pmd_flags2 = 0;
1013 
1014 		for (; addr != end; addr += PAGE_SIZE) {
1015 			unsigned long offset;
1016 
1017 			offset = (addr & ~PAGEMAP_WALK_MASK) >>
1018 					PAGE_SHIFT;
1019 			thp_pmd_to_pagemap_entry(&pme, pm, *pmd, offset, pmd_flags2);
1020 			err = add_to_pagemap(addr, &pme, pm);
1021 			if (err)
1022 				break;
1023 		}
1024 		spin_unlock(ptl);
1025 		return err;
1026 	}
1027 
1028 	if (pmd_trans_unstable(pmd))
1029 		return 0;
1030 	for (; addr != end; addr += PAGE_SIZE) {
1031 		int flags2;
1032 
1033 		/* check to see if we've left 'vma' behind
1034 		 * and need a new, higher one */
1035 		if (vma && (addr >= vma->vm_end)) {
1036 			vma = find_vma(walk->mm, addr);
1037 			if (vma && (vma->vm_flags & VM_SOFTDIRTY))
1038 				flags2 = __PM_SOFT_DIRTY;
1039 			else
1040 				flags2 = 0;
1041 			pme = make_pme(PM_NOT_PRESENT(pm->v2) | PM_STATUS2(pm->v2, flags2));
1042 		}
1043 
1044 		/* check that 'vma' actually covers this address,
1045 		 * and that it isn't a huge page vma */
1046 		if (vma && (vma->vm_start <= addr) &&
1047 		    !is_vm_hugetlb_page(vma)) {
1048 			pte = pte_offset_map(pmd, addr);
1049 			pte_to_pagemap_entry(&pme, pm, vma, addr, *pte);
1050 			/* unmap before userspace copy */
1051 			pte_unmap(pte);
1052 		}
1053 		err = add_to_pagemap(addr, &pme, pm);
1054 		if (err)
1055 			return err;
1056 	}
1057 
1058 	cond_resched();
1059 
1060 	return err;
1061 }
1062 
1063 #ifdef CONFIG_HUGETLB_PAGE
1064 static void huge_pte_to_pagemap_entry(pagemap_entry_t *pme, struct pagemapread *pm,
1065 					pte_t pte, int offset, int flags2)
1066 {
1067 	if (pte_present(pte))
1068 		*pme = make_pme(PM_PFRAME(pte_pfn(pte) + offset)	|
1069 				PM_STATUS2(pm->v2, flags2)		|
1070 				PM_PRESENT);
1071 	else
1072 		*pme = make_pme(PM_NOT_PRESENT(pm->v2)			|
1073 				PM_STATUS2(pm->v2, flags2));
1074 }
1075 
1076 /* This function walks within one hugetlb entry in the single call */
1077 static int pagemap_hugetlb_range(pte_t *pte, unsigned long hmask,
1078 				 unsigned long addr, unsigned long end,
1079 				 struct mm_walk *walk)
1080 {
1081 	struct pagemapread *pm = walk->private;
1082 	struct vm_area_struct *vma;
1083 	int err = 0;
1084 	int flags2;
1085 	pagemap_entry_t pme;
1086 
1087 	vma = find_vma(walk->mm, addr);
1088 	WARN_ON_ONCE(!vma);
1089 
1090 	if (vma && (vma->vm_flags & VM_SOFTDIRTY))
1091 		flags2 = __PM_SOFT_DIRTY;
1092 	else
1093 		flags2 = 0;
1094 
1095 	for (; addr != end; addr += PAGE_SIZE) {
1096 		int offset = (addr & ~hmask) >> PAGE_SHIFT;
1097 		huge_pte_to_pagemap_entry(&pme, pm, *pte, offset, flags2);
1098 		err = add_to_pagemap(addr, &pme, pm);
1099 		if (err)
1100 			return err;
1101 	}
1102 
1103 	cond_resched();
1104 
1105 	return err;
1106 }
1107 #endif /* HUGETLB_PAGE */
1108 
1109 /*
1110  * /proc/pid/pagemap - an array mapping virtual pages to pfns
1111  *
1112  * For each page in the address space, this file contains one 64-bit entry
1113  * consisting of the following:
1114  *
1115  * Bits 0-54  page frame number (PFN) if present
1116  * Bits 0-4   swap type if swapped
1117  * Bits 5-54  swap offset if swapped
1118  * Bits 55-60 page shift (page size = 1<<page shift)
1119  * Bit  61    page is file-page or shared-anon
1120  * Bit  62    page swapped
1121  * Bit  63    page present
1122  *
1123  * If the page is not present but in swap, then the PFN contains an
1124  * encoding of the swap file number and the page's offset into the
1125  * swap. Unmapped pages return a null PFN. This allows determining
1126  * precisely which pages are mapped (or in swap) and comparing mapped
1127  * pages between processes.
1128  *
1129  * Efficient users of this interface will use /proc/pid/maps to
1130  * determine which areas of memory are actually mapped and llseek to
1131  * skip over unmapped regions.
1132  */
1133 static ssize_t pagemap_read(struct file *file, char __user *buf,
1134 			    size_t count, loff_t *ppos)
1135 {
1136 	struct task_struct *task = get_proc_task(file_inode(file));
1137 	struct mm_struct *mm;
1138 	struct pagemapread pm;
1139 	int ret = -ESRCH;
1140 	struct mm_walk pagemap_walk = {};
1141 	unsigned long src;
1142 	unsigned long svpfn;
1143 	unsigned long start_vaddr;
1144 	unsigned long end_vaddr;
1145 	int copied = 0;
1146 
1147 	if (!task)
1148 		goto out;
1149 
1150 	ret = -EINVAL;
1151 	/* file position must be aligned */
1152 	if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES))
1153 		goto out_task;
1154 
1155 	ret = 0;
1156 	if (!count)
1157 		goto out_task;
1158 
1159 	pm.v2 = soft_dirty_cleared;
1160 	pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT);
1161 	pm.buffer = kmalloc(pm.len * PM_ENTRY_BYTES, GFP_TEMPORARY);
1162 	ret = -ENOMEM;
1163 	if (!pm.buffer)
1164 		goto out_task;
1165 
1166 	mm = mm_access(task, PTRACE_MODE_READ);
1167 	ret = PTR_ERR(mm);
1168 	if (!mm || IS_ERR(mm))
1169 		goto out_free;
1170 
1171 	pagemap_walk.pmd_entry = pagemap_pte_range;
1172 	pagemap_walk.pte_hole = pagemap_pte_hole;
1173 #ifdef CONFIG_HUGETLB_PAGE
1174 	pagemap_walk.hugetlb_entry = pagemap_hugetlb_range;
1175 #endif
1176 	pagemap_walk.mm = mm;
1177 	pagemap_walk.private = &pm;
1178 
1179 	src = *ppos;
1180 	svpfn = src / PM_ENTRY_BYTES;
1181 	start_vaddr = svpfn << PAGE_SHIFT;
1182 	end_vaddr = TASK_SIZE_OF(task);
1183 
1184 	/* watch out for wraparound */
1185 	if (svpfn > TASK_SIZE_OF(task) >> PAGE_SHIFT)
1186 		start_vaddr = end_vaddr;
1187 
1188 	/*
1189 	 * The odds are that this will stop walking way
1190 	 * before end_vaddr, because the length of the
1191 	 * user buffer is tracked in "pm", and the walk
1192 	 * will stop when we hit the end of the buffer.
1193 	 */
1194 	ret = 0;
1195 	while (count && (start_vaddr < end_vaddr)) {
1196 		int len;
1197 		unsigned long end;
1198 
1199 		pm.pos = 0;
1200 		end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK;
1201 		/* overflow ? */
1202 		if (end < start_vaddr || end > end_vaddr)
1203 			end = end_vaddr;
1204 		down_read(&mm->mmap_sem);
1205 		ret = walk_page_range(start_vaddr, end, &pagemap_walk);
1206 		up_read(&mm->mmap_sem);
1207 		start_vaddr = end;
1208 
1209 		len = min(count, PM_ENTRY_BYTES * pm.pos);
1210 		if (copy_to_user(buf, pm.buffer, len)) {
1211 			ret = -EFAULT;
1212 			goto out_mm;
1213 		}
1214 		copied += len;
1215 		buf += len;
1216 		count -= len;
1217 	}
1218 	*ppos += copied;
1219 	if (!ret || ret == PM_END_OF_BUFFER)
1220 		ret = copied;
1221 
1222 out_mm:
1223 	mmput(mm);
1224 out_free:
1225 	kfree(pm.buffer);
1226 out_task:
1227 	put_task_struct(task);
1228 out:
1229 	return ret;
1230 }
1231 
1232 static int pagemap_open(struct inode *inode, struct file *file)
1233 {
1234 	pr_warn_once("Bits 55-60 of /proc/PID/pagemap entries are about "
1235 			"to stop being page-shift some time soon. See the "
1236 			"linux/Documentation/vm/pagemap.txt for details.\n");
1237 	return 0;
1238 }
1239 
1240 const struct file_operations proc_pagemap_operations = {
1241 	.llseek		= mem_lseek, /* borrow this */
1242 	.read		= pagemap_read,
1243 	.open		= pagemap_open,
1244 };
1245 #endif /* CONFIG_PROC_PAGE_MONITOR */
1246 
1247 #ifdef CONFIG_NUMA
1248 
1249 struct numa_maps {
1250 	struct vm_area_struct *vma;
1251 	unsigned long pages;
1252 	unsigned long anon;
1253 	unsigned long active;
1254 	unsigned long writeback;
1255 	unsigned long mapcount_max;
1256 	unsigned long dirty;
1257 	unsigned long swapcache;
1258 	unsigned long node[MAX_NUMNODES];
1259 };
1260 
1261 struct numa_maps_private {
1262 	struct proc_maps_private proc_maps;
1263 	struct numa_maps md;
1264 };
1265 
1266 static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty,
1267 			unsigned long nr_pages)
1268 {
1269 	int count = page_mapcount(page);
1270 
1271 	md->pages += nr_pages;
1272 	if (pte_dirty || PageDirty(page))
1273 		md->dirty += nr_pages;
1274 
1275 	if (PageSwapCache(page))
1276 		md->swapcache += nr_pages;
1277 
1278 	if (PageActive(page) || PageUnevictable(page))
1279 		md->active += nr_pages;
1280 
1281 	if (PageWriteback(page))
1282 		md->writeback += nr_pages;
1283 
1284 	if (PageAnon(page))
1285 		md->anon += nr_pages;
1286 
1287 	if (count > md->mapcount_max)
1288 		md->mapcount_max = count;
1289 
1290 	md->node[page_to_nid(page)] += nr_pages;
1291 }
1292 
1293 static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma,
1294 		unsigned long addr)
1295 {
1296 	struct page *page;
1297 	int nid;
1298 
1299 	if (!pte_present(pte))
1300 		return NULL;
1301 
1302 	page = vm_normal_page(vma, addr, pte);
1303 	if (!page)
1304 		return NULL;
1305 
1306 	if (PageReserved(page))
1307 		return NULL;
1308 
1309 	nid = page_to_nid(page);
1310 	if (!node_isset(nid, node_states[N_MEMORY]))
1311 		return NULL;
1312 
1313 	return page;
1314 }
1315 
1316 static int gather_pte_stats(pmd_t *pmd, unsigned long addr,
1317 		unsigned long end, struct mm_walk *walk)
1318 {
1319 	struct numa_maps *md;
1320 	spinlock_t *ptl;
1321 	pte_t *orig_pte;
1322 	pte_t *pte;
1323 
1324 	md = walk->private;
1325 
1326 	if (pmd_trans_huge_lock(pmd, md->vma, &ptl) == 1) {
1327 		pte_t huge_pte = *(pte_t *)pmd;
1328 		struct page *page;
1329 
1330 		page = can_gather_numa_stats(huge_pte, md->vma, addr);
1331 		if (page)
1332 			gather_stats(page, md, pte_dirty(huge_pte),
1333 				     HPAGE_PMD_SIZE/PAGE_SIZE);
1334 		spin_unlock(ptl);
1335 		return 0;
1336 	}
1337 
1338 	if (pmd_trans_unstable(pmd))
1339 		return 0;
1340 	orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
1341 	do {
1342 		struct page *page = can_gather_numa_stats(*pte, md->vma, addr);
1343 		if (!page)
1344 			continue;
1345 		gather_stats(page, md, pte_dirty(*pte), 1);
1346 
1347 	} while (pte++, addr += PAGE_SIZE, addr != end);
1348 	pte_unmap_unlock(orig_pte, ptl);
1349 	return 0;
1350 }
1351 #ifdef CONFIG_HUGETLB_PAGE
1352 static int gather_hugetbl_stats(pte_t *pte, unsigned long hmask,
1353 		unsigned long addr, unsigned long end, struct mm_walk *walk)
1354 {
1355 	struct numa_maps *md;
1356 	struct page *page;
1357 
1358 	if (pte_none(*pte))
1359 		return 0;
1360 
1361 	page = pte_page(*pte);
1362 	if (!page)
1363 		return 0;
1364 
1365 	md = walk->private;
1366 	gather_stats(page, md, pte_dirty(*pte), 1);
1367 	return 0;
1368 }
1369 
1370 #else
1371 static int gather_hugetbl_stats(pte_t *pte, unsigned long hmask,
1372 		unsigned long addr, unsigned long end, struct mm_walk *walk)
1373 {
1374 	return 0;
1375 }
1376 #endif
1377 
1378 /*
1379  * Display pages allocated per node and memory policy via /proc.
1380  */
1381 static int show_numa_map(struct seq_file *m, void *v, int is_pid)
1382 {
1383 	struct numa_maps_private *numa_priv = m->private;
1384 	struct proc_maps_private *proc_priv = &numa_priv->proc_maps;
1385 	struct vm_area_struct *vma = v;
1386 	struct numa_maps *md = &numa_priv->md;
1387 	struct file *file = vma->vm_file;
1388 	struct task_struct *task = proc_priv->task;
1389 	struct mm_struct *mm = vma->vm_mm;
1390 	struct mm_walk walk = {};
1391 	struct mempolicy *pol;
1392 	char buffer[64];
1393 	int nid;
1394 
1395 	if (!mm)
1396 		return 0;
1397 
1398 	/* Ensure we start with an empty set of numa_maps statistics. */
1399 	memset(md, 0, sizeof(*md));
1400 
1401 	md->vma = vma;
1402 
1403 	walk.hugetlb_entry = gather_hugetbl_stats;
1404 	walk.pmd_entry = gather_pte_stats;
1405 	walk.private = md;
1406 	walk.mm = mm;
1407 
1408 	pol = get_vma_policy(task, vma, vma->vm_start);
1409 	mpol_to_str(buffer, sizeof(buffer), pol);
1410 	mpol_cond_put(pol);
1411 
1412 	seq_printf(m, "%08lx %s", vma->vm_start, buffer);
1413 
1414 	if (file) {
1415 		seq_printf(m, " file=");
1416 		seq_path(m, &file->f_path, "\n\t= ");
1417 	} else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
1418 		seq_printf(m, " heap");
1419 	} else {
1420 		pid_t tid = vm_is_stack(task, vma, is_pid);
1421 		if (tid != 0) {
1422 			/*
1423 			 * Thread stack in /proc/PID/task/TID/maps or
1424 			 * the main process stack.
1425 			 */
1426 			if (!is_pid || (vma->vm_start <= mm->start_stack &&
1427 			    vma->vm_end >= mm->start_stack))
1428 				seq_printf(m, " stack");
1429 			else
1430 				seq_printf(m, " stack:%d", tid);
1431 		}
1432 	}
1433 
1434 	if (is_vm_hugetlb_page(vma))
1435 		seq_printf(m, " huge");
1436 
1437 	walk_page_range(vma->vm_start, vma->vm_end, &walk);
1438 
1439 	if (!md->pages)
1440 		goto out;
1441 
1442 	if (md->anon)
1443 		seq_printf(m, " anon=%lu", md->anon);
1444 
1445 	if (md->dirty)
1446 		seq_printf(m, " dirty=%lu", md->dirty);
1447 
1448 	if (md->pages != md->anon && md->pages != md->dirty)
1449 		seq_printf(m, " mapped=%lu", md->pages);
1450 
1451 	if (md->mapcount_max > 1)
1452 		seq_printf(m, " mapmax=%lu", md->mapcount_max);
1453 
1454 	if (md->swapcache)
1455 		seq_printf(m, " swapcache=%lu", md->swapcache);
1456 
1457 	if (md->active < md->pages && !is_vm_hugetlb_page(vma))
1458 		seq_printf(m, " active=%lu", md->active);
1459 
1460 	if (md->writeback)
1461 		seq_printf(m, " writeback=%lu", md->writeback);
1462 
1463 	for_each_node_state(nid, N_MEMORY)
1464 		if (md->node[nid])
1465 			seq_printf(m, " N%d=%lu", nid, md->node[nid]);
1466 out:
1467 	seq_putc(m, '\n');
1468 
1469 	if (m->count < m->size)
1470 		m->version = (vma != proc_priv->tail_vma) ? vma->vm_start : 0;
1471 	return 0;
1472 }
1473 
1474 static int show_pid_numa_map(struct seq_file *m, void *v)
1475 {
1476 	return show_numa_map(m, v, 1);
1477 }
1478 
1479 static int show_tid_numa_map(struct seq_file *m, void *v)
1480 {
1481 	return show_numa_map(m, v, 0);
1482 }
1483 
1484 static const struct seq_operations proc_pid_numa_maps_op = {
1485 	.start  = m_start,
1486 	.next   = m_next,
1487 	.stop   = m_stop,
1488 	.show   = show_pid_numa_map,
1489 };
1490 
1491 static const struct seq_operations proc_tid_numa_maps_op = {
1492 	.start  = m_start,
1493 	.next   = m_next,
1494 	.stop   = m_stop,
1495 	.show   = show_tid_numa_map,
1496 };
1497 
1498 static int numa_maps_open(struct inode *inode, struct file *file,
1499 			  const struct seq_operations *ops)
1500 {
1501 	struct numa_maps_private *priv;
1502 	int ret = -ENOMEM;
1503 	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
1504 	if (priv) {
1505 		priv->proc_maps.pid = proc_pid(inode);
1506 		ret = seq_open(file, ops);
1507 		if (!ret) {
1508 			struct seq_file *m = file->private_data;
1509 			m->private = priv;
1510 		} else {
1511 			kfree(priv);
1512 		}
1513 	}
1514 	return ret;
1515 }
1516 
1517 static int pid_numa_maps_open(struct inode *inode, struct file *file)
1518 {
1519 	return numa_maps_open(inode, file, &proc_pid_numa_maps_op);
1520 }
1521 
1522 static int tid_numa_maps_open(struct inode *inode, struct file *file)
1523 {
1524 	return numa_maps_open(inode, file, &proc_tid_numa_maps_op);
1525 }
1526 
1527 const struct file_operations proc_pid_numa_maps_operations = {
1528 	.open		= pid_numa_maps_open,
1529 	.read		= seq_read,
1530 	.llseek		= seq_lseek,
1531 	.release	= seq_release_private,
1532 };
1533 
1534 const struct file_operations proc_tid_numa_maps_operations = {
1535 	.open		= tid_numa_maps_open,
1536 	.read		= seq_read,
1537 	.llseek		= seq_lseek,
1538 	.release	= seq_release_private,
1539 };
1540 #endif /* CONFIG_NUMA */
1541