1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * page-types: Tool for querying page flags 4 * 5 * Copyright (C) 2009 Intel corporation 6 * 7 * Authors: Wu Fengguang <fengguang.wu@intel.com> 8 */ 9 10 #define _FILE_OFFSET_BITS 64 11 #define _GNU_SOURCE 12 #include <stdio.h> 13 #include <stdlib.h> 14 #include <unistd.h> 15 #include <stdint.h> 16 #include <stdarg.h> 17 #include <string.h> 18 #include <getopt.h> 19 #include <limits.h> 20 #include <assert.h> 21 #include <ftw.h> 22 #include <time.h> 23 #include <setjmp.h> 24 #include <signal.h> 25 #include <inttypes.h> 26 #include <sys/types.h> 27 #include <sys/errno.h> 28 #include <sys/fcntl.h> 29 #include <sys/mount.h> 30 #include <sys/statfs.h> 31 #include <sys/mman.h> 32 #include "../../include/uapi/linux/magic.h" 33 #include "../../include/uapi/linux/kernel-page-flags.h" 34 #include <api/fs/fs.h> 35 36 #ifndef MAX_PATH 37 # define MAX_PATH 256 38 #endif 39 40 #ifndef STR 41 # define _STR(x) #x 42 # define STR(x) _STR(x) 43 #endif 44 45 /* 46 * pagemap kernel ABI bits 47 */ 48 49 #define PM_ENTRY_BYTES 8 50 #define PM_PFRAME_BITS 55 51 #define PM_PFRAME_MASK ((1LL << PM_PFRAME_BITS) - 1) 52 #define PM_PFRAME(x) ((x) & PM_PFRAME_MASK) 53 #define MAX_SWAPFILES_SHIFT 5 54 #define PM_SWAP_OFFSET(x) (((x) & PM_PFRAME_MASK) >> MAX_SWAPFILES_SHIFT) 55 #define PM_SOFT_DIRTY (1ULL << 55) 56 #define PM_MMAP_EXCLUSIVE (1ULL << 56) 57 #define PM_FILE (1ULL << 61) 58 #define PM_SWAP (1ULL << 62) 59 #define PM_PRESENT (1ULL << 63) 60 61 /* 62 * kernel page flags 63 */ 64 65 #define KPF_BYTES 8 66 #define PROC_KPAGEFLAGS "/proc/kpageflags" 67 #define PROC_KPAGECOUNT "/proc/kpagecount" 68 #define PROC_KPAGECGROUP "/proc/kpagecgroup" 69 70 #define SYS_KERNEL_MM_PAGE_IDLE "/sys/kernel/mm/page_idle/bitmap" 71 72 /* [32-] kernel hacking assistances */ 73 #define KPF_RESERVED 32 74 #define KPF_MLOCKED 33 75 #define KPF_MAPPEDTODISK 34 76 #define KPF_PRIVATE 35 77 #define KPF_PRIVATE_2 36 78 #define KPF_OWNER_PRIVATE 37 79 #define KPF_ARCH 38 80 #define KPF_UNCACHED 39 81 #define KPF_SOFTDIRTY 40 82 #define KPF_ARCH_2 41 83 84 /* [47-] take some arbitrary free slots for expanding overloaded flags 85 * not part of kernel API 86 */ 87 #define KPF_ANON_EXCLUSIVE 47 88 #define KPF_READAHEAD 48 89 #define KPF_SLUB_FROZEN 50 90 #define KPF_SLUB_DEBUG 51 91 #define KPF_FILE 61 92 #define KPF_SWAP 62 93 #define KPF_MMAP_EXCLUSIVE 63 94 95 #define KPF_ALL_BITS ((uint64_t)~0ULL) 96 #define KPF_HACKERS_BITS (0xffffULL << 32) 97 #define KPF_OVERLOADED_BITS (0xffffULL << 48) 98 #define BIT(name) (1ULL << KPF_##name) 99 #define BITS_COMPOUND (BIT(COMPOUND_HEAD) | BIT(COMPOUND_TAIL)) 100 101 static const char * const page_flag_names[] = { 102 [KPF_LOCKED] = "L:locked", 103 [KPF_ERROR] = "E:error", 104 [KPF_REFERENCED] = "R:referenced", 105 [KPF_UPTODATE] = "U:uptodate", 106 [KPF_DIRTY] = "D:dirty", 107 [KPF_LRU] = "l:lru", 108 [KPF_ACTIVE] = "A:active", 109 [KPF_SLAB] = "S:slab", 110 [KPF_WRITEBACK] = "W:writeback", 111 [KPF_RECLAIM] = "I:reclaim", 112 [KPF_BUDDY] = "B:buddy", 113 114 [KPF_MMAP] = "M:mmap", 115 [KPF_ANON] = "a:anonymous", 116 [KPF_SWAPCACHE] = "s:swapcache", 117 [KPF_SWAPBACKED] = "b:swapbacked", 118 [KPF_COMPOUND_HEAD] = "H:compound_head", 119 [KPF_COMPOUND_TAIL] = "T:compound_tail", 120 [KPF_HUGE] = "G:huge", 121 [KPF_UNEVICTABLE] = "u:unevictable", 122 [KPF_HWPOISON] = "X:hwpoison", 123 [KPF_NOPAGE] = "n:nopage", 124 [KPF_KSM] = "x:ksm", 125 [KPF_THP] = "t:thp", 126 [KPF_OFFLINE] = "o:offline", 127 [KPF_PGTABLE] = "g:pgtable", 128 [KPF_ZERO_PAGE] = "z:zero_page", 129 [KPF_IDLE] = "i:idle_page", 130 131 [KPF_RESERVED] = "r:reserved", 132 [KPF_MLOCKED] = "m:mlocked", 133 [KPF_MAPPEDTODISK] = "d:mappedtodisk", 134 [KPF_PRIVATE] = "P:private", 135 [KPF_PRIVATE_2] = "p:private_2", 136 [KPF_OWNER_PRIVATE] = "O:owner_private", 137 [KPF_ARCH] = "h:arch", 138 [KPF_UNCACHED] = "c:uncached", 139 [KPF_SOFTDIRTY] = "f:softdirty", 140 [KPF_ARCH_2] = "H:arch_2", 141 142 [KPF_ANON_EXCLUSIVE] = "d:anon_exclusive", 143 [KPF_READAHEAD] = "I:readahead", 144 [KPF_SLUB_FROZEN] = "A:slub_frozen", 145 [KPF_SLUB_DEBUG] = "E:slub_debug", 146 147 [KPF_FILE] = "F:file", 148 [KPF_SWAP] = "w:swap", 149 [KPF_MMAP_EXCLUSIVE] = "1:mmap_exclusive", 150 }; 151 152 153 /* 154 * data structures 155 */ 156 157 static int opt_raw; /* for kernel developers */ 158 static int opt_list; /* list pages (in ranges) */ 159 static int opt_mark_idle; /* set accessed bit */ 160 static int opt_no_summary; /* don't show summary */ 161 static pid_t opt_pid; /* process to walk */ 162 const char *opt_file; /* file or directory path */ 163 static uint64_t opt_cgroup; /* cgroup inode */ 164 static int opt_list_cgroup;/* list page cgroup */ 165 static int opt_list_mapcnt;/* list page map count */ 166 static const char *opt_kpageflags;/* kpageflags file to parse */ 167 168 #define MAX_ADDR_RANGES 1024 169 static int nr_addr_ranges; 170 static unsigned long opt_offset[MAX_ADDR_RANGES]; 171 static unsigned long opt_size[MAX_ADDR_RANGES]; 172 173 #define MAX_VMAS 10240 174 static int nr_vmas; 175 static unsigned long pg_start[MAX_VMAS]; 176 static unsigned long pg_end[MAX_VMAS]; 177 178 #define MAX_BIT_FILTERS 64 179 static int nr_bit_filters; 180 static uint64_t opt_mask[MAX_BIT_FILTERS]; 181 static uint64_t opt_bits[MAX_BIT_FILTERS]; 182 183 static int page_size; 184 185 static int pagemap_fd; 186 static int kpageflags_fd; 187 static int kpagecount_fd = -1; 188 static int kpagecgroup_fd = -1; 189 static int page_idle_fd = -1; 190 191 static int opt_hwpoison; 192 static int opt_unpoison; 193 194 static const char *hwpoison_debug_fs; 195 static int hwpoison_inject_fd; 196 static int hwpoison_forget_fd; 197 198 #define HASH_SHIFT 13 199 #define HASH_SIZE (1 << HASH_SHIFT) 200 #define HASH_MASK (HASH_SIZE - 1) 201 #define HASH_KEY(flags) (flags & HASH_MASK) 202 203 static unsigned long total_pages; 204 static unsigned long nr_pages[HASH_SIZE]; 205 static uint64_t page_flags[HASH_SIZE]; 206 207 208 /* 209 * helper functions 210 */ 211 212 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) 213 214 #define min_t(type, x, y) ({ \ 215 type __min1 = (x); \ 216 type __min2 = (y); \ 217 __min1 < __min2 ? __min1 : __min2; }) 218 219 #define max_t(type, x, y) ({ \ 220 type __max1 = (x); \ 221 type __max2 = (y); \ 222 __max1 > __max2 ? __max1 : __max2; }) 223 224 static unsigned long pages2mb(unsigned long pages) 225 { 226 return (pages * page_size) >> 20; 227 } 228 229 static void fatal(const char *x, ...) 230 { 231 va_list ap; 232 233 va_start(ap, x); 234 vfprintf(stderr, x, ap); 235 va_end(ap); 236 exit(EXIT_FAILURE); 237 } 238 239 static int checked_open(const char *pathname, int flags) 240 { 241 int fd = open(pathname, flags); 242 243 if (fd < 0) { 244 perror(pathname); 245 exit(EXIT_FAILURE); 246 } 247 248 return fd; 249 } 250 251 /* 252 * pagemap/kpageflags routines 253 */ 254 255 static unsigned long do_u64_read(int fd, const char *name, 256 uint64_t *buf, 257 unsigned long index, 258 unsigned long count) 259 { 260 long bytes; 261 262 if (index > ULONG_MAX / 8) 263 fatal("index overflow: %lu\n", index); 264 265 bytes = pread(fd, buf, count * 8, (off_t)index * 8); 266 if (bytes < 0) { 267 perror(name); 268 exit(EXIT_FAILURE); 269 } 270 if (bytes % 8) 271 fatal("partial read: %lu bytes\n", bytes); 272 273 return bytes / 8; 274 } 275 276 static unsigned long kpageflags_read(uint64_t *buf, 277 unsigned long index, 278 unsigned long pages) 279 { 280 return do_u64_read(kpageflags_fd, opt_kpageflags, buf, index, pages); 281 } 282 283 static unsigned long kpagecgroup_read(uint64_t *buf, 284 unsigned long index, 285 unsigned long pages) 286 { 287 if (kpagecgroup_fd < 0) 288 return pages; 289 290 return do_u64_read(kpagecgroup_fd, opt_kpageflags, buf, index, pages); 291 } 292 293 static unsigned long kpagecount_read(uint64_t *buf, 294 unsigned long index, 295 unsigned long pages) 296 { 297 return kpagecount_fd < 0 ? pages : 298 do_u64_read(kpagecount_fd, PROC_KPAGECOUNT, 299 buf, index, pages); 300 } 301 302 static unsigned long pagemap_read(uint64_t *buf, 303 unsigned long index, 304 unsigned long pages) 305 { 306 return do_u64_read(pagemap_fd, "/proc/pid/pagemap", buf, index, pages); 307 } 308 309 static unsigned long pagemap_pfn(uint64_t val) 310 { 311 unsigned long pfn; 312 313 if (val & PM_PRESENT) 314 pfn = PM_PFRAME(val); 315 else 316 pfn = 0; 317 318 return pfn; 319 } 320 321 static unsigned long pagemap_swap_offset(uint64_t val) 322 { 323 return val & PM_SWAP ? PM_SWAP_OFFSET(val) : 0; 324 } 325 326 /* 327 * page flag names 328 */ 329 330 static char *page_flag_name(uint64_t flags) 331 { 332 static char buf[65]; 333 int present; 334 size_t i, j; 335 336 for (i = 0, j = 0; i < ARRAY_SIZE(page_flag_names); i++) { 337 present = (flags >> i) & 1; 338 if (!page_flag_names[i]) { 339 if (present) 340 fatal("unknown flag bit %d\n", i); 341 continue; 342 } 343 buf[j++] = present ? page_flag_names[i][0] : '_'; 344 } 345 346 return buf; 347 } 348 349 static char *page_flag_longname(uint64_t flags) 350 { 351 static char buf[1024]; 352 size_t i, n; 353 354 for (i = 0, n = 0; i < ARRAY_SIZE(page_flag_names); i++) { 355 if (!page_flag_names[i]) 356 continue; 357 if ((flags >> i) & 1) 358 n += snprintf(buf + n, sizeof(buf) - n, "%s,", 359 page_flag_names[i] + 2); 360 } 361 if (n) 362 n--; 363 buf[n] = '\0'; 364 365 return buf; 366 } 367 368 369 /* 370 * page list and summary 371 */ 372 373 static void show_page_range(unsigned long voffset, unsigned long offset, 374 unsigned long size, uint64_t flags, 375 uint64_t cgroup, uint64_t mapcnt) 376 { 377 static uint64_t flags0; 378 static uint64_t cgroup0; 379 static uint64_t mapcnt0; 380 static unsigned long voff; 381 static unsigned long index; 382 static unsigned long count; 383 384 if (flags == flags0 && cgroup == cgroup0 && mapcnt == mapcnt0 && 385 offset == index + count && size && voffset == voff + count) { 386 count += size; 387 return; 388 } 389 390 if (count) { 391 if (opt_pid) 392 printf("%lx\t", voff); 393 if (opt_file) 394 printf("%lx\t", voff); 395 if (opt_list_cgroup) 396 printf("@%" PRIu64 "\t", cgroup0); 397 if (opt_list_mapcnt) 398 printf("%" PRIu64 "\t", mapcnt0); 399 printf("%lx\t%lx\t%s\n", 400 index, count, page_flag_name(flags0)); 401 } 402 403 flags0 = flags; 404 cgroup0 = cgroup; 405 mapcnt0 = mapcnt; 406 index = offset; 407 voff = voffset; 408 count = size; 409 } 410 411 static void flush_page_range(void) 412 { 413 show_page_range(0, 0, 0, 0, 0, 0); 414 } 415 416 static void show_page(unsigned long voffset, unsigned long offset, 417 uint64_t flags, uint64_t cgroup, uint64_t mapcnt) 418 { 419 if (opt_pid) 420 printf("%lx\t", voffset); 421 if (opt_file) 422 printf("%lx\t", voffset); 423 if (opt_list_cgroup) 424 printf("@%" PRIu64 "\t", cgroup) 425 if (opt_list_mapcnt) 426 printf("%" PRIu64 "\t", mapcnt); 427 428 printf("%lx\t%s\n", offset, page_flag_name(flags)); 429 } 430 431 static void show_summary(void) 432 { 433 size_t i; 434 435 printf(" flags\tpage-count MB" 436 " symbolic-flags\t\t\tlong-symbolic-flags\n"); 437 438 for (i = 0; i < ARRAY_SIZE(nr_pages); i++) { 439 if (nr_pages[i]) 440 printf("0x%016llx\t%10lu %8lu %s\t%s\n", 441 (unsigned long long)page_flags[i], 442 nr_pages[i], 443 pages2mb(nr_pages[i]), 444 page_flag_name(page_flags[i]), 445 page_flag_longname(page_flags[i])); 446 } 447 448 printf(" total\t%10lu %8lu\n", 449 total_pages, pages2mb(total_pages)); 450 } 451 452 453 /* 454 * page flag filters 455 */ 456 457 static int bit_mask_ok(uint64_t flags) 458 { 459 int i; 460 461 for (i = 0; i < nr_bit_filters; i++) { 462 if (opt_bits[i] == KPF_ALL_BITS) { 463 if ((flags & opt_mask[i]) == 0) 464 return 0; 465 } else { 466 if ((flags & opt_mask[i]) != opt_bits[i]) 467 return 0; 468 } 469 } 470 471 return 1; 472 } 473 474 static uint64_t expand_overloaded_flags(uint64_t flags, uint64_t pme) 475 { 476 /* Anonymous pages overload PG_mappedtodisk */ 477 if ((flags & BIT(ANON)) && (flags & BIT(MAPPEDTODISK))) 478 flags ^= BIT(MAPPEDTODISK) | BIT(ANON_EXCLUSIVE); 479 480 /* SLUB overloads several page flags */ 481 if (flags & BIT(SLAB)) { 482 if (flags & BIT(ACTIVE)) 483 flags ^= BIT(ACTIVE) | BIT(SLUB_FROZEN); 484 if (flags & BIT(ERROR)) 485 flags ^= BIT(ERROR) | BIT(SLUB_DEBUG); 486 } 487 488 /* PG_reclaim is overloaded as PG_readahead in the read path */ 489 if ((flags & (BIT(RECLAIM) | BIT(WRITEBACK))) == BIT(RECLAIM)) 490 flags ^= BIT(RECLAIM) | BIT(READAHEAD); 491 492 if (pme & PM_SOFT_DIRTY) 493 flags |= BIT(SOFTDIRTY); 494 if (pme & PM_FILE) 495 flags |= BIT(FILE); 496 if (pme & PM_SWAP) 497 flags |= BIT(SWAP); 498 if (pme & PM_MMAP_EXCLUSIVE) 499 flags |= BIT(MMAP_EXCLUSIVE); 500 501 return flags; 502 } 503 504 static uint64_t well_known_flags(uint64_t flags) 505 { 506 /* hide flags intended only for kernel hacker */ 507 flags &= ~KPF_HACKERS_BITS; 508 509 /* hide non-hugeTLB compound pages */ 510 if ((flags & BITS_COMPOUND) && !(flags & BIT(HUGE))) 511 flags &= ~BITS_COMPOUND; 512 513 return flags; 514 } 515 516 static uint64_t kpageflags_flags(uint64_t flags, uint64_t pme) 517 { 518 if (opt_raw) 519 flags = expand_overloaded_flags(flags, pme); 520 else 521 flags = well_known_flags(flags); 522 523 return flags; 524 } 525 526 /* 527 * page actions 528 */ 529 530 static void prepare_hwpoison_fd(void) 531 { 532 char buf[MAX_PATH + 1]; 533 534 hwpoison_debug_fs = debugfs__mount(); 535 if (!hwpoison_debug_fs) { 536 perror("mount debugfs"); 537 exit(EXIT_FAILURE); 538 } 539 540 if (opt_hwpoison && !hwpoison_inject_fd) { 541 snprintf(buf, MAX_PATH, "%s/hwpoison/corrupt-pfn", 542 hwpoison_debug_fs); 543 hwpoison_inject_fd = checked_open(buf, O_WRONLY); 544 } 545 546 if (opt_unpoison && !hwpoison_forget_fd) { 547 snprintf(buf, MAX_PATH, "%s/hwpoison/unpoison-pfn", 548 hwpoison_debug_fs); 549 hwpoison_forget_fd = checked_open(buf, O_WRONLY); 550 } 551 } 552 553 static int hwpoison_page(unsigned long offset) 554 { 555 char buf[100]; 556 int len; 557 558 len = sprintf(buf, "0x%lx\n", offset); 559 len = write(hwpoison_inject_fd, buf, len); 560 if (len < 0) { 561 perror("hwpoison inject"); 562 return len; 563 } 564 return 0; 565 } 566 567 static int unpoison_page(unsigned long offset) 568 { 569 char buf[100]; 570 int len; 571 572 len = sprintf(buf, "0x%lx\n", offset); 573 len = write(hwpoison_forget_fd, buf, len); 574 if (len < 0) { 575 perror("hwpoison forget"); 576 return len; 577 } 578 return 0; 579 } 580 581 static int mark_page_idle(unsigned long offset) 582 { 583 static unsigned long off; 584 static uint64_t buf; 585 int len; 586 587 if ((offset / 64 == off / 64) || buf == 0) { 588 buf |= 1UL << (offset % 64); 589 off = offset; 590 return 0; 591 } 592 593 len = pwrite(page_idle_fd, &buf, 8, 8 * (off / 64)); 594 if (len < 0) { 595 perror("mark page idle"); 596 return len; 597 } 598 599 buf = 1UL << (offset % 64); 600 off = offset; 601 602 return 0; 603 } 604 605 /* 606 * page frame walker 607 */ 608 609 static size_t hash_slot(uint64_t flags) 610 { 611 size_t k = HASH_KEY(flags); 612 size_t i; 613 614 /* Explicitly reserve slot 0 for flags 0: the following logic 615 * cannot distinguish an unoccupied slot from slot (flags==0). 616 */ 617 if (flags == 0) 618 return 0; 619 620 /* search through the remaining (HASH_SIZE-1) slots */ 621 for (i = 1; i < ARRAY_SIZE(page_flags); i++, k++) { 622 if (!k || k >= ARRAY_SIZE(page_flags)) 623 k = 1; 624 if (page_flags[k] == 0) { 625 page_flags[k] = flags; 626 return k; 627 } 628 if (page_flags[k] == flags) 629 return k; 630 } 631 632 fatal("hash table full: bump up HASH_SHIFT?\n"); 633 exit(EXIT_FAILURE); 634 } 635 636 static void add_page(unsigned long voffset, unsigned long offset, 637 uint64_t flags, uint64_t cgroup, uint64_t mapcnt, 638 uint64_t pme) 639 { 640 flags = kpageflags_flags(flags, pme); 641 642 if (!bit_mask_ok(flags)) 643 return; 644 645 if (opt_cgroup && cgroup != (uint64_t)opt_cgroup) 646 return; 647 648 if (opt_hwpoison) 649 hwpoison_page(offset); 650 if (opt_unpoison) 651 unpoison_page(offset); 652 653 if (opt_mark_idle) 654 mark_page_idle(offset); 655 656 if (opt_list == 1) 657 show_page_range(voffset, offset, 1, flags, cgroup, mapcnt); 658 else if (opt_list == 2) 659 show_page(voffset, offset, flags, cgroup, mapcnt); 660 661 nr_pages[hash_slot(flags)]++; 662 total_pages++; 663 } 664 665 #define KPAGEFLAGS_BATCH (64 << 10) /* 64k pages */ 666 static void walk_pfn(unsigned long voffset, 667 unsigned long index, 668 unsigned long count, 669 uint64_t pme) 670 { 671 uint64_t buf[KPAGEFLAGS_BATCH]; 672 uint64_t cgi[KPAGEFLAGS_BATCH]; 673 uint64_t cnt[KPAGEFLAGS_BATCH]; 674 unsigned long batch; 675 unsigned long pages; 676 unsigned long i; 677 678 /* 679 * kpagecgroup_read() reads only if kpagecgroup were opened, but 680 * /proc/kpagecgroup might even not exist, so it's better to fill 681 * them with zeros here. 682 */ 683 if (count == 1) 684 cgi[0] = 0; 685 else 686 memset(cgi, 0, sizeof cgi); 687 688 while (count) { 689 batch = min_t(unsigned long, count, KPAGEFLAGS_BATCH); 690 pages = kpageflags_read(buf, index, batch); 691 if (pages == 0) 692 break; 693 694 if (kpagecgroup_read(cgi, index, pages) != pages) 695 fatal("kpagecgroup returned fewer pages than expected"); 696 697 if (kpagecount_read(cnt, index, pages) != pages) 698 fatal("kpagecount returned fewer pages than expected"); 699 700 for (i = 0; i < pages; i++) 701 add_page(voffset + i, index + i, 702 buf[i], cgi[i], cnt[i], pme); 703 704 index += pages; 705 count -= pages; 706 } 707 } 708 709 static void walk_swap(unsigned long voffset, uint64_t pme) 710 { 711 uint64_t flags = kpageflags_flags(0, pme); 712 713 if (!bit_mask_ok(flags)) 714 return; 715 716 if (opt_cgroup) 717 return; 718 719 if (opt_list == 1) 720 show_page_range(voffset, pagemap_swap_offset(pme), 721 1, flags, 0, 0); 722 else if (opt_list == 2) 723 show_page(voffset, pagemap_swap_offset(pme), flags, 0, 0); 724 725 nr_pages[hash_slot(flags)]++; 726 total_pages++; 727 } 728 729 #define PAGEMAP_BATCH (64 << 10) 730 static void walk_vma(unsigned long index, unsigned long count) 731 { 732 uint64_t buf[PAGEMAP_BATCH]; 733 unsigned long batch; 734 unsigned long pages; 735 unsigned long pfn; 736 unsigned long i; 737 738 while (count) { 739 batch = min_t(unsigned long, count, PAGEMAP_BATCH); 740 pages = pagemap_read(buf, index, batch); 741 if (pages == 0) 742 break; 743 744 for (i = 0; i < pages; i++) { 745 pfn = pagemap_pfn(buf[i]); 746 if (pfn) 747 walk_pfn(index + i, pfn, 1, buf[i]); 748 if (buf[i] & PM_SWAP) 749 walk_swap(index + i, buf[i]); 750 } 751 752 index += pages; 753 count -= pages; 754 } 755 } 756 757 static void walk_task(unsigned long index, unsigned long count) 758 { 759 const unsigned long end = index + count; 760 unsigned long start; 761 int i = 0; 762 763 while (index < end) { 764 765 while (pg_end[i] <= index) 766 if (++i >= nr_vmas) 767 return; 768 if (pg_start[i] >= end) 769 return; 770 771 start = max_t(unsigned long, pg_start[i], index); 772 index = min_t(unsigned long, pg_end[i], end); 773 774 assert(start < index); 775 walk_vma(start, index - start); 776 } 777 } 778 779 static void add_addr_range(unsigned long offset, unsigned long size) 780 { 781 if (nr_addr_ranges >= MAX_ADDR_RANGES) 782 fatal("too many addr ranges\n"); 783 784 opt_offset[nr_addr_ranges] = offset; 785 opt_size[nr_addr_ranges] = min_t(unsigned long, size, ULONG_MAX-offset); 786 nr_addr_ranges++; 787 } 788 789 static void walk_addr_ranges(void) 790 { 791 int i; 792 793 kpageflags_fd = checked_open(opt_kpageflags, O_RDONLY); 794 795 if (!nr_addr_ranges) 796 add_addr_range(0, ULONG_MAX); 797 798 for (i = 0; i < nr_addr_ranges; i++) 799 if (!opt_pid) 800 walk_pfn(opt_offset[i], opt_offset[i], opt_size[i], 0); 801 else 802 walk_task(opt_offset[i], opt_size[i]); 803 804 if (opt_mark_idle) 805 mark_page_idle(0); 806 807 close(kpageflags_fd); 808 } 809 810 811 /* 812 * user interface 813 */ 814 815 static const char *page_flag_type(uint64_t flag) 816 { 817 if (flag & KPF_HACKERS_BITS) 818 return "(r)"; 819 if (flag & KPF_OVERLOADED_BITS) 820 return "(o)"; 821 return " "; 822 } 823 824 static void usage(void) 825 { 826 size_t i, j; 827 828 printf( 829 "page-types [options]\n" 830 " -r|--raw Raw mode, for kernel developers\n" 831 " -d|--describe flags Describe flags\n" 832 " -a|--addr addr-spec Walk a range of pages\n" 833 " -b|--bits bits-spec Walk pages with specified bits\n" 834 " -c|--cgroup path|@inode Walk pages within memory cgroup\n" 835 " -p|--pid pid Walk process address space\n" 836 " -f|--file filename Walk file address space\n" 837 " -i|--mark-idle Mark pages idle\n" 838 " -l|--list Show page details in ranges\n" 839 " -L|--list-each Show page details one by one\n" 840 " -C|--list-cgroup Show cgroup inode for pages\n" 841 " -M|--list-mapcnt Show page map count\n" 842 " -N|--no-summary Don't show summary info\n" 843 " -X|--hwpoison hwpoison pages\n" 844 " -x|--unpoison unpoison pages\n" 845 " -F|--kpageflags filename kpageflags file to parse\n" 846 " -h|--help Show this usage message\n" 847 "flags:\n" 848 " 0x10 bitfield format, e.g.\n" 849 " anon bit-name, e.g.\n" 850 " 0x10,anon comma-separated list, e.g.\n" 851 "addr-spec:\n" 852 " N one page at offset N (unit: pages)\n" 853 " N+M pages range from N to N+M-1\n" 854 " N,M pages range from N to M-1\n" 855 " N, pages range from N to end\n" 856 " ,M pages range from 0 to M-1\n" 857 "bits-spec:\n" 858 " bit1,bit2 (flags & (bit1|bit2)) != 0\n" 859 " bit1,bit2=bit1 (flags & (bit1|bit2)) == bit1\n" 860 " bit1,~bit2 (flags & (bit1|bit2)) == bit1\n" 861 " =bit1,bit2 flags == (bit1|bit2)\n" 862 "bit-names:\n" 863 ); 864 865 for (i = 0, j = 0; i < ARRAY_SIZE(page_flag_names); i++) { 866 if (!page_flag_names[i]) 867 continue; 868 printf("%16s%s", page_flag_names[i] + 2, 869 page_flag_type(1ULL << i)); 870 if (++j > 3) { 871 j = 0; 872 putchar('\n'); 873 } 874 } 875 printf("\n " 876 "(r) raw mode bits (o) overloaded bits\n"); 877 } 878 879 static unsigned long long parse_number(const char *str) 880 { 881 unsigned long long n; 882 883 n = strtoll(str, NULL, 0); 884 885 if (n == 0 && str[0] != '0') 886 fatal("invalid name or number: %s\n", str); 887 888 return n; 889 } 890 891 static void parse_pid(const char *str) 892 { 893 FILE *file; 894 char buf[5000]; 895 896 opt_pid = parse_number(str); 897 898 sprintf(buf, "/proc/%d/pagemap", opt_pid); 899 pagemap_fd = checked_open(buf, O_RDONLY); 900 901 sprintf(buf, "/proc/%d/maps", opt_pid); 902 file = fopen(buf, "r"); 903 if (!file) { 904 perror(buf); 905 exit(EXIT_FAILURE); 906 } 907 908 while (fgets(buf, sizeof(buf), file) != NULL) { 909 unsigned long vm_start; 910 unsigned long vm_end; 911 unsigned long long pgoff; 912 int major, minor; 913 char r, w, x, s; 914 unsigned long ino; 915 int n; 916 917 n = sscanf(buf, "%lx-%lx %c%c%c%c %llx %x:%x %lu", 918 &vm_start, 919 &vm_end, 920 &r, &w, &x, &s, 921 &pgoff, 922 &major, &minor, 923 &ino); 924 if (n < 10) { 925 fprintf(stderr, "unexpected line: %s\n", buf); 926 continue; 927 } 928 pg_start[nr_vmas] = vm_start / page_size; 929 pg_end[nr_vmas] = vm_end / page_size; 930 if (++nr_vmas >= MAX_VMAS) { 931 fprintf(stderr, "too many VMAs\n"); 932 break; 933 } 934 } 935 fclose(file); 936 } 937 938 static void show_file(const char *name, const struct stat *st) 939 { 940 unsigned long long size = st->st_size; 941 char atime[64], mtime[64]; 942 long now = time(NULL); 943 944 printf("%s\tInode: %u\tSize: %llu (%llu pages)\n", 945 name, (unsigned)st->st_ino, 946 size, (size + page_size - 1) / page_size); 947 948 strftime(atime, sizeof(atime), "%c", localtime(&st->st_atime)); 949 strftime(mtime, sizeof(mtime), "%c", localtime(&st->st_mtime)); 950 951 printf("Modify: %s (%ld seconds ago)\nAccess: %s (%ld seconds ago)\n", 952 mtime, now - st->st_mtime, 953 atime, now - st->st_atime); 954 } 955 956 static sigjmp_buf sigbus_jmp; 957 958 static void * volatile sigbus_addr; 959 960 static void sigbus_handler(int sig, siginfo_t *info, void *ucontex) 961 { 962 (void)sig; 963 (void)ucontex; 964 sigbus_addr = info ? info->si_addr : NULL; 965 siglongjmp(sigbus_jmp, 1); 966 } 967 968 static struct sigaction sigbus_action = { 969 .sa_sigaction = sigbus_handler, 970 .sa_flags = SA_SIGINFO, 971 }; 972 973 static void walk_file_range(const char *name, int fd, 974 unsigned long off, unsigned long end) 975 { 976 uint8_t vec[PAGEMAP_BATCH]; 977 uint64_t buf[PAGEMAP_BATCH], flags; 978 uint64_t cgroup = 0; 979 uint64_t mapcnt = 0; 980 unsigned long nr_pages, pfn, i; 981 ssize_t len; 982 void *ptr; 983 int first = 1; 984 985 for (; off < end; off += len) { 986 nr_pages = (end - off + page_size - 1) / page_size; 987 if (nr_pages > PAGEMAP_BATCH) 988 nr_pages = PAGEMAP_BATCH; 989 len = nr_pages * page_size; 990 991 ptr = mmap(NULL, len, PROT_READ, MAP_SHARED, fd, off); 992 if (ptr == MAP_FAILED) 993 fatal("mmap failed: %s", name); 994 995 /* determine cached pages */ 996 if (mincore(ptr, len, vec)) 997 fatal("mincore failed: %s", name); 998 999 /* turn off readahead */ 1000 if (madvise(ptr, len, MADV_RANDOM)) 1001 fatal("madvice failed: %s", name); 1002 1003 if (sigsetjmp(sigbus_jmp, 1)) { 1004 end = off + sigbus_addr ? sigbus_addr - ptr : 0; 1005 fprintf(stderr, "got sigbus at offset %lld: %s\n", 1006 (long long)end, name); 1007 goto got_sigbus; 1008 } 1009 1010 /* populate ptes */ 1011 for (i = 0; i < nr_pages ; i++) { 1012 if (vec[i] & 1) 1013 (void)*(volatile int *)(ptr + i * page_size); 1014 } 1015 got_sigbus: 1016 1017 /* turn off harvesting reference bits */ 1018 if (madvise(ptr, len, MADV_SEQUENTIAL)) 1019 fatal("madvice failed: %s", name); 1020 1021 if (pagemap_read(buf, (unsigned long)ptr / page_size, 1022 nr_pages) != nr_pages) 1023 fatal("cannot read pagemap"); 1024 1025 munmap(ptr, len); 1026 1027 for (i = 0; i < nr_pages; i++) { 1028 pfn = pagemap_pfn(buf[i]); 1029 if (!pfn) 1030 continue; 1031 if (!kpageflags_read(&flags, pfn, 1)) 1032 continue; 1033 if (!kpagecgroup_read(&cgroup, pfn, 1)) 1034 fatal("kpagecgroup_read failed"); 1035 if (!kpagecount_read(&mapcnt, pfn, 1)) 1036 fatal("kpagecount_read failed"); 1037 if (first && opt_list) { 1038 first = 0; 1039 flush_page_range(); 1040 } 1041 add_page(off / page_size + i, pfn, 1042 flags, cgroup, mapcnt, buf[i]); 1043 } 1044 } 1045 } 1046 1047 static void walk_file(const char *name, const struct stat *st) 1048 { 1049 int i; 1050 int fd; 1051 1052 fd = checked_open(name, O_RDONLY|O_NOATIME|O_NOFOLLOW); 1053 1054 if (!nr_addr_ranges) 1055 add_addr_range(0, st->st_size / page_size); 1056 1057 for (i = 0; i < nr_addr_ranges; i++) 1058 walk_file_range(name, fd, opt_offset[i] * page_size, 1059 (opt_offset[i] + opt_size[i]) * page_size); 1060 1061 close(fd); 1062 } 1063 1064 int walk_tree(const char *name, const struct stat *st, int type, struct FTW *f) 1065 { 1066 (void)f; 1067 switch (type) { 1068 case FTW_F: 1069 if (S_ISREG(st->st_mode)) 1070 walk_file(name, st); 1071 break; 1072 case FTW_DNR: 1073 fprintf(stderr, "cannot read dir: %s\n", name); 1074 break; 1075 } 1076 return 0; 1077 } 1078 1079 struct stat st; 1080 1081 static void walk_page_cache(void) 1082 { 1083 kpageflags_fd = checked_open(opt_kpageflags, O_RDONLY); 1084 pagemap_fd = checked_open("/proc/self/pagemap", O_RDONLY); 1085 sigaction(SIGBUS, &sigbus_action, NULL); 1086 1087 if (stat(opt_file, &st)) 1088 fatal("stat failed: %s\n", opt_file); 1089 1090 if (S_ISREG(st.st_mode)) { 1091 walk_file(opt_file, &st); 1092 } else if (S_ISDIR(st.st_mode)) { 1093 /* do not follow symlinks and mountpoints */ 1094 if (nftw(opt_file, walk_tree, 64, FTW_MOUNT | FTW_PHYS) < 0) 1095 fatal("nftw failed: %s\n", opt_file); 1096 } else 1097 fatal("unhandled file type: %s\n", opt_file); 1098 1099 close(kpageflags_fd); 1100 close(pagemap_fd); 1101 signal(SIGBUS, SIG_DFL); 1102 } 1103 1104 static void parse_file(const char *name) 1105 { 1106 opt_file = name; 1107 } 1108 1109 static void parse_cgroup(const char *path) 1110 { 1111 if (path[0] == '@') { 1112 opt_cgroup = parse_number(path + 1); 1113 return; 1114 } 1115 1116 struct stat st; 1117 1118 if (stat(path, &st)) 1119 fatal("stat failed: %s: %m\n", path); 1120 1121 if (!S_ISDIR(st.st_mode)) 1122 fatal("cgroup supposed to be a directory: %s\n", path); 1123 1124 opt_cgroup = st.st_ino; 1125 } 1126 1127 static void parse_addr_range(const char *optarg) 1128 { 1129 unsigned long offset; 1130 unsigned long size; 1131 char *p; 1132 1133 p = strchr(optarg, ','); 1134 if (!p) 1135 p = strchr(optarg, '+'); 1136 1137 if (p == optarg) { 1138 offset = 0; 1139 size = parse_number(p + 1); 1140 } else if (p) { 1141 offset = parse_number(optarg); 1142 if (p[1] == '\0') 1143 size = ULONG_MAX; 1144 else { 1145 size = parse_number(p + 1); 1146 if (*p == ',') { 1147 if (size < offset) 1148 fatal("invalid range: %lu,%lu\n", 1149 offset, size); 1150 size -= offset; 1151 } 1152 } 1153 } else { 1154 offset = parse_number(optarg); 1155 size = 1; 1156 } 1157 1158 add_addr_range(offset, size); 1159 } 1160 1161 static void add_bits_filter(uint64_t mask, uint64_t bits) 1162 { 1163 if (nr_bit_filters >= MAX_BIT_FILTERS) 1164 fatal("too much bit filters\n"); 1165 1166 opt_mask[nr_bit_filters] = mask; 1167 opt_bits[nr_bit_filters] = bits; 1168 nr_bit_filters++; 1169 } 1170 1171 static uint64_t parse_flag_name(const char *str, int len) 1172 { 1173 size_t i; 1174 1175 if (!*str || !len) 1176 return 0; 1177 1178 if (len <= 8 && !strncmp(str, "compound", len)) 1179 return BITS_COMPOUND; 1180 1181 for (i = 0; i < ARRAY_SIZE(page_flag_names); i++) { 1182 if (!page_flag_names[i]) 1183 continue; 1184 if (!strncmp(str, page_flag_names[i] + 2, len)) 1185 return 1ULL << i; 1186 } 1187 1188 return parse_number(str); 1189 } 1190 1191 static uint64_t parse_flag_names(const char *str, int all) 1192 { 1193 const char *p = str; 1194 uint64_t flags = 0; 1195 1196 while (1) { 1197 if (*p == ',' || *p == '=' || *p == '\0') { 1198 if ((*str != '~') || (*str == '~' && all && *++str)) 1199 flags |= parse_flag_name(str, p - str); 1200 if (*p != ',') 1201 break; 1202 str = p + 1; 1203 } 1204 p++; 1205 } 1206 1207 return flags; 1208 } 1209 1210 static void parse_bits_mask(const char *optarg) 1211 { 1212 uint64_t mask; 1213 uint64_t bits; 1214 const char *p; 1215 1216 p = strchr(optarg, '='); 1217 if (p == optarg) { 1218 mask = KPF_ALL_BITS; 1219 bits = parse_flag_names(p + 1, 0); 1220 } else if (p) { 1221 mask = parse_flag_names(optarg, 0); 1222 bits = parse_flag_names(p + 1, 0); 1223 } else if (strchr(optarg, '~')) { 1224 mask = parse_flag_names(optarg, 1); 1225 bits = parse_flag_names(optarg, 0); 1226 } else { 1227 mask = parse_flag_names(optarg, 0); 1228 bits = KPF_ALL_BITS; 1229 } 1230 1231 add_bits_filter(mask, bits); 1232 } 1233 1234 static void parse_kpageflags(const char *name) 1235 { 1236 opt_kpageflags = name; 1237 } 1238 1239 static void describe_flags(const char *optarg) 1240 { 1241 uint64_t flags = parse_flag_names(optarg, 0); 1242 1243 printf("0x%016llx\t%s\t%s\n", 1244 (unsigned long long)flags, 1245 page_flag_name(flags), 1246 page_flag_longname(flags)); 1247 } 1248 1249 static const struct option opts[] = { 1250 { "raw" , 0, NULL, 'r' }, 1251 { "pid" , 1, NULL, 'p' }, 1252 { "file" , 1, NULL, 'f' }, 1253 { "addr" , 1, NULL, 'a' }, 1254 { "bits" , 1, NULL, 'b' }, 1255 { "cgroup" , 1, NULL, 'c' }, 1256 { "describe" , 1, NULL, 'd' }, 1257 { "mark-idle" , 0, NULL, 'i' }, 1258 { "list" , 0, NULL, 'l' }, 1259 { "list-each" , 0, NULL, 'L' }, 1260 { "list-cgroup", 0, NULL, 'C' }, 1261 { "list-mapcnt", 0, NULL, 'M' }, 1262 { "no-summary", 0, NULL, 'N' }, 1263 { "hwpoison" , 0, NULL, 'X' }, 1264 { "unpoison" , 0, NULL, 'x' }, 1265 { "kpageflags", 0, NULL, 'F' }, 1266 { "help" , 0, NULL, 'h' }, 1267 { NULL , 0, NULL, 0 } 1268 }; 1269 1270 int main(int argc, char *argv[]) 1271 { 1272 int c; 1273 1274 page_size = getpagesize(); 1275 1276 while ((c = getopt_long(argc, argv, 1277 "rp:f:a:b:d:c:CilLMNXxF:h", 1278 opts, NULL)) != -1) { 1279 switch (c) { 1280 case 'r': 1281 opt_raw = 1; 1282 break; 1283 case 'p': 1284 parse_pid(optarg); 1285 break; 1286 case 'f': 1287 parse_file(optarg); 1288 break; 1289 case 'a': 1290 parse_addr_range(optarg); 1291 break; 1292 case 'b': 1293 parse_bits_mask(optarg); 1294 break; 1295 case 'c': 1296 parse_cgroup(optarg); 1297 break; 1298 case 'C': 1299 opt_list_cgroup = 1; 1300 break; 1301 case 'd': 1302 describe_flags(optarg); 1303 exit(0); 1304 case 'i': 1305 opt_mark_idle = 1; 1306 break; 1307 case 'l': 1308 opt_list = 1; 1309 break; 1310 case 'L': 1311 opt_list = 2; 1312 break; 1313 case 'M': 1314 opt_list_mapcnt = 1; 1315 break; 1316 case 'N': 1317 opt_no_summary = 1; 1318 break; 1319 case 'X': 1320 opt_hwpoison = 1; 1321 prepare_hwpoison_fd(); 1322 break; 1323 case 'x': 1324 opt_unpoison = 1; 1325 prepare_hwpoison_fd(); 1326 break; 1327 case 'F': 1328 parse_kpageflags(optarg); 1329 break; 1330 case 'h': 1331 usage(); 1332 exit(0); 1333 default: 1334 usage(); 1335 exit(1); 1336 } 1337 } 1338 1339 if (!opt_kpageflags) 1340 opt_kpageflags = PROC_KPAGEFLAGS; 1341 1342 if (opt_cgroup || opt_list_cgroup) 1343 kpagecgroup_fd = checked_open(PROC_KPAGECGROUP, O_RDONLY); 1344 1345 if (opt_list && opt_list_mapcnt) 1346 kpagecount_fd = checked_open(PROC_KPAGECOUNT, O_RDONLY); 1347 1348 if (opt_mark_idle) 1349 page_idle_fd = checked_open(SYS_KERNEL_MM_PAGE_IDLE, O_RDWR); 1350 1351 if (opt_list && opt_pid) 1352 printf("voffset\t"); 1353 if (opt_list && opt_file) 1354 printf("foffset\t"); 1355 if (opt_list && opt_list_cgroup) 1356 printf("cgroup\t"); 1357 if (opt_list && opt_list_mapcnt) 1358 printf("map-cnt\t"); 1359 1360 if (opt_list == 1) 1361 printf("offset\tlen\tflags\n"); 1362 if (opt_list == 2) 1363 printf("offset\tflags\n"); 1364 1365 if (opt_file) 1366 walk_page_cache(); 1367 else 1368 walk_addr_ranges(); 1369 1370 if (opt_list == 1) 1371 flush_page_range(); 1372 1373 if (opt_no_summary) 1374 return 0; 1375 1376 if (opt_list) 1377 printf("\n\n"); 1378 1379 if (opt_file) { 1380 show_file(opt_file, &st); 1381 printf("\n"); 1382 } 1383 1384 show_summary(); 1385 1386 if (opt_list_mapcnt) 1387 close(kpagecount_fd); 1388 1389 if (page_idle_fd >= 0) 1390 close(page_idle_fd); 1391 1392 return 0; 1393 } 1394