1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/debugfs.h> 3 #include <linux/mm.h> 4 #include <linux/slab.h> 5 #include <linux/uaccess.h> 6 #include <linux/memblock.h> 7 #include <linux/stacktrace.h> 8 #include <linux/page_owner.h> 9 #include <linux/jump_label.h> 10 #include <linux/migrate.h> 11 #include <linux/stackdepot.h> 12 #include <linux/seq_file.h> 13 #include <linux/sched/clock.h> 14 15 #include "internal.h" 16 17 /* 18 * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack) 19 * to use off stack temporal storage 20 */ 21 #define PAGE_OWNER_STACK_DEPTH (16) 22 23 struct page_owner { 24 unsigned short order; 25 short last_migrate_reason; 26 gfp_t gfp_mask; 27 depot_stack_handle_t handle; 28 depot_stack_handle_t free_handle; 29 u64 ts_nsec; 30 pid_t pid; 31 }; 32 33 static bool page_owner_enabled = false; 34 DEFINE_STATIC_KEY_FALSE(page_owner_inited); 35 36 static depot_stack_handle_t dummy_handle; 37 static depot_stack_handle_t failure_handle; 38 static depot_stack_handle_t early_handle; 39 40 static void init_early_allocated_pages(void); 41 42 static int __init early_page_owner_param(char *buf) 43 { 44 if (!buf) 45 return -EINVAL; 46 47 if (strcmp(buf, "on") == 0) 48 page_owner_enabled = true; 49 50 return 0; 51 } 52 early_param("page_owner", early_page_owner_param); 53 54 static bool need_page_owner(void) 55 { 56 return page_owner_enabled; 57 } 58 59 static __always_inline depot_stack_handle_t create_dummy_stack(void) 60 { 61 unsigned long entries[4]; 62 unsigned int nr_entries; 63 64 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0); 65 return stack_depot_save(entries, nr_entries, GFP_KERNEL); 66 } 67 68 static noinline void register_dummy_stack(void) 69 { 70 dummy_handle = create_dummy_stack(); 71 } 72 73 static noinline void register_failure_stack(void) 74 { 75 failure_handle = create_dummy_stack(); 76 } 77 78 static noinline void register_early_stack(void) 79 { 80 early_handle = create_dummy_stack(); 81 } 82 83 static void init_page_owner(void) 84 { 85 if (!page_owner_enabled) 86 return; 87 88 register_dummy_stack(); 89 register_failure_stack(); 90 register_early_stack(); 91 static_branch_enable(&page_owner_inited); 92 init_early_allocated_pages(); 93 } 94 95 struct page_ext_operations page_owner_ops = { 96 .size = sizeof(struct page_owner), 97 .need = need_page_owner, 98 .init = init_page_owner, 99 }; 100 101 static inline struct page_owner *get_page_owner(struct page_ext *page_ext) 102 { 103 return (void *)page_ext + page_owner_ops.offset; 104 } 105 106 static inline bool check_recursive_alloc(unsigned long *entries, 107 unsigned int nr_entries, 108 unsigned long ip) 109 { 110 unsigned int i; 111 112 for (i = 0; i < nr_entries; i++) { 113 if (entries[i] == ip) 114 return true; 115 } 116 return false; 117 } 118 119 static noinline depot_stack_handle_t save_stack(gfp_t flags) 120 { 121 unsigned long entries[PAGE_OWNER_STACK_DEPTH]; 122 depot_stack_handle_t handle; 123 unsigned int nr_entries; 124 125 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 2); 126 127 /* 128 * We need to check recursion here because our request to 129 * stackdepot could trigger memory allocation to save new 130 * entry. New memory allocation would reach here and call 131 * stack_depot_save_entries() again if we don't catch it. There is 132 * still not enough memory in stackdepot so it would try to 133 * allocate memory again and loop forever. 134 */ 135 if (check_recursive_alloc(entries, nr_entries, _RET_IP_)) 136 return dummy_handle; 137 138 handle = stack_depot_save(entries, nr_entries, flags); 139 if (!handle) 140 handle = failure_handle; 141 142 return handle; 143 } 144 145 void __reset_page_owner(struct page *page, unsigned int order) 146 { 147 int i; 148 struct page_ext *page_ext; 149 depot_stack_handle_t handle = 0; 150 struct page_owner *page_owner; 151 152 handle = save_stack(GFP_NOWAIT | __GFP_NOWARN); 153 154 page_ext = lookup_page_ext(page); 155 if (unlikely(!page_ext)) 156 return; 157 for (i = 0; i < (1 << order); i++) { 158 __clear_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags); 159 page_owner = get_page_owner(page_ext); 160 page_owner->free_handle = handle; 161 page_ext = page_ext_next(page_ext); 162 } 163 } 164 165 static inline void __set_page_owner_handle(struct page *page, 166 struct page_ext *page_ext, depot_stack_handle_t handle, 167 unsigned int order, gfp_t gfp_mask) 168 { 169 struct page_owner *page_owner; 170 int i; 171 172 for (i = 0; i < (1 << order); i++) { 173 page_owner = get_page_owner(page_ext); 174 page_owner->handle = handle; 175 page_owner->order = order; 176 page_owner->gfp_mask = gfp_mask; 177 page_owner->last_migrate_reason = -1; 178 page_owner->pid = current->pid; 179 page_owner->ts_nsec = local_clock(); 180 __set_bit(PAGE_EXT_OWNER, &page_ext->flags); 181 __set_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags); 182 183 page_ext = page_ext_next(page_ext); 184 } 185 } 186 187 noinline void __set_page_owner(struct page *page, unsigned int order, 188 gfp_t gfp_mask) 189 { 190 struct page_ext *page_ext = lookup_page_ext(page); 191 depot_stack_handle_t handle; 192 193 if (unlikely(!page_ext)) 194 return; 195 196 handle = save_stack(gfp_mask); 197 __set_page_owner_handle(page, page_ext, handle, order, gfp_mask); 198 } 199 200 void __set_page_owner_migrate_reason(struct page *page, int reason) 201 { 202 struct page_ext *page_ext = lookup_page_ext(page); 203 struct page_owner *page_owner; 204 205 if (unlikely(!page_ext)) 206 return; 207 208 page_owner = get_page_owner(page_ext); 209 page_owner->last_migrate_reason = reason; 210 } 211 212 void __split_page_owner(struct page *page, unsigned int nr) 213 { 214 int i; 215 struct page_ext *page_ext = lookup_page_ext(page); 216 struct page_owner *page_owner; 217 218 if (unlikely(!page_ext)) 219 return; 220 221 for (i = 0; i < nr; i++) { 222 page_owner = get_page_owner(page_ext); 223 page_owner->order = 0; 224 page_ext = page_ext_next(page_ext); 225 } 226 } 227 228 void __copy_page_owner(struct page *oldpage, struct page *newpage) 229 { 230 struct page_ext *old_ext = lookup_page_ext(oldpage); 231 struct page_ext *new_ext = lookup_page_ext(newpage); 232 struct page_owner *old_page_owner, *new_page_owner; 233 234 if (unlikely(!old_ext || !new_ext)) 235 return; 236 237 old_page_owner = get_page_owner(old_ext); 238 new_page_owner = get_page_owner(new_ext); 239 new_page_owner->order = old_page_owner->order; 240 new_page_owner->gfp_mask = old_page_owner->gfp_mask; 241 new_page_owner->last_migrate_reason = 242 old_page_owner->last_migrate_reason; 243 new_page_owner->handle = old_page_owner->handle; 244 new_page_owner->pid = old_page_owner->pid; 245 new_page_owner->ts_nsec = old_page_owner->ts_nsec; 246 247 /* 248 * We don't clear the bit on the oldpage as it's going to be freed 249 * after migration. Until then, the info can be useful in case of 250 * a bug, and the overal stats will be off a bit only temporarily. 251 * Also, migrate_misplaced_transhuge_page() can still fail the 252 * migration and then we want the oldpage to retain the info. But 253 * in that case we also don't need to explicitly clear the info from 254 * the new page, which will be freed. 255 */ 256 __set_bit(PAGE_EXT_OWNER, &new_ext->flags); 257 __set_bit(PAGE_EXT_OWNER_ALLOCATED, &new_ext->flags); 258 } 259 260 void pagetypeinfo_showmixedcount_print(struct seq_file *m, 261 pg_data_t *pgdat, struct zone *zone) 262 { 263 struct page *page; 264 struct page_ext *page_ext; 265 struct page_owner *page_owner; 266 unsigned long pfn, block_end_pfn; 267 unsigned long end_pfn = zone_end_pfn(zone); 268 unsigned long count[MIGRATE_TYPES] = { 0, }; 269 int pageblock_mt, page_mt; 270 int i; 271 272 /* Scan block by block. First and last block may be incomplete */ 273 pfn = zone->zone_start_pfn; 274 275 /* 276 * Walk the zone in pageblock_nr_pages steps. If a page block spans 277 * a zone boundary, it will be double counted between zones. This does 278 * not matter as the mixed block count will still be correct 279 */ 280 for (; pfn < end_pfn; ) { 281 page = pfn_to_online_page(pfn); 282 if (!page) { 283 pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); 284 continue; 285 } 286 287 block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); 288 block_end_pfn = min(block_end_pfn, end_pfn); 289 290 pageblock_mt = get_pageblock_migratetype(page); 291 292 for (; pfn < block_end_pfn; pfn++) { 293 if (!pfn_valid_within(pfn)) 294 continue; 295 296 /* The pageblock is online, no need to recheck. */ 297 page = pfn_to_page(pfn); 298 299 if (page_zone(page) != zone) 300 continue; 301 302 if (PageBuddy(page)) { 303 unsigned long freepage_order; 304 305 freepage_order = buddy_order_unsafe(page); 306 if (freepage_order < MAX_ORDER) 307 pfn += (1UL << freepage_order) - 1; 308 continue; 309 } 310 311 if (PageReserved(page)) 312 continue; 313 314 page_ext = lookup_page_ext(page); 315 if (unlikely(!page_ext)) 316 continue; 317 318 if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) 319 continue; 320 321 page_owner = get_page_owner(page_ext); 322 page_mt = gfp_migratetype(page_owner->gfp_mask); 323 if (pageblock_mt != page_mt) { 324 if (is_migrate_cma(pageblock_mt)) 325 count[MIGRATE_MOVABLE]++; 326 else 327 count[pageblock_mt]++; 328 329 pfn = block_end_pfn; 330 break; 331 } 332 pfn += (1UL << page_owner->order) - 1; 333 } 334 } 335 336 /* Print counts */ 337 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 338 for (i = 0; i < MIGRATE_TYPES; i++) 339 seq_printf(m, "%12lu ", count[i]); 340 seq_putc(m, '\n'); 341 } 342 343 static ssize_t 344 print_page_owner(char __user *buf, size_t count, unsigned long pfn, 345 struct page *page, struct page_owner *page_owner, 346 depot_stack_handle_t handle) 347 { 348 int ret, pageblock_mt, page_mt; 349 unsigned long *entries; 350 unsigned int nr_entries; 351 char *kbuf; 352 353 count = min_t(size_t, count, PAGE_SIZE); 354 kbuf = kmalloc(count, GFP_KERNEL); 355 if (!kbuf) 356 return -ENOMEM; 357 358 ret = snprintf(kbuf, count, 359 "Page allocated via order %u, mask %#x(%pGg), pid %d, ts %llu ns\n", 360 page_owner->order, page_owner->gfp_mask, 361 &page_owner->gfp_mask, page_owner->pid, 362 page_owner->ts_nsec); 363 364 if (ret >= count) 365 goto err; 366 367 /* Print information relevant to grouping pages by mobility */ 368 pageblock_mt = get_pageblock_migratetype(page); 369 page_mt = gfp_migratetype(page_owner->gfp_mask); 370 ret += snprintf(kbuf + ret, count - ret, 371 "PFN %lu type %s Block %lu type %s Flags %#lx(%pGp)\n", 372 pfn, 373 migratetype_names[page_mt], 374 pfn >> pageblock_order, 375 migratetype_names[pageblock_mt], 376 page->flags, &page->flags); 377 378 if (ret >= count) 379 goto err; 380 381 nr_entries = stack_depot_fetch(handle, &entries); 382 ret += stack_trace_snprint(kbuf + ret, count - ret, entries, nr_entries, 0); 383 if (ret >= count) 384 goto err; 385 386 if (page_owner->last_migrate_reason != -1) { 387 ret += snprintf(kbuf + ret, count - ret, 388 "Page has been migrated, last migrate reason: %s\n", 389 migrate_reason_names[page_owner->last_migrate_reason]); 390 if (ret >= count) 391 goto err; 392 } 393 394 ret += snprintf(kbuf + ret, count - ret, "\n"); 395 if (ret >= count) 396 goto err; 397 398 if (copy_to_user(buf, kbuf, ret)) 399 ret = -EFAULT; 400 401 kfree(kbuf); 402 return ret; 403 404 err: 405 kfree(kbuf); 406 return -ENOMEM; 407 } 408 409 void __dump_page_owner(struct page *page) 410 { 411 struct page_ext *page_ext = lookup_page_ext(page); 412 struct page_owner *page_owner; 413 depot_stack_handle_t handle; 414 unsigned long *entries; 415 unsigned int nr_entries; 416 gfp_t gfp_mask; 417 int mt; 418 419 if (unlikely(!page_ext)) { 420 pr_alert("There is not page extension available.\n"); 421 return; 422 } 423 424 page_owner = get_page_owner(page_ext); 425 gfp_mask = page_owner->gfp_mask; 426 mt = gfp_migratetype(gfp_mask); 427 428 if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) { 429 pr_alert("page_owner info is not present (never set?)\n"); 430 return; 431 } 432 433 if (test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) 434 pr_alert("page_owner tracks the page as allocated\n"); 435 else 436 pr_alert("page_owner tracks the page as freed\n"); 437 438 pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, ts %llu\n", 439 page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask, 440 page_owner->pid, page_owner->ts_nsec); 441 442 handle = READ_ONCE(page_owner->handle); 443 if (!handle) { 444 pr_alert("page_owner allocation stack trace missing\n"); 445 } else { 446 nr_entries = stack_depot_fetch(handle, &entries); 447 stack_trace_print(entries, nr_entries, 0); 448 } 449 450 handle = READ_ONCE(page_owner->free_handle); 451 if (!handle) { 452 pr_alert("page_owner free stack trace missing\n"); 453 } else { 454 nr_entries = stack_depot_fetch(handle, &entries); 455 pr_alert("page last free stack trace:\n"); 456 stack_trace_print(entries, nr_entries, 0); 457 } 458 459 if (page_owner->last_migrate_reason != -1) 460 pr_alert("page has been migrated, last migrate reason: %s\n", 461 migrate_reason_names[page_owner->last_migrate_reason]); 462 } 463 464 static ssize_t 465 read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) 466 { 467 unsigned long pfn; 468 struct page *page; 469 struct page_ext *page_ext; 470 struct page_owner *page_owner; 471 depot_stack_handle_t handle; 472 473 if (!static_branch_unlikely(&page_owner_inited)) 474 return -EINVAL; 475 476 page = NULL; 477 pfn = min_low_pfn + *ppos; 478 479 /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */ 480 while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0) 481 pfn++; 482 483 drain_all_pages(NULL); 484 485 /* Find an allocated page */ 486 for (; pfn < max_pfn; pfn++) { 487 /* 488 * If the new page is in a new MAX_ORDER_NR_PAGES area, 489 * validate the area as existing, skip it if not 490 */ 491 if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) { 492 pfn += MAX_ORDER_NR_PAGES - 1; 493 continue; 494 } 495 496 /* Check for holes within a MAX_ORDER area */ 497 if (!pfn_valid_within(pfn)) 498 continue; 499 500 page = pfn_to_page(pfn); 501 if (PageBuddy(page)) { 502 unsigned long freepage_order = buddy_order_unsafe(page); 503 504 if (freepage_order < MAX_ORDER) 505 pfn += (1UL << freepage_order) - 1; 506 continue; 507 } 508 509 page_ext = lookup_page_ext(page); 510 if (unlikely(!page_ext)) 511 continue; 512 513 /* 514 * Some pages could be missed by concurrent allocation or free, 515 * because we don't hold the zone lock. 516 */ 517 if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) 518 continue; 519 520 /* 521 * Although we do have the info about past allocation of free 522 * pages, it's not relevant for current memory usage. 523 */ 524 if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) 525 continue; 526 527 page_owner = get_page_owner(page_ext); 528 529 /* 530 * Don't print "tail" pages of high-order allocations as that 531 * would inflate the stats. 532 */ 533 if (!IS_ALIGNED(pfn, 1 << page_owner->order)) 534 continue; 535 536 /* 537 * Access to page_ext->handle isn't synchronous so we should 538 * be careful to access it. 539 */ 540 handle = READ_ONCE(page_owner->handle); 541 if (!handle) 542 continue; 543 544 /* Record the next PFN to read in the file offset */ 545 *ppos = (pfn - min_low_pfn) + 1; 546 547 return print_page_owner(buf, count, pfn, page, 548 page_owner, handle); 549 } 550 551 return 0; 552 } 553 554 static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone) 555 { 556 unsigned long pfn = zone->zone_start_pfn; 557 unsigned long end_pfn = zone_end_pfn(zone); 558 unsigned long count = 0; 559 560 /* 561 * Walk the zone in pageblock_nr_pages steps. If a page block spans 562 * a zone boundary, it will be double counted between zones. This does 563 * not matter as the mixed block count will still be correct 564 */ 565 for (; pfn < end_pfn; ) { 566 unsigned long block_end_pfn; 567 568 if (!pfn_valid(pfn)) { 569 pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); 570 continue; 571 } 572 573 block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); 574 block_end_pfn = min(block_end_pfn, end_pfn); 575 576 for (; pfn < block_end_pfn; pfn++) { 577 struct page *page; 578 struct page_ext *page_ext; 579 580 if (!pfn_valid_within(pfn)) 581 continue; 582 583 page = pfn_to_page(pfn); 584 585 if (page_zone(page) != zone) 586 continue; 587 588 /* 589 * To avoid having to grab zone->lock, be a little 590 * careful when reading buddy page order. The only 591 * danger is that we skip too much and potentially miss 592 * some early allocated pages, which is better than 593 * heavy lock contention. 594 */ 595 if (PageBuddy(page)) { 596 unsigned long order = buddy_order_unsafe(page); 597 598 if (order > 0 && order < MAX_ORDER) 599 pfn += (1UL << order) - 1; 600 continue; 601 } 602 603 if (PageReserved(page)) 604 continue; 605 606 page_ext = lookup_page_ext(page); 607 if (unlikely(!page_ext)) 608 continue; 609 610 /* Maybe overlapping zone */ 611 if (test_bit(PAGE_EXT_OWNER, &page_ext->flags)) 612 continue; 613 614 /* Found early allocated page */ 615 __set_page_owner_handle(page, page_ext, early_handle, 616 0, 0); 617 count++; 618 } 619 cond_resched(); 620 } 621 622 pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n", 623 pgdat->node_id, zone->name, count); 624 } 625 626 static void init_zones_in_node(pg_data_t *pgdat) 627 { 628 struct zone *zone; 629 struct zone *node_zones = pgdat->node_zones; 630 631 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { 632 if (!populated_zone(zone)) 633 continue; 634 635 init_pages_in_zone(pgdat, zone); 636 } 637 } 638 639 static void init_early_allocated_pages(void) 640 { 641 pg_data_t *pgdat; 642 643 for_each_online_pgdat(pgdat) 644 init_zones_in_node(pgdat); 645 } 646 647 static const struct file_operations proc_page_owner_operations = { 648 .read = read_page_owner, 649 }; 650 651 static int __init pageowner_init(void) 652 { 653 if (!static_branch_unlikely(&page_owner_inited)) { 654 pr_info("page_owner is disabled\n"); 655 return 0; 656 } 657 658 debugfs_create_file("page_owner", 0400, NULL, NULL, 659 &proc_page_owner_operations); 660 661 return 0; 662 } 663 late_initcall(pageowner_init) 664