1 /* 2 * linux/mm/vmstat.c 3 * 4 * Manages VM statistics 5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 6 * 7 * zoned VM statistics 8 * Copyright (C) 2006 Silicon Graphics, Inc., 9 * Christoph Lameter <christoph@lameter.com> 10 */ 11 #include <linux/fs.h> 12 #include <linux/mm.h> 13 #include <linux/err.h> 14 #include <linux/module.h> 15 #include <linux/slab.h> 16 #include <linux/cpu.h> 17 #include <linux/vmstat.h> 18 #include <linux/sched.h> 19 #include <linux/math64.h> 20 #include <linux/writeback.h> 21 #include <linux/compaction.h> 22 23 #ifdef CONFIG_VM_EVENT_COUNTERS 24 DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}}; 25 EXPORT_PER_CPU_SYMBOL(vm_event_states); 26 27 static void sum_vm_events(unsigned long *ret) 28 { 29 int cpu; 30 int i; 31 32 memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long)); 33 34 for_each_online_cpu(cpu) { 35 struct vm_event_state *this = &per_cpu(vm_event_states, cpu); 36 37 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) 38 ret[i] += this->event[i]; 39 } 40 } 41 42 /* 43 * Accumulate the vm event counters across all CPUs. 44 * The result is unavoidably approximate - it can change 45 * during and after execution of this function. 46 */ 47 void all_vm_events(unsigned long *ret) 48 { 49 get_online_cpus(); 50 sum_vm_events(ret); 51 put_online_cpus(); 52 } 53 EXPORT_SYMBOL_GPL(all_vm_events); 54 55 #ifdef CONFIG_HOTPLUG 56 /* 57 * Fold the foreign cpu events into our own. 58 * 59 * This is adding to the events on one processor 60 * but keeps the global counts constant. 61 */ 62 void vm_events_fold_cpu(int cpu) 63 { 64 struct vm_event_state *fold_state = &per_cpu(vm_event_states, cpu); 65 int i; 66 67 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) { 68 count_vm_events(i, fold_state->event[i]); 69 fold_state->event[i] = 0; 70 } 71 } 72 #endif /* CONFIG_HOTPLUG */ 73 74 #endif /* CONFIG_VM_EVENT_COUNTERS */ 75 76 /* 77 * Manage combined zone based / global counters 78 * 79 * vm_stat contains the global counters 80 */ 81 atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp; 82 EXPORT_SYMBOL(vm_stat); 83 84 #ifdef CONFIG_SMP 85 86 int calculate_pressure_threshold(struct zone *zone) 87 { 88 int threshold; 89 int watermark_distance; 90 91 /* 92 * As vmstats are not up to date, there is drift between the estimated 93 * and real values. For high thresholds and a high number of CPUs, it 94 * is possible for the min watermark to be breached while the estimated 95 * value looks fine. The pressure threshold is a reduced value such 96 * that even the maximum amount of drift will not accidentally breach 97 * the min watermark 98 */ 99 watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone); 100 threshold = max(1, (int)(watermark_distance / num_online_cpus())); 101 102 /* 103 * Maximum threshold is 125 104 */ 105 threshold = min(125, threshold); 106 107 return threshold; 108 } 109 110 int calculate_normal_threshold(struct zone *zone) 111 { 112 int threshold; 113 int mem; /* memory in 128 MB units */ 114 115 /* 116 * The threshold scales with the number of processors and the amount 117 * of memory per zone. More memory means that we can defer updates for 118 * longer, more processors could lead to more contention. 119 * fls() is used to have a cheap way of logarithmic scaling. 120 * 121 * Some sample thresholds: 122 * 123 * Threshold Processors (fls) Zonesize fls(mem+1) 124 * ------------------------------------------------------------------ 125 * 8 1 1 0.9-1 GB 4 126 * 16 2 2 0.9-1 GB 4 127 * 20 2 2 1-2 GB 5 128 * 24 2 2 2-4 GB 6 129 * 28 2 2 4-8 GB 7 130 * 32 2 2 8-16 GB 8 131 * 4 2 2 <128M 1 132 * 30 4 3 2-4 GB 5 133 * 48 4 3 8-16 GB 8 134 * 32 8 4 1-2 GB 4 135 * 32 8 4 0.9-1GB 4 136 * 10 16 5 <128M 1 137 * 40 16 5 900M 4 138 * 70 64 7 2-4 GB 5 139 * 84 64 7 4-8 GB 6 140 * 108 512 9 4-8 GB 6 141 * 125 1024 10 8-16 GB 8 142 * 125 1024 10 16-32 GB 9 143 */ 144 145 mem = zone->present_pages >> (27 - PAGE_SHIFT); 146 147 threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem)); 148 149 /* 150 * Maximum threshold is 125 151 */ 152 threshold = min(125, threshold); 153 154 return threshold; 155 } 156 157 /* 158 * Refresh the thresholds for each zone. 159 */ 160 void refresh_zone_stat_thresholds(void) 161 { 162 struct zone *zone; 163 int cpu; 164 int threshold; 165 166 for_each_populated_zone(zone) { 167 unsigned long max_drift, tolerate_drift; 168 169 threshold = calculate_normal_threshold(zone); 170 171 for_each_online_cpu(cpu) 172 per_cpu_ptr(zone->pageset, cpu)->stat_threshold 173 = threshold; 174 175 /* 176 * Only set percpu_drift_mark if there is a danger that 177 * NR_FREE_PAGES reports the low watermark is ok when in fact 178 * the min watermark could be breached by an allocation 179 */ 180 tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone); 181 max_drift = num_online_cpus() * threshold; 182 if (max_drift > tolerate_drift) 183 zone->percpu_drift_mark = high_wmark_pages(zone) + 184 max_drift; 185 } 186 } 187 188 void set_pgdat_percpu_threshold(pg_data_t *pgdat, 189 int (*calculate_pressure)(struct zone *)) 190 { 191 struct zone *zone; 192 int cpu; 193 int threshold; 194 int i; 195 196 for (i = 0; i < pgdat->nr_zones; i++) { 197 zone = &pgdat->node_zones[i]; 198 if (!zone->percpu_drift_mark) 199 continue; 200 201 threshold = (*calculate_pressure)(zone); 202 for_each_possible_cpu(cpu) 203 per_cpu_ptr(zone->pageset, cpu)->stat_threshold 204 = threshold; 205 } 206 } 207 208 /* 209 * For use when we know that interrupts are disabled. 210 */ 211 void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 212 int delta) 213 { 214 struct per_cpu_pageset __percpu *pcp = zone->pageset; 215 s8 __percpu *p = pcp->vm_stat_diff + item; 216 long x; 217 long t; 218 219 x = delta + __this_cpu_read(*p); 220 221 t = __this_cpu_read(pcp->stat_threshold); 222 223 if (unlikely(x > t || x < -t)) { 224 zone_page_state_add(x, zone, item); 225 x = 0; 226 } 227 __this_cpu_write(*p, x); 228 } 229 EXPORT_SYMBOL(__mod_zone_page_state); 230 231 /* 232 * Optimized increment and decrement functions. 233 * 234 * These are only for a single page and therefore can take a struct page * 235 * argument instead of struct zone *. This allows the inclusion of the code 236 * generated for page_zone(page) into the optimized functions. 237 * 238 * No overflow check is necessary and therefore the differential can be 239 * incremented or decremented in place which may allow the compilers to 240 * generate better code. 241 * The increment or decrement is known and therefore one boundary check can 242 * be omitted. 243 * 244 * NOTE: These functions are very performance sensitive. Change only 245 * with care. 246 * 247 * Some processors have inc/dec instructions that are atomic vs an interrupt. 248 * However, the code must first determine the differential location in a zone 249 * based on the processor number and then inc/dec the counter. There is no 250 * guarantee without disabling preemption that the processor will not change 251 * in between and therefore the atomicity vs. interrupt cannot be exploited 252 * in a useful way here. 253 */ 254 void __inc_zone_state(struct zone *zone, enum zone_stat_item item) 255 { 256 struct per_cpu_pageset __percpu *pcp = zone->pageset; 257 s8 __percpu *p = pcp->vm_stat_diff + item; 258 s8 v, t; 259 260 v = __this_cpu_inc_return(*p); 261 t = __this_cpu_read(pcp->stat_threshold); 262 if (unlikely(v > t)) { 263 s8 overstep = t >> 1; 264 265 zone_page_state_add(v + overstep, zone, item); 266 __this_cpu_write(*p, -overstep); 267 } 268 } 269 270 void __inc_zone_page_state(struct page *page, enum zone_stat_item item) 271 { 272 __inc_zone_state(page_zone(page), item); 273 } 274 EXPORT_SYMBOL(__inc_zone_page_state); 275 276 void __dec_zone_state(struct zone *zone, enum zone_stat_item item) 277 { 278 struct per_cpu_pageset __percpu *pcp = zone->pageset; 279 s8 __percpu *p = pcp->vm_stat_diff + item; 280 s8 v, t; 281 282 v = __this_cpu_dec_return(*p); 283 t = __this_cpu_read(pcp->stat_threshold); 284 if (unlikely(v < - t)) { 285 s8 overstep = t >> 1; 286 287 zone_page_state_add(v - overstep, zone, item); 288 __this_cpu_write(*p, overstep); 289 } 290 } 291 292 void __dec_zone_page_state(struct page *page, enum zone_stat_item item) 293 { 294 __dec_zone_state(page_zone(page), item); 295 } 296 EXPORT_SYMBOL(__dec_zone_page_state); 297 298 #ifdef CONFIG_HAVE_CMPXCHG_LOCAL 299 /* 300 * If we have cmpxchg_local support then we do not need to incur the overhead 301 * that comes with local_irq_save/restore if we use this_cpu_cmpxchg. 302 * 303 * mod_state() modifies the zone counter state through atomic per cpu 304 * operations. 305 * 306 * Overstep mode specifies how overstep should handled: 307 * 0 No overstepping 308 * 1 Overstepping half of threshold 309 * -1 Overstepping minus half of threshold 310 */ 311 static inline void mod_state(struct zone *zone, 312 enum zone_stat_item item, int delta, int overstep_mode) 313 { 314 struct per_cpu_pageset __percpu *pcp = zone->pageset; 315 s8 __percpu *p = pcp->vm_stat_diff + item; 316 long o, n, t, z; 317 318 do { 319 z = 0; /* overflow to zone counters */ 320 321 /* 322 * The fetching of the stat_threshold is racy. We may apply 323 * a counter threshold to the wrong the cpu if we get 324 * rescheduled while executing here. However, the next 325 * counter update will apply the threshold again and 326 * therefore bring the counter under the threshold again. 327 * 328 * Most of the time the thresholds are the same anyways 329 * for all cpus in a zone. 330 */ 331 t = this_cpu_read(pcp->stat_threshold); 332 333 o = this_cpu_read(*p); 334 n = delta + o; 335 336 if (n > t || n < -t) { 337 int os = overstep_mode * (t >> 1) ; 338 339 /* Overflow must be added to zone counters */ 340 z = n + os; 341 n = -os; 342 } 343 } while (this_cpu_cmpxchg(*p, o, n) != o); 344 345 if (z) 346 zone_page_state_add(z, zone, item); 347 } 348 349 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 350 int delta) 351 { 352 mod_state(zone, item, delta, 0); 353 } 354 EXPORT_SYMBOL(mod_zone_page_state); 355 356 void inc_zone_state(struct zone *zone, enum zone_stat_item item) 357 { 358 mod_state(zone, item, 1, 1); 359 } 360 361 void inc_zone_page_state(struct page *page, enum zone_stat_item item) 362 { 363 mod_state(page_zone(page), item, 1, 1); 364 } 365 EXPORT_SYMBOL(inc_zone_page_state); 366 367 void dec_zone_page_state(struct page *page, enum zone_stat_item item) 368 { 369 mod_state(page_zone(page), item, -1, -1); 370 } 371 EXPORT_SYMBOL(dec_zone_page_state); 372 #else 373 /* 374 * Use interrupt disable to serialize counter updates 375 */ 376 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item, 377 int delta) 378 { 379 unsigned long flags; 380 381 local_irq_save(flags); 382 __mod_zone_page_state(zone, item, delta); 383 local_irq_restore(flags); 384 } 385 EXPORT_SYMBOL(mod_zone_page_state); 386 387 void inc_zone_state(struct zone *zone, enum zone_stat_item item) 388 { 389 unsigned long flags; 390 391 local_irq_save(flags); 392 __inc_zone_state(zone, item); 393 local_irq_restore(flags); 394 } 395 396 void inc_zone_page_state(struct page *page, enum zone_stat_item item) 397 { 398 unsigned long flags; 399 struct zone *zone; 400 401 zone = page_zone(page); 402 local_irq_save(flags); 403 __inc_zone_state(zone, item); 404 local_irq_restore(flags); 405 } 406 EXPORT_SYMBOL(inc_zone_page_state); 407 408 void dec_zone_page_state(struct page *page, enum zone_stat_item item) 409 { 410 unsigned long flags; 411 412 local_irq_save(flags); 413 __dec_zone_page_state(page, item); 414 local_irq_restore(flags); 415 } 416 EXPORT_SYMBOL(dec_zone_page_state); 417 #endif 418 419 /* 420 * Update the zone counters for one cpu. 421 * 422 * The cpu specified must be either the current cpu or a processor that 423 * is not online. If it is the current cpu then the execution thread must 424 * be pinned to the current cpu. 425 * 426 * Note that refresh_cpu_vm_stats strives to only access 427 * node local memory. The per cpu pagesets on remote zones are placed 428 * in the memory local to the processor using that pageset. So the 429 * loop over all zones will access a series of cachelines local to 430 * the processor. 431 * 432 * The call to zone_page_state_add updates the cachelines with the 433 * statistics in the remote zone struct as well as the global cachelines 434 * with the global counters. These could cause remote node cache line 435 * bouncing and will have to be only done when necessary. 436 */ 437 void refresh_cpu_vm_stats(int cpu) 438 { 439 struct zone *zone; 440 int i; 441 int global_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, }; 442 443 for_each_populated_zone(zone) { 444 struct per_cpu_pageset *p; 445 446 p = per_cpu_ptr(zone->pageset, cpu); 447 448 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 449 if (p->vm_stat_diff[i]) { 450 unsigned long flags; 451 int v; 452 453 local_irq_save(flags); 454 v = p->vm_stat_diff[i]; 455 p->vm_stat_diff[i] = 0; 456 local_irq_restore(flags); 457 atomic_long_add(v, &zone->vm_stat[i]); 458 global_diff[i] += v; 459 #ifdef CONFIG_NUMA 460 /* 3 seconds idle till flush */ 461 p->expire = 3; 462 #endif 463 } 464 cond_resched(); 465 #ifdef CONFIG_NUMA 466 /* 467 * Deal with draining the remote pageset of this 468 * processor 469 * 470 * Check if there are pages remaining in this pageset 471 * if not then there is nothing to expire. 472 */ 473 if (!p->expire || !p->pcp.count) 474 continue; 475 476 /* 477 * We never drain zones local to this processor. 478 */ 479 if (zone_to_nid(zone) == numa_node_id()) { 480 p->expire = 0; 481 continue; 482 } 483 484 p->expire--; 485 if (p->expire) 486 continue; 487 488 if (p->pcp.count) 489 drain_zone_pages(zone, &p->pcp); 490 #endif 491 } 492 493 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 494 if (global_diff[i]) 495 atomic_long_add(global_diff[i], &vm_stat[i]); 496 } 497 498 #endif 499 500 #ifdef CONFIG_NUMA 501 /* 502 * zonelist = the list of zones passed to the allocator 503 * z = the zone from which the allocation occurred. 504 * 505 * Must be called with interrupts disabled. 506 * 507 * When __GFP_OTHER_NODE is set assume the node of the preferred 508 * zone is the local node. This is useful for daemons who allocate 509 * memory on behalf of other processes. 510 */ 511 void zone_statistics(struct zone *preferred_zone, struct zone *z, gfp_t flags) 512 { 513 if (z->zone_pgdat == preferred_zone->zone_pgdat) { 514 __inc_zone_state(z, NUMA_HIT); 515 } else { 516 __inc_zone_state(z, NUMA_MISS); 517 __inc_zone_state(preferred_zone, NUMA_FOREIGN); 518 } 519 if (z->node == ((flags & __GFP_OTHER_NODE) ? 520 preferred_zone->node : numa_node_id())) 521 __inc_zone_state(z, NUMA_LOCAL); 522 else 523 __inc_zone_state(z, NUMA_OTHER); 524 } 525 #endif 526 527 #ifdef CONFIG_COMPACTION 528 529 struct contig_page_info { 530 unsigned long free_pages; 531 unsigned long free_blocks_total; 532 unsigned long free_blocks_suitable; 533 }; 534 535 /* 536 * Calculate the number of free pages in a zone, how many contiguous 537 * pages are free and how many are large enough to satisfy an allocation of 538 * the target size. Note that this function makes no attempt to estimate 539 * how many suitable free blocks there *might* be if MOVABLE pages were 540 * migrated. Calculating that is possible, but expensive and can be 541 * figured out from userspace 542 */ 543 static void fill_contig_page_info(struct zone *zone, 544 unsigned int suitable_order, 545 struct contig_page_info *info) 546 { 547 unsigned int order; 548 549 info->free_pages = 0; 550 info->free_blocks_total = 0; 551 info->free_blocks_suitable = 0; 552 553 for (order = 0; order < MAX_ORDER; order++) { 554 unsigned long blocks; 555 556 /* Count number of free blocks */ 557 blocks = zone->free_area[order].nr_free; 558 info->free_blocks_total += blocks; 559 560 /* Count free base pages */ 561 info->free_pages += blocks << order; 562 563 /* Count the suitable free blocks */ 564 if (order >= suitable_order) 565 info->free_blocks_suitable += blocks << 566 (order - suitable_order); 567 } 568 } 569 570 /* 571 * A fragmentation index only makes sense if an allocation of a requested 572 * size would fail. If that is true, the fragmentation index indicates 573 * whether external fragmentation or a lack of memory was the problem. 574 * The value can be used to determine if page reclaim or compaction 575 * should be used 576 */ 577 static int __fragmentation_index(unsigned int order, struct contig_page_info *info) 578 { 579 unsigned long requested = 1UL << order; 580 581 if (!info->free_blocks_total) 582 return 0; 583 584 /* Fragmentation index only makes sense when a request would fail */ 585 if (info->free_blocks_suitable) 586 return -1000; 587 588 /* 589 * Index is between 0 and 1 so return within 3 decimal places 590 * 591 * 0 => allocation would fail due to lack of memory 592 * 1 => allocation would fail due to fragmentation 593 */ 594 return 1000 - div_u64( (1000+(div_u64(info->free_pages * 1000ULL, requested))), info->free_blocks_total); 595 } 596 597 /* Same as __fragmentation index but allocs contig_page_info on stack */ 598 int fragmentation_index(struct zone *zone, unsigned int order) 599 { 600 struct contig_page_info info; 601 602 fill_contig_page_info(zone, order, &info); 603 return __fragmentation_index(order, &info); 604 } 605 #endif 606 607 #if defined(CONFIG_PROC_FS) || defined(CONFIG_COMPACTION) 608 #include <linux/proc_fs.h> 609 #include <linux/seq_file.h> 610 611 static char * const migratetype_names[MIGRATE_TYPES] = { 612 "Unmovable", 613 "Reclaimable", 614 "Movable", 615 "Reserve", 616 "Isolate", 617 }; 618 619 static void *frag_start(struct seq_file *m, loff_t *pos) 620 { 621 pg_data_t *pgdat; 622 loff_t node = *pos; 623 for (pgdat = first_online_pgdat(); 624 pgdat && node; 625 pgdat = next_online_pgdat(pgdat)) 626 --node; 627 628 return pgdat; 629 } 630 631 static void *frag_next(struct seq_file *m, void *arg, loff_t *pos) 632 { 633 pg_data_t *pgdat = (pg_data_t *)arg; 634 635 (*pos)++; 636 return next_online_pgdat(pgdat); 637 } 638 639 static void frag_stop(struct seq_file *m, void *arg) 640 { 641 } 642 643 /* Walk all the zones in a node and print using a callback */ 644 static void walk_zones_in_node(struct seq_file *m, pg_data_t *pgdat, 645 void (*print)(struct seq_file *m, pg_data_t *, struct zone *)) 646 { 647 struct zone *zone; 648 struct zone *node_zones = pgdat->node_zones; 649 unsigned long flags; 650 651 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { 652 if (!populated_zone(zone)) 653 continue; 654 655 spin_lock_irqsave(&zone->lock, flags); 656 print(m, pgdat, zone); 657 spin_unlock_irqrestore(&zone->lock, flags); 658 } 659 } 660 #endif 661 662 #if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || defined(CONFIG_NUMA) 663 #ifdef CONFIG_ZONE_DMA 664 #define TEXT_FOR_DMA(xx) xx "_dma", 665 #else 666 #define TEXT_FOR_DMA(xx) 667 #endif 668 669 #ifdef CONFIG_ZONE_DMA32 670 #define TEXT_FOR_DMA32(xx) xx "_dma32", 671 #else 672 #define TEXT_FOR_DMA32(xx) 673 #endif 674 675 #ifdef CONFIG_HIGHMEM 676 #define TEXT_FOR_HIGHMEM(xx) xx "_high", 677 #else 678 #define TEXT_FOR_HIGHMEM(xx) 679 #endif 680 681 #define TEXTS_FOR_ZONES(xx) TEXT_FOR_DMA(xx) TEXT_FOR_DMA32(xx) xx "_normal", \ 682 TEXT_FOR_HIGHMEM(xx) xx "_movable", 683 684 const char * const vmstat_text[] = { 685 /* Zoned VM counters */ 686 "nr_free_pages", 687 "nr_inactive_anon", 688 "nr_active_anon", 689 "nr_inactive_file", 690 "nr_active_file", 691 "nr_unevictable", 692 "nr_mlock", 693 "nr_anon_pages", 694 "nr_mapped", 695 "nr_file_pages", 696 "nr_dirty", 697 "nr_writeback", 698 "nr_slab_reclaimable", 699 "nr_slab_unreclaimable", 700 "nr_page_table_pages", 701 "nr_kernel_stack", 702 "nr_unstable", 703 "nr_bounce", 704 "nr_vmscan_write", 705 "nr_vmscan_immediate_reclaim", 706 "nr_writeback_temp", 707 "nr_isolated_anon", 708 "nr_isolated_file", 709 "nr_shmem", 710 "nr_dirtied", 711 "nr_written", 712 713 #ifdef CONFIG_NUMA 714 "numa_hit", 715 "numa_miss", 716 "numa_foreign", 717 "numa_interleave", 718 "numa_local", 719 "numa_other", 720 #endif 721 "nr_anon_transparent_hugepages", 722 "nr_dirty_threshold", 723 "nr_dirty_background_threshold", 724 725 #ifdef CONFIG_VM_EVENT_COUNTERS 726 "pgpgin", 727 "pgpgout", 728 "pswpin", 729 "pswpout", 730 731 TEXTS_FOR_ZONES("pgalloc") 732 733 "pgfree", 734 "pgactivate", 735 "pgdeactivate", 736 737 "pgfault", 738 "pgmajfault", 739 740 TEXTS_FOR_ZONES("pgrefill") 741 TEXTS_FOR_ZONES("pgsteal") 742 TEXTS_FOR_ZONES("pgscan_kswapd") 743 TEXTS_FOR_ZONES("pgscan_direct") 744 745 #ifdef CONFIG_NUMA 746 "zone_reclaim_failed", 747 #endif 748 "pginodesteal", 749 "slabs_scanned", 750 "kswapd_steal", 751 "kswapd_inodesteal", 752 "kswapd_low_wmark_hit_quickly", 753 "kswapd_high_wmark_hit_quickly", 754 "kswapd_skip_congestion_wait", 755 "pageoutrun", 756 "allocstall", 757 758 "pgrotated", 759 760 #ifdef CONFIG_COMPACTION 761 "compact_blocks_moved", 762 "compact_pages_moved", 763 "compact_pagemigrate_failed", 764 "compact_stall", 765 "compact_fail", 766 "compact_success", 767 #endif 768 769 #ifdef CONFIG_HUGETLB_PAGE 770 "htlb_buddy_alloc_success", 771 "htlb_buddy_alloc_fail", 772 #endif 773 "unevictable_pgs_culled", 774 "unevictable_pgs_scanned", 775 "unevictable_pgs_rescued", 776 "unevictable_pgs_mlocked", 777 "unevictable_pgs_munlocked", 778 "unevictable_pgs_cleared", 779 "unevictable_pgs_stranded", 780 "unevictable_pgs_mlockfreed", 781 782 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 783 "thp_fault_alloc", 784 "thp_fault_fallback", 785 "thp_collapse_alloc", 786 "thp_collapse_alloc_failed", 787 "thp_split", 788 #endif 789 790 #endif /* CONFIG_VM_EVENTS_COUNTERS */ 791 }; 792 #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA */ 793 794 795 #ifdef CONFIG_PROC_FS 796 static void frag_show_print(struct seq_file *m, pg_data_t *pgdat, 797 struct zone *zone) 798 { 799 int order; 800 801 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 802 for (order = 0; order < MAX_ORDER; ++order) 803 seq_printf(m, "%6lu ", zone->free_area[order].nr_free); 804 seq_putc(m, '\n'); 805 } 806 807 /* 808 * This walks the free areas for each zone. 809 */ 810 static int frag_show(struct seq_file *m, void *arg) 811 { 812 pg_data_t *pgdat = (pg_data_t *)arg; 813 walk_zones_in_node(m, pgdat, frag_show_print); 814 return 0; 815 } 816 817 static void pagetypeinfo_showfree_print(struct seq_file *m, 818 pg_data_t *pgdat, struct zone *zone) 819 { 820 int order, mtype; 821 822 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) { 823 seq_printf(m, "Node %4d, zone %8s, type %12s ", 824 pgdat->node_id, 825 zone->name, 826 migratetype_names[mtype]); 827 for (order = 0; order < MAX_ORDER; ++order) { 828 unsigned long freecount = 0; 829 struct free_area *area; 830 struct list_head *curr; 831 832 area = &(zone->free_area[order]); 833 834 list_for_each(curr, &area->free_list[mtype]) 835 freecount++; 836 seq_printf(m, "%6lu ", freecount); 837 } 838 seq_putc(m, '\n'); 839 } 840 } 841 842 /* Print out the free pages at each order for each migatetype */ 843 static int pagetypeinfo_showfree(struct seq_file *m, void *arg) 844 { 845 int order; 846 pg_data_t *pgdat = (pg_data_t *)arg; 847 848 /* Print header */ 849 seq_printf(m, "%-43s ", "Free pages count per migrate type at order"); 850 for (order = 0; order < MAX_ORDER; ++order) 851 seq_printf(m, "%6d ", order); 852 seq_putc(m, '\n'); 853 854 walk_zones_in_node(m, pgdat, pagetypeinfo_showfree_print); 855 856 return 0; 857 } 858 859 static void pagetypeinfo_showblockcount_print(struct seq_file *m, 860 pg_data_t *pgdat, struct zone *zone) 861 { 862 int mtype; 863 unsigned long pfn; 864 unsigned long start_pfn = zone->zone_start_pfn; 865 unsigned long end_pfn = start_pfn + zone->spanned_pages; 866 unsigned long count[MIGRATE_TYPES] = { 0, }; 867 868 for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { 869 struct page *page; 870 871 if (!pfn_valid(pfn)) 872 continue; 873 874 page = pfn_to_page(pfn); 875 876 /* Watch for unexpected holes punched in the memmap */ 877 if (!memmap_valid_within(pfn, page, zone)) 878 continue; 879 880 mtype = get_pageblock_migratetype(page); 881 882 if (mtype < MIGRATE_TYPES) 883 count[mtype]++; 884 } 885 886 /* Print counts */ 887 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 888 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 889 seq_printf(m, "%12lu ", count[mtype]); 890 seq_putc(m, '\n'); 891 } 892 893 /* Print out the free pages at each order for each migratetype */ 894 static int pagetypeinfo_showblockcount(struct seq_file *m, void *arg) 895 { 896 int mtype; 897 pg_data_t *pgdat = (pg_data_t *)arg; 898 899 seq_printf(m, "\n%-23s", "Number of blocks type "); 900 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) 901 seq_printf(m, "%12s ", migratetype_names[mtype]); 902 seq_putc(m, '\n'); 903 walk_zones_in_node(m, pgdat, pagetypeinfo_showblockcount_print); 904 905 return 0; 906 } 907 908 /* 909 * This prints out statistics in relation to grouping pages by mobility. 910 * It is expensive to collect so do not constantly read the file. 911 */ 912 static int pagetypeinfo_show(struct seq_file *m, void *arg) 913 { 914 pg_data_t *pgdat = (pg_data_t *)arg; 915 916 /* check memoryless node */ 917 if (!node_state(pgdat->node_id, N_HIGH_MEMORY)) 918 return 0; 919 920 seq_printf(m, "Page block order: %d\n", pageblock_order); 921 seq_printf(m, "Pages per block: %lu\n", pageblock_nr_pages); 922 seq_putc(m, '\n'); 923 pagetypeinfo_showfree(m, pgdat); 924 pagetypeinfo_showblockcount(m, pgdat); 925 926 return 0; 927 } 928 929 static const struct seq_operations fragmentation_op = { 930 .start = frag_start, 931 .next = frag_next, 932 .stop = frag_stop, 933 .show = frag_show, 934 }; 935 936 static int fragmentation_open(struct inode *inode, struct file *file) 937 { 938 return seq_open(file, &fragmentation_op); 939 } 940 941 static const struct file_operations fragmentation_file_operations = { 942 .open = fragmentation_open, 943 .read = seq_read, 944 .llseek = seq_lseek, 945 .release = seq_release, 946 }; 947 948 static const struct seq_operations pagetypeinfo_op = { 949 .start = frag_start, 950 .next = frag_next, 951 .stop = frag_stop, 952 .show = pagetypeinfo_show, 953 }; 954 955 static int pagetypeinfo_open(struct inode *inode, struct file *file) 956 { 957 return seq_open(file, &pagetypeinfo_op); 958 } 959 960 static const struct file_operations pagetypeinfo_file_ops = { 961 .open = pagetypeinfo_open, 962 .read = seq_read, 963 .llseek = seq_lseek, 964 .release = seq_release, 965 }; 966 967 static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, 968 struct zone *zone) 969 { 970 int i; 971 seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name); 972 seq_printf(m, 973 "\n pages free %lu" 974 "\n min %lu" 975 "\n low %lu" 976 "\n high %lu" 977 "\n scanned %lu" 978 "\n spanned %lu" 979 "\n present %lu", 980 zone_page_state(zone, NR_FREE_PAGES), 981 min_wmark_pages(zone), 982 low_wmark_pages(zone), 983 high_wmark_pages(zone), 984 zone->pages_scanned, 985 zone->spanned_pages, 986 zone->present_pages); 987 988 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 989 seq_printf(m, "\n %-12s %lu", vmstat_text[i], 990 zone_page_state(zone, i)); 991 992 seq_printf(m, 993 "\n protection: (%lu", 994 zone->lowmem_reserve[0]); 995 for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++) 996 seq_printf(m, ", %lu", zone->lowmem_reserve[i]); 997 seq_printf(m, 998 ")" 999 "\n pagesets"); 1000 for_each_online_cpu(i) { 1001 struct per_cpu_pageset *pageset; 1002 1003 pageset = per_cpu_ptr(zone->pageset, i); 1004 seq_printf(m, 1005 "\n cpu: %i" 1006 "\n count: %i" 1007 "\n high: %i" 1008 "\n batch: %i", 1009 i, 1010 pageset->pcp.count, 1011 pageset->pcp.high, 1012 pageset->pcp.batch); 1013 #ifdef CONFIG_SMP 1014 seq_printf(m, "\n vm stats threshold: %d", 1015 pageset->stat_threshold); 1016 #endif 1017 } 1018 seq_printf(m, 1019 "\n all_unreclaimable: %u" 1020 "\n start_pfn: %lu" 1021 "\n inactive_ratio: %u", 1022 zone->all_unreclaimable, 1023 zone->zone_start_pfn, 1024 zone->inactive_ratio); 1025 seq_putc(m, '\n'); 1026 } 1027 1028 /* 1029 * Output information about zones in @pgdat. 1030 */ 1031 static int zoneinfo_show(struct seq_file *m, void *arg) 1032 { 1033 pg_data_t *pgdat = (pg_data_t *)arg; 1034 walk_zones_in_node(m, pgdat, zoneinfo_show_print); 1035 return 0; 1036 } 1037 1038 static const struct seq_operations zoneinfo_op = { 1039 .start = frag_start, /* iterate over all zones. The same as in 1040 * fragmentation. */ 1041 .next = frag_next, 1042 .stop = frag_stop, 1043 .show = zoneinfo_show, 1044 }; 1045 1046 static int zoneinfo_open(struct inode *inode, struct file *file) 1047 { 1048 return seq_open(file, &zoneinfo_op); 1049 } 1050 1051 static const struct file_operations proc_zoneinfo_file_operations = { 1052 .open = zoneinfo_open, 1053 .read = seq_read, 1054 .llseek = seq_lseek, 1055 .release = seq_release, 1056 }; 1057 1058 enum writeback_stat_item { 1059 NR_DIRTY_THRESHOLD, 1060 NR_DIRTY_BG_THRESHOLD, 1061 NR_VM_WRITEBACK_STAT_ITEMS, 1062 }; 1063 1064 static void *vmstat_start(struct seq_file *m, loff_t *pos) 1065 { 1066 unsigned long *v; 1067 int i, stat_items_size; 1068 1069 if (*pos >= ARRAY_SIZE(vmstat_text)) 1070 return NULL; 1071 stat_items_size = NR_VM_ZONE_STAT_ITEMS * sizeof(unsigned long) + 1072 NR_VM_WRITEBACK_STAT_ITEMS * sizeof(unsigned long); 1073 1074 #ifdef CONFIG_VM_EVENT_COUNTERS 1075 stat_items_size += sizeof(struct vm_event_state); 1076 #endif 1077 1078 v = kmalloc(stat_items_size, GFP_KERNEL); 1079 m->private = v; 1080 if (!v) 1081 return ERR_PTR(-ENOMEM); 1082 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) 1083 v[i] = global_page_state(i); 1084 v += NR_VM_ZONE_STAT_ITEMS; 1085 1086 global_dirty_limits(v + NR_DIRTY_BG_THRESHOLD, 1087 v + NR_DIRTY_THRESHOLD); 1088 v += NR_VM_WRITEBACK_STAT_ITEMS; 1089 1090 #ifdef CONFIG_VM_EVENT_COUNTERS 1091 all_vm_events(v); 1092 v[PGPGIN] /= 2; /* sectors -> kbytes */ 1093 v[PGPGOUT] /= 2; 1094 #endif 1095 return (unsigned long *)m->private + *pos; 1096 } 1097 1098 static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos) 1099 { 1100 (*pos)++; 1101 if (*pos >= ARRAY_SIZE(vmstat_text)) 1102 return NULL; 1103 return (unsigned long *)m->private + *pos; 1104 } 1105 1106 static int vmstat_show(struct seq_file *m, void *arg) 1107 { 1108 unsigned long *l = arg; 1109 unsigned long off = l - (unsigned long *)m->private; 1110 1111 seq_printf(m, "%s %lu\n", vmstat_text[off], *l); 1112 return 0; 1113 } 1114 1115 static void vmstat_stop(struct seq_file *m, void *arg) 1116 { 1117 kfree(m->private); 1118 m->private = NULL; 1119 } 1120 1121 static const struct seq_operations vmstat_op = { 1122 .start = vmstat_start, 1123 .next = vmstat_next, 1124 .stop = vmstat_stop, 1125 .show = vmstat_show, 1126 }; 1127 1128 static int vmstat_open(struct inode *inode, struct file *file) 1129 { 1130 return seq_open(file, &vmstat_op); 1131 } 1132 1133 static const struct file_operations proc_vmstat_file_operations = { 1134 .open = vmstat_open, 1135 .read = seq_read, 1136 .llseek = seq_lseek, 1137 .release = seq_release, 1138 }; 1139 #endif /* CONFIG_PROC_FS */ 1140 1141 #ifdef CONFIG_SMP 1142 static DEFINE_PER_CPU(struct delayed_work, vmstat_work); 1143 int sysctl_stat_interval __read_mostly = HZ; 1144 1145 static void vmstat_update(struct work_struct *w) 1146 { 1147 refresh_cpu_vm_stats(smp_processor_id()); 1148 schedule_delayed_work(&__get_cpu_var(vmstat_work), 1149 round_jiffies_relative(sysctl_stat_interval)); 1150 } 1151 1152 static void __cpuinit start_cpu_timer(int cpu) 1153 { 1154 struct delayed_work *work = &per_cpu(vmstat_work, cpu); 1155 1156 INIT_DELAYED_WORK_DEFERRABLE(work, vmstat_update); 1157 schedule_delayed_work_on(cpu, work, __round_jiffies_relative(HZ, cpu)); 1158 } 1159 1160 /* 1161 * Use the cpu notifier to insure that the thresholds are recalculated 1162 * when necessary. 1163 */ 1164 static int __cpuinit vmstat_cpuup_callback(struct notifier_block *nfb, 1165 unsigned long action, 1166 void *hcpu) 1167 { 1168 long cpu = (long)hcpu; 1169 1170 switch (action) { 1171 case CPU_ONLINE: 1172 case CPU_ONLINE_FROZEN: 1173 refresh_zone_stat_thresholds(); 1174 start_cpu_timer(cpu); 1175 node_set_state(cpu_to_node(cpu), N_CPU); 1176 break; 1177 case CPU_DOWN_PREPARE: 1178 case CPU_DOWN_PREPARE_FROZEN: 1179 cancel_delayed_work_sync(&per_cpu(vmstat_work, cpu)); 1180 per_cpu(vmstat_work, cpu).work.func = NULL; 1181 break; 1182 case CPU_DOWN_FAILED: 1183 case CPU_DOWN_FAILED_FROZEN: 1184 start_cpu_timer(cpu); 1185 break; 1186 case CPU_DEAD: 1187 case CPU_DEAD_FROZEN: 1188 refresh_zone_stat_thresholds(); 1189 break; 1190 default: 1191 break; 1192 } 1193 return NOTIFY_OK; 1194 } 1195 1196 static struct notifier_block __cpuinitdata vmstat_notifier = 1197 { &vmstat_cpuup_callback, NULL, 0 }; 1198 #endif 1199 1200 static int __init setup_vmstat(void) 1201 { 1202 #ifdef CONFIG_SMP 1203 int cpu; 1204 1205 register_cpu_notifier(&vmstat_notifier); 1206 1207 for_each_online_cpu(cpu) 1208 start_cpu_timer(cpu); 1209 #endif 1210 #ifdef CONFIG_PROC_FS 1211 proc_create("buddyinfo", S_IRUGO, NULL, &fragmentation_file_operations); 1212 proc_create("pagetypeinfo", S_IRUGO, NULL, &pagetypeinfo_file_ops); 1213 proc_create("vmstat", S_IRUGO, NULL, &proc_vmstat_file_operations); 1214 proc_create("zoneinfo", S_IRUGO, NULL, &proc_zoneinfo_file_operations); 1215 #endif 1216 return 0; 1217 } 1218 module_init(setup_vmstat) 1219 1220 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION) 1221 #include <linux/debugfs.h> 1222 1223 static struct dentry *extfrag_debug_root; 1224 1225 /* 1226 * Return an index indicating how much of the available free memory is 1227 * unusable for an allocation of the requested size. 1228 */ 1229 static int unusable_free_index(unsigned int order, 1230 struct contig_page_info *info) 1231 { 1232 /* No free memory is interpreted as all free memory is unusable */ 1233 if (info->free_pages == 0) 1234 return 1000; 1235 1236 /* 1237 * Index should be a value between 0 and 1. Return a value to 3 1238 * decimal places. 1239 * 1240 * 0 => no fragmentation 1241 * 1 => high fragmentation 1242 */ 1243 return div_u64((info->free_pages - (info->free_blocks_suitable << order)) * 1000ULL, info->free_pages); 1244 1245 } 1246 1247 static void unusable_show_print(struct seq_file *m, 1248 pg_data_t *pgdat, struct zone *zone) 1249 { 1250 unsigned int order; 1251 int index; 1252 struct contig_page_info info; 1253 1254 seq_printf(m, "Node %d, zone %8s ", 1255 pgdat->node_id, 1256 zone->name); 1257 for (order = 0; order < MAX_ORDER; ++order) { 1258 fill_contig_page_info(zone, order, &info); 1259 index = unusable_free_index(order, &info); 1260 seq_printf(m, "%d.%03d ", index / 1000, index % 1000); 1261 } 1262 1263 seq_putc(m, '\n'); 1264 } 1265 1266 /* 1267 * Display unusable free space index 1268 * 1269 * The unusable free space index measures how much of the available free 1270 * memory cannot be used to satisfy an allocation of a given size and is a 1271 * value between 0 and 1. The higher the value, the more of free memory is 1272 * unusable and by implication, the worse the external fragmentation is. This 1273 * can be expressed as a percentage by multiplying by 100. 1274 */ 1275 static int unusable_show(struct seq_file *m, void *arg) 1276 { 1277 pg_data_t *pgdat = (pg_data_t *)arg; 1278 1279 /* check memoryless node */ 1280 if (!node_state(pgdat->node_id, N_HIGH_MEMORY)) 1281 return 0; 1282 1283 walk_zones_in_node(m, pgdat, unusable_show_print); 1284 1285 return 0; 1286 } 1287 1288 static const struct seq_operations unusable_op = { 1289 .start = frag_start, 1290 .next = frag_next, 1291 .stop = frag_stop, 1292 .show = unusable_show, 1293 }; 1294 1295 static int unusable_open(struct inode *inode, struct file *file) 1296 { 1297 return seq_open(file, &unusable_op); 1298 } 1299 1300 static const struct file_operations unusable_file_ops = { 1301 .open = unusable_open, 1302 .read = seq_read, 1303 .llseek = seq_lseek, 1304 .release = seq_release, 1305 }; 1306 1307 static void extfrag_show_print(struct seq_file *m, 1308 pg_data_t *pgdat, struct zone *zone) 1309 { 1310 unsigned int order; 1311 int index; 1312 1313 /* Alloc on stack as interrupts are disabled for zone walk */ 1314 struct contig_page_info info; 1315 1316 seq_printf(m, "Node %d, zone %8s ", 1317 pgdat->node_id, 1318 zone->name); 1319 for (order = 0; order < MAX_ORDER; ++order) { 1320 fill_contig_page_info(zone, order, &info); 1321 index = __fragmentation_index(order, &info); 1322 seq_printf(m, "%d.%03d ", index / 1000, index % 1000); 1323 } 1324 1325 seq_putc(m, '\n'); 1326 } 1327 1328 /* 1329 * Display fragmentation index for orders that allocations would fail for 1330 */ 1331 static int extfrag_show(struct seq_file *m, void *arg) 1332 { 1333 pg_data_t *pgdat = (pg_data_t *)arg; 1334 1335 walk_zones_in_node(m, pgdat, extfrag_show_print); 1336 1337 return 0; 1338 } 1339 1340 static const struct seq_operations extfrag_op = { 1341 .start = frag_start, 1342 .next = frag_next, 1343 .stop = frag_stop, 1344 .show = extfrag_show, 1345 }; 1346 1347 static int extfrag_open(struct inode *inode, struct file *file) 1348 { 1349 return seq_open(file, &extfrag_op); 1350 } 1351 1352 static const struct file_operations extfrag_file_ops = { 1353 .open = extfrag_open, 1354 .read = seq_read, 1355 .llseek = seq_lseek, 1356 .release = seq_release, 1357 }; 1358 1359 static int __init extfrag_debug_init(void) 1360 { 1361 extfrag_debug_root = debugfs_create_dir("extfrag", NULL); 1362 if (!extfrag_debug_root) 1363 return -ENOMEM; 1364 1365 if (!debugfs_create_file("unusable_index", 0444, 1366 extfrag_debug_root, NULL, &unusable_file_ops)) 1367 return -ENOMEM; 1368 1369 if (!debugfs_create_file("extfrag_index", 0444, 1370 extfrag_debug_root, NULL, &extfrag_file_ops)) 1371 return -ENOMEM; 1372 1373 return 0; 1374 } 1375 1376 module_init(extfrag_debug_init); 1377 #endif 1378