1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_VMSTAT_H 3 #define _LINUX_VMSTAT_H 4 5 #include <linux/types.h> 6 #include <linux/percpu.h> 7 #include <linux/mmzone.h> 8 #include <linux/vm_event_item.h> 9 #include <linux/atomic.h> 10 #include <linux/static_key.h> 11 #include <linux/mmdebug.h> 12 13 extern int sysctl_stat_interval; 14 15 #ifdef CONFIG_NUMA 16 #define ENABLE_NUMA_STAT 1 17 #define DISABLE_NUMA_STAT 0 18 extern int sysctl_vm_numa_stat; 19 DECLARE_STATIC_KEY_TRUE(vm_numa_stat_key); 20 int sysctl_vm_numa_stat_handler(struct ctl_table *table, int write, 21 void *buffer, size_t *length, loff_t *ppos); 22 #endif 23 24 struct reclaim_stat { 25 unsigned nr_dirty; 26 unsigned nr_unqueued_dirty; 27 unsigned nr_congested; 28 unsigned nr_writeback; 29 unsigned nr_immediate; 30 unsigned nr_pageout; 31 unsigned nr_activate[ANON_AND_FILE]; 32 unsigned nr_ref_keep; 33 unsigned nr_unmap_fail; 34 unsigned nr_lazyfree_fail; 35 }; 36 37 enum writeback_stat_item { 38 NR_DIRTY_THRESHOLD, 39 NR_DIRTY_BG_THRESHOLD, 40 NR_VM_WRITEBACK_STAT_ITEMS, 41 }; 42 43 #ifdef CONFIG_VM_EVENT_COUNTERS 44 /* 45 * Light weight per cpu counter implementation. 46 * 47 * Counters should only be incremented and no critical kernel component 48 * should rely on the counter values. 49 * 50 * Counters are handled completely inline. On many platforms the code 51 * generated will simply be the increment of a global address. 52 */ 53 54 struct vm_event_state { 55 unsigned long event[NR_VM_EVENT_ITEMS]; 56 }; 57 58 DECLARE_PER_CPU(struct vm_event_state, vm_event_states); 59 60 /* 61 * vm counters are allowed to be racy. Use raw_cpu_ops to avoid the 62 * local_irq_disable overhead. 63 */ 64 static inline void __count_vm_event(enum vm_event_item item) 65 { 66 raw_cpu_inc(vm_event_states.event[item]); 67 } 68 69 static inline void count_vm_event(enum vm_event_item item) 70 { 71 this_cpu_inc(vm_event_states.event[item]); 72 } 73 74 static inline void __count_vm_events(enum vm_event_item item, long delta) 75 { 76 raw_cpu_add(vm_event_states.event[item], delta); 77 } 78 79 static inline void count_vm_events(enum vm_event_item item, long delta) 80 { 81 this_cpu_add(vm_event_states.event[item], delta); 82 } 83 84 extern void all_vm_events(unsigned long *); 85 86 extern void vm_events_fold_cpu(int cpu); 87 88 #else 89 90 /* Disable counters */ 91 static inline void count_vm_event(enum vm_event_item item) 92 { 93 } 94 static inline void count_vm_events(enum vm_event_item item, long delta) 95 { 96 } 97 static inline void __count_vm_event(enum vm_event_item item) 98 { 99 } 100 static inline void __count_vm_events(enum vm_event_item item, long delta) 101 { 102 } 103 static inline void all_vm_events(unsigned long *ret) 104 { 105 } 106 static inline void vm_events_fold_cpu(int cpu) 107 { 108 } 109 110 #endif /* CONFIG_VM_EVENT_COUNTERS */ 111 112 #ifdef CONFIG_NUMA_BALANCING 113 #define count_vm_numa_event(x) count_vm_event(x) 114 #define count_vm_numa_events(x, y) count_vm_events(x, y) 115 #else 116 #define count_vm_numa_event(x) do {} while (0) 117 #define count_vm_numa_events(x, y) do { (void)(y); } while (0) 118 #endif /* CONFIG_NUMA_BALANCING */ 119 120 #ifdef CONFIG_DEBUG_TLBFLUSH 121 #define count_vm_tlb_event(x) count_vm_event(x) 122 #define count_vm_tlb_events(x, y) count_vm_events(x, y) 123 #else 124 #define count_vm_tlb_event(x) do {} while (0) 125 #define count_vm_tlb_events(x, y) do { (void)(y); } while (0) 126 #endif 127 128 #define __count_zid_vm_events(item, zid, delta) \ 129 __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta) 130 131 /* 132 * Zone and node-based page accounting with per cpu differentials. 133 */ 134 extern atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS]; 135 extern atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS]; 136 extern atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS]; 137 138 #ifdef CONFIG_NUMA 139 static inline void zone_numa_event_add(long x, struct zone *zone, 140 enum numa_stat_item item) 141 { 142 atomic_long_add(x, &zone->vm_numa_event[item]); 143 atomic_long_add(x, &vm_numa_event[item]); 144 } 145 146 static inline unsigned long zone_numa_event_state(struct zone *zone, 147 enum numa_stat_item item) 148 { 149 return atomic_long_read(&zone->vm_numa_event[item]); 150 } 151 152 static inline unsigned long 153 global_numa_event_state(enum numa_stat_item item) 154 { 155 return atomic_long_read(&vm_numa_event[item]); 156 } 157 #endif /* CONFIG_NUMA */ 158 159 static inline void zone_page_state_add(long x, struct zone *zone, 160 enum zone_stat_item item) 161 { 162 atomic_long_add(x, &zone->vm_stat[item]); 163 atomic_long_add(x, &vm_zone_stat[item]); 164 } 165 166 static inline void node_page_state_add(long x, struct pglist_data *pgdat, 167 enum node_stat_item item) 168 { 169 atomic_long_add(x, &pgdat->vm_stat[item]); 170 atomic_long_add(x, &vm_node_stat[item]); 171 } 172 173 static inline unsigned long global_zone_page_state(enum zone_stat_item item) 174 { 175 long x = atomic_long_read(&vm_zone_stat[item]); 176 #ifdef CONFIG_SMP 177 if (x < 0) 178 x = 0; 179 #endif 180 return x; 181 } 182 183 static inline 184 unsigned long global_node_page_state_pages(enum node_stat_item item) 185 { 186 long x = atomic_long_read(&vm_node_stat[item]); 187 #ifdef CONFIG_SMP 188 if (x < 0) 189 x = 0; 190 #endif 191 return x; 192 } 193 194 static inline unsigned long global_node_page_state(enum node_stat_item item) 195 { 196 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); 197 198 return global_node_page_state_pages(item); 199 } 200 201 static inline unsigned long zone_page_state(struct zone *zone, 202 enum zone_stat_item item) 203 { 204 long x = atomic_long_read(&zone->vm_stat[item]); 205 #ifdef CONFIG_SMP 206 if (x < 0) 207 x = 0; 208 #endif 209 return x; 210 } 211 212 /* 213 * More accurate version that also considers the currently pending 214 * deltas. For that we need to loop over all cpus to find the current 215 * deltas. There is no synchronization so the result cannot be 216 * exactly accurate either. 217 */ 218 static inline unsigned long zone_page_state_snapshot(struct zone *zone, 219 enum zone_stat_item item) 220 { 221 long x = atomic_long_read(&zone->vm_stat[item]); 222 223 #ifdef CONFIG_SMP 224 int cpu; 225 for_each_online_cpu(cpu) 226 x += per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_stat_diff[item]; 227 228 if (x < 0) 229 x = 0; 230 #endif 231 return x; 232 } 233 234 #ifdef CONFIG_NUMA 235 /* See __count_vm_event comment on why raw_cpu_inc is used. */ 236 static inline void 237 __count_numa_event(struct zone *zone, enum numa_stat_item item) 238 { 239 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats; 240 241 raw_cpu_inc(pzstats->vm_numa_event[item]); 242 } 243 244 static inline void 245 __count_numa_events(struct zone *zone, enum numa_stat_item item, long delta) 246 { 247 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats; 248 249 raw_cpu_add(pzstats->vm_numa_event[item], delta); 250 } 251 252 extern unsigned long sum_zone_node_page_state(int node, 253 enum zone_stat_item item); 254 extern unsigned long sum_zone_numa_event_state(int node, enum numa_stat_item item); 255 extern unsigned long node_page_state(struct pglist_data *pgdat, 256 enum node_stat_item item); 257 extern unsigned long node_page_state_pages(struct pglist_data *pgdat, 258 enum node_stat_item item); 259 extern void fold_vm_numa_events(void); 260 #else 261 #define sum_zone_node_page_state(node, item) global_zone_page_state(item) 262 #define node_page_state(node, item) global_node_page_state(item) 263 #define node_page_state_pages(node, item) global_node_page_state_pages(item) 264 static inline void fold_vm_numa_events(void) 265 { 266 } 267 #endif /* CONFIG_NUMA */ 268 269 #ifdef CONFIG_SMP 270 void __mod_zone_page_state(struct zone *, enum zone_stat_item item, long); 271 void __inc_zone_page_state(struct page *, enum zone_stat_item); 272 void __dec_zone_page_state(struct page *, enum zone_stat_item); 273 274 void __mod_node_page_state(struct pglist_data *, enum node_stat_item item, long); 275 void __inc_node_page_state(struct page *, enum node_stat_item); 276 void __dec_node_page_state(struct page *, enum node_stat_item); 277 278 void mod_zone_page_state(struct zone *, enum zone_stat_item, long); 279 void inc_zone_page_state(struct page *, enum zone_stat_item); 280 void dec_zone_page_state(struct page *, enum zone_stat_item); 281 282 void mod_node_page_state(struct pglist_data *, enum node_stat_item, long); 283 void inc_node_page_state(struct page *, enum node_stat_item); 284 void dec_node_page_state(struct page *, enum node_stat_item); 285 286 extern void inc_node_state(struct pglist_data *, enum node_stat_item); 287 extern void __inc_zone_state(struct zone *, enum zone_stat_item); 288 extern void __inc_node_state(struct pglist_data *, enum node_stat_item); 289 extern void dec_zone_state(struct zone *, enum zone_stat_item); 290 extern void __dec_zone_state(struct zone *, enum zone_stat_item); 291 extern void __dec_node_state(struct pglist_data *, enum node_stat_item); 292 293 void quiet_vmstat(void); 294 void cpu_vm_stats_fold(int cpu); 295 void refresh_zone_stat_thresholds(void); 296 297 struct ctl_table; 298 int vmstat_refresh(struct ctl_table *, int write, void *buffer, size_t *lenp, 299 loff_t *ppos); 300 301 void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *); 302 303 int calculate_pressure_threshold(struct zone *zone); 304 int calculate_normal_threshold(struct zone *zone); 305 void set_pgdat_percpu_threshold(pg_data_t *pgdat, 306 int (*calculate_pressure)(struct zone *)); 307 #else /* CONFIG_SMP */ 308 309 /* 310 * We do not maintain differentials in a single processor configuration. 311 * The functions directly modify the zone and global counters. 312 */ 313 static inline void __mod_zone_page_state(struct zone *zone, 314 enum zone_stat_item item, long delta) 315 { 316 zone_page_state_add(delta, zone, item); 317 } 318 319 static inline void __mod_node_page_state(struct pglist_data *pgdat, 320 enum node_stat_item item, int delta) 321 { 322 if (vmstat_item_in_bytes(item)) { 323 /* 324 * Only cgroups use subpage accounting right now; at 325 * the global level, these items still change in 326 * multiples of whole pages. Store them as pages 327 * internally to keep the per-cpu counters compact. 328 */ 329 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1)); 330 delta >>= PAGE_SHIFT; 331 } 332 333 node_page_state_add(delta, pgdat, item); 334 } 335 336 static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item) 337 { 338 atomic_long_inc(&zone->vm_stat[item]); 339 atomic_long_inc(&vm_zone_stat[item]); 340 } 341 342 static inline void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) 343 { 344 atomic_long_inc(&pgdat->vm_stat[item]); 345 atomic_long_inc(&vm_node_stat[item]); 346 } 347 348 static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item) 349 { 350 atomic_long_dec(&zone->vm_stat[item]); 351 atomic_long_dec(&vm_zone_stat[item]); 352 } 353 354 static inline void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item) 355 { 356 atomic_long_dec(&pgdat->vm_stat[item]); 357 atomic_long_dec(&vm_node_stat[item]); 358 } 359 360 static inline void __inc_zone_page_state(struct page *page, 361 enum zone_stat_item item) 362 { 363 __inc_zone_state(page_zone(page), item); 364 } 365 366 static inline void __inc_node_page_state(struct page *page, 367 enum node_stat_item item) 368 { 369 __inc_node_state(page_pgdat(page), item); 370 } 371 372 373 static inline void __dec_zone_page_state(struct page *page, 374 enum zone_stat_item item) 375 { 376 __dec_zone_state(page_zone(page), item); 377 } 378 379 static inline void __dec_node_page_state(struct page *page, 380 enum node_stat_item item) 381 { 382 __dec_node_state(page_pgdat(page), item); 383 } 384 385 386 /* 387 * We only use atomic operations to update counters. So there is no need to 388 * disable interrupts. 389 */ 390 #define inc_zone_page_state __inc_zone_page_state 391 #define dec_zone_page_state __dec_zone_page_state 392 #define mod_zone_page_state __mod_zone_page_state 393 394 #define inc_node_page_state __inc_node_page_state 395 #define dec_node_page_state __dec_node_page_state 396 #define mod_node_page_state __mod_node_page_state 397 398 #define inc_zone_state __inc_zone_state 399 #define inc_node_state __inc_node_state 400 #define dec_zone_state __dec_zone_state 401 402 #define set_pgdat_percpu_threshold(pgdat, callback) { } 403 404 static inline void refresh_zone_stat_thresholds(void) { } 405 static inline void cpu_vm_stats_fold(int cpu) { } 406 static inline void quiet_vmstat(void) { } 407 408 static inline void drain_zonestat(struct zone *zone, 409 struct per_cpu_zonestat *pzstats) { } 410 #endif /* CONFIG_SMP */ 411 412 static inline void __zone_stat_mod_folio(struct folio *folio, 413 enum zone_stat_item item, long nr) 414 { 415 __mod_zone_page_state(folio_zone(folio), item, nr); 416 } 417 418 static inline void __zone_stat_add_folio(struct folio *folio, 419 enum zone_stat_item item) 420 { 421 __mod_zone_page_state(folio_zone(folio), item, folio_nr_pages(folio)); 422 } 423 424 static inline void __zone_stat_sub_folio(struct folio *folio, 425 enum zone_stat_item item) 426 { 427 __mod_zone_page_state(folio_zone(folio), item, -folio_nr_pages(folio)); 428 } 429 430 static inline void zone_stat_mod_folio(struct folio *folio, 431 enum zone_stat_item item, long nr) 432 { 433 mod_zone_page_state(folio_zone(folio), item, nr); 434 } 435 436 static inline void zone_stat_add_folio(struct folio *folio, 437 enum zone_stat_item item) 438 { 439 mod_zone_page_state(folio_zone(folio), item, folio_nr_pages(folio)); 440 } 441 442 static inline void zone_stat_sub_folio(struct folio *folio, 443 enum zone_stat_item item) 444 { 445 mod_zone_page_state(folio_zone(folio), item, -folio_nr_pages(folio)); 446 } 447 448 static inline void __node_stat_mod_folio(struct folio *folio, 449 enum node_stat_item item, long nr) 450 { 451 __mod_node_page_state(folio_pgdat(folio), item, nr); 452 } 453 454 static inline void __node_stat_add_folio(struct folio *folio, 455 enum node_stat_item item) 456 { 457 __mod_node_page_state(folio_pgdat(folio), item, folio_nr_pages(folio)); 458 } 459 460 static inline void __node_stat_sub_folio(struct folio *folio, 461 enum node_stat_item item) 462 { 463 __mod_node_page_state(folio_pgdat(folio), item, -folio_nr_pages(folio)); 464 } 465 466 static inline void node_stat_mod_folio(struct folio *folio, 467 enum node_stat_item item, long nr) 468 { 469 mod_node_page_state(folio_pgdat(folio), item, nr); 470 } 471 472 static inline void node_stat_add_folio(struct folio *folio, 473 enum node_stat_item item) 474 { 475 mod_node_page_state(folio_pgdat(folio), item, folio_nr_pages(folio)); 476 } 477 478 static inline void node_stat_sub_folio(struct folio *folio, 479 enum node_stat_item item) 480 { 481 mod_node_page_state(folio_pgdat(folio), item, -folio_nr_pages(folio)); 482 } 483 484 static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages, 485 int migratetype) 486 { 487 __mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages); 488 if (is_migrate_cma(migratetype)) 489 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages); 490 } 491 492 extern const char * const vmstat_text[]; 493 494 static inline const char *zone_stat_name(enum zone_stat_item item) 495 { 496 return vmstat_text[item]; 497 } 498 499 #ifdef CONFIG_NUMA 500 static inline const char *numa_stat_name(enum numa_stat_item item) 501 { 502 return vmstat_text[NR_VM_ZONE_STAT_ITEMS + 503 item]; 504 } 505 #endif /* CONFIG_NUMA */ 506 507 static inline const char *node_stat_name(enum node_stat_item item) 508 { 509 return vmstat_text[NR_VM_ZONE_STAT_ITEMS + 510 NR_VM_NUMA_EVENT_ITEMS + 511 item]; 512 } 513 514 static inline const char *lru_list_name(enum lru_list lru) 515 { 516 return node_stat_name(NR_LRU_BASE + lru) + 3; // skip "nr_" 517 } 518 519 static inline const char *writeback_stat_name(enum writeback_stat_item item) 520 { 521 return vmstat_text[NR_VM_ZONE_STAT_ITEMS + 522 NR_VM_NUMA_EVENT_ITEMS + 523 NR_VM_NODE_STAT_ITEMS + 524 item]; 525 } 526 527 #if defined(CONFIG_VM_EVENT_COUNTERS) || defined(CONFIG_MEMCG) 528 static inline const char *vm_event_name(enum vm_event_item item) 529 { 530 return vmstat_text[NR_VM_ZONE_STAT_ITEMS + 531 NR_VM_NUMA_EVENT_ITEMS + 532 NR_VM_NODE_STAT_ITEMS + 533 NR_VM_WRITEBACK_STAT_ITEMS + 534 item]; 535 } 536 #endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_MEMCG */ 537 538 #ifdef CONFIG_MEMCG 539 540 void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, 541 int val); 542 543 static inline void mod_lruvec_state(struct lruvec *lruvec, 544 enum node_stat_item idx, int val) 545 { 546 unsigned long flags; 547 548 local_irq_save(flags); 549 __mod_lruvec_state(lruvec, idx, val); 550 local_irq_restore(flags); 551 } 552 553 void __mod_lruvec_page_state(struct page *page, 554 enum node_stat_item idx, int val); 555 556 static inline void mod_lruvec_page_state(struct page *page, 557 enum node_stat_item idx, int val) 558 { 559 unsigned long flags; 560 561 local_irq_save(flags); 562 __mod_lruvec_page_state(page, idx, val); 563 local_irq_restore(flags); 564 } 565 566 #else 567 568 static inline void __mod_lruvec_state(struct lruvec *lruvec, 569 enum node_stat_item idx, int val) 570 { 571 __mod_node_page_state(lruvec_pgdat(lruvec), idx, val); 572 } 573 574 static inline void mod_lruvec_state(struct lruvec *lruvec, 575 enum node_stat_item idx, int val) 576 { 577 mod_node_page_state(lruvec_pgdat(lruvec), idx, val); 578 } 579 580 static inline void __mod_lruvec_page_state(struct page *page, 581 enum node_stat_item idx, int val) 582 { 583 __mod_node_page_state(page_pgdat(page), idx, val); 584 } 585 586 static inline void mod_lruvec_page_state(struct page *page, 587 enum node_stat_item idx, int val) 588 { 589 mod_node_page_state(page_pgdat(page), idx, val); 590 } 591 592 #endif /* CONFIG_MEMCG */ 593 594 static inline void __inc_lruvec_page_state(struct page *page, 595 enum node_stat_item idx) 596 { 597 __mod_lruvec_page_state(page, idx, 1); 598 } 599 600 static inline void __dec_lruvec_page_state(struct page *page, 601 enum node_stat_item idx) 602 { 603 __mod_lruvec_page_state(page, idx, -1); 604 } 605 606 static inline void __lruvec_stat_mod_folio(struct folio *folio, 607 enum node_stat_item idx, int val) 608 { 609 __mod_lruvec_page_state(&folio->page, idx, val); 610 } 611 612 static inline void __lruvec_stat_add_folio(struct folio *folio, 613 enum node_stat_item idx) 614 { 615 __lruvec_stat_mod_folio(folio, idx, folio_nr_pages(folio)); 616 } 617 618 static inline void __lruvec_stat_sub_folio(struct folio *folio, 619 enum node_stat_item idx) 620 { 621 __lruvec_stat_mod_folio(folio, idx, -folio_nr_pages(folio)); 622 } 623 624 static inline void inc_lruvec_page_state(struct page *page, 625 enum node_stat_item idx) 626 { 627 mod_lruvec_page_state(page, idx, 1); 628 } 629 630 static inline void dec_lruvec_page_state(struct page *page, 631 enum node_stat_item idx) 632 { 633 mod_lruvec_page_state(page, idx, -1); 634 } 635 636 static inline void lruvec_stat_mod_folio(struct folio *folio, 637 enum node_stat_item idx, int val) 638 { 639 mod_lruvec_page_state(&folio->page, idx, val); 640 } 641 642 static inline void lruvec_stat_add_folio(struct folio *folio, 643 enum node_stat_item idx) 644 { 645 lruvec_stat_mod_folio(folio, idx, folio_nr_pages(folio)); 646 } 647 648 static inline void lruvec_stat_sub_folio(struct folio *folio, 649 enum node_stat_item idx) 650 { 651 lruvec_stat_mod_folio(folio, idx, -folio_nr_pages(folio)); 652 } 653 #endif /* _LINUX_VMSTAT_H */ 654