38 static void zero_zone_numa_counters(struct zone *zone)  in zero_zone_numa_counters()  argument
43 		atomic_long_set(&zone->vm_numa_event[item], 0);  in zero_zone_numa_counters()
45 			per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_numa_event[item]  in zero_zone_numa_counters()
54 	struct zone *zone;  in zero_zones_numa_counters()  local
56 	for_each_populated_zone(zone)  in zero_zones_numa_counters()
57 		zero_zone_numa_counters(zone);  in zero_zones_numa_counters()
169 static void fold_vm_zone_numa_events(struct zone *zone)  in fold_vm_zone_numa_events()  argument
178 		pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);  in fold_vm_zone_numa_events()
184 		zone_numa_event_add(zone_numa_events[item], zone, item);  in fold_vm_zone_numa_events()
189 	struct zone *zone;  in fold_vm_numa_events()  local
191 	for_each_populated_zone(zone)  in fold_vm_numa_events()
192 		fold_vm_zone_numa_events(zone);  in fold_vm_numa_events()
198 int calculate_pressure_threshold(struct zone *zone)  in calculate_pressure_threshold()  argument
211 	watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone);  in calculate_pressure_threshold()
222 int calculate_normal_threshold(struct zone *zone)  in calculate_normal_threshold()  argument
257 	mem = zone_managed_pages(zone) >> (27 - PAGE_SHIFT);  in calculate_normal_threshold()
275 	struct zone *zone;  in refresh_zone_stat_thresholds()  local
286 	for_each_populated_zone(zone) {  in refresh_zone_stat_thresholds()
287 		struct pglist_data *pgdat = zone->zone_pgdat;  in refresh_zone_stat_thresholds()
290 		threshold = calculate_normal_threshold(zone);  in refresh_zone_stat_thresholds()
295 			per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold  in refresh_zone_stat_thresholds()
309 		tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone);  in refresh_zone_stat_thresholds()
312 			zone->percpu_drift_mark = high_wmark_pages(zone) +  in refresh_zone_stat_thresholds()
318 				int (*calculate_pressure)(struct zone *))  in set_pgdat_percpu_threshold()  argument
320 	struct zone *zone;  in set_pgdat_percpu_threshold()  local
326 		zone = &pgdat->node_zones[i];  in set_pgdat_percpu_threshold()
327 		if (!zone->percpu_drift_mark)  in set_pgdat_percpu_threshold()
330 		threshold = (*calculate_pressure)(zone);  in set_pgdat_percpu_threshold()
332 			per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold  in set_pgdat_percpu_threshold()
342 void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,  in __mod_zone_page_state()  argument
345 	struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;  in __mod_zone_page_state()
364 		zone_page_state_add(x, zone, item);  in __mod_zone_page_state()
432 void __inc_zone_state(struct zone *zone, enum zone_stat_item item)  in __inc_zone_state()  argument
434 	struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;  in __inc_zone_state()
446 		zone_page_state_add(v + overstep, zone, item);  in __inc_zone_state()
488 void __dec_zone_state(struct zone *zone, enum zone_stat_item item)  in __dec_zone_state()  argument
490 	struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;  in __dec_zone_state()
502 		zone_page_state_add(v - overstep, zone, item);  in __dec_zone_state()
557 static inline void mod_zone_state(struct zone *zone,  in mod_zone_state()  argument
560 	struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;  in mod_zone_state()
592 		zone_page_state_add(z, zone, item);  in mod_zone_state()
595 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,  in mod_zone_page_state()  argument
598 	mod_zone_state(zone, item, delta, 0);  in mod_zone_page_state()
690 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,  in mod_zone_page_state()  argument
696 	__mod_zone_page_state(zone, item, delta);  in mod_zone_page_state()
704 	struct zone *zone;  in inc_zone_page_state()  local
706 	zone = page_zone(page);  in inc_zone_page_state()
708 	__inc_zone_state(zone, item);  in inc_zone_page_state()
809 	struct zone *zone;  in refresh_cpu_vm_stats()  local
815 	for_each_populated_zone(zone) {  in refresh_cpu_vm_stats()
816 		struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;  in refresh_cpu_vm_stats()
818 		struct per_cpu_pages __percpu *pcp = zone->per_cpu_pageset;  in refresh_cpu_vm_stats()
827 				atomic_long_add(v, &zone->vm_stat[i]);  in refresh_cpu_vm_stats()
853 			if (zone_to_nid(zone) == numa_node_id()) {  in refresh_cpu_vm_stats()
862 				drain_zone_pages(zone, this_cpu_ptr(pcp));  in refresh_cpu_vm_stats()
895 	struct zone *zone;  in cpu_vm_stats_fold()  local
900 	for_each_populated_zone(zone) {  in cpu_vm_stats_fold()
903 		pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);  in cpu_vm_stats_fold()
911 				atomic_long_add(v, &zone->vm_stat[i]);  in cpu_vm_stats_fold()
922 				zone_numa_event_add(v, zone, i);  in cpu_vm_stats_fold()
951 void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *pzstats)  in drain_zonestat()  argument
960 			zone_page_state_add(v, zone, i);  in drain_zonestat()
969 			zone_numa_event_add(v, zone, i);  in drain_zonestat()
985 	struct zone *zones = NODE_DATA(node)->node_zones;  in sum_zone_node_page_state()
999 	struct zone *zones = NODE_DATA(node)->node_zones;  in sum_zone_numa_event_state()
1048 static void fill_contig_page_info(struct zone *zone,  in fill_contig_page_info()  argument
1067 		blocks = data_race(zone->free_area[order].nr_free);  in fill_contig_page_info()
1115 unsigned int extfrag_for_order(struct zone *zone, unsigned int order)  in extfrag_for_order()  argument
1119 	fill_contig_page_info(zone, order, &info);  in extfrag_for_order()
1129 int fragmentation_index(struct zone *zone, unsigned int order)  in fragmentation_index()  argument
1133 	fill_contig_page_info(zone, order, &info);  in fragmentation_index()
1448 		void (*print)(struct seq_file *m, pg_data_t *, struct zone *))  in walk_zones_in_node()  argument
1450 	struct zone *zone;  in walk_zones_in_node()  local
1451 	struct zone *node_zones = pgdat->node_zones;  in walk_zones_in_node()
1454 	for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {  in walk_zones_in_node()
1455 		if (assert_populated && !populated_zone(zone))  in walk_zones_in_node()
1459 			spin_lock_irqsave(&zone->lock, flags);  in walk_zones_in_node()
1460 		print(m, pgdat, zone);  in walk_zones_in_node()
1462 			spin_unlock_irqrestore(&zone->lock, flags);  in walk_zones_in_node()
1469 						struct zone *zone)  in frag_show_print()  argument
1473 	seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);  in frag_show_print()
1479 		seq_printf(m, "%6lu ", data_race(zone->free_area[order].nr_free));  in frag_show_print()
1494 					pg_data_t *pgdat, struct zone *zone)  in pagetypeinfo_showfree_print()  argument
1501 					zone->name,  in pagetypeinfo_showfree_print()
1509 			area = &(zone->free_area[order]);  in pagetypeinfo_showfree_print()
1527 			spin_unlock_irq(&zone->lock);  in pagetypeinfo_showfree_print()
1529 			spin_lock_irq(&zone->lock);  in pagetypeinfo_showfree_print()
1551 					pg_data_t *pgdat, struct zone *zone)  in pagetypeinfo_showblockcount_print()  argument
1555 	unsigned long start_pfn = zone->zone_start_pfn;  in pagetypeinfo_showblockcount_print()
1556 	unsigned long end_pfn = zone_end_pfn(zone);  in pagetypeinfo_showblockcount_print()
1566 		if (page_zone(page) != zone)  in pagetypeinfo_showblockcount_print()
1576 	seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);  in pagetypeinfo_showblockcount_print()
1658 static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone)  in is_zone_first_populated()  argument
1663 		struct zone *compare = &pgdat->node_zones[zid];  in is_zone_first_populated()
1666 			return zone == compare;  in is_zone_first_populated()
1673 							struct zone *zone)  in zoneinfo_show_print()  argument
1676 	seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name);  in zoneinfo_show_print()
1677 	if (is_zone_first_populated(pgdat, zone)) {  in zoneinfo_show_print()
1698 		   zone_page_state(zone, NR_FREE_PAGES),  in zoneinfo_show_print()
1699 		   zone->watermark_boost,  in zoneinfo_show_print()
1700 		   min_wmark_pages(zone),  in zoneinfo_show_print()
1701 		   low_wmark_pages(zone),  in zoneinfo_show_print()
1702 		   high_wmark_pages(zone),  in zoneinfo_show_print()
1703 		   zone->spanned_pages,  in zoneinfo_show_print()
1704 		   zone->present_pages,  in zoneinfo_show_print()
1705 		   zone_managed_pages(zone),  in zoneinfo_show_print()
1706 		   zone_cma_pages(zone));  in zoneinfo_show_print()
1710 		   zone->lowmem_reserve[0]);  in zoneinfo_show_print()
1711 	for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++)  in zoneinfo_show_print()
1712 		seq_printf(m, ", %ld", zone->lowmem_reserve[i]);  in zoneinfo_show_print()
1716 	if (!populated_zone(zone)) {  in zoneinfo_show_print()
1723 			   zone_page_state(zone, i));  in zoneinfo_show_print()
1726 	fold_vm_zone_numa_events(zone);  in zoneinfo_show_print()
1729 			   zone_numa_event_state(zone, i));  in zoneinfo_show_print()
1737 		pcp = per_cpu_ptr(zone->per_cpu_pageset, i);  in zoneinfo_show_print()
1748 		pzstats = per_cpu_ptr(zone->per_cpu_zonestats, i);  in zoneinfo_show_print()
1757 		   zone->zone_start_pfn);  in zoneinfo_show_print()
1963 	struct zone *zone;  in need_update()  local
1965 	for_each_populated_zone(zone) {  in need_update()
1966 		struct per_cpu_zonestat *pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);  in need_update()
1975 		if (last_pgdat == zone->zone_pgdat)  in need_update()
1977 		last_pgdat = zone->zone_pgdat;  in need_update()
1978 		n = per_cpu_ptr(zone->zone_pgdat->per_cpu_nodestats, cpu);  in need_update()
2171 					pg_data_t *pgdat, struct zone *zone)  in unusable_show_print()  argument
2179 				zone->name);  in unusable_show_print()
2181 		fill_contig_page_info(zone, order, &info);  in unusable_show_print()
2221 					pg_data_t *pgdat, struct zone *zone)  in extfrag_show_print()  argument
2231 				zone->name);  in extfrag_show_print()
2233 		fill_contig_page_info(zone, order, &info);  in extfrag_show_print()