153 static void defer_compaction(struct zone *zone, int order)  in defer_compaction()  argument
155 	zone->compact_considered = 0;  in defer_compaction()
156 	zone->compact_defer_shift++;  in defer_compaction()
158 	if (order < zone->compact_order_failed)  in defer_compaction()
159 		zone->compact_order_failed = order;  in defer_compaction()
161 	if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)  in defer_compaction()
162 		zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT;  in defer_compaction()
164 	trace_mm_compaction_defer_compaction(zone, order);  in defer_compaction()
168 static bool compaction_deferred(struct zone *zone, int order)  in compaction_deferred()  argument
170 	unsigned long defer_limit = 1UL << zone->compact_defer_shift;  in compaction_deferred()
172 	if (order < zone->compact_order_failed)  in compaction_deferred()
176 	if (++zone->compact_considered >= defer_limit) {  in compaction_deferred()
177 		zone->compact_considered = defer_limit;  in compaction_deferred()
181 	trace_mm_compaction_deferred(zone, order);  in compaction_deferred()
191 void compaction_defer_reset(struct zone *zone, int order,  in compaction_defer_reset()  argument
195 		zone->compact_considered = 0;  in compaction_defer_reset()
196 		zone->compact_defer_shift = 0;  in compaction_defer_reset()
198 	if (order >= zone->compact_order_failed)  in compaction_defer_reset()
199 		zone->compact_order_failed = order + 1;  in compaction_defer_reset()
201 	trace_mm_compaction_defer_reset(zone, order);  in compaction_defer_reset()
205 static bool compaction_restarting(struct zone *zone, int order)  in compaction_restarting()  argument
207 	if (order < zone->compact_order_failed)  in compaction_restarting()
210 	return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&  in compaction_restarting()
211 		zone->compact_considered >= 1UL << zone->compact_defer_shift;  in compaction_restarting()
224 static void reset_cached_positions(struct zone *zone)  in reset_cached_positions()  argument
226 	zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;  in reset_cached_positions()
227 	zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;  in reset_cached_positions()
228 	zone->compact_cached_free_pfn =  in reset_cached_positions()
229 				pageblock_start_pfn(zone_end_pfn(zone) - 1);  in reset_cached_positions()
303 __reset_isolation_pfn(struct zone *zone, unsigned long pfn, bool check_source,  in __reset_isolation_pfn()  argument
313 	if (zone != page_zone(page))  in __reset_isolation_pfn()
335 	block_pfn = max(block_pfn, zone->zone_start_pfn);  in __reset_isolation_pfn()
344 	block_pfn = min(block_pfn, zone_end_pfn(zone) - 1);  in __reset_isolation_pfn()
376 static void __reset_isolation_suitable(struct zone *zone)  in __reset_isolation_suitable()  argument
378 	unsigned long migrate_pfn = zone->zone_start_pfn;  in __reset_isolation_suitable()
379 	unsigned long free_pfn = zone_end_pfn(zone) - 1;  in __reset_isolation_suitable()
385 	if (!zone->compact_blockskip_flush)  in __reset_isolation_suitable()
388 	zone->compact_blockskip_flush = false;  in __reset_isolation_suitable()
401 		if (__reset_isolation_pfn(zone, migrate_pfn, true, source_set) &&  in __reset_isolation_suitable()
405 			zone->compact_init_migrate_pfn = reset_migrate;  in __reset_isolation_suitable()
406 			zone->compact_cached_migrate_pfn[0] = reset_migrate;  in __reset_isolation_suitable()
407 			zone->compact_cached_migrate_pfn[1] = reset_migrate;  in __reset_isolation_suitable()
411 		if (__reset_isolation_pfn(zone, free_pfn, free_set, true) &&  in __reset_isolation_suitable()
415 			zone->compact_init_free_pfn = reset_free;  in __reset_isolation_suitable()
416 			zone->compact_cached_free_pfn = reset_free;  in __reset_isolation_suitable()
422 		zone->compact_cached_migrate_pfn[0] = migrate_pfn;  in __reset_isolation_suitable()
423 		zone->compact_cached_migrate_pfn[1] = migrate_pfn;  in __reset_isolation_suitable()
424 		zone->compact_cached_free_pfn = free_pfn;  in __reset_isolation_suitable()
433 		struct zone *zone = &pgdat->node_zones[zoneid];  in reset_isolation_suitable()  local
434 		if (!populated_zone(zone))  in reset_isolation_suitable()
438 		if (zone->compact_blockskip_flush)  in reset_isolation_suitable()
439 			__reset_isolation_suitable(zone);  in reset_isolation_suitable()
464 	struct zone *zone = cc->zone;  in update_cached_migrate()  local
473 	if (pfn > zone->compact_cached_migrate_pfn[0])  in update_cached_migrate()
474 		zone->compact_cached_migrate_pfn[0] = pfn;  in update_cached_migrate()
476 	    pfn > zone->compact_cached_migrate_pfn[1])  in update_cached_migrate()
477 		zone->compact_cached_migrate_pfn[1] = pfn;  in update_cached_migrate()
487 	struct zone *zone = cc->zone;  in update_pageblock_skip()  local
494 	if (pfn < zone->compact_cached_free_pfn)  in update_pageblock_skip()
495 		zone->compact_cached_free_pfn = pfn;  in update_pageblock_skip()
614 		    && compact_unlock_should_abort(&cc->zone->lock, flags,  in isolate_freepages_block()
642 			locked = compact_lock_irqsave(&cc->zone->lock,  in isolate_freepages_block()
678 		spin_unlock_irqrestore(&cc->zone->lock, flags);  in isolate_freepages_block()
730 	if (block_start_pfn < cc->zone->zone_start_pfn)  in isolate_freepages_range()
731 		block_start_pfn = cc->zone->zone_start_pfn;  in isolate_freepages_range()
753 					block_end_pfn, cc->zone))  in isolate_freepages_range()
790 	pg_data_t *pgdat = cc->zone->zone_pgdat;  in too_many_isolated()
841 	pg_data_t *pgdat = cc->zone->zone_pgdat;  in isolate_migratepages_block()
941 				    low_pfn == cc->zone->zone_start_pfn)) {  in isolate_migratepages_block()
1295 	if (block_start_pfn < cc->zone->zone_start_pfn)  in isolate_migratepages_range()
1296 		block_start_pfn = cc->zone->zone_start_pfn;  in isolate_migratepages_range()
1306 					block_end_pfn, cc->zone))  in isolate_migratepages_range()
1436 	start_pfn = max(pageblock_start_pfn(pfn), cc->zone->zone_start_pfn);  in fast_isolate_around()
1437 	end_pfn = min(pageblock_end_pfn(pfn), zone_end_pfn(cc->zone));  in fast_isolate_around()
1439 	page = pageblock_pfn_to_page(start_pfn, end_pfn, cc->zone);  in fast_isolate_around()
1487 	if (cc->free_pfn >= cc->zone->compact_init_free_pfn) {  in fast_isolate_freepages()
1512 		struct free_area *area = &cc->zone->free_area[order];  in fast_isolate_freepages()
1522 		spin_lock_irqsave(&cc->zone->lock, flags);  in fast_isolate_freepages()
1533 					      cc->zone->zone_start_pfn);  in fast_isolate_freepages()
1581 		spin_unlock_irqrestore(&cc->zone->lock, flags);  in fast_isolate_freepages()
1613 						    zone_end_pfn(cc->zone)),  in fast_isolate_freepages()
1614 						cc->zone);  in fast_isolate_freepages()
1621 	if (highest && highest >= cc->zone->compact_cached_free_pfn) {  in fast_isolate_freepages()
1623 		cc->zone->compact_cached_free_pfn = highest;  in fast_isolate_freepages()
1640 	struct zone *zone = cc->zone;  in isolate_freepages()  local
1668 						zone_end_pfn(zone));  in isolate_freepages()
1691 									zone);  in isolate_freepages()
1871 	if (pfn != cc->zone->zone_start_pfn && pfn != pageblock_start_pfn(pfn))  in fast_find_migrateblock()
1898 	if (cc->migrate_pfn != cc->zone->zone_start_pfn)  in fast_find_migrateblock()
1905 		struct free_area *area = &cc->zone->free_area[order];  in fast_find_migrateblock()
1913 		spin_lock_irqsave(&cc->zone->lock, flags);  in fast_find_migrateblock()
1939 				if (pfn < cc->zone->zone_start_pfn)  in fast_find_migrateblock()
1940 					pfn = cc->zone->zone_start_pfn;  in fast_find_migrateblock()
1946 		spin_unlock_irqrestore(&cc->zone->lock, flags);  in fast_find_migrateblock()
1985 	if (block_start_pfn < cc->zone->zone_start_pfn)  in isolate_migratepages()
1986 		block_start_pfn = cc->zone->zone_start_pfn;  in isolate_migratepages()
2017 						block_end_pfn, cc->zone);  in isolate_migratepages()
2035 		     low_pfn == cc->zone->zone_start_pfn) &&  in isolate_migratepages()
2098 static unsigned int fragmentation_score_zone(struct zone *zone)  in fragmentation_score_zone()  argument
2100 	return extfrag_for_order(zone, COMPACTION_HPAGE_ORDER);  in fragmentation_score_zone()
2113 static unsigned int fragmentation_score_zone_weighted(struct zone *zone)  in fragmentation_score_zone_weighted()  argument
2117 	score = zone->present_pages * fragmentation_score_zone(zone);  in fragmentation_score_zone_weighted()
2118 	return div64_ul(score, zone->zone_pgdat->node_present_pages + 1);  in fragmentation_score_zone_weighted()
2134 		struct zone *zone;  in fragmentation_score_node()  local
2136 		zone = &pgdat->node_zones[zoneid];  in fragmentation_score_node()
2137 		if (!populated_zone(zone))  in fragmentation_score_node()
2139 		score += fragmentation_score_zone_weighted(zone);  in fragmentation_score_node()
2178 		reset_cached_positions(cc->zone);  in __compact_finished()
2187 			cc->zone->compact_blockskip_flush = true;  in __compact_finished()
2199 		pgdat = cc->zone->zone_pgdat;  in __compact_finished()
2203 		score = fragmentation_score_zone(cc->zone);  in __compact_finished()
2229 		struct free_area *area = &cc->zone->free_area[order];  in __compact_finished()
2271 	trace_mm_compaction_finished(cc->zone, cc->order, ret);  in compact_finished()
2278 static bool __compaction_suitable(struct zone *zone, int order,  in __compaction_suitable()  argument
2298 				low_wmark_pages(zone) : min_wmark_pages(zone);  in __compaction_suitable()
2300 	return __zone_watermark_ok(zone, 0, watermark, highest_zoneidx,  in __compaction_suitable()
2307 bool compaction_suitable(struct zone *zone, int order, int highest_zoneidx)  in compaction_suitable()  argument
2312 	suitable = __compaction_suitable(zone, order, highest_zoneidx,  in compaction_suitable()
2313 					 zone_page_state(zone, NR_FREE_PAGES));  in compaction_suitable()
2333 			int fragindex = fragmentation_index(zone, order);  in compaction_suitable()
2345 	trace_mm_compaction_suitable(zone, order, compact_result);  in compaction_suitable()
2353 	struct zone *zone;  in compaction_zonelist_suitable()  local
2360 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,  in compaction_zonelist_suitable()
2370 		available = zone_reclaimable_pages(zone) / order;  in compaction_zonelist_suitable()
2371 		available += zone_page_state_snapshot(zone, NR_FREE_PAGES);  in compaction_zonelist_suitable()
2372 		if (__compaction_suitable(zone, order, ac->highest_zoneidx,  in compaction_zonelist_suitable()
2384 	unsigned long start_pfn = cc->zone->zone_start_pfn;  in compact_zone()
2385 	unsigned long end_pfn = zone_end_pfn(cc->zone);  in compact_zone()
2408 		watermark = wmark_pages(cc->zone,  in compact_zone()
2410 		if (zone_watermark_ok(cc->zone, cc->order, watermark,  in compact_zone()
2415 		if (!compaction_suitable(cc->zone, cc->order,  in compact_zone()
2424 	if (compaction_restarting(cc->zone, cc->order))  in compact_zone()
2425 		__reset_isolation_suitable(cc->zone);  in compact_zone()
2438 		cc->migrate_pfn = cc->zone->compact_cached_migrate_pfn[sync];  in compact_zone()
2439 		cc->free_pfn = cc->zone->compact_cached_free_pfn;  in compact_zone()
2442 			cc->zone->compact_cached_free_pfn = cc->free_pfn;  in compact_zone()
2446 			cc->zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;  in compact_zone()
2447 			cc->zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;  in compact_zone()
2450 		if (cc->migrate_pfn <= cc->zone->compact_init_migrate_pfn)  in compact_zone()
2465 		cc->zone->compact_cached_migrate_pfn[0] == cc->zone->compact_cached_migrate_pfn[1];  in compact_zone()
2499 				cc->zone->compact_cached_migrate_pfn[1] =  in compact_zone()
2500 					cc->zone->compact_cached_migrate_pfn[0];  in compact_zone()
2511 			last_migrated_pfn = max(cc->zone->zone_start_pfn,  in compact_zone()
2580 				lru_add_drain_cpu_zone(cc->zone);  in compact_zone()
2603 		if (free_pfn > cc->zone->compact_cached_free_pfn)  in compact_zone()
2604 			cc->zone->compact_cached_free_pfn = free_pfn;  in compact_zone()
2618 static enum compact_result compact_zone_order(struct zone *zone, int order,  in compact_zone_order()  argument
2628 		.zone = zone,  in compact_zone_order()
2688 	struct zone *zone;  in try_to_compact_pages()  local
2697 	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,  in try_to_compact_pages()
2702 					&& compaction_deferred(zone, order)) {  in try_to_compact_pages()
2707 		status = compact_zone_order(zone, order, gfp_mask, prio,  in try_to_compact_pages()
2719 			compaction_defer_reset(zone, order, false);  in try_to_compact_pages()
2731 			defer_compaction(zone, order);  in try_to_compact_pages()
2758 	struct zone *zone;  in proactive_compact_node()  local
2769 		zone = &pgdat->node_zones[zoneid];  in proactive_compact_node()
2770 		if (!populated_zone(zone))  in proactive_compact_node()
2773 		cc.zone = zone;  in proactive_compact_node()
2789 	struct zone *zone;  in compact_node()  local
2801 		zone = &pgdat->node_zones[zoneid];  in compact_node()
2802 		if (!populated_zone(zone))  in compact_node()
2805 		cc.zone = zone;  in compact_node()
2909 	struct zone *zone;  in kcompactd_node_suitable()  local
2913 		zone = &pgdat->node_zones[zoneid];  in kcompactd_node_suitable()
2915 		if (!populated_zone(zone))  in kcompactd_node_suitable()
2919 		if (zone_watermark_ok(zone, pgdat->kcompactd_max_order,  in kcompactd_node_suitable()
2920 				      min_wmark_pages(zone),  in kcompactd_node_suitable()
2924 		if (compaction_suitable(zone, pgdat->kcompactd_max_order,  in kcompactd_node_suitable()
2939 	struct zone *zone;  in kcompactd_do_work()  local
2955 		zone = &pgdat->node_zones[zoneid];  in kcompactd_do_work()
2956 		if (!populated_zone(zone))  in kcompactd_do_work()
2959 		if (compaction_deferred(zone, cc.order))  in kcompactd_do_work()
2963 		if (zone_watermark_ok(zone, cc.order,  in kcompactd_do_work()
2964 				      min_wmark_pages(zone), zoneid, 0))  in kcompactd_do_work()
2967 		if (!compaction_suitable(zone, cc.order, zoneid))  in kcompactd_do_work()
2973 		cc.zone = zone;  in kcompactd_do_work()
2977 			compaction_defer_reset(zone, cc.order, false);  in kcompactd_do_work()
2985 			drain_all_pages(zone);  in kcompactd_do_work()
2991 			defer_compaction(zone, cc.order);  in kcompactd_do_work()