compaction.c (566e54e113eb2b669f9300db2c2df400cbb06646) compaction.c (40cacbcb324036233a927418441323459d28d19b)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * linux/mm/compaction.c
4 *
5 * Memory compaction for the reduction of external fragmentation. Note that
6 * this heavily depends upon page migration to do all the real heavy
7 * lifting
8 *

--- 1286 unchanged lines hidden (view full) ---

1295 * order == -1 is expected when compacting via
1296 * /proc/sys/vm/compact_memory
1297 */
1298static inline bool is_via_compact_memory(int order)
1299{
1300 return order == -1;
1301}
1302
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * linux/mm/compaction.c
4 *
5 * Memory compaction for the reduction of external fragmentation. Note that
6 * this heavily depends upon page migration to do all the real heavy
7 * lifting
8 *

--- 1286 unchanged lines hidden (view full) ---

1295 * order == -1 is expected when compacting via
1296 * /proc/sys/vm/compact_memory
1297 */
1298static inline bool is_via_compact_memory(int order)
1299{
1300 return order == -1;
1301}
1302
1303static enum compact_result __compact_finished(struct zone *zone,
1304 struct compact_control *cc)
1303static enum compact_result __compact_finished(struct compact_control *cc)
1305{
1306 unsigned int order;
1307 const int migratetype = cc->migratetype;
1308
1309 if (cc->contended || fatal_signal_pending(current))
1310 return COMPACT_CONTENDED;
1311
1312 /* Compaction run completes if the migrate and free scanner meet */
1313 if (compact_scanners_met(cc)) {
1314 /* Let the next compaction start anew. */
1304{
1305 unsigned int order;
1306 const int migratetype = cc->migratetype;
1307
1308 if (cc->contended || fatal_signal_pending(current))
1309 return COMPACT_CONTENDED;
1310
1311 /* Compaction run completes if the migrate and free scanner meet */
1312 if (compact_scanners_met(cc)) {
1313 /* Let the next compaction start anew. */
1315 reset_cached_positions(zone);
1314 reset_cached_positions(cc->zone);
1316
1317 /*
1318 * Mark that the PG_migrate_skip information should be cleared
1319 * by kswapd when it goes to sleep. kcompactd does not set the
1320 * flag itself as the decision to be clear should be directly
1321 * based on an allocation request.
1322 */
1323 if (cc->direct_compaction)
1315
1316 /*
1317 * Mark that the PG_migrate_skip information should be cleared
1318 * by kswapd when it goes to sleep. kcompactd does not set the
1319 * flag itself as the decision to be clear should be directly
1320 * based on an allocation request.
1321 */
1322 if (cc->direct_compaction)
1324 zone->compact_blockskip_flush = true;
1323 cc->zone->compact_blockskip_flush = true;
1325
1326 if (cc->whole_zone)
1327 return COMPACT_COMPLETE;
1328 else
1329 return COMPACT_PARTIAL_SKIPPED;
1330 }
1331
1332 if (is_via_compact_memory(cc->order))

--- 7 unchanged lines hidden (view full) ---

1340 if (IS_ALIGNED(cc->migrate_pfn, pageblock_nr_pages))
1341 cc->finishing_block = false;
1342 else
1343 return COMPACT_CONTINUE;
1344 }
1345
1346 /* Direct compactor: Is a suitable page free? */
1347 for (order = cc->order; order < MAX_ORDER; order++) {
1324
1325 if (cc->whole_zone)
1326 return COMPACT_COMPLETE;
1327 else
1328 return COMPACT_PARTIAL_SKIPPED;
1329 }
1330
1331 if (is_via_compact_memory(cc->order))

--- 7 unchanged lines hidden (view full) ---

1339 if (IS_ALIGNED(cc->migrate_pfn, pageblock_nr_pages))
1340 cc->finishing_block = false;
1341 else
1342 return COMPACT_CONTINUE;
1343 }
1344
1345 /* Direct compactor: Is a suitable page free? */
1346 for (order = cc->order; order < MAX_ORDER; order++) {
1348 struct free_area *area = &zone->free_area[order];
1347 struct free_area *area = &cc->zone->free_area[order];
1349 bool can_steal;
1350
1351 /* Job done if page is free of the right migratetype */
1352 if (!list_empty(&area->free_list[migratetype]))
1353 return COMPACT_SUCCESS;
1354
1355#ifdef CONFIG_CMA
1356 /* MIGRATE_MOVABLE can fallback on MIGRATE_CMA */

--- 29 unchanged lines hidden (view full) ---

1386 cc->finishing_block = true;
1387 return COMPACT_CONTINUE;
1388 }
1389 }
1390
1391 return COMPACT_NO_SUITABLE_PAGE;
1392}
1393
1348 bool can_steal;
1349
1350 /* Job done if page is free of the right migratetype */
1351 if (!list_empty(&area->free_list[migratetype]))
1352 return COMPACT_SUCCESS;
1353
1354#ifdef CONFIG_CMA
1355 /* MIGRATE_MOVABLE can fallback on MIGRATE_CMA */

--- 29 unchanged lines hidden (view full) ---

1385 cc->finishing_block = true;
1386 return COMPACT_CONTINUE;
1387 }
1388 }
1389
1390 return COMPACT_NO_SUITABLE_PAGE;
1391}
1392
1394static enum compact_result compact_finished(struct zone *zone,
1395 struct compact_control *cc)
1393static enum compact_result compact_finished(struct compact_control *cc)
1396{
1397 int ret;
1398
1394{
1395 int ret;
1396
1399 ret = __compact_finished(zone, cc);
1400 trace_mm_compaction_finished(zone, cc->order, ret);
1397 ret = __compact_finished(cc);
1398 trace_mm_compaction_finished(cc->zone, cc->order, ret);
1401 if (ret == COMPACT_NO_SUITABLE_PAGE)
1402 ret = COMPACT_CONTINUE;
1403
1404 return ret;
1405}
1406
1407/*
1408 * compaction_suitable: Is this suitable to run compaction on this zone now?

--- 110 unchanged lines hidden (view full) ---

1519 ac_classzone_idx(ac), available);
1520 if (compact_result != COMPACT_SKIPPED)
1521 return true;
1522 }
1523
1524 return false;
1525}
1526
1399 if (ret == COMPACT_NO_SUITABLE_PAGE)
1400 ret = COMPACT_CONTINUE;
1401
1402 return ret;
1403}
1404
1405/*
1406 * compaction_suitable: Is this suitable to run compaction on this zone now?

--- 110 unchanged lines hidden (view full) ---

1517 ac_classzone_idx(ac), available);
1518 if (compact_result != COMPACT_SKIPPED)
1519 return true;
1520 }
1521
1522 return false;
1523}
1524
1527static enum compact_result compact_zone(struct zone *zone, struct compact_control *cc)
1525static enum compact_result compact_zone(struct compact_control *cc)
1528{
1529 enum compact_result ret;
1526{
1527 enum compact_result ret;
1530 unsigned long start_pfn = zone->zone_start_pfn;
1531 unsigned long end_pfn = zone_end_pfn(zone);
1528 unsigned long start_pfn = cc->zone->zone_start_pfn;
1529 unsigned long end_pfn = zone_end_pfn(cc->zone);
1532 unsigned long last_migrated_pfn;
1533 const bool sync = cc->mode != MIGRATE_ASYNC;
1534
1535 cc->migratetype = gfpflags_to_migratetype(cc->gfp_mask);
1530 unsigned long last_migrated_pfn;
1531 const bool sync = cc->mode != MIGRATE_ASYNC;
1532
1533 cc->migratetype = gfpflags_to_migratetype(cc->gfp_mask);
1536 ret = compaction_suitable(zone, cc->order, cc->alloc_flags,
1534 ret = compaction_suitable(cc->zone, cc->order, cc->alloc_flags,
1537 cc->classzone_idx);
1538 /* Compaction is likely to fail */
1539 if (ret == COMPACT_SUCCESS || ret == COMPACT_SKIPPED)
1540 return ret;
1541
1542 /* huh, compaction_suitable is returning something unexpected */
1543 VM_BUG_ON(ret != COMPACT_CONTINUE);
1544
1545 /*
1546 * Clear pageblock skip if there were failures recently and compaction
1547 * is about to be retried after being deferred.
1548 */
1535 cc->classzone_idx);
1536 /* Compaction is likely to fail */
1537 if (ret == COMPACT_SUCCESS || ret == COMPACT_SKIPPED)
1538 return ret;
1539
1540 /* huh, compaction_suitable is returning something unexpected */
1541 VM_BUG_ON(ret != COMPACT_CONTINUE);
1542
1543 /*
1544 * Clear pageblock skip if there were failures recently and compaction
1545 * is about to be retried after being deferred.
1546 */
1549 if (compaction_restarting(zone, cc->order))
1550 __reset_isolation_suitable(zone);
1547 if (compaction_restarting(cc->zone, cc->order))
1548 __reset_isolation_suitable(cc->zone);
1551
1552 /*
1553 * Setup to move all movable pages to the end of the zone. Used cached
1554 * information on where the scanners should start (unless we explicitly
1555 * want to compact the whole zone), but check that it is initialised
1556 * by ensuring the values are within zone boundaries.
1557 */
1558 if (cc->whole_zone) {
1559 cc->migrate_pfn = start_pfn;
1560 cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
1561 } else {
1549
1550 /*
1551 * Setup to move all movable pages to the end of the zone. Used cached
1552 * information on where the scanners should start (unless we explicitly
1553 * want to compact the whole zone), but check that it is initialised
1554 * by ensuring the values are within zone boundaries.
1555 */
1556 if (cc->whole_zone) {
1557 cc->migrate_pfn = start_pfn;
1558 cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
1559 } else {
1562 cc->migrate_pfn = zone->compact_cached_migrate_pfn[sync];
1563 cc->free_pfn = zone->compact_cached_free_pfn;
1560 cc->migrate_pfn = cc->zone->compact_cached_migrate_pfn[sync];
1561 cc->free_pfn = cc->zone->compact_cached_free_pfn;
1564 if (cc->free_pfn < start_pfn || cc->free_pfn >= end_pfn) {
1565 cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
1562 if (cc->free_pfn < start_pfn || cc->free_pfn >= end_pfn) {
1563 cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
1566 zone->compact_cached_free_pfn = cc->free_pfn;
1564 cc->zone->compact_cached_free_pfn = cc->free_pfn;
1567 }
1568 if (cc->migrate_pfn < start_pfn || cc->migrate_pfn >= end_pfn) {
1569 cc->migrate_pfn = start_pfn;
1565 }
1566 if (cc->migrate_pfn < start_pfn || cc->migrate_pfn >= end_pfn) {
1567 cc->migrate_pfn = start_pfn;
1570 zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;
1571 zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;
1568 cc->zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;
1569 cc->zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;
1572 }
1573
1574 if (cc->migrate_pfn == start_pfn)
1575 cc->whole_zone = true;
1576 }
1577
1578 last_migrated_pfn = 0;
1579
1580 trace_mm_compaction_begin(start_pfn, cc->migrate_pfn,
1581 cc->free_pfn, end_pfn, sync);
1582
1583 migrate_prep_local();
1584
1570 }
1571
1572 if (cc->migrate_pfn == start_pfn)
1573 cc->whole_zone = true;
1574 }
1575
1576 last_migrated_pfn = 0;
1577
1578 trace_mm_compaction_begin(start_pfn, cc->migrate_pfn,
1579 cc->free_pfn, end_pfn, sync);
1580
1581 migrate_prep_local();
1582
1585 while ((ret = compact_finished(zone, cc)) == COMPACT_CONTINUE) {
1583 while ((ret = compact_finished(cc)) == COMPACT_CONTINUE) {
1586 int err;
1587 unsigned long start_pfn = cc->migrate_pfn;
1588
1584 int err;
1585 unsigned long start_pfn = cc->migrate_pfn;
1586
1589 switch (isolate_migratepages(zone, cc)) {
1587 switch (isolate_migratepages(cc->zone, cc)) {
1590 case ISOLATE_ABORT:
1591 ret = COMPACT_CONTENDED;
1592 putback_movable_pages(&cc->migratepages);
1593 cc->nr_migratepages = 0;
1594 last_migrated_pfn = 0;
1595 goto out;
1596 case ISOLATE_NONE:
1597 /*

--- 50 unchanged lines hidden (view full) ---

1648 if (cc->order > 0 && last_migrated_pfn) {
1649 int cpu;
1650 unsigned long current_block_start =
1651 block_start_pfn(cc->migrate_pfn, cc->order);
1652
1653 if (last_migrated_pfn < current_block_start) {
1654 cpu = get_cpu();
1655 lru_add_drain_cpu(cpu);
1588 case ISOLATE_ABORT:
1589 ret = COMPACT_CONTENDED;
1590 putback_movable_pages(&cc->migratepages);
1591 cc->nr_migratepages = 0;
1592 last_migrated_pfn = 0;
1593 goto out;
1594 case ISOLATE_NONE:
1595 /*

--- 50 unchanged lines hidden (view full) ---

1646 if (cc->order > 0 && last_migrated_pfn) {
1647 int cpu;
1648 unsigned long current_block_start =
1649 block_start_pfn(cc->migrate_pfn, cc->order);
1650
1651 if (last_migrated_pfn < current_block_start) {
1652 cpu = get_cpu();
1653 lru_add_drain_cpu(cpu);
1656 drain_local_pages(zone);
1654 drain_local_pages(cc->zone);
1657 put_cpu();
1658 /* No more flushing until we migrate again */
1659 last_migrated_pfn = 0;
1660 }
1661 }
1662
1663 }
1664

--- 8 unchanged lines hidden (view full) ---

1673 cc->nr_freepages = 0;
1674 VM_BUG_ON(free_pfn == 0);
1675 /* The cached pfn is always the first in a pageblock */
1676 free_pfn = pageblock_start_pfn(free_pfn);
1677 /*
1678 * Only go back, not forward. The cached pfn might have been
1679 * already reset to zone end in compact_finished()
1680 */
1655 put_cpu();
1656 /* No more flushing until we migrate again */
1657 last_migrated_pfn = 0;
1658 }
1659 }
1660
1661 }
1662

--- 8 unchanged lines hidden (view full) ---

1671 cc->nr_freepages = 0;
1672 VM_BUG_ON(free_pfn == 0);
1673 /* The cached pfn is always the first in a pageblock */
1674 free_pfn = pageblock_start_pfn(free_pfn);
1675 /*
1676 * Only go back, not forward. The cached pfn might have been
1677 * already reset to zone end in compact_finished()
1678 */
1681 if (free_pfn > zone->compact_cached_free_pfn)
1682 zone->compact_cached_free_pfn = free_pfn;
1679 if (free_pfn > cc->zone->compact_cached_free_pfn)
1680 cc->zone->compact_cached_free_pfn = free_pfn;
1683 }
1684
1685 count_compact_events(COMPACTMIGRATE_SCANNED, cc->total_migrate_scanned);
1686 count_compact_events(COMPACTFREE_SCANNED, cc->total_free_scanned);
1687
1688 trace_mm_compaction_end(start_pfn, cc->migrate_pfn,
1689 cc->free_pfn, end_pfn, sync, ret);
1690

--- 20 unchanged lines hidden (view full) ---

1711 .direct_compaction = true,
1712 .whole_zone = (prio == MIN_COMPACT_PRIORITY),
1713 .ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY),
1714 .ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY)
1715 };
1716 INIT_LIST_HEAD(&cc.freepages);
1717 INIT_LIST_HEAD(&cc.migratepages);
1718
1681 }
1682
1683 count_compact_events(COMPACTMIGRATE_SCANNED, cc->total_migrate_scanned);
1684 count_compact_events(COMPACTFREE_SCANNED, cc->total_free_scanned);
1685
1686 trace_mm_compaction_end(start_pfn, cc->migrate_pfn,
1687 cc->free_pfn, end_pfn, sync, ret);
1688

--- 20 unchanged lines hidden (view full) ---

1709 .direct_compaction = true,
1710 .whole_zone = (prio == MIN_COMPACT_PRIORITY),
1711 .ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY),
1712 .ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY)
1713 };
1714 INIT_LIST_HEAD(&cc.freepages);
1715 INIT_LIST_HEAD(&cc.migratepages);
1716
1719 ret = compact_zone(zone, &cc);
1717 ret = compact_zone(&cc);
1720
1721 VM_BUG_ON(!list_empty(&cc.freepages));
1722 VM_BUG_ON(!list_empty(&cc.migratepages));
1723
1724 return ret;
1725}
1726
1727int sysctl_extfrag_threshold = 500;

--- 101 unchanged lines hidden (view full) ---

1829 continue;
1830
1831 cc.nr_freepages = 0;
1832 cc.nr_migratepages = 0;
1833 cc.zone = zone;
1834 INIT_LIST_HEAD(&cc.freepages);
1835 INIT_LIST_HEAD(&cc.migratepages);
1836
1718
1719 VM_BUG_ON(!list_empty(&cc.freepages));
1720 VM_BUG_ON(!list_empty(&cc.migratepages));
1721
1722 return ret;
1723}
1724
1725int sysctl_extfrag_threshold = 500;

--- 101 unchanged lines hidden (view full) ---

1827 continue;
1828
1829 cc.nr_freepages = 0;
1830 cc.nr_migratepages = 0;
1831 cc.zone = zone;
1832 INIT_LIST_HEAD(&cc.freepages);
1833 INIT_LIST_HEAD(&cc.migratepages);
1834
1837 compact_zone(zone, &cc);
1835 compact_zone(&cc);
1838
1839 VM_BUG_ON(!list_empty(&cc.freepages));
1840 VM_BUG_ON(!list_empty(&cc.migratepages));
1841 }
1842}
1843
1844/* Compact all nodes in the system */
1845static void compact_nodes(void)

--- 117 unchanged lines hidden (view full) ---

1963 cc.total_migrate_scanned = 0;
1964 cc.total_free_scanned = 0;
1965 cc.zone = zone;
1966 INIT_LIST_HEAD(&cc.freepages);
1967 INIT_LIST_HEAD(&cc.migratepages);
1968
1969 if (kthread_should_stop())
1970 return;
1836
1837 VM_BUG_ON(!list_empty(&cc.freepages));
1838 VM_BUG_ON(!list_empty(&cc.migratepages));
1839 }
1840}
1841
1842/* Compact all nodes in the system */
1843static void compact_nodes(void)

--- 117 unchanged lines hidden (view full) ---

1961 cc.total_migrate_scanned = 0;
1962 cc.total_free_scanned = 0;
1963 cc.zone = zone;
1964 INIT_LIST_HEAD(&cc.freepages);
1965 INIT_LIST_HEAD(&cc.migratepages);
1966
1967 if (kthread_should_stop())
1968 return;
1971 status = compact_zone(zone, &cc);
1969 status = compact_zone(&cc);
1972
1973 if (status == COMPACT_SUCCESS) {
1974 compaction_defer_reset(zone, cc.order, false);
1975 } else if (status == COMPACT_PARTIAL_SKIPPED || status == COMPACT_COMPLETE) {
1976 /*
1977 * Buddy pages may become stranded on pcps that could
1978 * otherwise coalesce on the zone's free area for
1979 * order >= cc.order. This is ratelimited by the

--- 169 unchanged lines hidden --- 		1970
1971 if (status == COMPACT_SUCCESS) {
1972 compaction_defer_reset(zone, cc.order, false);
1973 } else if (status == COMPACT_PARTIAL_SKIPPED || status == COMPACT_COMPLETE) {
1974 /*
1975 * Buddy pages may become stranded on pcps that could
1976 * otherwise coalesce on the zone's free area for
1977 * order >= cc.order. This is ratelimited by the

--- 169 unchanged lines hidden ---