migrate.c (0744f2807a4ffb111a31c0769878ec8b125b5241) migrate.c (1a5bae25e3cf95c4e83a97f87a6b5280d9acbb22)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Memory Migration functionality - linux/mm/migrate.c
4 *
5 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
6 *
7 * Page migration was first developed in the context of the memory hotplug
8 * project. The main authors of the migration code are:

--- 1404 unchanged lines hidden (view full) ---

1413 *
1414 * Returns the number of pages that were not migrated, or an error code.
1415 */
1416int migrate_pages(struct list_head *from, new_page_t get_new_page,
1417 free_page_t put_new_page, unsigned long private,
1418 enum migrate_mode mode, int reason)
1419{
1420 int retry = 1;
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Memory Migration functionality - linux/mm/migrate.c
4 *
5 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
6 *
7 * Page migration was first developed in the context of the memory hotplug
8 * project. The main authors of the migration code are:

--- 1404 unchanged lines hidden (view full) ---

1413 *
1414 * Returns the number of pages that were not migrated, or an error code.
1415 */
1416int migrate_pages(struct list_head *from, new_page_t get_new_page,
1417 free_page_t put_new_page, unsigned long private,
1418 enum migrate_mode mode, int reason)
1419{
1420 int retry = 1;
1421 int thp_retry = 1;
1421 int nr_failed = 0;
1422 int nr_succeeded = 0;
1422 int nr_failed = 0;
1423 int nr_succeeded = 0;
1424 int nr_thp_succeeded = 0;
1425 int nr_thp_failed = 0;
1426 int nr_thp_split = 0;
1423 int pass = 0;
1427 int pass = 0;
1428 bool is_thp = false;
1424 struct page *page;
1425 struct page *page2;
1426 int swapwrite = current->flags & PF_SWAPWRITE;
1429 struct page *page;
1430 struct page *page2;
1431 int swapwrite = current->flags & PF_SWAPWRITE;
1427 int rc;
1432 int rc, nr_subpages;
1428
1429 if (!swapwrite)
1430 current->flags |= PF_SWAPWRITE;
1431
1433
1434 if (!swapwrite)
1435 current->flags |= PF_SWAPWRITE;
1436
1432 for(pass = 0; pass < 10 && retry; pass++) {
1437 for (pass = 0; pass < 10 && (retry || thp_retry); pass++) {
1433 retry = 0;
1438 retry = 0;
1439 thp_retry = 0;
1434
1435 list_for_each_entry_safe(page, page2, from, lru) {
1436retry:
1440
1441 list_for_each_entry_safe(page, page2, from, lru) {
1442retry:
1443 /*
1444 * THP statistics is based on the source huge page.
1445 * Capture required information that might get lost
1446 * during migration.
1447 */
1448 is_thp = PageTransHuge(page);
1449 nr_subpages = hpage_nr_pages(page);
1437 cond_resched();
1438
1439 if (PageHuge(page))
1440 rc = unmap_and_move_huge_page(get_new_page,
1441 put_new_page, private, page,
1442 pass > 2, mode, reason);
1443 else
1444 rc = unmap_and_move(get_new_page, put_new_page,

--- 14 unchanged lines hidden (view full) ---

1459 * is processed.
1460 */
1461 if (PageTransHuge(page) && !PageHuge(page)) {
1462 lock_page(page);
1463 rc = split_huge_page_to_list(page, from);
1464 unlock_page(page);
1465 if (!rc) {
1466 list_safe_reset_next(page, page2, lru);
1450 cond_resched();
1451
1452 if (PageHuge(page))
1453 rc = unmap_and_move_huge_page(get_new_page,
1454 put_new_page, private, page,
1455 pass > 2, mode, reason);
1456 else
1457 rc = unmap_and_move(get_new_page, put_new_page,

--- 14 unchanged lines hidden (view full) ---

1472 * is processed.
1473 */
1474 if (PageTransHuge(page) && !PageHuge(page)) {
1475 lock_page(page);
1476 rc = split_huge_page_to_list(page, from);
1477 unlock_page(page);
1478 if (!rc) {
1479 list_safe_reset_next(page, page2, lru);
1480 nr_thp_split++;
1467 goto retry;
1468 }
1469 }
1481 goto retry;
1482 }
1483 }
1484 if (is_thp) {
1485 nr_thp_failed++;
1486 nr_failed += nr_subpages;
1487 goto out;
1488 }
1470 nr_failed++;
1471 goto out;
1472 case -EAGAIN:
1489 nr_failed++;
1490 goto out;
1491 case -EAGAIN:
1492 if (is_thp) {
1493 thp_retry++;
1494 break;
1495 }
1473 retry++;
1474 break;
1475 case MIGRATEPAGE_SUCCESS:
1496 retry++;
1497 break;
1498 case MIGRATEPAGE_SUCCESS:
1499 if (is_thp) {
1500 nr_thp_succeeded++;
1501 nr_succeeded += nr_subpages;
1502 break;
1503 }
1476 nr_succeeded++;
1477 break;
1478 default:
1479 /*
1480 * Permanent failure (-EBUSY, -ENOSYS, etc.):
1481 * unlike -EAGAIN case, the failed page is
1482 * removed from migration page list and not
1483 * retried in the next outer loop.
1484 */
1504 nr_succeeded++;
1505 break;
1506 default:
1507 /*
1508 * Permanent failure (-EBUSY, -ENOSYS, etc.):
1509 * unlike -EAGAIN case, the failed page is
1510 * removed from migration page list and not
1511 * retried in the next outer loop.
1512 */
1513 if (is_thp) {
1514 nr_thp_failed++;
1515 nr_failed += nr_subpages;
1516 break;
1517 }
1485 nr_failed++;
1486 break;
1487 }
1488 }
1489 }
1518 nr_failed++;
1519 break;
1520 }
1521 }
1522 }
1490 nr_failed += retry;
1523 nr_failed += retry + thp_retry;
1524 nr_thp_failed += thp_retry;
1491 rc = nr_failed;
1492out:
1525 rc = nr_failed;
1526out:
1493 if (nr_succeeded)
1494 count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);
1495 if (nr_failed)
1496 count_vm_events(PGMIGRATE_FAIL, nr_failed);
1497 trace_mm_migrate_pages(nr_succeeded, nr_failed, mode, reason);
1527 count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);
1528 count_vm_events(PGMIGRATE_FAIL, nr_failed);
1529 count_vm_events(THP_MIGRATION_SUCCESS, nr_thp_succeeded);
1530 count_vm_events(THP_MIGRATION_FAIL, nr_thp_failed);
1531 count_vm_events(THP_MIGRATION_SPLIT, nr_thp_split);
1532 trace_mm_migrate_pages(nr_succeeded, nr_failed, nr_thp_succeeded,
1533 nr_thp_failed, nr_thp_split, mode, reason);
1498
1499 if (!swapwrite)
1500 current->flags &= ~PF_SWAPWRITE;
1501
1502 return rc;
1503}
1504
1505#ifdef CONFIG_NUMA

--- 1502 unchanged lines hidden --- 		1534
1535 if (!swapwrite)
1536 current->flags &= ~PF_SWAPWRITE;
1537
1538 return rc;
1539}
1540
1541#ifdef CONFIG_NUMA

--- 1502 unchanged lines hidden ---