1 /* 2 * linux/mm/compaction.c 3 * 4 * Memory compaction for the reduction of external fragmentation. Note that 5 * this heavily depends upon page migration to do all the real heavy 6 * lifting 7 * 8 * Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie> 9 */ 10 #include <linux/swap.h> 11 #include <linux/migrate.h> 12 #include <linux/compaction.h> 13 #include <linux/mm_inline.h> 14 #include <linux/backing-dev.h> 15 #include <linux/sysctl.h> 16 #include <linux/sysfs.h> 17 #include "internal.h" 18 19 #if defined CONFIG_COMPACTION || defined CONFIG_CMA 20 21 #define CREATE_TRACE_POINTS 22 #include <trace/events/compaction.h> 23 24 static unsigned long release_freepages(struct list_head *freelist) 25 { 26 struct page *page, *next; 27 unsigned long count = 0; 28 29 list_for_each_entry_safe(page, next, freelist, lru) { 30 list_del(&page->lru); 31 __free_page(page); 32 count++; 33 } 34 35 return count; 36 } 37 38 static void map_pages(struct list_head *list) 39 { 40 struct page *page; 41 42 list_for_each_entry(page, list, lru) { 43 arch_alloc_page(page, 0); 44 kernel_map_pages(page, 1, 1); 45 } 46 } 47 48 static inline bool migrate_async_suitable(int migratetype) 49 { 50 return is_migrate_cma(migratetype) || migratetype == MIGRATE_MOVABLE; 51 } 52 53 /* 54 * Compaction requires the taking of some coarse locks that are potentially 55 * very heavily contended. Check if the process needs to be scheduled or 56 * if the lock is contended. For async compaction, back out in the event 57 * if contention is severe. For sync compaction, schedule. 58 * 59 * Returns true if the lock is held. 60 * Returns false if the lock is released and compaction should abort 61 */ 62 static bool compact_checklock_irqsave(spinlock_t *lock, unsigned long *flags, 63 bool locked, struct compact_control *cc) 64 { 65 if (need_resched() || spin_is_contended(lock)) { 66 if (locked) { 67 spin_unlock_irqrestore(lock, *flags); 68 locked = false; 69 } 70 71 /* async aborts if taking too long or contended */ 72 if (!cc->sync) { 73 if (cc->contended) 74 *cc->contended = true; 75 return false; 76 } 77 78 cond_resched(); 79 if (fatal_signal_pending(current)) 80 return false; 81 } 82 83 if (!locked) 84 spin_lock_irqsave(lock, *flags); 85 return true; 86 } 87 88 static inline bool compact_trylock_irqsave(spinlock_t *lock, 89 unsigned long *flags, struct compact_control *cc) 90 { 91 return compact_checklock_irqsave(lock, flags, false, cc); 92 } 93 94 /* 95 * Isolate free pages onto a private freelist. Caller must hold zone->lock. 96 * If @strict is true, will abort returning 0 on any invalid PFNs or non-free 97 * pages inside of the pageblock (even though it may still end up isolating 98 * some pages). 99 */ 100 static unsigned long isolate_freepages_block(unsigned long blockpfn, 101 unsigned long end_pfn, 102 struct list_head *freelist, 103 bool strict) 104 { 105 int nr_scanned = 0, total_isolated = 0; 106 struct page *cursor; 107 108 cursor = pfn_to_page(blockpfn); 109 110 /* Isolate free pages. This assumes the block is valid */ 111 for (; blockpfn < end_pfn; blockpfn++, cursor++) { 112 int isolated, i; 113 struct page *page = cursor; 114 115 if (!pfn_valid_within(blockpfn)) { 116 if (strict) 117 return 0; 118 continue; 119 } 120 nr_scanned++; 121 122 if (!PageBuddy(page)) { 123 if (strict) 124 return 0; 125 continue; 126 } 127 128 /* Found a free page, break it into order-0 pages */ 129 isolated = split_free_page(page); 130 if (!isolated && strict) 131 return 0; 132 total_isolated += isolated; 133 for (i = 0; i < isolated; i++) { 134 list_add(&page->lru, freelist); 135 page++; 136 } 137 138 /* If a page was split, advance to the end of it */ 139 if (isolated) { 140 blockpfn += isolated - 1; 141 cursor += isolated - 1; 142 } 143 } 144 145 trace_mm_compaction_isolate_freepages(nr_scanned, total_isolated); 146 return total_isolated; 147 } 148 149 /** 150 * isolate_freepages_range() - isolate free pages. 151 * @start_pfn: The first PFN to start isolating. 152 * @end_pfn: The one-past-last PFN. 153 * 154 * Non-free pages, invalid PFNs, or zone boundaries within the 155 * [start_pfn, end_pfn) range are considered errors, cause function to 156 * undo its actions and return zero. 157 * 158 * Otherwise, function returns one-past-the-last PFN of isolated page 159 * (which may be greater then end_pfn if end fell in a middle of 160 * a free page). 161 */ 162 unsigned long 163 isolate_freepages_range(unsigned long start_pfn, unsigned long end_pfn) 164 { 165 unsigned long isolated, pfn, block_end_pfn, flags; 166 struct zone *zone = NULL; 167 LIST_HEAD(freelist); 168 169 if (pfn_valid(start_pfn)) 170 zone = page_zone(pfn_to_page(start_pfn)); 171 172 for (pfn = start_pfn; pfn < end_pfn; pfn += isolated) { 173 if (!pfn_valid(pfn) || zone != page_zone(pfn_to_page(pfn))) 174 break; 175 176 /* 177 * On subsequent iterations ALIGN() is actually not needed, 178 * but we keep it that we not to complicate the code. 179 */ 180 block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); 181 block_end_pfn = min(block_end_pfn, end_pfn); 182 183 spin_lock_irqsave(&zone->lock, flags); 184 isolated = isolate_freepages_block(pfn, block_end_pfn, 185 &freelist, true); 186 spin_unlock_irqrestore(&zone->lock, flags); 187 188 /* 189 * In strict mode, isolate_freepages_block() returns 0 if 190 * there are any holes in the block (ie. invalid PFNs or 191 * non-free pages). 192 */ 193 if (!isolated) 194 break; 195 196 /* 197 * If we managed to isolate pages, it is always (1 << n) * 198 * pageblock_nr_pages for some non-negative n. (Max order 199 * page may span two pageblocks). 200 */ 201 } 202 203 /* split_free_page does not map the pages */ 204 map_pages(&freelist); 205 206 if (pfn < end_pfn) { 207 /* Loop terminated early, cleanup. */ 208 release_freepages(&freelist); 209 return 0; 210 } 211 212 /* We don't use freelists for anything. */ 213 return pfn; 214 } 215 216 /* Update the number of anon and file isolated pages in the zone */ 217 static void acct_isolated(struct zone *zone, bool locked, struct compact_control *cc) 218 { 219 struct page *page; 220 unsigned int count[2] = { 0, }; 221 222 list_for_each_entry(page, &cc->migratepages, lru) 223 count[!!page_is_file_cache(page)]++; 224 225 /* If locked we can use the interrupt unsafe versions */ 226 if (locked) { 227 __mod_zone_page_state(zone, NR_ISOLATED_ANON, count[0]); 228 __mod_zone_page_state(zone, NR_ISOLATED_FILE, count[1]); 229 } else { 230 mod_zone_page_state(zone, NR_ISOLATED_ANON, count[0]); 231 mod_zone_page_state(zone, NR_ISOLATED_FILE, count[1]); 232 } 233 } 234 235 /* Similar to reclaim, but different enough that they don't share logic */ 236 static bool too_many_isolated(struct zone *zone) 237 { 238 unsigned long active, inactive, isolated; 239 240 inactive = zone_page_state(zone, NR_INACTIVE_FILE) + 241 zone_page_state(zone, NR_INACTIVE_ANON); 242 active = zone_page_state(zone, NR_ACTIVE_FILE) + 243 zone_page_state(zone, NR_ACTIVE_ANON); 244 isolated = zone_page_state(zone, NR_ISOLATED_FILE) + 245 zone_page_state(zone, NR_ISOLATED_ANON); 246 247 return isolated > (inactive + active) / 2; 248 } 249 250 /** 251 * isolate_migratepages_range() - isolate all migrate-able pages in range. 252 * @zone: Zone pages are in. 253 * @cc: Compaction control structure. 254 * @low_pfn: The first PFN of the range. 255 * @end_pfn: The one-past-the-last PFN of the range. 256 * 257 * Isolate all pages that can be migrated from the range specified by 258 * [low_pfn, end_pfn). Returns zero if there is a fatal signal 259 * pending), otherwise PFN of the first page that was not scanned 260 * (which may be both less, equal to or more then end_pfn). 261 * 262 * Assumes that cc->migratepages is empty and cc->nr_migratepages is 263 * zero. 264 * 265 * Apart from cc->migratepages and cc->nr_migratetypes this function 266 * does not modify any cc's fields, in particular it does not modify 267 * (or read for that matter) cc->migrate_pfn. 268 */ 269 unsigned long 270 isolate_migratepages_range(struct zone *zone, struct compact_control *cc, 271 unsigned long low_pfn, unsigned long end_pfn) 272 { 273 unsigned long last_pageblock_nr = 0, pageblock_nr; 274 unsigned long nr_scanned = 0, nr_isolated = 0; 275 struct list_head *migratelist = &cc->migratepages; 276 isolate_mode_t mode = 0; 277 struct lruvec *lruvec; 278 unsigned long flags; 279 bool locked; 280 281 /* 282 * Ensure that there are not too many pages isolated from the LRU 283 * list by either parallel reclaimers or compaction. If there are, 284 * delay for some time until fewer pages are isolated 285 */ 286 while (unlikely(too_many_isolated(zone))) { 287 /* async migration should just abort */ 288 if (!cc->sync) 289 return 0; 290 291 congestion_wait(BLK_RW_ASYNC, HZ/10); 292 293 if (fatal_signal_pending(current)) 294 return 0; 295 } 296 297 /* Time to isolate some pages for migration */ 298 cond_resched(); 299 spin_lock_irqsave(&zone->lru_lock, flags); 300 locked = true; 301 for (; low_pfn < end_pfn; low_pfn++) { 302 struct page *page; 303 304 /* give a chance to irqs before checking need_resched() */ 305 if (!((low_pfn+1) % SWAP_CLUSTER_MAX)) { 306 spin_unlock_irqrestore(&zone->lru_lock, flags); 307 locked = false; 308 } 309 310 /* Check if it is ok to still hold the lock */ 311 locked = compact_checklock_irqsave(&zone->lru_lock, &flags, 312 locked, cc); 313 if (!locked) 314 break; 315 316 /* 317 * migrate_pfn does not necessarily start aligned to a 318 * pageblock. Ensure that pfn_valid is called when moving 319 * into a new MAX_ORDER_NR_PAGES range in case of large 320 * memory holes within the zone 321 */ 322 if ((low_pfn & (MAX_ORDER_NR_PAGES - 1)) == 0) { 323 if (!pfn_valid(low_pfn)) { 324 low_pfn += MAX_ORDER_NR_PAGES - 1; 325 continue; 326 } 327 } 328 329 if (!pfn_valid_within(low_pfn)) 330 continue; 331 nr_scanned++; 332 333 /* 334 * Get the page and ensure the page is within the same zone. 335 * See the comment in isolate_freepages about overlapping 336 * nodes. It is deliberate that the new zone lock is not taken 337 * as memory compaction should not move pages between nodes. 338 */ 339 page = pfn_to_page(low_pfn); 340 if (page_zone(page) != zone) 341 continue; 342 343 /* Skip if free */ 344 if (PageBuddy(page)) 345 continue; 346 347 /* 348 * For async migration, also only scan in MOVABLE blocks. Async 349 * migration is optimistic to see if the minimum amount of work 350 * satisfies the allocation 351 */ 352 pageblock_nr = low_pfn >> pageblock_order; 353 if (!cc->sync && last_pageblock_nr != pageblock_nr && 354 !migrate_async_suitable(get_pageblock_migratetype(page))) { 355 low_pfn += pageblock_nr_pages; 356 low_pfn = ALIGN(low_pfn, pageblock_nr_pages) - 1; 357 last_pageblock_nr = pageblock_nr; 358 continue; 359 } 360 361 if (!PageLRU(page)) 362 continue; 363 364 /* 365 * PageLRU is set, and lru_lock excludes isolation, 366 * splitting and collapsing (collapsing has already 367 * happened if PageLRU is set). 368 */ 369 if (PageTransHuge(page)) { 370 low_pfn += (1 << compound_order(page)) - 1; 371 continue; 372 } 373 374 if (!cc->sync) 375 mode |= ISOLATE_ASYNC_MIGRATE; 376 377 lruvec = mem_cgroup_page_lruvec(page, zone); 378 379 /* Try isolate the page */ 380 if (__isolate_lru_page(page, mode) != 0) 381 continue; 382 383 VM_BUG_ON(PageTransCompound(page)); 384 385 /* Successfully isolated */ 386 del_page_from_lru_list(page, lruvec, page_lru(page)); 387 list_add(&page->lru, migratelist); 388 cc->nr_migratepages++; 389 nr_isolated++; 390 391 /* Avoid isolating too much */ 392 if (cc->nr_migratepages == COMPACT_CLUSTER_MAX) { 393 ++low_pfn; 394 break; 395 } 396 } 397 398 acct_isolated(zone, locked, cc); 399 400 if (locked) 401 spin_unlock_irqrestore(&zone->lru_lock, flags); 402 403 trace_mm_compaction_isolate_migratepages(nr_scanned, nr_isolated); 404 405 return low_pfn; 406 } 407 408 #endif /* CONFIG_COMPACTION || CONFIG_CMA */ 409 #ifdef CONFIG_COMPACTION 410 411 /* Returns true if the page is within a block suitable for migration to */ 412 static bool suitable_migration_target(struct page *page) 413 { 414 415 int migratetype = get_pageblock_migratetype(page); 416 417 /* Don't interfere with memory hot-remove or the min_free_kbytes blocks */ 418 if (migratetype == MIGRATE_ISOLATE || migratetype == MIGRATE_RESERVE) 419 return false; 420 421 /* If the page is a large free page, then allow migration */ 422 if (PageBuddy(page) && page_order(page) >= pageblock_order) 423 return true; 424 425 /* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */ 426 if (migrate_async_suitable(migratetype)) 427 return true; 428 429 /* Otherwise skip the block */ 430 return false; 431 } 432 433 /* 434 * Returns the start pfn of the last page block in a zone. This is the starting 435 * point for full compaction of a zone. Compaction searches for free pages from 436 * the end of each zone, while isolate_freepages_block scans forward inside each 437 * page block. 438 */ 439 static unsigned long start_free_pfn(struct zone *zone) 440 { 441 unsigned long free_pfn; 442 free_pfn = zone->zone_start_pfn + zone->spanned_pages; 443 free_pfn &= ~(pageblock_nr_pages-1); 444 return free_pfn; 445 } 446 447 /* 448 * Based on information in the current compact_control, find blocks 449 * suitable for isolating free pages from and then isolate them. 450 */ 451 static void isolate_freepages(struct zone *zone, 452 struct compact_control *cc) 453 { 454 struct page *page; 455 unsigned long high_pfn, low_pfn, pfn, zone_end_pfn, end_pfn; 456 unsigned long flags; 457 int nr_freepages = cc->nr_freepages; 458 struct list_head *freelist = &cc->freepages; 459 460 /* 461 * Initialise the free scanner. The starting point is where we last 462 * scanned from (or the end of the zone if starting). The low point 463 * is the end of the pageblock the migration scanner is using. 464 */ 465 pfn = cc->free_pfn; 466 low_pfn = cc->migrate_pfn + pageblock_nr_pages; 467 468 /* 469 * Take care that if the migration scanner is at the end of the zone 470 * that the free scanner does not accidentally move to the next zone 471 * in the next isolation cycle. 472 */ 473 high_pfn = min(low_pfn, pfn); 474 475 zone_end_pfn = zone->zone_start_pfn + zone->spanned_pages; 476 477 /* 478 * Isolate free pages until enough are available to migrate the 479 * pages on cc->migratepages. We stop searching if the migrate 480 * and free page scanners meet or enough free pages are isolated. 481 */ 482 for (; pfn > low_pfn && cc->nr_migratepages > nr_freepages; 483 pfn -= pageblock_nr_pages) { 484 unsigned long isolated; 485 486 if (!pfn_valid(pfn)) 487 continue; 488 489 /* 490 * Check for overlapping nodes/zones. It's possible on some 491 * configurations to have a setup like 492 * node0 node1 node0 493 * i.e. it's possible that all pages within a zones range of 494 * pages do not belong to a single zone. 495 */ 496 page = pfn_to_page(pfn); 497 if (page_zone(page) != zone) 498 continue; 499 500 /* Check the block is suitable for migration */ 501 if (!suitable_migration_target(page)) 502 continue; 503 504 /* 505 * Found a block suitable for isolating free pages from. Now 506 * we disabled interrupts, double check things are ok and 507 * isolate the pages. This is to minimise the time IRQs 508 * are disabled 509 */ 510 isolated = 0; 511 512 /* 513 * The zone lock must be held to isolate freepages. This 514 * unfortunately this is a very coarse lock and can be 515 * heavily contended if there are parallel allocations 516 * or parallel compactions. For async compaction do not 517 * spin on the lock 518 */ 519 if (!compact_trylock_irqsave(&zone->lock, &flags, cc)) 520 break; 521 if (suitable_migration_target(page)) { 522 end_pfn = min(pfn + pageblock_nr_pages, zone_end_pfn); 523 isolated = isolate_freepages_block(pfn, end_pfn, 524 freelist, false); 525 nr_freepages += isolated; 526 } 527 spin_unlock_irqrestore(&zone->lock, flags); 528 529 /* 530 * Record the highest PFN we isolated pages from. When next 531 * looking for free pages, the search will restart here as 532 * page migration may have returned some pages to the allocator 533 */ 534 if (isolated) { 535 high_pfn = max(high_pfn, pfn); 536 537 /* 538 * If the free scanner has wrapped, update 539 * compact_cached_free_pfn to point to the highest 540 * pageblock with free pages. This reduces excessive 541 * scanning of full pageblocks near the end of the 542 * zone 543 */ 544 if (cc->order > 0 && cc->wrapped) 545 zone->compact_cached_free_pfn = high_pfn; 546 } 547 } 548 549 /* split_free_page does not map the pages */ 550 map_pages(freelist); 551 552 cc->free_pfn = high_pfn; 553 cc->nr_freepages = nr_freepages; 554 555 /* If compact_cached_free_pfn is reset then set it now */ 556 if (cc->order > 0 && !cc->wrapped && 557 zone->compact_cached_free_pfn == start_free_pfn(zone)) 558 zone->compact_cached_free_pfn = high_pfn; 559 } 560 561 /* 562 * This is a migrate-callback that "allocates" freepages by taking pages 563 * from the isolated freelists in the block we are migrating to. 564 */ 565 static struct page *compaction_alloc(struct page *migratepage, 566 unsigned long data, 567 int **result) 568 { 569 struct compact_control *cc = (struct compact_control *)data; 570 struct page *freepage; 571 572 /* Isolate free pages if necessary */ 573 if (list_empty(&cc->freepages)) { 574 isolate_freepages(cc->zone, cc); 575 576 if (list_empty(&cc->freepages)) 577 return NULL; 578 } 579 580 freepage = list_entry(cc->freepages.next, struct page, lru); 581 list_del(&freepage->lru); 582 cc->nr_freepages--; 583 584 return freepage; 585 } 586 587 /* 588 * We cannot control nr_migratepages and nr_freepages fully when migration is 589 * running as migrate_pages() has no knowledge of compact_control. When 590 * migration is complete, we count the number of pages on the lists by hand. 591 */ 592 static void update_nr_listpages(struct compact_control *cc) 593 { 594 int nr_migratepages = 0; 595 int nr_freepages = 0; 596 struct page *page; 597 598 list_for_each_entry(page, &cc->migratepages, lru) 599 nr_migratepages++; 600 list_for_each_entry(page, &cc->freepages, lru) 601 nr_freepages++; 602 603 cc->nr_migratepages = nr_migratepages; 604 cc->nr_freepages = nr_freepages; 605 } 606 607 /* possible outcome of isolate_migratepages */ 608 typedef enum { 609 ISOLATE_ABORT, /* Abort compaction now */ 610 ISOLATE_NONE, /* No pages isolated, continue scanning */ 611 ISOLATE_SUCCESS, /* Pages isolated, migrate */ 612 } isolate_migrate_t; 613 614 /* 615 * Isolate all pages that can be migrated from the block pointed to by 616 * the migrate scanner within compact_control. 617 */ 618 static isolate_migrate_t isolate_migratepages(struct zone *zone, 619 struct compact_control *cc) 620 { 621 unsigned long low_pfn, end_pfn; 622 623 /* Do not scan outside zone boundaries */ 624 low_pfn = max(cc->migrate_pfn, zone->zone_start_pfn); 625 626 /* Only scan within a pageblock boundary */ 627 end_pfn = ALIGN(low_pfn + pageblock_nr_pages, pageblock_nr_pages); 628 629 /* Do not cross the free scanner or scan within a memory hole */ 630 if (end_pfn > cc->free_pfn || !pfn_valid(low_pfn)) { 631 cc->migrate_pfn = end_pfn; 632 return ISOLATE_NONE; 633 } 634 635 /* Perform the isolation */ 636 low_pfn = isolate_migratepages_range(zone, cc, low_pfn, end_pfn); 637 if (!low_pfn) 638 return ISOLATE_ABORT; 639 640 cc->migrate_pfn = low_pfn; 641 642 return ISOLATE_SUCCESS; 643 } 644 645 static int compact_finished(struct zone *zone, 646 struct compact_control *cc) 647 { 648 unsigned int order; 649 unsigned long watermark; 650 651 if (fatal_signal_pending(current)) 652 return COMPACT_PARTIAL; 653 654 /* 655 * A full (order == -1) compaction run starts at the beginning and 656 * end of a zone; it completes when the migrate and free scanner meet. 657 * A partial (order > 0) compaction can start with the free scanner 658 * at a random point in the zone, and may have to restart. 659 */ 660 if (cc->free_pfn <= cc->migrate_pfn) { 661 if (cc->order > 0 && !cc->wrapped) { 662 /* We started partway through; restart at the end. */ 663 unsigned long free_pfn = start_free_pfn(zone); 664 zone->compact_cached_free_pfn = free_pfn; 665 cc->free_pfn = free_pfn; 666 cc->wrapped = 1; 667 return COMPACT_CONTINUE; 668 } 669 return COMPACT_COMPLETE; 670 } 671 672 /* We wrapped around and ended up where we started. */ 673 if (cc->wrapped && cc->free_pfn <= cc->start_free_pfn) 674 return COMPACT_COMPLETE; 675 676 /* 677 * order == -1 is expected when compacting via 678 * /proc/sys/vm/compact_memory 679 */ 680 if (cc->order == -1) 681 return COMPACT_CONTINUE; 682 683 /* Compaction run is not finished if the watermark is not met */ 684 watermark = low_wmark_pages(zone); 685 watermark += (1 << cc->order); 686 687 if (!zone_watermark_ok(zone, cc->order, watermark, 0, 0)) 688 return COMPACT_CONTINUE; 689 690 /* Direct compactor: Is a suitable page free? */ 691 for (order = cc->order; order < MAX_ORDER; order++) { 692 /* Job done if page is free of the right migratetype */ 693 if (!list_empty(&zone->free_area[order].free_list[cc->migratetype])) 694 return COMPACT_PARTIAL; 695 696 /* Job done if allocation would set block type */ 697 if (order >= pageblock_order && zone->free_area[order].nr_free) 698 return COMPACT_PARTIAL; 699 } 700 701 return COMPACT_CONTINUE; 702 } 703 704 /* 705 * compaction_suitable: Is this suitable to run compaction on this zone now? 706 * Returns 707 * COMPACT_SKIPPED - If there are too few free pages for compaction 708 * COMPACT_PARTIAL - If the allocation would succeed without compaction 709 * COMPACT_CONTINUE - If compaction should run now 710 */ 711 unsigned long compaction_suitable(struct zone *zone, int order) 712 { 713 int fragindex; 714 unsigned long watermark; 715 716 /* 717 * order == -1 is expected when compacting via 718 * /proc/sys/vm/compact_memory 719 */ 720 if (order == -1) 721 return COMPACT_CONTINUE; 722 723 /* 724 * Watermarks for order-0 must be met for compaction. Note the 2UL. 725 * This is because during migration, copies of pages need to be 726 * allocated and for a short time, the footprint is higher 727 */ 728 watermark = low_wmark_pages(zone) + (2UL << order); 729 if (!zone_watermark_ok(zone, 0, watermark, 0, 0)) 730 return COMPACT_SKIPPED; 731 732 /* 733 * fragmentation index determines if allocation failures are due to 734 * low memory or external fragmentation 735 * 736 * index of -1000 implies allocations might succeed depending on 737 * watermarks 738 * index towards 0 implies failure is due to lack of memory 739 * index towards 1000 implies failure is due to fragmentation 740 * 741 * Only compact if a failure would be due to fragmentation. 742 */ 743 fragindex = fragmentation_index(zone, order); 744 if (fragindex >= 0 && fragindex <= sysctl_extfrag_threshold) 745 return COMPACT_SKIPPED; 746 747 if (fragindex == -1000 && zone_watermark_ok(zone, order, watermark, 748 0, 0)) 749 return COMPACT_PARTIAL; 750 751 return COMPACT_CONTINUE; 752 } 753 754 static int compact_zone(struct zone *zone, struct compact_control *cc) 755 { 756 int ret; 757 758 ret = compaction_suitable(zone, cc->order); 759 switch (ret) { 760 case COMPACT_PARTIAL: 761 case COMPACT_SKIPPED: 762 /* Compaction is likely to fail */ 763 return ret; 764 case COMPACT_CONTINUE: 765 /* Fall through to compaction */ 766 ; 767 } 768 769 /* Setup to move all movable pages to the end of the zone */ 770 cc->migrate_pfn = zone->zone_start_pfn; 771 772 if (cc->order > 0) { 773 /* Incremental compaction. Start where the last one stopped. */ 774 cc->free_pfn = zone->compact_cached_free_pfn; 775 cc->start_free_pfn = cc->free_pfn; 776 } else { 777 /* Order == -1 starts at the end of the zone. */ 778 cc->free_pfn = start_free_pfn(zone); 779 } 780 781 migrate_prep_local(); 782 783 while ((ret = compact_finished(zone, cc)) == COMPACT_CONTINUE) { 784 unsigned long nr_migrate, nr_remaining; 785 int err; 786 787 switch (isolate_migratepages(zone, cc)) { 788 case ISOLATE_ABORT: 789 ret = COMPACT_PARTIAL; 790 goto out; 791 case ISOLATE_NONE: 792 continue; 793 case ISOLATE_SUCCESS: 794 ; 795 } 796 797 nr_migrate = cc->nr_migratepages; 798 err = migrate_pages(&cc->migratepages, compaction_alloc, 799 (unsigned long)cc, false, 800 cc->sync ? MIGRATE_SYNC_LIGHT : MIGRATE_ASYNC); 801 update_nr_listpages(cc); 802 nr_remaining = cc->nr_migratepages; 803 804 count_vm_event(COMPACTBLOCKS); 805 count_vm_events(COMPACTPAGES, nr_migrate - nr_remaining); 806 if (nr_remaining) 807 count_vm_events(COMPACTPAGEFAILED, nr_remaining); 808 trace_mm_compaction_migratepages(nr_migrate - nr_remaining, 809 nr_remaining); 810 811 /* Release LRU pages not migrated */ 812 if (err) { 813 putback_lru_pages(&cc->migratepages); 814 cc->nr_migratepages = 0; 815 if (err == -ENOMEM) { 816 ret = COMPACT_PARTIAL; 817 goto out; 818 } 819 } 820 } 821 822 out: 823 /* Release free pages and check accounting */ 824 cc->nr_freepages -= release_freepages(&cc->freepages); 825 VM_BUG_ON(cc->nr_freepages != 0); 826 827 return ret; 828 } 829 830 static unsigned long compact_zone_order(struct zone *zone, 831 int order, gfp_t gfp_mask, 832 bool sync, bool *contended) 833 { 834 struct compact_control cc = { 835 .nr_freepages = 0, 836 .nr_migratepages = 0, 837 .order = order, 838 .migratetype = allocflags_to_migratetype(gfp_mask), 839 .zone = zone, 840 .sync = sync, 841 .contended = contended, 842 }; 843 INIT_LIST_HEAD(&cc.freepages); 844 INIT_LIST_HEAD(&cc.migratepages); 845 846 return compact_zone(zone, &cc); 847 } 848 849 int sysctl_extfrag_threshold = 500; 850 851 /** 852 * try_to_compact_pages - Direct compact to satisfy a high-order allocation 853 * @zonelist: The zonelist used for the current allocation 854 * @order: The order of the current allocation 855 * @gfp_mask: The GFP mask of the current allocation 856 * @nodemask: The allowed nodes to allocate from 857 * @sync: Whether migration is synchronous or not 858 * 859 * This is the main entry point for direct page compaction. 860 */ 861 unsigned long try_to_compact_pages(struct zonelist *zonelist, 862 int order, gfp_t gfp_mask, nodemask_t *nodemask, 863 bool sync, bool *contended) 864 { 865 enum zone_type high_zoneidx = gfp_zone(gfp_mask); 866 int may_enter_fs = gfp_mask & __GFP_FS; 867 int may_perform_io = gfp_mask & __GFP_IO; 868 struct zoneref *z; 869 struct zone *zone; 870 int rc = COMPACT_SKIPPED; 871 872 /* 873 * Check whether it is worth even starting compaction. The order check is 874 * made because an assumption is made that the page allocator can satisfy 875 * the "cheaper" orders without taking special steps 876 */ 877 if (!order || !may_enter_fs || !may_perform_io) 878 return rc; 879 880 count_vm_event(COMPACTSTALL); 881 882 /* Compact each zone in the list */ 883 for_each_zone_zonelist_nodemask(zone, z, zonelist, high_zoneidx, 884 nodemask) { 885 int status; 886 887 status = compact_zone_order(zone, order, gfp_mask, sync, 888 contended); 889 rc = max(status, rc); 890 891 /* If a normal allocation would succeed, stop compacting */ 892 if (zone_watermark_ok(zone, order, low_wmark_pages(zone), 0, 0)) 893 break; 894 } 895 896 return rc; 897 } 898 899 900 /* Compact all zones within a node */ 901 static int __compact_pgdat(pg_data_t *pgdat, struct compact_control *cc) 902 { 903 int zoneid; 904 struct zone *zone; 905 906 for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) { 907 908 zone = &pgdat->node_zones[zoneid]; 909 if (!populated_zone(zone)) 910 continue; 911 912 cc->nr_freepages = 0; 913 cc->nr_migratepages = 0; 914 cc->zone = zone; 915 INIT_LIST_HEAD(&cc->freepages); 916 INIT_LIST_HEAD(&cc->migratepages); 917 918 if (cc->order == -1 || !compaction_deferred(zone, cc->order)) 919 compact_zone(zone, cc); 920 921 if (cc->order > 0) { 922 int ok = zone_watermark_ok(zone, cc->order, 923 low_wmark_pages(zone), 0, 0); 924 if (ok && cc->order >= zone->compact_order_failed) 925 zone->compact_order_failed = cc->order + 1; 926 /* Currently async compaction is never deferred. */ 927 else if (!ok && cc->sync) 928 defer_compaction(zone, cc->order); 929 } 930 931 VM_BUG_ON(!list_empty(&cc->freepages)); 932 VM_BUG_ON(!list_empty(&cc->migratepages)); 933 } 934 935 return 0; 936 } 937 938 int compact_pgdat(pg_data_t *pgdat, int order) 939 { 940 struct compact_control cc = { 941 .order = order, 942 .sync = false, 943 }; 944 945 return __compact_pgdat(pgdat, &cc); 946 } 947 948 static int compact_node(int nid) 949 { 950 struct compact_control cc = { 951 .order = -1, 952 .sync = true, 953 }; 954 955 return __compact_pgdat(NODE_DATA(nid), &cc); 956 } 957 958 /* Compact all nodes in the system */ 959 static int compact_nodes(void) 960 { 961 int nid; 962 963 /* Flush pending updates to the LRU lists */ 964 lru_add_drain_all(); 965 966 for_each_online_node(nid) 967 compact_node(nid); 968 969 return COMPACT_COMPLETE; 970 } 971 972 /* The written value is actually unused, all memory is compacted */ 973 int sysctl_compact_memory; 974 975 /* This is the entry point for compacting all nodes via /proc/sys/vm */ 976 int sysctl_compaction_handler(struct ctl_table *table, int write, 977 void __user *buffer, size_t *length, loff_t *ppos) 978 { 979 if (write) 980 return compact_nodes(); 981 982 return 0; 983 } 984 985 int sysctl_extfrag_handler(struct ctl_table *table, int write, 986 void __user *buffer, size_t *length, loff_t *ppos) 987 { 988 proc_dointvec_minmax(table, write, buffer, length, ppos); 989 990 return 0; 991 } 992 993 #if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA) 994 ssize_t sysfs_compact_node(struct device *dev, 995 struct device_attribute *attr, 996 const char *buf, size_t count) 997 { 998 int nid = dev->id; 999 1000 if (nid >= 0 && nid < nr_node_ids && node_online(nid)) { 1001 /* Flush pending updates to the LRU lists */ 1002 lru_add_drain_all(); 1003 1004 compact_node(nid); 1005 } 1006 1007 return count; 1008 } 1009 static DEVICE_ATTR(compact, S_IWUSR, NULL, sysfs_compact_node); 1010 1011 int compaction_register_node(struct node *node) 1012 { 1013 return device_create_file(&node->dev, &dev_attr_compact); 1014 } 1015 1016 void compaction_unregister_node(struct node *node) 1017 { 1018 return device_remove_file(&node->dev, &dev_attr_compact); 1019 } 1020 #endif /* CONFIG_SYSFS && CONFIG_NUMA */ 1021 1022 #endif /* CONFIG_COMPACTION */ 1023