page_alloc.c (624f58d8f4639676d2fa1238425ab0148d501c4a) page_alloc.c (36e66c554b5c6a9d17a229faca7a61693527b0bd)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * linux/mm/page_alloc.c
4 *
5 * Manages the free list, the system allocates free pages here.
6 * Note that kmalloc() lives in slab.c
7 *
8 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds

--- 60 unchanged lines hidden (view full) ---

69#include <linux/nmi.h>
70#include <linux/psi.h>
71
72#include <asm/sections.h>
73#include <asm/tlbflush.h>
74#include <asm/div64.h>
75#include "internal.h"
76#include "shuffle.h"
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * linux/mm/page_alloc.c
4 *
5 * Manages the free list, the system allocates free pages here.
6 * Note that kmalloc() lives in slab.c
7 *
8 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds

--- 60 unchanged lines hidden (view full) ---

69#include <linux/nmi.h>
70#include <linux/psi.h>
71
72#include <asm/sections.h>
73#include <asm/tlbflush.h>
74#include <asm/div64.h>
75#include "internal.h"
76#include "shuffle.h"
77#include "page_reporting.h"
77
78/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
79static DEFINE_MUTEX(pcp_batch_high_lock);
80#define MIN_PERCPU_PAGELIST_FRACTION (8)
81
82#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
83DEFINE_PER_CPU(int, numa_node);
84EXPORT_PER_CPU_SYMBOL(numa_node);

--- 806 unchanged lines hidden (view full) ---

891 struct free_area *area = &zone->free_area[order];
892
893 list_move(&page->lru, &area->free_list[migratetype]);
894}
895
896static inline void del_page_from_free_list(struct page *page, struct zone *zone,
897 unsigned int order)
898{
78
79/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
80static DEFINE_MUTEX(pcp_batch_high_lock);
81#define MIN_PERCPU_PAGELIST_FRACTION (8)
82
83#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
84DEFINE_PER_CPU(int, numa_node);
85EXPORT_PER_CPU_SYMBOL(numa_node);

--- 806 unchanged lines hidden (view full) ---

892 struct free_area *area = &zone->free_area[order];
893
894 list_move(&page->lru, &area->free_list[migratetype]);
895}
896
897static inline void del_page_from_free_list(struct page *page, struct zone *zone,
898 unsigned int order)
899{
900 /* clear reported state and update reported page count */
901 if (page_reported(page))
902 __ClearPageReported(page);
903
899 list_del(&page->lru);
900 __ClearPageBuddy(page);
901 set_page_private(page, 0);
902 zone->free_area[order].nr_free--;
903}
904
905/*
906 * If this is not the largest possible page, check if the buddy

--- 47 unchanged lines hidden (view full) ---

954 * triggers coalescing into a block of larger size.
955 *
956 * -- nyc
957 */
958
959static inline void __free_one_page(struct page *page,
960 unsigned long pfn,
961 struct zone *zone, unsigned int order,
904 list_del(&page->lru);
905 __ClearPageBuddy(page);
906 set_page_private(page, 0);
907 zone->free_area[order].nr_free--;
908}
909
910/*
911 * If this is not the largest possible page, check if the buddy

--- 47 unchanged lines hidden (view full) ---

959 * triggers coalescing into a block of larger size.
960 *
961 * -- nyc
962 */
963
964static inline void __free_one_page(struct page *page,
965 unsigned long pfn,
966 struct zone *zone, unsigned int order,
962 int migratetype)
967 int migratetype, bool report)
963{
964 struct capture_control *capc = task_capc(zone);
965 unsigned long uninitialized_var(buddy_pfn);
966 unsigned long combined_pfn;
967 unsigned int max_order;
968 struct page *buddy;
969 bool to_tail;
970

--- 68 unchanged lines hidden (view full) ---

1039 to_tail = shuffle_pick_tail();
1040 else
1041 to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order);
1042
1043 if (to_tail)
1044 add_to_free_list_tail(page, zone, order, migratetype);
1045 else
1046 add_to_free_list(page, zone, order, migratetype);
968{
969 struct capture_control *capc = task_capc(zone);
970 unsigned long uninitialized_var(buddy_pfn);
971 unsigned long combined_pfn;
972 unsigned int max_order;
973 struct page *buddy;
974 bool to_tail;
975

--- 68 unchanged lines hidden (view full) ---

1044 to_tail = shuffle_pick_tail();
1045 else
1046 to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order);
1047
1048 if (to_tail)
1049 add_to_free_list_tail(page, zone, order, migratetype);
1050 else
1051 add_to_free_list(page, zone, order, migratetype);
1052
1053 /* Notify page reporting subsystem of freed page */
1054 if (report)
1055 page_reporting_notify_free(order);
1047}
1048
1049/*
1050 * A bad page could be due to a number of fields. Instead of multiple branches,
1051 * try and check multiple fields with one check. The caller must do a detailed
1052 * check if necessary.
1053 */
1054static inline bool page_expected_state(struct page *page,

--- 300 unchanged lines hidden (view full) ---

1355 list_for_each_entry_safe(page, tmp, &head, lru) {
1356 int mt = get_pcppage_migratetype(page);
1357 /* MIGRATE_ISOLATE page should not go to pcplists */
1358 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
1359 /* Pageblock could have been isolated meanwhile */
1360 if (unlikely(isolated_pageblocks))
1361 mt = get_pageblock_migratetype(page);
1362
1056}
1057
1058/*
1059 * A bad page could be due to a number of fields. Instead of multiple branches,
1060 * try and check multiple fields with one check. The caller must do a detailed
1061 * check if necessary.
1062 */
1063static inline bool page_expected_state(struct page *page,

--- 300 unchanged lines hidden (view full) ---

1364 list_for_each_entry_safe(page, tmp, &head, lru) {
1365 int mt = get_pcppage_migratetype(page);
1366 /* MIGRATE_ISOLATE page should not go to pcplists */
1367 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
1368 /* Pageblock could have been isolated meanwhile */
1369 if (unlikely(isolated_pageblocks))
1370 mt = get_pageblock_migratetype(page);
1371
1363 __free_one_page(page, page_to_pfn(page), zone, 0, mt);
1372 __free_one_page(page, page_to_pfn(page), zone, 0, mt, true);
1364 trace_mm_page_pcpu_drain(page, 0, mt);
1365 }
1366 spin_unlock(&zone->lock);
1367}
1368
1369static void free_one_page(struct zone *zone,
1370 struct page *page, unsigned long pfn,
1371 unsigned int order,
1372 int migratetype)
1373{
1374 spin_lock(&zone->lock);
1375 if (unlikely(has_isolate_pageblock(zone) ||
1376 is_migrate_isolate(migratetype))) {
1377 migratetype = get_pfnblock_migratetype(page, pfn);
1378 }
1373 trace_mm_page_pcpu_drain(page, 0, mt);
1374 }
1375 spin_unlock(&zone->lock);
1376}
1377
1378static void free_one_page(struct zone *zone,
1379 struct page *page, unsigned long pfn,
1380 unsigned int order,
1381 int migratetype)
1382{
1383 spin_lock(&zone->lock);
1384 if (unlikely(has_isolate_pageblock(zone) ||
1385 is_migrate_isolate(migratetype))) {
1386 migratetype = get_pfnblock_migratetype(page, pfn);
1387 }
1379 __free_one_page(page, pfn, zone, order, migratetype);
1388 __free_one_page(page, pfn, zone, order, migratetype, true);
1380 spin_unlock(&zone->lock);
1381}
1382
1383static void __meminit __init_single_page(struct page *page, unsigned long pfn,
1384 unsigned long zone, int nid)
1385{
1386 mm_zero_struct_page(page);
1387 set_page_links(page, zone, nid, pfn);

--- 1834 unchanged lines hidden (view full) ---

3222void __putback_isolated_page(struct page *page, unsigned int order, int mt)
3223{
3224 struct zone *zone = page_zone(page);
3225
3226 /* zone lock should be held when this function is called */
3227 lockdep_assert_held(&zone->lock);
3228
3229 /* Return isolated page to tail of freelist. */
1389 spin_unlock(&zone->lock);
1390}
1391
1392static void __meminit __init_single_page(struct page *page, unsigned long pfn,
1393 unsigned long zone, int nid)
1394{
1395 mm_zero_struct_page(page);
1396 set_page_links(page, zone, nid, pfn);

--- 1834 unchanged lines hidden (view full) ---

3231void __putback_isolated_page(struct page *page, unsigned int order, int mt)
3232{
3233 struct zone *zone = page_zone(page);
3234
3235 /* zone lock should be held when this function is called */
3236 lockdep_assert_held(&zone->lock);
3237
3238 /* Return isolated page to tail of freelist. */
3230 __free_one_page(page, page_to_pfn(page), zone, order, mt);
3239 __free_one_page(page, page_to_pfn(page), zone, order, mt, false);
3231}
3232
3233/*
3234 * Update NUMA hit/miss statistics
3235 *
3236 * Must be called with interrupts disabled.
3237 */
3238static inline void zone_statistics(struct zone *preferred_zone, struct zone *z)

--- 5597 unchanged lines hidden --- 		3240}
3241
3242/*
3243 * Update NUMA hit/miss statistics
3244 *
3245 * Must be called with interrupts disabled.
3246 */
3247static inline void zone_statistics(struct zone *preferred_zone, struct zone *z)

--- 5597 unchanged lines hidden ---