1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_MMZONE_H 3 #define _LINUX_MMZONE_H 4 5 #ifndef __ASSEMBLY__ 6 #ifndef __GENERATING_BOUNDS_H 7 8 #include <linux/spinlock.h> 9 #include <linux/list.h> 10 #include <linux/wait.h> 11 #include <linux/bitops.h> 12 #include <linux/cache.h> 13 #include <linux/threads.h> 14 #include <linux/numa.h> 15 #include <linux/init.h> 16 #include <linux/seqlock.h> 17 #include <linux/nodemask.h> 18 #include <linux/pageblock-flags.h> 19 #include <linux/page-flags-layout.h> 20 #include <linux/atomic.h> 21 #include <asm/page.h> 22 23 /* Free memory management - zoned buddy allocator. */ 24 #ifndef CONFIG_FORCE_MAX_ZONEORDER 25 #define MAX_ORDER 11 26 #else 27 #define MAX_ORDER CONFIG_FORCE_MAX_ZONEORDER 28 #endif 29 #define MAX_ORDER_NR_PAGES (1 << (MAX_ORDER - 1)) 30 31 /* 32 * PAGE_ALLOC_COSTLY_ORDER is the order at which allocations are deemed 33 * costly to service. That is between allocation orders which should 34 * coalesce naturally under reasonable reclaim pressure and those which 35 * will not. 36 */ 37 #define PAGE_ALLOC_COSTLY_ORDER 3 38 39 enum migratetype { 40 MIGRATE_UNMOVABLE, 41 MIGRATE_MOVABLE, 42 MIGRATE_RECLAIMABLE, 43 MIGRATE_PCPTYPES, /* the number of types on the pcp lists */ 44 MIGRATE_HIGHATOMIC = MIGRATE_PCPTYPES, 45 #ifdef CONFIG_CMA 46 /* 47 * MIGRATE_CMA migration type is designed to mimic the way 48 * ZONE_MOVABLE works. Only movable pages can be allocated 49 * from MIGRATE_CMA pageblocks and page allocator never 50 * implicitly change migration type of MIGRATE_CMA pageblock. 51 * 52 * The way to use it is to change migratetype of a range of 53 * pageblocks to MIGRATE_CMA which can be done by 54 * __free_pageblock_cma() function. What is important though 55 * is that a range of pageblocks must be aligned to 56 * MAX_ORDER_NR_PAGES should biggest page be bigger then 57 * a single pageblock. 58 */ 59 MIGRATE_CMA, 60 #endif 61 #ifdef CONFIG_MEMORY_ISOLATION 62 MIGRATE_ISOLATE, /* can't allocate from here */ 63 #endif 64 MIGRATE_TYPES 65 }; 66 67 /* In mm/page_alloc.c; keep in sync also with show_migration_types() there */ 68 extern char * const migratetype_names[MIGRATE_TYPES]; 69 70 #ifdef CONFIG_CMA 71 # define is_migrate_cma(migratetype) unlikely((migratetype) == MIGRATE_CMA) 72 # define is_migrate_cma_page(_page) (get_pageblock_migratetype(_page) == MIGRATE_CMA) 73 #else 74 # define is_migrate_cma(migratetype) false 75 # define is_migrate_cma_page(_page) false 76 #endif 77 78 static inline bool is_migrate_movable(int mt) 79 { 80 return is_migrate_cma(mt) || mt == MIGRATE_MOVABLE; 81 } 82 83 #define for_each_migratetype_order(order, type) \ 84 for (order = 0; order < MAX_ORDER; order++) \ 85 for (type = 0; type < MIGRATE_TYPES; type++) 86 87 extern int page_group_by_mobility_disabled; 88 89 #define NR_MIGRATETYPE_BITS (PB_migrate_end - PB_migrate + 1) 90 #define MIGRATETYPE_MASK ((1UL << NR_MIGRATETYPE_BITS) - 1) 91 92 #define get_pageblock_migratetype(page) \ 93 get_pfnblock_flags_mask(page, page_to_pfn(page), \ 94 PB_migrate_end, MIGRATETYPE_MASK) 95 96 struct free_area { 97 struct list_head free_list[MIGRATE_TYPES]; 98 unsigned long nr_free; 99 }; 100 101 struct pglist_data; 102 103 /* 104 * zone->lock and the zone lru_lock are two of the hottest locks in the kernel. 105 * So add a wild amount of padding here to ensure that they fall into separate 106 * cachelines. There are very few zone structures in the machine, so space 107 * consumption is not a concern here. 108 */ 109 #if defined(CONFIG_SMP) 110 struct zone_padding { 111 char x[0]; 112 } ____cacheline_internodealigned_in_smp; 113 #define ZONE_PADDING(name) struct zone_padding name; 114 #else 115 #define ZONE_PADDING(name) 116 #endif 117 118 #ifdef CONFIG_NUMA 119 enum numa_stat_item { 120 NUMA_HIT, /* allocated in intended node */ 121 NUMA_MISS, /* allocated in non intended node */ 122 NUMA_FOREIGN, /* was intended here, hit elsewhere */ 123 NUMA_INTERLEAVE_HIT, /* interleaver preferred this zone */ 124 NUMA_LOCAL, /* allocation from local node */ 125 NUMA_OTHER, /* allocation from other node */ 126 NR_VM_NUMA_STAT_ITEMS 127 }; 128 #else 129 #define NR_VM_NUMA_STAT_ITEMS 0 130 #endif 131 132 enum zone_stat_item { 133 /* First 128 byte cacheline (assuming 64 bit words) */ 134 NR_FREE_PAGES, 135 NR_ZONE_LRU_BASE, /* Used only for compaction and reclaim retry */ 136 NR_ZONE_INACTIVE_ANON = NR_ZONE_LRU_BASE, 137 NR_ZONE_ACTIVE_ANON, 138 NR_ZONE_INACTIVE_FILE, 139 NR_ZONE_ACTIVE_FILE, 140 NR_ZONE_UNEVICTABLE, 141 NR_ZONE_WRITE_PENDING, /* Count of dirty, writeback and unstable pages */ 142 NR_MLOCK, /* mlock()ed pages found and moved off LRU */ 143 NR_PAGETABLE, /* used for pagetables */ 144 NR_KERNEL_STACK_KB, /* measured in KiB */ 145 /* Second 128 byte cacheline */ 146 NR_BOUNCE, 147 #if IS_ENABLED(CONFIG_ZSMALLOC) 148 NR_ZSPAGES, /* allocated in zsmalloc */ 149 #endif 150 NR_FREE_CMA_PAGES, 151 NR_VM_ZONE_STAT_ITEMS }; 152 153 enum node_stat_item { 154 NR_LRU_BASE, 155 NR_INACTIVE_ANON = NR_LRU_BASE, /* must match order of LRU_[IN]ACTIVE */ 156 NR_ACTIVE_ANON, /* " " " " " */ 157 NR_INACTIVE_FILE, /* " " " " " */ 158 NR_ACTIVE_FILE, /* " " " " " */ 159 NR_UNEVICTABLE, /* " " " " " */ 160 NR_SLAB_RECLAIMABLE, 161 NR_SLAB_UNRECLAIMABLE, 162 NR_ISOLATED_ANON, /* Temporary isolated pages from anon lru */ 163 NR_ISOLATED_FILE, /* Temporary isolated pages from file lru */ 164 WORKINGSET_REFAULT, 165 WORKINGSET_ACTIVATE, 166 WORKINGSET_NODERECLAIM, 167 NR_ANON_MAPPED, /* Mapped anonymous pages */ 168 NR_FILE_MAPPED, /* pagecache pages mapped into pagetables. 169 only modified from process context */ 170 NR_FILE_PAGES, 171 NR_FILE_DIRTY, 172 NR_WRITEBACK, 173 NR_WRITEBACK_TEMP, /* Writeback using temporary buffers */ 174 NR_SHMEM, /* shmem pages (included tmpfs/GEM pages) */ 175 NR_SHMEM_THPS, 176 NR_SHMEM_PMDMAPPED, 177 NR_ANON_THPS, 178 NR_UNSTABLE_NFS, /* NFS unstable pages */ 179 NR_VMSCAN_WRITE, 180 NR_VMSCAN_IMMEDIATE, /* Prioritise for reclaim when writeback ends */ 181 NR_DIRTIED, /* page dirtyings since bootup */ 182 NR_WRITTEN, /* page writings since bootup */ 183 NR_INDIRECTLY_RECLAIMABLE_BYTES, /* measured in bytes */ 184 NR_VM_NODE_STAT_ITEMS 185 }; 186 187 /* 188 * We do arithmetic on the LRU lists in various places in the code, 189 * so it is important to keep the active lists LRU_ACTIVE higher in 190 * the array than the corresponding inactive lists, and to keep 191 * the *_FILE lists LRU_FILE higher than the corresponding _ANON lists. 192 * 193 * This has to be kept in sync with the statistics in zone_stat_item 194 * above and the descriptions in vmstat_text in mm/vmstat.c 195 */ 196 #define LRU_BASE 0 197 #define LRU_ACTIVE 1 198 #define LRU_FILE 2 199 200 enum lru_list { 201 LRU_INACTIVE_ANON = LRU_BASE, 202 LRU_ACTIVE_ANON = LRU_BASE + LRU_ACTIVE, 203 LRU_INACTIVE_FILE = LRU_BASE + LRU_FILE, 204 LRU_ACTIVE_FILE = LRU_BASE + LRU_FILE + LRU_ACTIVE, 205 LRU_UNEVICTABLE, 206 NR_LRU_LISTS 207 }; 208 209 #define for_each_lru(lru) for (lru = 0; lru < NR_LRU_LISTS; lru++) 210 211 #define for_each_evictable_lru(lru) for (lru = 0; lru <= LRU_ACTIVE_FILE; lru++) 212 213 static inline int is_file_lru(enum lru_list lru) 214 { 215 return (lru == LRU_INACTIVE_FILE || lru == LRU_ACTIVE_FILE); 216 } 217 218 static inline int is_active_lru(enum lru_list lru) 219 { 220 return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE); 221 } 222 223 struct zone_reclaim_stat { 224 /* 225 * The pageout code in vmscan.c keeps track of how many of the 226 * mem/swap backed and file backed pages are referenced. 227 * The higher the rotated/scanned ratio, the more valuable 228 * that cache is. 229 * 230 * The anon LRU stats live in [0], file LRU stats in [1] 231 */ 232 unsigned long recent_rotated[2]; 233 unsigned long recent_scanned[2]; 234 }; 235 236 struct lruvec { 237 struct list_head lists[NR_LRU_LISTS]; 238 struct zone_reclaim_stat reclaim_stat; 239 /* Evictions & activations on the inactive file list */ 240 atomic_long_t inactive_age; 241 /* Refaults at the time of last reclaim cycle */ 242 unsigned long refaults; 243 #ifdef CONFIG_MEMCG 244 struct pglist_data *pgdat; 245 #endif 246 }; 247 248 /* Mask used at gathering information at once (see memcontrol.c) */ 249 #define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE)) 250 #define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON)) 251 #define LRU_ALL ((1 << NR_LRU_LISTS) - 1) 252 253 /* Isolate unmapped file */ 254 #define ISOLATE_UNMAPPED ((__force isolate_mode_t)0x2) 255 /* Isolate for asynchronous migration */ 256 #define ISOLATE_ASYNC_MIGRATE ((__force isolate_mode_t)0x4) 257 /* Isolate unevictable pages */ 258 #define ISOLATE_UNEVICTABLE ((__force isolate_mode_t)0x8) 259 260 /* LRU Isolation modes. */ 261 typedef unsigned __bitwise isolate_mode_t; 262 263 enum zone_watermarks { 264 WMARK_MIN, 265 WMARK_LOW, 266 WMARK_HIGH, 267 NR_WMARK 268 }; 269 270 #define min_wmark_pages(z) (z->watermark[WMARK_MIN]) 271 #define low_wmark_pages(z) (z->watermark[WMARK_LOW]) 272 #define high_wmark_pages(z) (z->watermark[WMARK_HIGH]) 273 274 struct per_cpu_pages { 275 int count; /* number of pages in the list */ 276 int high; /* high watermark, emptying needed */ 277 int batch; /* chunk size for buddy add/remove */ 278 279 /* Lists of pages, one per migrate type stored on the pcp-lists */ 280 struct list_head lists[MIGRATE_PCPTYPES]; 281 }; 282 283 struct per_cpu_pageset { 284 struct per_cpu_pages pcp; 285 #ifdef CONFIG_NUMA 286 s8 expire; 287 u16 vm_numa_stat_diff[NR_VM_NUMA_STAT_ITEMS]; 288 #endif 289 #ifdef CONFIG_SMP 290 s8 stat_threshold; 291 s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS]; 292 #endif 293 }; 294 295 struct per_cpu_nodestat { 296 s8 stat_threshold; 297 s8 vm_node_stat_diff[NR_VM_NODE_STAT_ITEMS]; 298 }; 299 300 #endif /* !__GENERATING_BOUNDS.H */ 301 302 enum zone_type { 303 #ifdef CONFIG_ZONE_DMA 304 /* 305 * ZONE_DMA is used when there are devices that are not able 306 * to do DMA to all of addressable memory (ZONE_NORMAL). Then we 307 * carve out the portion of memory that is needed for these devices. 308 * The range is arch specific. 309 * 310 * Some examples 311 * 312 * Architecture Limit 313 * --------------------------- 314 * parisc, ia64, sparc <4G 315 * s390 <2G 316 * arm Various 317 * alpha Unlimited or 0-16MB. 318 * 319 * i386, x86_64 and multiple other arches 320 * <16M. 321 */ 322 ZONE_DMA, 323 #endif 324 #ifdef CONFIG_ZONE_DMA32 325 /* 326 * x86_64 needs two ZONE_DMAs because it supports devices that are 327 * only able to do DMA to the lower 16M but also 32 bit devices that 328 * can only do DMA areas below 4G. 329 */ 330 ZONE_DMA32, 331 #endif 332 /* 333 * Normal addressable memory is in ZONE_NORMAL. DMA operations can be 334 * performed on pages in ZONE_NORMAL if the DMA devices support 335 * transfers to all addressable memory. 336 */ 337 ZONE_NORMAL, 338 #ifdef CONFIG_HIGHMEM 339 /* 340 * A memory area that is only addressable by the kernel through 341 * mapping portions into its own address space. This is for example 342 * used by i386 to allow the kernel to address the memory beyond 343 * 900MB. The kernel will set up special mappings (page 344 * table entries on i386) for each page that the kernel needs to 345 * access. 346 */ 347 ZONE_HIGHMEM, 348 #endif 349 ZONE_MOVABLE, 350 #ifdef CONFIG_ZONE_DEVICE 351 ZONE_DEVICE, 352 #endif 353 __MAX_NR_ZONES 354 355 }; 356 357 #ifndef __GENERATING_BOUNDS_H 358 359 struct zone { 360 /* Read-mostly fields */ 361 362 /* zone watermarks, access with *_wmark_pages(zone) macros */ 363 unsigned long watermark[NR_WMARK]; 364 365 unsigned long nr_reserved_highatomic; 366 367 /* 368 * We don't know if the memory that we're going to allocate will be 369 * freeable or/and it will be released eventually, so to avoid totally 370 * wasting several GB of ram we must reserve some of the lower zone 371 * memory (otherwise we risk to run OOM on the lower zones despite 372 * there being tons of freeable ram on the higher zones). This array is 373 * recalculated at runtime if the sysctl_lowmem_reserve_ratio sysctl 374 * changes. 375 */ 376 long lowmem_reserve[MAX_NR_ZONES]; 377 378 #ifdef CONFIG_NUMA 379 int node; 380 #endif 381 struct pglist_data *zone_pgdat; 382 struct per_cpu_pageset __percpu *pageset; 383 384 #ifndef CONFIG_SPARSEMEM 385 /* 386 * Flags for a pageblock_nr_pages block. See pageblock-flags.h. 387 * In SPARSEMEM, this map is stored in struct mem_section 388 */ 389 unsigned long *pageblock_flags; 390 #endif /* CONFIG_SPARSEMEM */ 391 392 /* zone_start_pfn == zone_start_paddr >> PAGE_SHIFT */ 393 unsigned long zone_start_pfn; 394 395 /* 396 * spanned_pages is the total pages spanned by the zone, including 397 * holes, which is calculated as: 398 * spanned_pages = zone_end_pfn - zone_start_pfn; 399 * 400 * present_pages is physical pages existing within the zone, which 401 * is calculated as: 402 * present_pages = spanned_pages - absent_pages(pages in holes); 403 * 404 * managed_pages is present pages managed by the buddy system, which 405 * is calculated as (reserved_pages includes pages allocated by the 406 * bootmem allocator): 407 * managed_pages = present_pages - reserved_pages; 408 * 409 * So present_pages may be used by memory hotplug or memory power 410 * management logic to figure out unmanaged pages by checking 411 * (present_pages - managed_pages). And managed_pages should be used 412 * by page allocator and vm scanner to calculate all kinds of watermarks 413 * and thresholds. 414 * 415 * Locking rules: 416 * 417 * zone_start_pfn and spanned_pages are protected by span_seqlock. 418 * It is a seqlock because it has to be read outside of zone->lock, 419 * and it is done in the main allocator path. But, it is written 420 * quite infrequently. 421 * 422 * The span_seq lock is declared along with zone->lock because it is 423 * frequently read in proximity to zone->lock. It's good to 424 * give them a chance of being in the same cacheline. 425 * 426 * Write access to present_pages at runtime should be protected by 427 * mem_hotplug_begin/end(). Any reader who can't tolerant drift of 428 * present_pages should get_online_mems() to get a stable value. 429 * 430 * Read access to managed_pages should be safe because it's unsigned 431 * long. Write access to zone->managed_pages and totalram_pages are 432 * protected by managed_page_count_lock at runtime. Idealy only 433 * adjust_managed_page_count() should be used instead of directly 434 * touching zone->managed_pages and totalram_pages. 435 */ 436 unsigned long managed_pages; 437 unsigned long spanned_pages; 438 unsigned long present_pages; 439 440 const char *name; 441 442 #ifdef CONFIG_MEMORY_ISOLATION 443 /* 444 * Number of isolated pageblock. It is used to solve incorrect 445 * freepage counting problem due to racy retrieving migratetype 446 * of pageblock. Protected by zone->lock. 447 */ 448 unsigned long nr_isolate_pageblock; 449 #endif 450 451 #ifdef CONFIG_MEMORY_HOTPLUG 452 /* see spanned/present_pages for more description */ 453 seqlock_t span_seqlock; 454 #endif 455 456 int initialized; 457 458 /* Write-intensive fields used from the page allocator */ 459 ZONE_PADDING(_pad1_) 460 461 /* free areas of different sizes */ 462 struct free_area free_area[MAX_ORDER]; 463 464 /* zone flags, see below */ 465 unsigned long flags; 466 467 /* Primarily protects free_area */ 468 spinlock_t lock; 469 470 /* Write-intensive fields used by compaction and vmstats. */ 471 ZONE_PADDING(_pad2_) 472 473 /* 474 * When free pages are below this point, additional steps are taken 475 * when reading the number of free pages to avoid per-cpu counter 476 * drift allowing watermarks to be breached 477 */ 478 unsigned long percpu_drift_mark; 479 480 #if defined CONFIG_COMPACTION || defined CONFIG_CMA 481 /* pfn where compaction free scanner should start */ 482 unsigned long compact_cached_free_pfn; 483 /* pfn where async and sync compaction migration scanner should start */ 484 unsigned long compact_cached_migrate_pfn[2]; 485 #endif 486 487 #ifdef CONFIG_COMPACTION 488 /* 489 * On compaction failure, 1<<compact_defer_shift compactions 490 * are skipped before trying again. The number attempted since 491 * last failure is tracked with compact_considered. 492 */ 493 unsigned int compact_considered; 494 unsigned int compact_defer_shift; 495 int compact_order_failed; 496 #endif 497 498 #if defined CONFIG_COMPACTION || defined CONFIG_CMA 499 /* Set to true when the PG_migrate_skip bits should be cleared */ 500 bool compact_blockskip_flush; 501 #endif 502 503 bool contiguous; 504 505 ZONE_PADDING(_pad3_) 506 /* Zone statistics */ 507 atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; 508 atomic_long_t vm_numa_stat[NR_VM_NUMA_STAT_ITEMS]; 509 } ____cacheline_internodealigned_in_smp; 510 511 enum pgdat_flags { 512 PGDAT_CONGESTED, /* pgdat has many dirty pages backed by 513 * a congested BDI 514 */ 515 PGDAT_DIRTY, /* reclaim scanning has recently found 516 * many dirty file pages at the tail 517 * of the LRU. 518 */ 519 PGDAT_WRITEBACK, /* reclaim scanning has recently found 520 * many pages under writeback 521 */ 522 PGDAT_RECLAIM_LOCKED, /* prevents concurrent reclaim */ 523 }; 524 525 static inline unsigned long zone_end_pfn(const struct zone *zone) 526 { 527 return zone->zone_start_pfn + zone->spanned_pages; 528 } 529 530 static inline bool zone_spans_pfn(const struct zone *zone, unsigned long pfn) 531 { 532 return zone->zone_start_pfn <= pfn && pfn < zone_end_pfn(zone); 533 } 534 535 static inline bool zone_is_initialized(struct zone *zone) 536 { 537 return zone->initialized; 538 } 539 540 static inline bool zone_is_empty(struct zone *zone) 541 { 542 return zone->spanned_pages == 0; 543 } 544 545 /* 546 * Return true if [start_pfn, start_pfn + nr_pages) range has a non-empty 547 * intersection with the given zone 548 */ 549 static inline bool zone_intersects(struct zone *zone, 550 unsigned long start_pfn, unsigned long nr_pages) 551 { 552 if (zone_is_empty(zone)) 553 return false; 554 if (start_pfn >= zone_end_pfn(zone) || 555 start_pfn + nr_pages <= zone->zone_start_pfn) 556 return false; 557 558 return true; 559 } 560 561 /* 562 * The "priority" of VM scanning is how much of the queues we will scan in one 563 * go. A value of 12 for DEF_PRIORITY implies that we will scan 1/4096th of the 564 * queues ("queue_length >> 12") during an aging round. 565 */ 566 #define DEF_PRIORITY 12 567 568 /* Maximum number of zones on a zonelist */ 569 #define MAX_ZONES_PER_ZONELIST (MAX_NUMNODES * MAX_NR_ZONES) 570 571 enum { 572 ZONELIST_FALLBACK, /* zonelist with fallback */ 573 #ifdef CONFIG_NUMA 574 /* 575 * The NUMA zonelists are doubled because we need zonelists that 576 * restrict the allocations to a single node for __GFP_THISNODE. 577 */ 578 ZONELIST_NOFALLBACK, /* zonelist without fallback (__GFP_THISNODE) */ 579 #endif 580 MAX_ZONELISTS 581 }; 582 583 /* 584 * This struct contains information about a zone in a zonelist. It is stored 585 * here to avoid dereferences into large structures and lookups of tables 586 */ 587 struct zoneref { 588 struct zone *zone; /* Pointer to actual zone */ 589 int zone_idx; /* zone_idx(zoneref->zone) */ 590 }; 591 592 /* 593 * One allocation request operates on a zonelist. A zonelist 594 * is a list of zones, the first one is the 'goal' of the 595 * allocation, the other zones are fallback zones, in decreasing 596 * priority. 597 * 598 * To speed the reading of the zonelist, the zonerefs contain the zone index 599 * of the entry being read. Helper functions to access information given 600 * a struct zoneref are 601 * 602 * zonelist_zone() - Return the struct zone * for an entry in _zonerefs 603 * zonelist_zone_idx() - Return the index of the zone for an entry 604 * zonelist_node_idx() - Return the index of the node for an entry 605 */ 606 struct zonelist { 607 struct zoneref _zonerefs[MAX_ZONES_PER_ZONELIST + 1]; 608 }; 609 610 #ifndef CONFIG_DISCONTIGMEM 611 /* The array of struct pages - for discontigmem use pgdat->lmem_map */ 612 extern struct page *mem_map; 613 #endif 614 615 /* 616 * On NUMA machines, each NUMA node would have a pg_data_t to describe 617 * it's memory layout. On UMA machines there is a single pglist_data which 618 * describes the whole memory. 619 * 620 * Memory statistics and page replacement data structures are maintained on a 621 * per-zone basis. 622 */ 623 struct bootmem_data; 624 typedef struct pglist_data { 625 struct zone node_zones[MAX_NR_ZONES]; 626 struct zonelist node_zonelists[MAX_ZONELISTS]; 627 int nr_zones; 628 #ifdef CONFIG_FLAT_NODE_MEM_MAP /* means !SPARSEMEM */ 629 struct page *node_mem_map; 630 #ifdef CONFIG_PAGE_EXTENSION 631 struct page_ext *node_page_ext; 632 #endif 633 #endif 634 #ifndef CONFIG_NO_BOOTMEM 635 struct bootmem_data *bdata; 636 #endif 637 #if defined(CONFIG_MEMORY_HOTPLUG) || defined(CONFIG_DEFERRED_STRUCT_PAGE_INIT) 638 /* 639 * Must be held any time you expect node_start_pfn, node_present_pages 640 * or node_spanned_pages stay constant. Holding this will also 641 * guarantee that any pfn_valid() stays that way. 642 * 643 * pgdat_resize_lock() and pgdat_resize_unlock() are provided to 644 * manipulate node_size_lock without checking for CONFIG_MEMORY_HOTPLUG 645 * or CONFIG_DEFERRED_STRUCT_PAGE_INIT. 646 * 647 * Nests above zone->lock and zone->span_seqlock 648 */ 649 spinlock_t node_size_lock; 650 #endif 651 unsigned long node_start_pfn; 652 unsigned long node_present_pages; /* total number of physical pages */ 653 unsigned long node_spanned_pages; /* total size of physical page 654 range, including holes */ 655 int node_id; 656 wait_queue_head_t kswapd_wait; 657 wait_queue_head_t pfmemalloc_wait; 658 struct task_struct *kswapd; /* Protected by 659 mem_hotplug_begin/end() */ 660 int kswapd_order; 661 enum zone_type kswapd_classzone_idx; 662 663 int kswapd_failures; /* Number of 'reclaimed == 0' runs */ 664 665 #ifdef CONFIG_COMPACTION 666 int kcompactd_max_order; 667 enum zone_type kcompactd_classzone_idx; 668 wait_queue_head_t kcompactd_wait; 669 struct task_struct *kcompactd; 670 #endif 671 #ifdef CONFIG_NUMA_BALANCING 672 /* Lock serializing the migrate rate limiting window */ 673 spinlock_t numabalancing_migrate_lock; 674 675 /* Rate limiting time interval */ 676 unsigned long numabalancing_migrate_next_window; 677 678 /* Number of pages migrated during the rate limiting time interval */ 679 unsigned long numabalancing_migrate_nr_pages; 680 #endif 681 /* 682 * This is a per-node reserve of pages that are not available 683 * to userspace allocations. 684 */ 685 unsigned long totalreserve_pages; 686 687 #ifdef CONFIG_NUMA 688 /* 689 * zone reclaim becomes active if more unmapped pages exist. 690 */ 691 unsigned long min_unmapped_pages; 692 unsigned long min_slab_pages; 693 #endif /* CONFIG_NUMA */ 694 695 /* Write-intensive fields used by page reclaim */ 696 ZONE_PADDING(_pad1_) 697 spinlock_t lru_lock; 698 699 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 700 /* 701 * If memory initialisation on large machines is deferred then this 702 * is the first PFN that needs to be initialised. 703 */ 704 unsigned long first_deferred_pfn; 705 /* Number of non-deferred pages */ 706 unsigned long static_init_pgcnt; 707 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 708 709 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 710 spinlock_t split_queue_lock; 711 struct list_head split_queue; 712 unsigned long split_queue_len; 713 #endif 714 715 /* Fields commonly accessed by the page reclaim scanner */ 716 struct lruvec lruvec; 717 718 unsigned long flags; 719 720 ZONE_PADDING(_pad2_) 721 722 /* Per-node vmstats */ 723 struct per_cpu_nodestat __percpu *per_cpu_nodestats; 724 atomic_long_t vm_stat[NR_VM_NODE_STAT_ITEMS]; 725 } pg_data_t; 726 727 #define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages) 728 #define node_spanned_pages(nid) (NODE_DATA(nid)->node_spanned_pages) 729 #ifdef CONFIG_FLAT_NODE_MEM_MAP 730 #define pgdat_page_nr(pgdat, pagenr) ((pgdat)->node_mem_map + (pagenr)) 731 #else 732 #define pgdat_page_nr(pgdat, pagenr) pfn_to_page((pgdat)->node_start_pfn + (pagenr)) 733 #endif 734 #define nid_page_nr(nid, pagenr) pgdat_page_nr(NODE_DATA(nid),(pagenr)) 735 736 #define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn) 737 #define node_end_pfn(nid) pgdat_end_pfn(NODE_DATA(nid)) 738 static inline spinlock_t *zone_lru_lock(struct zone *zone) 739 { 740 return &zone->zone_pgdat->lru_lock; 741 } 742 743 static inline struct lruvec *node_lruvec(struct pglist_data *pgdat) 744 { 745 return &pgdat->lruvec; 746 } 747 748 static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat) 749 { 750 return pgdat->node_start_pfn + pgdat->node_spanned_pages; 751 } 752 753 static inline bool pgdat_is_empty(pg_data_t *pgdat) 754 { 755 return !pgdat->node_start_pfn && !pgdat->node_spanned_pages; 756 } 757 758 static inline int zone_id(const struct zone *zone) 759 { 760 struct pglist_data *pgdat = zone->zone_pgdat; 761 762 return zone - pgdat->node_zones; 763 } 764 765 #ifdef CONFIG_ZONE_DEVICE 766 static inline bool is_dev_zone(const struct zone *zone) 767 { 768 return zone_id(zone) == ZONE_DEVICE; 769 } 770 #else 771 static inline bool is_dev_zone(const struct zone *zone) 772 { 773 return false; 774 } 775 #endif 776 777 #include <linux/memory_hotplug.h> 778 779 void build_all_zonelists(pg_data_t *pgdat); 780 void wakeup_kswapd(struct zone *zone, gfp_t gfp_mask, int order, 781 enum zone_type classzone_idx); 782 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, 783 int classzone_idx, unsigned int alloc_flags, 784 long free_pages); 785 bool zone_watermark_ok(struct zone *z, unsigned int order, 786 unsigned long mark, int classzone_idx, 787 unsigned int alloc_flags); 788 bool zone_watermark_ok_safe(struct zone *z, unsigned int order, 789 unsigned long mark, int classzone_idx); 790 enum memmap_context { 791 MEMMAP_EARLY, 792 MEMMAP_HOTPLUG, 793 }; 794 extern void init_currently_empty_zone(struct zone *zone, unsigned long start_pfn, 795 unsigned long size); 796 797 extern void lruvec_init(struct lruvec *lruvec); 798 799 static inline struct pglist_data *lruvec_pgdat(struct lruvec *lruvec) 800 { 801 #ifdef CONFIG_MEMCG 802 return lruvec->pgdat; 803 #else 804 return container_of(lruvec, struct pglist_data, lruvec); 805 #endif 806 } 807 808 extern unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx); 809 810 #ifdef CONFIG_HAVE_MEMORY_PRESENT 811 void memory_present(int nid, unsigned long start, unsigned long end); 812 #else 813 static inline void memory_present(int nid, unsigned long start, unsigned long end) {} 814 #endif 815 816 #ifdef CONFIG_HAVE_MEMORYLESS_NODES 817 int local_memory_node(int node_id); 818 #else 819 static inline int local_memory_node(int node_id) { return node_id; }; 820 #endif 821 822 /* 823 * zone_idx() returns 0 for the ZONE_DMA zone, 1 for the ZONE_NORMAL zone, etc. 824 */ 825 #define zone_idx(zone) ((zone) - (zone)->zone_pgdat->node_zones) 826 827 /* 828 * Returns true if a zone has pages managed by the buddy allocator. 829 * All the reclaim decisions have to use this function rather than 830 * populated_zone(). If the whole zone is reserved then we can easily 831 * end up with populated_zone() && !managed_zone(). 832 */ 833 static inline bool managed_zone(struct zone *zone) 834 { 835 return zone->managed_pages; 836 } 837 838 /* Returns true if a zone has memory */ 839 static inline bool populated_zone(struct zone *zone) 840 { 841 return zone->present_pages; 842 } 843 844 extern int movable_zone; 845 846 #ifdef CONFIG_HIGHMEM 847 static inline int zone_movable_is_highmem(void) 848 { 849 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 850 return movable_zone == ZONE_HIGHMEM; 851 #else 852 return (ZONE_MOVABLE - 1) == ZONE_HIGHMEM; 853 #endif 854 } 855 #endif 856 857 static inline int is_highmem_idx(enum zone_type idx) 858 { 859 #ifdef CONFIG_HIGHMEM 860 return (idx == ZONE_HIGHMEM || 861 (idx == ZONE_MOVABLE && zone_movable_is_highmem())); 862 #else 863 return 0; 864 #endif 865 } 866 867 /** 868 * is_highmem - helper function to quickly check if a struct zone is a 869 * highmem zone or not. This is an attempt to keep references 870 * to ZONE_{DMA/NORMAL/HIGHMEM/etc} in general code to a minimum. 871 * @zone - pointer to struct zone variable 872 */ 873 static inline int is_highmem(struct zone *zone) 874 { 875 #ifdef CONFIG_HIGHMEM 876 return is_highmem_idx(zone_idx(zone)); 877 #else 878 return 0; 879 #endif 880 } 881 882 /* These two functions are used to setup the per zone pages min values */ 883 struct ctl_table; 884 int min_free_kbytes_sysctl_handler(struct ctl_table *, int, 885 void __user *, size_t *, loff_t *); 886 int watermark_scale_factor_sysctl_handler(struct ctl_table *, int, 887 void __user *, size_t *, loff_t *); 888 extern int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES]; 889 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *, int, 890 void __user *, size_t *, loff_t *); 891 int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *, int, 892 void __user *, size_t *, loff_t *); 893 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *, int, 894 void __user *, size_t *, loff_t *); 895 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *, int, 896 void __user *, size_t *, loff_t *); 897 898 extern int numa_zonelist_order_handler(struct ctl_table *, int, 899 void __user *, size_t *, loff_t *); 900 extern char numa_zonelist_order[]; 901 #define NUMA_ZONELIST_ORDER_LEN 16 902 903 #ifndef CONFIG_NEED_MULTIPLE_NODES 904 905 extern struct pglist_data contig_page_data; 906 #define NODE_DATA(nid) (&contig_page_data) 907 #define NODE_MEM_MAP(nid) mem_map 908 909 #else /* CONFIG_NEED_MULTIPLE_NODES */ 910 911 #include <asm/mmzone.h> 912 913 #endif /* !CONFIG_NEED_MULTIPLE_NODES */ 914 915 extern struct pglist_data *first_online_pgdat(void); 916 extern struct pglist_data *next_online_pgdat(struct pglist_data *pgdat); 917 extern struct zone *next_zone(struct zone *zone); 918 919 /** 920 * for_each_online_pgdat - helper macro to iterate over all online nodes 921 * @pgdat - pointer to a pg_data_t variable 922 */ 923 #define for_each_online_pgdat(pgdat) \ 924 for (pgdat = first_online_pgdat(); \ 925 pgdat; \ 926 pgdat = next_online_pgdat(pgdat)) 927 /** 928 * for_each_zone - helper macro to iterate over all memory zones 929 * @zone - pointer to struct zone variable 930 * 931 * The user only needs to declare the zone variable, for_each_zone 932 * fills it in. 933 */ 934 #define for_each_zone(zone) \ 935 for (zone = (first_online_pgdat())->node_zones; \ 936 zone; \ 937 zone = next_zone(zone)) 938 939 #define for_each_populated_zone(zone) \ 940 for (zone = (first_online_pgdat())->node_zones; \ 941 zone; \ 942 zone = next_zone(zone)) \ 943 if (!populated_zone(zone)) \ 944 ; /* do nothing */ \ 945 else 946 947 static inline struct zone *zonelist_zone(struct zoneref *zoneref) 948 { 949 return zoneref->zone; 950 } 951 952 static inline int zonelist_zone_idx(struct zoneref *zoneref) 953 { 954 return zoneref->zone_idx; 955 } 956 957 static inline int zonelist_node_idx(struct zoneref *zoneref) 958 { 959 #ifdef CONFIG_NUMA 960 /* zone_to_nid not available in this context */ 961 return zoneref->zone->node; 962 #else 963 return 0; 964 #endif /* CONFIG_NUMA */ 965 } 966 967 struct zoneref *__next_zones_zonelist(struct zoneref *z, 968 enum zone_type highest_zoneidx, 969 nodemask_t *nodes); 970 971 /** 972 * next_zones_zonelist - Returns the next zone at or below highest_zoneidx within the allowed nodemask using a cursor within a zonelist as a starting point 973 * @z - The cursor used as a starting point for the search 974 * @highest_zoneidx - The zone index of the highest zone to return 975 * @nodes - An optional nodemask to filter the zonelist with 976 * 977 * This function returns the next zone at or below a given zone index that is 978 * within the allowed nodemask using a cursor as the starting point for the 979 * search. The zoneref returned is a cursor that represents the current zone 980 * being examined. It should be advanced by one before calling 981 * next_zones_zonelist again. 982 */ 983 static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z, 984 enum zone_type highest_zoneidx, 985 nodemask_t *nodes) 986 { 987 if (likely(!nodes && zonelist_zone_idx(z) <= highest_zoneidx)) 988 return z; 989 return __next_zones_zonelist(z, highest_zoneidx, nodes); 990 } 991 992 /** 993 * first_zones_zonelist - Returns the first zone at or below highest_zoneidx within the allowed nodemask in a zonelist 994 * @zonelist - The zonelist to search for a suitable zone 995 * @highest_zoneidx - The zone index of the highest zone to return 996 * @nodes - An optional nodemask to filter the zonelist with 997 * @return - Zoneref pointer for the first suitable zone found (see below) 998 * 999 * This function returns the first zone at or below a given zone index that is 1000 * within the allowed nodemask. The zoneref returned is a cursor that can be 1001 * used to iterate the zonelist with next_zones_zonelist by advancing it by 1002 * one before calling. 1003 * 1004 * When no eligible zone is found, zoneref->zone is NULL (zoneref itself is 1005 * never NULL). This may happen either genuinely, or due to concurrent nodemask 1006 * update due to cpuset modification. 1007 */ 1008 static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist, 1009 enum zone_type highest_zoneidx, 1010 nodemask_t *nodes) 1011 { 1012 return next_zones_zonelist(zonelist->_zonerefs, 1013 highest_zoneidx, nodes); 1014 } 1015 1016 /** 1017 * for_each_zone_zonelist_nodemask - helper macro to iterate over valid zones in a zonelist at or below a given zone index and within a nodemask 1018 * @zone - The current zone in the iterator 1019 * @z - The current pointer within zonelist->zones being iterated 1020 * @zlist - The zonelist being iterated 1021 * @highidx - The zone index of the highest zone to return 1022 * @nodemask - Nodemask allowed by the allocator 1023 * 1024 * This iterator iterates though all zones at or below a given zone index and 1025 * within a given nodemask 1026 */ 1027 #define for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, nodemask) \ 1028 for (z = first_zones_zonelist(zlist, highidx, nodemask), zone = zonelist_zone(z); \ 1029 zone; \ 1030 z = next_zones_zonelist(++z, highidx, nodemask), \ 1031 zone = zonelist_zone(z)) 1032 1033 #define for_next_zone_zonelist_nodemask(zone, z, zlist, highidx, nodemask) \ 1034 for (zone = z->zone; \ 1035 zone; \ 1036 z = next_zones_zonelist(++z, highidx, nodemask), \ 1037 zone = zonelist_zone(z)) 1038 1039 1040 /** 1041 * for_each_zone_zonelist - helper macro to iterate over valid zones in a zonelist at or below a given zone index 1042 * @zone - The current zone in the iterator 1043 * @z - The current pointer within zonelist->zones being iterated 1044 * @zlist - The zonelist being iterated 1045 * @highidx - The zone index of the highest zone to return 1046 * 1047 * This iterator iterates though all zones at or below a given zone index. 1048 */ 1049 #define for_each_zone_zonelist(zone, z, zlist, highidx) \ 1050 for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, NULL) 1051 1052 #ifdef CONFIG_SPARSEMEM 1053 #include <asm/sparsemem.h> 1054 #endif 1055 1056 #if !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) && \ 1057 !defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) 1058 static inline unsigned long early_pfn_to_nid(unsigned long pfn) 1059 { 1060 BUILD_BUG_ON(IS_ENABLED(CONFIG_NUMA)); 1061 return 0; 1062 } 1063 #endif 1064 1065 #ifdef CONFIG_FLATMEM 1066 #define pfn_to_nid(pfn) (0) 1067 #endif 1068 1069 #ifdef CONFIG_SPARSEMEM 1070 1071 /* 1072 * SECTION_SHIFT #bits space required to store a section # 1073 * 1074 * PA_SECTION_SHIFT physical address to/from section number 1075 * PFN_SECTION_SHIFT pfn to/from section number 1076 */ 1077 #define PA_SECTION_SHIFT (SECTION_SIZE_BITS) 1078 #define PFN_SECTION_SHIFT (SECTION_SIZE_BITS - PAGE_SHIFT) 1079 1080 #define NR_MEM_SECTIONS (1UL << SECTIONS_SHIFT) 1081 1082 #define PAGES_PER_SECTION (1UL << PFN_SECTION_SHIFT) 1083 #define PAGE_SECTION_MASK (~(PAGES_PER_SECTION-1)) 1084 1085 #define SECTION_BLOCKFLAGS_BITS \ 1086 ((1UL << (PFN_SECTION_SHIFT - pageblock_order)) * NR_PAGEBLOCK_BITS) 1087 1088 #if (MAX_ORDER - 1 + PAGE_SHIFT) > SECTION_SIZE_BITS 1089 #error Allocator MAX_ORDER exceeds SECTION_SIZE 1090 #endif 1091 1092 static inline unsigned long pfn_to_section_nr(unsigned long pfn) 1093 { 1094 return pfn >> PFN_SECTION_SHIFT; 1095 } 1096 static inline unsigned long section_nr_to_pfn(unsigned long sec) 1097 { 1098 return sec << PFN_SECTION_SHIFT; 1099 } 1100 1101 #define SECTION_ALIGN_UP(pfn) (((pfn) + PAGES_PER_SECTION - 1) & PAGE_SECTION_MASK) 1102 #define SECTION_ALIGN_DOWN(pfn) ((pfn) & PAGE_SECTION_MASK) 1103 1104 struct page; 1105 struct page_ext; 1106 struct mem_section { 1107 /* 1108 * This is, logically, a pointer to an array of struct 1109 * pages. However, it is stored with some other magic. 1110 * (see sparse.c::sparse_init_one_section()) 1111 * 1112 * Additionally during early boot we encode node id of 1113 * the location of the section here to guide allocation. 1114 * (see sparse.c::memory_present()) 1115 * 1116 * Making it a UL at least makes someone do a cast 1117 * before using it wrong. 1118 */ 1119 unsigned long section_mem_map; 1120 1121 /* See declaration of similar field in struct zone */ 1122 unsigned long *pageblock_flags; 1123 #ifdef CONFIG_PAGE_EXTENSION 1124 /* 1125 * If SPARSEMEM, pgdat doesn't have page_ext pointer. We use 1126 * section. (see page_ext.h about this.) 1127 */ 1128 struct page_ext *page_ext; 1129 unsigned long pad; 1130 #endif 1131 /* 1132 * WARNING: mem_section must be a power-of-2 in size for the 1133 * calculation and use of SECTION_ROOT_MASK to make sense. 1134 */ 1135 }; 1136 1137 #ifdef CONFIG_SPARSEMEM_EXTREME 1138 #define SECTIONS_PER_ROOT (PAGE_SIZE / sizeof (struct mem_section)) 1139 #else 1140 #define SECTIONS_PER_ROOT 1 1141 #endif 1142 1143 #define SECTION_NR_TO_ROOT(sec) ((sec) / SECTIONS_PER_ROOT) 1144 #define NR_SECTION_ROOTS DIV_ROUND_UP(NR_MEM_SECTIONS, SECTIONS_PER_ROOT) 1145 #define SECTION_ROOT_MASK (SECTIONS_PER_ROOT - 1) 1146 1147 #ifdef CONFIG_SPARSEMEM_EXTREME 1148 extern struct mem_section **mem_section; 1149 #else 1150 extern struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT]; 1151 #endif 1152 1153 static inline struct mem_section *__nr_to_section(unsigned long nr) 1154 { 1155 #ifdef CONFIG_SPARSEMEM_EXTREME 1156 if (!mem_section) 1157 return NULL; 1158 #endif 1159 if (!mem_section[SECTION_NR_TO_ROOT(nr)]) 1160 return NULL; 1161 return &mem_section[SECTION_NR_TO_ROOT(nr)][nr & SECTION_ROOT_MASK]; 1162 } 1163 extern int __section_nr(struct mem_section* ms); 1164 extern unsigned long usemap_size(void); 1165 1166 /* 1167 * We use the lower bits of the mem_map pointer to store 1168 * a little bit of information. The pointer is calculated 1169 * as mem_map - section_nr_to_pfn(pnum). The result is 1170 * aligned to the minimum alignment of the two values: 1171 * 1. All mem_map arrays are page-aligned. 1172 * 2. section_nr_to_pfn() always clears PFN_SECTION_SHIFT 1173 * lowest bits. PFN_SECTION_SHIFT is arch-specific 1174 * (equal SECTION_SIZE_BITS - PAGE_SHIFT), and the 1175 * worst combination is powerpc with 256k pages, 1176 * which results in PFN_SECTION_SHIFT equal 6. 1177 * To sum it up, at least 6 bits are available. 1178 */ 1179 #define SECTION_MARKED_PRESENT (1UL<<0) 1180 #define SECTION_HAS_MEM_MAP (1UL<<1) 1181 #define SECTION_IS_ONLINE (1UL<<2) 1182 #define SECTION_MAP_LAST_BIT (1UL<<3) 1183 #define SECTION_MAP_MASK (~(SECTION_MAP_LAST_BIT-1)) 1184 #define SECTION_NID_SHIFT 3 1185 1186 static inline struct page *__section_mem_map_addr(struct mem_section *section) 1187 { 1188 unsigned long map = section->section_mem_map; 1189 map &= SECTION_MAP_MASK; 1190 return (struct page *)map; 1191 } 1192 1193 static inline int present_section(struct mem_section *section) 1194 { 1195 return (section && (section->section_mem_map & SECTION_MARKED_PRESENT)); 1196 } 1197 1198 static inline int present_section_nr(unsigned long nr) 1199 { 1200 return present_section(__nr_to_section(nr)); 1201 } 1202 1203 static inline int valid_section(struct mem_section *section) 1204 { 1205 return (section && (section->section_mem_map & SECTION_HAS_MEM_MAP)); 1206 } 1207 1208 static inline int valid_section_nr(unsigned long nr) 1209 { 1210 return valid_section(__nr_to_section(nr)); 1211 } 1212 1213 static inline int online_section(struct mem_section *section) 1214 { 1215 return (section && (section->section_mem_map & SECTION_IS_ONLINE)); 1216 } 1217 1218 static inline int online_section_nr(unsigned long nr) 1219 { 1220 return online_section(__nr_to_section(nr)); 1221 } 1222 1223 #ifdef CONFIG_MEMORY_HOTPLUG 1224 void online_mem_sections(unsigned long start_pfn, unsigned long end_pfn); 1225 #ifdef CONFIG_MEMORY_HOTREMOVE 1226 void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn); 1227 #endif 1228 #endif 1229 1230 static inline struct mem_section *__pfn_to_section(unsigned long pfn) 1231 { 1232 return __nr_to_section(pfn_to_section_nr(pfn)); 1233 } 1234 1235 extern int __highest_present_section_nr; 1236 1237 #ifndef CONFIG_HAVE_ARCH_PFN_VALID 1238 static inline int pfn_valid(unsigned long pfn) 1239 { 1240 if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS) 1241 return 0; 1242 return valid_section(__nr_to_section(pfn_to_section_nr(pfn))); 1243 } 1244 #endif 1245 1246 static inline int pfn_present(unsigned long pfn) 1247 { 1248 if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS) 1249 return 0; 1250 return present_section(__nr_to_section(pfn_to_section_nr(pfn))); 1251 } 1252 1253 /* 1254 * These are _only_ used during initialisation, therefore they 1255 * can use __initdata ... They could have names to indicate 1256 * this restriction. 1257 */ 1258 #ifdef CONFIG_NUMA 1259 #define pfn_to_nid(pfn) \ 1260 ({ \ 1261 unsigned long __pfn_to_nid_pfn = (pfn); \ 1262 page_to_nid(pfn_to_page(__pfn_to_nid_pfn)); \ 1263 }) 1264 #else 1265 #define pfn_to_nid(pfn) (0) 1266 #endif 1267 1268 #define early_pfn_valid(pfn) pfn_valid(pfn) 1269 void sparse_init(void); 1270 #else 1271 #define sparse_init() do {} while (0) 1272 #define sparse_index_init(_sec, _nid) do {} while (0) 1273 #endif /* CONFIG_SPARSEMEM */ 1274 1275 /* 1276 * During memory init memblocks map pfns to nids. The search is expensive and 1277 * this caches recent lookups. The implementation of __early_pfn_to_nid 1278 * may treat start/end as pfns or sections. 1279 */ 1280 struct mminit_pfnnid_cache { 1281 unsigned long last_start; 1282 unsigned long last_end; 1283 int last_nid; 1284 }; 1285 1286 #ifndef early_pfn_valid 1287 #define early_pfn_valid(pfn) (1) 1288 #endif 1289 1290 void memory_present(int nid, unsigned long start, unsigned long end); 1291 1292 /* 1293 * If it is possible to have holes within a MAX_ORDER_NR_PAGES, then we 1294 * need to check pfn validility within that MAX_ORDER_NR_PAGES block. 1295 * pfn_valid_within() should be used in this case; we optimise this away 1296 * when we have no holes within a MAX_ORDER_NR_PAGES block. 1297 */ 1298 #ifdef CONFIG_HOLES_IN_ZONE 1299 #define pfn_valid_within(pfn) pfn_valid(pfn) 1300 #else 1301 #define pfn_valid_within(pfn) (1) 1302 #endif 1303 1304 #ifdef CONFIG_ARCH_HAS_HOLES_MEMORYMODEL 1305 /* 1306 * pfn_valid() is meant to be able to tell if a given PFN has valid memmap 1307 * associated with it or not. This means that a struct page exists for this 1308 * pfn. The caller cannot assume the page is fully initialized in general. 1309 * Hotplugable pages might not have been onlined yet. pfn_to_online_page() 1310 * will ensure the struct page is fully online and initialized. Special pages 1311 * (e.g. ZONE_DEVICE) are never onlined and should be treated accordingly. 1312 * 1313 * In FLATMEM, it is expected that holes always have valid memmap as long as 1314 * there is valid PFNs either side of the hole. In SPARSEMEM, it is assumed 1315 * that a valid section has a memmap for the entire section. 1316 * 1317 * However, an ARM, and maybe other embedded architectures in the future 1318 * free memmap backing holes to save memory on the assumption the memmap is 1319 * never used. The page_zone linkages are then broken even though pfn_valid() 1320 * returns true. A walker of the full memmap must then do this additional 1321 * check to ensure the memmap they are looking at is sane by making sure 1322 * the zone and PFN linkages are still valid. This is expensive, but walkers 1323 * of the full memmap are extremely rare. 1324 */ 1325 bool memmap_valid_within(unsigned long pfn, 1326 struct page *page, struct zone *zone); 1327 #else 1328 static inline bool memmap_valid_within(unsigned long pfn, 1329 struct page *page, struct zone *zone) 1330 { 1331 return true; 1332 } 1333 #endif /* CONFIG_ARCH_HAS_HOLES_MEMORYMODEL */ 1334 1335 #endif /* !__GENERATING_BOUNDS.H */ 1336 #endif /* !__ASSEMBLY__ */ 1337 #endif /* _LINUX_MMZONE_H */ 1338