12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later
295f72d1eSYinghai Lu /*
395f72d1eSYinghai Lu * Procedures for maintaining information about logical memory blocks.
495f72d1eSYinghai Lu *
595f72d1eSYinghai Lu * Peter Bergner, IBM Corp. June 2001.
695f72d1eSYinghai Lu * Copyright (C) 2001 Peter Bergner.
795f72d1eSYinghai Lu */
895f72d1eSYinghai Lu
995f72d1eSYinghai Lu #include <linux/kernel.h>
10142b45a7SBenjamin Herrenschmidt #include <linux/slab.h>
1195f72d1eSYinghai Lu #include <linux/init.h>
1295f72d1eSYinghai Lu #include <linux/bitops.h>
13449e8df3SBenjamin Herrenschmidt #include <linux/poison.h>
14c196f76fSBenjamin Herrenschmidt #include <linux/pfn.h>
156d03b885SBenjamin Herrenschmidt #include <linux/debugfs.h>
16514c6032SRandy Dunlap #include <linux/kmemleak.h>
176d03b885SBenjamin Herrenschmidt #include <linux/seq_file.h>
1895f72d1eSYinghai Lu #include <linux/memblock.h>
1995f72d1eSYinghai Lu
20c4c5ad6bSChristoph Hellwig #include <asm/sections.h>
2126f09e9bSSantosh Shilimkar #include <linux/io.h>
2226f09e9bSSantosh Shilimkar
2326f09e9bSSantosh Shilimkar #include "internal.h"
2479442ed1STang Chen
258a5b403dSArd Biesheuvel #define INIT_MEMBLOCK_REGIONS 128
268a5b403dSArd Biesheuvel #define INIT_PHYSMEM_REGIONS 4
278a5b403dSArd Biesheuvel
288a5b403dSArd Biesheuvel #ifndef INIT_MEMBLOCK_RESERVED_REGIONS
298a5b403dSArd Biesheuvel # define INIT_MEMBLOCK_RESERVED_REGIONS INIT_MEMBLOCK_REGIONS
308a5b403dSArd Biesheuvel #endif
318a5b403dSArd Biesheuvel
32450d0e74SZhou Guanghui #ifndef INIT_MEMBLOCK_MEMORY_REGIONS
33450d0e74SZhou Guanghui #define INIT_MEMBLOCK_MEMORY_REGIONS INIT_MEMBLOCK_REGIONS
34450d0e74SZhou Guanghui #endif
35450d0e74SZhou Guanghui
363e039c5cSMike Rapoport /**
373e039c5cSMike Rapoport * DOC: memblock overview
383e039c5cSMike Rapoport *
393e039c5cSMike Rapoport * Memblock is a method of managing memory regions during the early
403e039c5cSMike Rapoport * boot period when the usual kernel memory allocators are not up and
413e039c5cSMike Rapoport * running.
423e039c5cSMike Rapoport *
433e039c5cSMike Rapoport * Memblock views the system memory as collections of contiguous
443e039c5cSMike Rapoport * regions. There are several types of these collections:
453e039c5cSMike Rapoport *
463e039c5cSMike Rapoport * * ``memory`` - describes the physical memory available to the
473e039c5cSMike Rapoport * kernel; this may differ from the actual physical memory installed
483e039c5cSMike Rapoport * in the system, for instance when the memory is restricted with
493e039c5cSMike Rapoport * ``mem=`` command line parameter
503e039c5cSMike Rapoport * * ``reserved`` - describes the regions that were allocated
5177649905SDavid Hildenbrand * * ``physmem`` - describes the actual physical memory available during
5277649905SDavid Hildenbrand * boot regardless of the possible restrictions and memory hot(un)plug;
5377649905SDavid Hildenbrand * the ``physmem`` type is only available on some architectures.
543e039c5cSMike Rapoport *
559303c9d5SMauro Carvalho Chehab * Each region is represented by struct memblock_region that
563e039c5cSMike Rapoport * defines the region extents, its attributes and NUMA node id on NUMA
571bf162e4SMauro Carvalho Chehab * systems. Every memory type is described by the struct memblock_type
581bf162e4SMauro Carvalho Chehab * which contains an array of memory regions along with
5977649905SDavid Hildenbrand * the allocator metadata. The "memory" and "reserved" types are nicely
609303c9d5SMauro Carvalho Chehab * wrapped with struct memblock. This structure is statically
6177649905SDavid Hildenbrand * initialized at build time. The region arrays are initially sized to
62450d0e74SZhou Guanghui * %INIT_MEMBLOCK_MEMORY_REGIONS for "memory" and
63450d0e74SZhou Guanghui * %INIT_MEMBLOCK_RESERVED_REGIONS for "reserved". The region array
64450d0e74SZhou Guanghui * for "physmem" is initially sized to %INIT_PHYSMEM_REGIONS.
656e5af9a8SCao jin * The memblock_allow_resize() enables automatic resizing of the region
666e5af9a8SCao jin * arrays during addition of new regions. This feature should be used
676e5af9a8SCao jin * with care so that memory allocated for the region array will not
686e5af9a8SCao jin * overlap with areas that should be reserved, for example initrd.
693e039c5cSMike Rapoport *
703e039c5cSMike Rapoport * The early architecture setup should tell memblock what the physical
716e5af9a8SCao jin * memory layout is by using memblock_add() or memblock_add_node()
726e5af9a8SCao jin * functions. The first function does not assign the region to a NUMA
736e5af9a8SCao jin * node and it is appropriate for UMA systems. Yet, it is possible to
746e5af9a8SCao jin * use it on NUMA systems as well and assign the region to a NUMA node
756e5af9a8SCao jin * later in the setup process using memblock_set_node(). The
766e5af9a8SCao jin * memblock_add_node() performs such an assignment directly.
773e039c5cSMike Rapoport *
78a2974133SMike Rapoport * Once memblock is setup the memory can be allocated using one of the
79a2974133SMike Rapoport * API variants:
80a2974133SMike Rapoport *
816e5af9a8SCao jin * * memblock_phys_alloc*() - these functions return the **physical**
826e5af9a8SCao jin * address of the allocated memory
836e5af9a8SCao jin * * memblock_alloc*() - these functions return the **virtual** address
846e5af9a8SCao jin * of the allocated memory.
85a2974133SMike Rapoport *
86df1758d9SEthon Paul * Note, that both API variants use implicit assumptions about allowed
87a2974133SMike Rapoport * memory ranges and the fallback methods. Consult the documentation
886e5af9a8SCao jin * of memblock_alloc_internal() and memblock_alloc_range_nid()
896e5af9a8SCao jin * functions for more elaborate description.
903e039c5cSMike Rapoport *
916e5af9a8SCao jin * As the system boot progresses, the architecture specific mem_init()
926e5af9a8SCao jin * function frees all the memory to the buddy page allocator.
933e039c5cSMike Rapoport *
946e5af9a8SCao jin * Unless an architecture enables %CONFIG_ARCH_KEEP_MEMBLOCK, the
9577649905SDavid Hildenbrand * memblock data structures (except "physmem") will be discarded after the
9677649905SDavid Hildenbrand * system initialization completes.
973e039c5cSMike Rapoport */
983e039c5cSMike Rapoport
99a9ee6cf5SMike Rapoport #ifndef CONFIG_NUMA
100bda49a81SMike Rapoport struct pglist_data __refdata contig_page_data;
101bda49a81SMike Rapoport EXPORT_SYMBOL(contig_page_data);
102bda49a81SMike Rapoport #endif
103bda49a81SMike Rapoport
104bda49a81SMike Rapoport unsigned long max_low_pfn;
105bda49a81SMike Rapoport unsigned long min_low_pfn;
106bda49a81SMike Rapoport unsigned long max_pfn;
107bda49a81SMike Rapoport unsigned long long max_possible_pfn;
108bda49a81SMike Rapoport
109450d0e74SZhou Guanghui static struct memblock_region memblock_memory_init_regions[INIT_MEMBLOCK_MEMORY_REGIONS] __initdata_memblock;
1108a5b403dSArd Biesheuvel static struct memblock_region memblock_reserved_init_regions[INIT_MEMBLOCK_RESERVED_REGIONS] __initdata_memblock;
11170210ed9SPhilipp Hachtmann #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
11277649905SDavid Hildenbrand static struct memblock_region memblock_physmem_init_regions[INIT_PHYSMEM_REGIONS];
11370210ed9SPhilipp Hachtmann #endif
114fe091c20STejun Heo
115fe091c20STejun Heo struct memblock memblock __initdata_memblock = {
116fe091c20STejun Heo .memory.regions = memblock_memory_init_regions,
117fe091c20STejun Heo .memory.cnt = 1, /* empty dummy entry */
118450d0e74SZhou Guanghui .memory.max = INIT_MEMBLOCK_MEMORY_REGIONS,
1190262d9c8SHeiko Carstens .memory.name = "memory",
120fe091c20STejun Heo
121fe091c20STejun Heo .reserved.regions = memblock_reserved_init_regions,
122fe091c20STejun Heo .reserved.cnt = 1, /* empty dummy entry */
1238a5b403dSArd Biesheuvel .reserved.max = INIT_MEMBLOCK_RESERVED_REGIONS,
1240262d9c8SHeiko Carstens .reserved.name = "reserved",
125fe091c20STejun Heo
12679442ed1STang Chen .bottom_up = false,
127fe091c20STejun Heo .current_limit = MEMBLOCK_ALLOC_ANYWHERE,
128fe091c20STejun Heo };
12995f72d1eSYinghai Lu
13077649905SDavid Hildenbrand #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
13177649905SDavid Hildenbrand struct memblock_type physmem = {
13277649905SDavid Hildenbrand .regions = memblock_physmem_init_regions,
13377649905SDavid Hildenbrand .cnt = 1, /* empty dummy entry */
13477649905SDavid Hildenbrand .max = INIT_PHYSMEM_REGIONS,
13577649905SDavid Hildenbrand .name = "physmem",
13677649905SDavid Hildenbrand };
13777649905SDavid Hildenbrand #endif
13877649905SDavid Hildenbrand
1399f3d5eaaSMike Rapoport /*
1409f3d5eaaSMike Rapoport * keep a pointer to &memblock.memory in the text section to use it in
1419f3d5eaaSMike Rapoport * __next_mem_range() and its helpers.
1429f3d5eaaSMike Rapoport * For architectures that do not keep memblock data after init, this
1439f3d5eaaSMike Rapoport * pointer will be reset to NULL at memblock_discard()
1449f3d5eaaSMike Rapoport */
1459f3d5eaaSMike Rapoport static __refdata struct memblock_type *memblock_memory = &memblock.memory;
1469f3d5eaaSMike Rapoport
147cd991db8SMike Rapoport #define for_each_memblock_type(i, memblock_type, rgn) \
148cd991db8SMike Rapoport for (i = 0, rgn = &memblock_type->regions[0]; \
149cd991db8SMike Rapoport i < memblock_type->cnt; \
150cd991db8SMike Rapoport i++, rgn = &memblock_type->regions[i])
151cd991db8SMike Rapoport
15287c55870SMike Rapoport #define memblock_dbg(fmt, ...) \
15387c55870SMike Rapoport do { \
15487c55870SMike Rapoport if (memblock_debug) \
15587c55870SMike Rapoport pr_info(fmt, ##__VA_ARGS__); \
15687c55870SMike Rapoport } while (0)
15787c55870SMike Rapoport
15887c55870SMike Rapoport static int memblock_debug __initdata_memblock;
159fc493f83SClaudio Migliorelli static bool system_has_some_mirror __initdata_memblock;
1601aadc056STejun Heo static int memblock_can_resize __initdata_memblock;
161fc493f83SClaudio Migliorelli static int memblock_memory_in_slab __initdata_memblock;
162fc493f83SClaudio Migliorelli static int memblock_reserved_in_slab __initdata_memblock;
16395f72d1eSYinghai Lu
memblock_has_mirror(void)1640db31d63SMa Wupeng bool __init_memblock memblock_has_mirror(void)
1650db31d63SMa Wupeng {
1660db31d63SMa Wupeng return system_has_some_mirror;
1670db31d63SMa Wupeng }
1680db31d63SMa Wupeng
choose_memblock_flags(void)169c366ea89SMike Rapoport static enum memblock_flags __init_memblock choose_memblock_flags(void)
170a3f5bafcSTony Luck {
171a3f5bafcSTony Luck return system_has_some_mirror ? MEMBLOCK_MIRROR : MEMBLOCK_NONE;
172a3f5bafcSTony Luck }
173a3f5bafcSTony Luck
174eb18f1b5STejun Heo /* adjust *@size so that (@base + *@size) doesn't overflow, return new size */
memblock_cap_size(phys_addr_t base,phys_addr_t * size)175eb18f1b5STejun Heo static inline phys_addr_t memblock_cap_size(phys_addr_t base, phys_addr_t *size)
176eb18f1b5STejun Heo {
1771c4bc43dSStefan Agner return *size = min(*size, PHYS_ADDR_MAX - base);
178eb18f1b5STejun Heo }
179eb18f1b5STejun Heo
1806ed311b2SBenjamin Herrenschmidt /*
1816ed311b2SBenjamin Herrenschmidt * Address comparison utilities
1826ed311b2SBenjamin Herrenschmidt */
183*b5bf39cdSAlison Schofield unsigned long __init_memblock
memblock_addrs_overlap(phys_addr_t base1,phys_addr_t size1,phys_addr_t base2,phys_addr_t size2)184*b5bf39cdSAlison Schofield memblock_addrs_overlap(phys_addr_t base1, phys_addr_t size1, phys_addr_t base2,
185*b5bf39cdSAlison Schofield phys_addr_t size2)
18695f72d1eSYinghai Lu {
18795f72d1eSYinghai Lu return ((base1 < (base2 + size2)) && (base2 < (base1 + size1)));
18895f72d1eSYinghai Lu }
18995f72d1eSYinghai Lu
memblock_overlaps_region(struct memblock_type * type,phys_addr_t base,phys_addr_t size)19095cf82ecSTang Chen bool __init_memblock memblock_overlaps_region(struct memblock_type *type,
1912d7d3eb2SH Hartley Sweeten phys_addr_t base, phys_addr_t size)
1926ed311b2SBenjamin Herrenschmidt {
1936ed311b2SBenjamin Herrenschmidt unsigned long i;
1946ed311b2SBenjamin Herrenschmidt
195023accf5SMike Rapoport memblock_cap_size(base, &size);
196023accf5SMike Rapoport
197f14516fbSAlexander Kuleshov for (i = 0; i < type->cnt; i++)
198f14516fbSAlexander Kuleshov if (memblock_addrs_overlap(base, size, type->regions[i].base,
199f14516fbSAlexander Kuleshov type->regions[i].size))
2006ed311b2SBenjamin Herrenschmidt break;
201c5c5c9d1STang Chen return i < type->cnt;
2026ed311b2SBenjamin Herrenschmidt }
2036ed311b2SBenjamin Herrenschmidt
20447cec443SMike Rapoport /**
20579442ed1STang Chen * __memblock_find_range_bottom_up - find free area utility in bottom-up
20679442ed1STang Chen * @start: start of candidate range
20747cec443SMike Rapoport * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
20847cec443SMike Rapoport * %MEMBLOCK_ALLOC_ACCESSIBLE
20979442ed1STang Chen * @size: size of free area to find
21079442ed1STang Chen * @align: alignment of free area to find
211b1154233SGrygorii Strashko * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
212fc6daaf9STony Luck * @flags: pick from blocks based on memory attributes
21379442ed1STang Chen *
21479442ed1STang Chen * Utility called from memblock_find_in_range_node(), find free area bottom-up.
21579442ed1STang Chen *
21647cec443SMike Rapoport * Return:
21779442ed1STang Chen * Found address on success, 0 on failure.
21879442ed1STang Chen */
21979442ed1STang Chen static phys_addr_t __init_memblock
__memblock_find_range_bottom_up(phys_addr_t start,phys_addr_t end,phys_addr_t size,phys_addr_t align,int nid,enum memblock_flags flags)22079442ed1STang Chen __memblock_find_range_bottom_up(phys_addr_t start, phys_addr_t end,
221fc6daaf9STony Luck phys_addr_t size, phys_addr_t align, int nid,
222e1720feeSMike Rapoport enum memblock_flags flags)
22379442ed1STang Chen {
22479442ed1STang Chen phys_addr_t this_start, this_end, cand;
22579442ed1STang Chen u64 i;
22679442ed1STang Chen
227fc6daaf9STony Luck for_each_free_mem_range(i, nid, flags, &this_start, &this_end, NULL) {
22879442ed1STang Chen this_start = clamp(this_start, start, end);
22979442ed1STang Chen this_end = clamp(this_end, start, end);
23079442ed1STang Chen
23179442ed1STang Chen cand = round_up(this_start, align);
23279442ed1STang Chen if (cand < this_end && this_end - cand >= size)
23379442ed1STang Chen return cand;
23479442ed1STang Chen }
23579442ed1STang Chen
23679442ed1STang Chen return 0;
23779442ed1STang Chen }
23879442ed1STang Chen
2397bd0b0f0STejun Heo /**
2401402899eSTang Chen * __memblock_find_range_top_down - find free area utility, in top-down
2411402899eSTang Chen * @start: start of candidate range
24247cec443SMike Rapoport * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
24347cec443SMike Rapoport * %MEMBLOCK_ALLOC_ACCESSIBLE
2441402899eSTang Chen * @size: size of free area to find
2451402899eSTang Chen * @align: alignment of free area to find
246b1154233SGrygorii Strashko * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
247fc6daaf9STony Luck * @flags: pick from blocks based on memory attributes
2481402899eSTang Chen *
2491402899eSTang Chen * Utility called from memblock_find_in_range_node(), find free area top-down.
2501402899eSTang Chen *
25147cec443SMike Rapoport * Return:
25279442ed1STang Chen * Found address on success, 0 on failure.
2531402899eSTang Chen */
2541402899eSTang Chen static phys_addr_t __init_memblock
__memblock_find_range_top_down(phys_addr_t start,phys_addr_t end,phys_addr_t size,phys_addr_t align,int nid,enum memblock_flags flags)2551402899eSTang Chen __memblock_find_range_top_down(phys_addr_t start, phys_addr_t end,
256fc6daaf9STony Luck phys_addr_t size, phys_addr_t align, int nid,
257e1720feeSMike Rapoport enum memblock_flags flags)
2581402899eSTang Chen {
2591402899eSTang Chen phys_addr_t this_start, this_end, cand;
2601402899eSTang Chen u64 i;
2611402899eSTang Chen
262fc6daaf9STony Luck for_each_free_mem_range_reverse(i, nid, flags, &this_start, &this_end,
263fc6daaf9STony Luck NULL) {
2641402899eSTang Chen this_start = clamp(this_start, start, end);
2651402899eSTang Chen this_end = clamp(this_end, start, end);
2661402899eSTang Chen
2671402899eSTang Chen if (this_end < size)
2681402899eSTang Chen continue;
2691402899eSTang Chen
2701402899eSTang Chen cand = round_down(this_end - size, align);
2711402899eSTang Chen if (cand >= this_start)
2721402899eSTang Chen return cand;
2731402899eSTang Chen }
2741402899eSTang Chen
2751402899eSTang Chen return 0;
2761402899eSTang Chen }
2771402899eSTang Chen
2781402899eSTang Chen /**
2797bd0b0f0STejun Heo * memblock_find_in_range_node - find free area in given range and node
2807bd0b0f0STejun Heo * @size: size of free area to find
2817bd0b0f0STejun Heo * @align: alignment of free area to find
28287029ee9SGrygorii Strashko * @start: start of candidate range
28347cec443SMike Rapoport * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
28447cec443SMike Rapoport * %MEMBLOCK_ALLOC_ACCESSIBLE
285b1154233SGrygorii Strashko * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
286fc6daaf9STony Luck * @flags: pick from blocks based on memory attributes
2877bd0b0f0STejun Heo *
2887bd0b0f0STejun Heo * Find @size free area aligned to @align in the specified range and node.
2897bd0b0f0STejun Heo *
29047cec443SMike Rapoport * Return:
29179442ed1STang Chen * Found address on success, 0 on failure.
2926ed311b2SBenjamin Herrenschmidt */
memblock_find_in_range_node(phys_addr_t size,phys_addr_t align,phys_addr_t start,phys_addr_t end,int nid,enum memblock_flags flags)293c366ea89SMike Rapoport static phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size,
29487029ee9SGrygorii Strashko phys_addr_t align, phys_addr_t start,
295e1720feeSMike Rapoport phys_addr_t end, int nid,
296e1720feeSMike Rapoport enum memblock_flags flags)
297f7210e6cSTang Chen {
298f7210e6cSTang Chen /* pump up @end */
299fed84c78SQian Cai if (end == MEMBLOCK_ALLOC_ACCESSIBLE ||
300c6975d7cSQian Cai end == MEMBLOCK_ALLOC_NOLEAKTRACE)
301f7210e6cSTang Chen end = memblock.current_limit;
302f7210e6cSTang Chen
303f7210e6cSTang Chen /* avoid allocating the first page */
304f7210e6cSTang Chen start = max_t(phys_addr_t, start, PAGE_SIZE);
305f7210e6cSTang Chen end = max(start, end);
30679442ed1STang Chen
3072dcb3964SRoman Gushchin if (memblock_bottom_up())
3082dcb3964SRoman Gushchin return __memblock_find_range_bottom_up(start, end, size, align,
3092dcb3964SRoman Gushchin nid, flags);
3102dcb3964SRoman Gushchin else
3112dcb3964SRoman Gushchin return __memblock_find_range_top_down(start, end, size, align,
3122dcb3964SRoman Gushchin nid, flags);
313f7210e6cSTang Chen }
3146ed311b2SBenjamin Herrenschmidt
3157bd0b0f0STejun Heo /**
3167bd0b0f0STejun Heo * memblock_find_in_range - find free area in given range
3177bd0b0f0STejun Heo * @start: start of candidate range
31847cec443SMike Rapoport * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
31947cec443SMike Rapoport * %MEMBLOCK_ALLOC_ACCESSIBLE
3207bd0b0f0STejun Heo * @size: size of free area to find
3217bd0b0f0STejun Heo * @align: alignment of free area to find
3227bd0b0f0STejun Heo *
3237bd0b0f0STejun Heo * Find @size free area aligned to @align in the specified range.
3247bd0b0f0STejun Heo *
32547cec443SMike Rapoport * Return:
32679442ed1STang Chen * Found address on success, 0 on failure.
3277bd0b0f0STejun Heo */
memblock_find_in_range(phys_addr_t start,phys_addr_t end,phys_addr_t size,phys_addr_t align)328a7259df7SMike Rapoport static phys_addr_t __init_memblock memblock_find_in_range(phys_addr_t start,
3297bd0b0f0STejun Heo phys_addr_t end, phys_addr_t size,
3307bd0b0f0STejun Heo phys_addr_t align)
3317bd0b0f0STejun Heo {
332a3f5bafcSTony Luck phys_addr_t ret;
333e1720feeSMike Rapoport enum memblock_flags flags = choose_memblock_flags();
334a3f5bafcSTony Luck
335a3f5bafcSTony Luck again:
336a3f5bafcSTony Luck ret = memblock_find_in_range_node(size, align, start, end,
337a3f5bafcSTony Luck NUMA_NO_NODE, flags);
338a3f5bafcSTony Luck
339a3f5bafcSTony Luck if (!ret && (flags & MEMBLOCK_MIRROR)) {
34014d9a675SMa Wupeng pr_warn_ratelimited("Could not allocate %pap bytes of mirrored memory\n",
341a3f5bafcSTony Luck &size);
342a3f5bafcSTony Luck flags &= ~MEMBLOCK_MIRROR;
343a3f5bafcSTony Luck goto again;
344a3f5bafcSTony Luck }
345a3f5bafcSTony Luck
346a3f5bafcSTony Luck return ret;
3477bd0b0f0STejun Heo }
3487bd0b0f0STejun Heo
memblock_remove_region(struct memblock_type * type,unsigned long r)34910d06439SYinghai Lu static void __init_memblock memblock_remove_region(struct memblock_type *type, unsigned long r)
35095f72d1eSYinghai Lu {
3511440c4e2STejun Heo type->total_size -= type->regions[r].size;
3527c0caeb8STejun Heo memmove(&type->regions[r], &type->regions[r + 1],
3537c0caeb8STejun Heo (type->cnt - (r + 1)) * sizeof(type->regions[r]));
354e3239ff9SBenjamin Herrenschmidt type->cnt--;
35595f72d1eSYinghai Lu
3568f7a6605SBenjamin Herrenschmidt /* Special case for empty arrays */
3578f7a6605SBenjamin Herrenschmidt if (type->cnt == 0) {
3581440c4e2STejun Heo WARN_ON(type->total_size != 0);
3598f7a6605SBenjamin Herrenschmidt type->cnt = 1;
3608f7a6605SBenjamin Herrenschmidt type->regions[0].base = 0;
3618f7a6605SBenjamin Herrenschmidt type->regions[0].size = 0;
36266a20757STang Chen type->regions[0].flags = 0;
3637c0caeb8STejun Heo memblock_set_region_node(&type->regions[0], MAX_NUMNODES);
3648f7a6605SBenjamin Herrenschmidt }
36595f72d1eSYinghai Lu }
36695f72d1eSYinghai Lu
367350e88baSMike Rapoport #ifndef CONFIG_ARCH_KEEP_MEMBLOCK
3683010f876SPavel Tatashin /**
36947cec443SMike Rapoport * memblock_discard - discard memory and reserved arrays if they were allocated
3703010f876SPavel Tatashin */
memblock_discard(void)3713010f876SPavel Tatashin void __init memblock_discard(void)
37229f67386SYinghai Lu {
3733010f876SPavel Tatashin phys_addr_t addr, size;
37429f67386SYinghai Lu
3753010f876SPavel Tatashin if (memblock.reserved.regions != memblock_reserved_init_regions) {
3763010f876SPavel Tatashin addr = __pa(memblock.reserved.regions);
3773010f876SPavel Tatashin size = PAGE_ALIGN(sizeof(struct memblock_region) *
37829f67386SYinghai Lu memblock.reserved.max);
379c94afc46SMiaohe Lin if (memblock_reserved_in_slab)
380c94afc46SMiaohe Lin kfree(memblock.reserved.regions);
381c94afc46SMiaohe Lin else
382621d9739SMike Rapoport memblock_free_late(addr, size);
38329f67386SYinghai Lu }
38429f67386SYinghai Lu
38591b540f9SPavel Tatashin if (memblock.memory.regions != memblock_memory_init_regions) {
3863010f876SPavel Tatashin addr = __pa(memblock.memory.regions);
3873010f876SPavel Tatashin size = PAGE_ALIGN(sizeof(struct memblock_region) *
3885e270e25SPhilipp Hachtmann memblock.memory.max);
389c94afc46SMiaohe Lin if (memblock_memory_in_slab)
390c94afc46SMiaohe Lin kfree(memblock.memory.regions);
391c94afc46SMiaohe Lin else
392621d9739SMike Rapoport memblock_free_late(addr, size);
3935e270e25SPhilipp Hachtmann }
3949f3d5eaaSMike Rapoport
3959f3d5eaaSMike Rapoport memblock_memory = NULL;
3963010f876SPavel Tatashin }
3975e270e25SPhilipp Hachtmann #endif
3985e270e25SPhilipp Hachtmann
39948c3b583SGreg Pearson /**
40048c3b583SGreg Pearson * memblock_double_array - double the size of the memblock regions array
40148c3b583SGreg Pearson * @type: memblock type of the regions array being doubled
40248c3b583SGreg Pearson * @new_area_start: starting address of memory range to avoid overlap with
40348c3b583SGreg Pearson * @new_area_size: size of memory range to avoid overlap with
40448c3b583SGreg Pearson *
40548c3b583SGreg Pearson * Double the size of the @type regions array. If memblock is being used to
40648c3b583SGreg Pearson * allocate memory for a new reserved regions array and there is a previously
40748c3b583SGreg Pearson * allocated memory range [@new_area_start, @new_area_start + @new_area_size]
40848c3b583SGreg Pearson * waiting to be reserved, ensure the memory used by the new array does
40948c3b583SGreg Pearson * not overlap.
41048c3b583SGreg Pearson *
41147cec443SMike Rapoport * Return:
41248c3b583SGreg Pearson * 0 on success, -1 on failure.
41348c3b583SGreg Pearson */
memblock_double_array(struct memblock_type * type,phys_addr_t new_area_start,phys_addr_t new_area_size)41448c3b583SGreg Pearson static int __init_memblock memblock_double_array(struct memblock_type *type,
41548c3b583SGreg Pearson phys_addr_t new_area_start,
41648c3b583SGreg Pearson phys_addr_t new_area_size)
417142b45a7SBenjamin Herrenschmidt {
418142b45a7SBenjamin Herrenschmidt struct memblock_region *new_array, *old_array;
41929f67386SYinghai Lu phys_addr_t old_alloc_size, new_alloc_size;
420a36aab89SMike Rapoport phys_addr_t old_size, new_size, addr, new_end;
421142b45a7SBenjamin Herrenschmidt int use_slab = slab_is_available();
422181eb394SGavin Shan int *in_slab;
423142b45a7SBenjamin Herrenschmidt
424142b45a7SBenjamin Herrenschmidt /* We don't allow resizing until we know about the reserved regions
425142b45a7SBenjamin Herrenschmidt * of memory that aren't suitable for allocation
426142b45a7SBenjamin Herrenschmidt */
427142b45a7SBenjamin Herrenschmidt if (!memblock_can_resize)
428142b45a7SBenjamin Herrenschmidt return -1;
429142b45a7SBenjamin Herrenschmidt
430142b45a7SBenjamin Herrenschmidt /* Calculate new doubled size */
431142b45a7SBenjamin Herrenschmidt old_size = type->max * sizeof(struct memblock_region);
432142b45a7SBenjamin Herrenschmidt new_size = old_size << 1;
43329f67386SYinghai Lu /*
43429f67386SYinghai Lu * We need to allocated new one align to PAGE_SIZE,
43529f67386SYinghai Lu * so we can free them completely later.
43629f67386SYinghai Lu */
43729f67386SYinghai Lu old_alloc_size = PAGE_ALIGN(old_size);
43829f67386SYinghai Lu new_alloc_size = PAGE_ALIGN(new_size);
439142b45a7SBenjamin Herrenschmidt
440181eb394SGavin Shan /* Retrieve the slab flag */
441181eb394SGavin Shan if (type == &memblock.memory)
442181eb394SGavin Shan in_slab = &memblock_memory_in_slab;
443181eb394SGavin Shan else
444181eb394SGavin Shan in_slab = &memblock_reserved_in_slab;
445181eb394SGavin Shan
446a2974133SMike Rapoport /* Try to find some space for it */
447142b45a7SBenjamin Herrenschmidt if (use_slab) {
448142b45a7SBenjamin Herrenschmidt new_array = kmalloc(new_size, GFP_KERNEL);
4491f5026a7STejun Heo addr = new_array ? __pa(new_array) : 0;
4504e2f0775SGavin Shan } else {
45148c3b583SGreg Pearson /* only exclude range when trying to double reserved.regions */
45248c3b583SGreg Pearson if (type != &memblock.reserved)
45348c3b583SGreg Pearson new_area_start = new_area_size = 0;
45448c3b583SGreg Pearson
45548c3b583SGreg Pearson addr = memblock_find_in_range(new_area_start + new_area_size,
45648c3b583SGreg Pearson memblock.current_limit,
45729f67386SYinghai Lu new_alloc_size, PAGE_SIZE);
45848c3b583SGreg Pearson if (!addr && new_area_size)
45948c3b583SGreg Pearson addr = memblock_find_in_range(0,
46048c3b583SGreg Pearson min(new_area_start, memblock.current_limit),
46129f67386SYinghai Lu new_alloc_size, PAGE_SIZE);
46248c3b583SGreg Pearson
46315674868SSachin Kamat new_array = addr ? __va(addr) : NULL;
4644e2f0775SGavin Shan }
4651f5026a7STejun Heo if (!addr) {
466142b45a7SBenjamin Herrenschmidt pr_err("memblock: Failed to double %s array from %ld to %ld entries !\n",
4670262d9c8SHeiko Carstens type->name, type->max, type->max * 2);
468142b45a7SBenjamin Herrenschmidt return -1;
469142b45a7SBenjamin Herrenschmidt }
470142b45a7SBenjamin Herrenschmidt
471a36aab89SMike Rapoport new_end = addr + new_size - 1;
472a36aab89SMike Rapoport memblock_dbg("memblock: %s is doubled to %ld at [%pa-%pa]",
473a36aab89SMike Rapoport type->name, type->max * 2, &addr, &new_end);
474ea9e4376SYinghai Lu
475fd07383bSAndrew Morton /*
476fd07383bSAndrew Morton * Found space, we now need to move the array over before we add the
477fd07383bSAndrew Morton * reserved region since it may be our reserved array itself that is
478fd07383bSAndrew Morton * full.
479142b45a7SBenjamin Herrenschmidt */
480142b45a7SBenjamin Herrenschmidt memcpy(new_array, type->regions, old_size);
481142b45a7SBenjamin Herrenschmidt memset(new_array + type->max, 0, old_size);
482142b45a7SBenjamin Herrenschmidt old_array = type->regions;
483142b45a7SBenjamin Herrenschmidt type->regions = new_array;
484142b45a7SBenjamin Herrenschmidt type->max <<= 1;
485142b45a7SBenjamin Herrenschmidt
486fd07383bSAndrew Morton /* Free old array. We needn't free it if the array is the static one */
487181eb394SGavin Shan if (*in_slab)
488181eb394SGavin Shan kfree(old_array);
489181eb394SGavin Shan else if (old_array != memblock_memory_init_regions &&
490142b45a7SBenjamin Herrenschmidt old_array != memblock_reserved_init_regions)
4914421cca0SMike Rapoport memblock_free(old_array, old_alloc_size);
492142b45a7SBenjamin Herrenschmidt
493fd07383bSAndrew Morton /*
494fd07383bSAndrew Morton * Reserve the new array if that comes from the memblock. Otherwise, we
495fd07383bSAndrew Morton * needn't do it
496181eb394SGavin Shan */
497181eb394SGavin Shan if (!use_slab)
49829f67386SYinghai Lu BUG_ON(memblock_reserve(addr, new_alloc_size));
499181eb394SGavin Shan
500181eb394SGavin Shan /* Update slab flag */
501181eb394SGavin Shan *in_slab = use_slab;
502181eb394SGavin Shan
503142b45a7SBenjamin Herrenschmidt return 0;
504142b45a7SBenjamin Herrenschmidt }
505142b45a7SBenjamin Herrenschmidt
506784656f9STejun Heo /**
507784656f9STejun Heo * memblock_merge_regions - merge neighboring compatible regions
508784656f9STejun Heo * @type: memblock type to scan
5092fe03412SPeng Zhang * @start_rgn: start scanning from (@start_rgn - 1)
5102fe03412SPeng Zhang * @end_rgn: end scanning at (@end_rgn - 1)
5112fe03412SPeng Zhang * Scan @type and merge neighboring compatible regions in [@start_rgn - 1, @end_rgn)
512784656f9STejun Heo */
memblock_merge_regions(struct memblock_type * type,unsigned long start_rgn,unsigned long end_rgn)5132fe03412SPeng Zhang static void __init_memblock memblock_merge_regions(struct memblock_type *type,
5142fe03412SPeng Zhang unsigned long start_rgn,
5152fe03412SPeng Zhang unsigned long end_rgn)
516784656f9STejun Heo {
517784656f9STejun Heo int i = 0;
5182fe03412SPeng Zhang if (start_rgn)
5192fe03412SPeng Zhang i = start_rgn - 1;
5202fe03412SPeng Zhang end_rgn = min(end_rgn, type->cnt - 1);
5212fe03412SPeng Zhang while (i < end_rgn) {
522784656f9STejun Heo struct memblock_region *this = &type->regions[i];
523784656f9STejun Heo struct memblock_region *next = &type->regions[i + 1];
524784656f9STejun Heo
5257c0caeb8STejun Heo if (this->base + this->size != next->base ||
5267c0caeb8STejun Heo memblock_get_region_node(this) !=
52766a20757STang Chen memblock_get_region_node(next) ||
52866a20757STang Chen this->flags != next->flags) {
529784656f9STejun Heo BUG_ON(this->base + this->size > next->base);
530784656f9STejun Heo i++;
531784656f9STejun Heo continue;
532784656f9STejun Heo }
533784656f9STejun Heo
534784656f9STejun Heo this->size += next->size;
535c0232ae8SLin Feng /* move forward from next + 1, index of which is i + 2 */
536c0232ae8SLin Feng memmove(next, next + 1, (type->cnt - (i + 2)) * sizeof(*next));
537784656f9STejun Heo type->cnt--;
5382fe03412SPeng Zhang end_rgn--;
539784656f9STejun Heo }
540784656f9STejun Heo }
541784656f9STejun Heo
542784656f9STejun Heo /**
543784656f9STejun Heo * memblock_insert_region - insert new memblock region
544784656f9STejun Heo * @type: memblock type to insert into
545784656f9STejun Heo * @idx: index for the insertion point
546784656f9STejun Heo * @base: base address of the new region
547784656f9STejun Heo * @size: size of the new region
548209ff86dSTang Chen * @nid: node id of the new region
54966a20757STang Chen * @flags: flags of the new region
550784656f9STejun Heo *
551784656f9STejun Heo * Insert new memblock region [@base, @base + @size) into @type at @idx.
552412d0008SAlexander Kuleshov * @type must already have extra room to accommodate the new region.
553784656f9STejun Heo */
memblock_insert_region(struct memblock_type * type,int idx,phys_addr_t base,phys_addr_t size,int nid,enum memblock_flags flags)554784656f9STejun Heo static void __init_memblock memblock_insert_region(struct memblock_type *type,
555784656f9STejun Heo int idx, phys_addr_t base,
55666a20757STang Chen phys_addr_t size,
557e1720feeSMike Rapoport int nid,
558e1720feeSMike Rapoport enum memblock_flags flags)
559784656f9STejun Heo {
560784656f9STejun Heo struct memblock_region *rgn = &type->regions[idx];
561784656f9STejun Heo
562784656f9STejun Heo BUG_ON(type->cnt >= type->max);
563784656f9STejun Heo memmove(rgn + 1, rgn, (type->cnt - idx) * sizeof(*rgn));
564784656f9STejun Heo rgn->base = base;
565784656f9STejun Heo rgn->size = size;
56666a20757STang Chen rgn->flags = flags;
5677c0caeb8STejun Heo memblock_set_region_node(rgn, nid);
568784656f9STejun Heo type->cnt++;
5691440c4e2STejun Heo type->total_size += size;
570784656f9STejun Heo }
571784656f9STejun Heo
572784656f9STejun Heo /**
573f1af9d3aSPhilipp Hachtmann * memblock_add_range - add new memblock region
574784656f9STejun Heo * @type: memblock type to add new region into
575784656f9STejun Heo * @base: base address of the new region
576784656f9STejun Heo * @size: size of the new region
5777fb0bc3fSTejun Heo * @nid: nid of the new region
57866a20757STang Chen * @flags: flags of the new region
579784656f9STejun Heo *
580784656f9STejun Heo * Add new memblock region [@base, @base + @size) into @type. The new region
581784656f9STejun Heo * is allowed to overlap with existing ones - overlaps don't affect already
582784656f9STejun Heo * existing regions. @type is guaranteed to be minimal (all neighbouring
583784656f9STejun Heo * compatible regions are merged) after the addition.
584784656f9STejun Heo *
58547cec443SMike Rapoport * Return:
586784656f9STejun Heo * 0 on success, -errno on failure.
587784656f9STejun Heo */
memblock_add_range(struct memblock_type * type,phys_addr_t base,phys_addr_t size,int nid,enum memblock_flags flags)58802634a44SAnshuman Khandual static int __init_memblock memblock_add_range(struct memblock_type *type,
58966a20757STang Chen phys_addr_t base, phys_addr_t size,
590e1720feeSMike Rapoport int nid, enum memblock_flags flags)
59195f72d1eSYinghai Lu {
592784656f9STejun Heo bool insert = false;
593eb18f1b5STejun Heo phys_addr_t obase = base;
594eb18f1b5STejun Heo phys_addr_t end = base + memblock_cap_size(base, &size);
5952fe03412SPeng Zhang int idx, nr_new, start_rgn = -1, end_rgn;
5968c9c1701SAlexander Kuleshov struct memblock_region *rgn;
59795f72d1eSYinghai Lu
598b3dc627cSTejun Heo if (!size)
599b3dc627cSTejun Heo return 0;
600b3dc627cSTejun Heo
601784656f9STejun Heo /* special case for empty array */
602784656f9STejun Heo if (type->regions[0].size == 0) {
6031440c4e2STejun Heo WARN_ON(type->cnt != 1 || type->total_size);
604784656f9STejun Heo type->regions[0].base = base;
605784656f9STejun Heo type->regions[0].size = size;
60666a20757STang Chen type->regions[0].flags = flags;
6077fb0bc3fSTejun Heo memblock_set_region_node(&type->regions[0], nid);
6081440c4e2STejun Heo type->total_size = size;
609784656f9STejun Heo return 0;
610784656f9STejun Heo }
61128e1a8f4SJinyu Tang
61228e1a8f4SJinyu Tang /*
61328e1a8f4SJinyu Tang * The worst case is when new range overlaps all existing regions,
61428e1a8f4SJinyu Tang * then we'll need type->cnt + 1 empty regions in @type. So if
615ad500fb2SPeng Zhang * type->cnt * 2 + 1 is less than or equal to type->max, we know
61628e1a8f4SJinyu Tang * that there is enough empty regions in @type, and we can insert
61728e1a8f4SJinyu Tang * regions directly.
61828e1a8f4SJinyu Tang */
619ad500fb2SPeng Zhang if (type->cnt * 2 + 1 <= type->max)
62028e1a8f4SJinyu Tang insert = true;
62128e1a8f4SJinyu Tang
622784656f9STejun Heo repeat:
623784656f9STejun Heo /*
624784656f9STejun Heo * The following is executed twice. Once with %false @insert and
625784656f9STejun Heo * then with %true. The first counts the number of regions needed
626412d0008SAlexander Kuleshov * to accommodate the new area. The second actually inserts them.
627784656f9STejun Heo */
628784656f9STejun Heo base = obase;
629784656f9STejun Heo nr_new = 0;
630784656f9STejun Heo
63166e8b438SGioh Kim for_each_memblock_type(idx, type, rgn) {
632784656f9STejun Heo phys_addr_t rbase = rgn->base;
633784656f9STejun Heo phys_addr_t rend = rbase + rgn->size;
6348f7a6605SBenjamin Herrenschmidt
635784656f9STejun Heo if (rbase >= end)
6368f7a6605SBenjamin Herrenschmidt break;
637784656f9STejun Heo if (rend <= base)
638784656f9STejun Heo continue;
639784656f9STejun Heo /*
640784656f9STejun Heo * @rgn overlaps. If it separates the lower part of new
641784656f9STejun Heo * area, insert that portion.
6428f7a6605SBenjamin Herrenschmidt */
643784656f9STejun Heo if (rbase > base) {
644a9ee6cf5SMike Rapoport #ifdef CONFIG_NUMA
645c0a29498SWei Yang WARN_ON(nid != memblock_get_region_node(rgn));
646c0a29498SWei Yang #endif
6474fcab5f4SWei Yang WARN_ON(flags != rgn->flags);
648784656f9STejun Heo nr_new++;
6492fe03412SPeng Zhang if (insert) {
6502fe03412SPeng Zhang if (start_rgn == -1)
6512fe03412SPeng Zhang start_rgn = idx;
6522fe03412SPeng Zhang end_rgn = idx + 1;
6538c9c1701SAlexander Kuleshov memblock_insert_region(type, idx++, base,
65466a20757STang Chen rbase - base, nid,
65566a20757STang Chen flags);
656784656f9STejun Heo }
6572fe03412SPeng Zhang }
658784656f9STejun Heo /* area below @rend is dealt with, forget about it */
659784656f9STejun Heo base = min(rend, end);
6608f7a6605SBenjamin Herrenschmidt }
6618f7a6605SBenjamin Herrenschmidt
662784656f9STejun Heo /* insert the remaining portion */
663784656f9STejun Heo if (base < end) {
664784656f9STejun Heo nr_new++;
6652fe03412SPeng Zhang if (insert) {
6662fe03412SPeng Zhang if (start_rgn == -1)
6672fe03412SPeng Zhang start_rgn = idx;
6682fe03412SPeng Zhang end_rgn = idx + 1;
6698c9c1701SAlexander Kuleshov memblock_insert_region(type, idx, base, end - base,
67066a20757STang Chen nid, flags);
6718f7a6605SBenjamin Herrenschmidt }
6722fe03412SPeng Zhang }
6738f7a6605SBenjamin Herrenschmidt
674ef3cc4dbSnimisolo if (!nr_new)
675ef3cc4dbSnimisolo return 0;
676ef3cc4dbSnimisolo
677784656f9STejun Heo /*
678784656f9STejun Heo * If this was the first round, resize array and repeat for actual
679784656f9STejun Heo * insertions; otherwise, merge and return.
6808f7a6605SBenjamin Herrenschmidt */
681784656f9STejun Heo if (!insert) {
682784656f9STejun Heo while (type->cnt + nr_new > type->max)
68348c3b583SGreg Pearson if (memblock_double_array(type, obase, size) < 0)
684784656f9STejun Heo return -ENOMEM;
685784656f9STejun Heo insert = true;
686784656f9STejun Heo goto repeat;
68795f72d1eSYinghai Lu } else {
6882fe03412SPeng Zhang memblock_merge_regions(type, start_rgn, end_rgn);
68995f72d1eSYinghai Lu return 0;
69095f72d1eSYinghai Lu }
691784656f9STejun Heo }
69295f72d1eSYinghai Lu
69348a833ccSMike Rapoport /**
69448a833ccSMike Rapoport * memblock_add_node - add new memblock region within a NUMA node
69548a833ccSMike Rapoport * @base: base address of the new region
69648a833ccSMike Rapoport * @size: size of the new region
69748a833ccSMike Rapoport * @nid: nid of the new region
698952eea9bSDavid Hildenbrand * @flags: flags of the new region
69948a833ccSMike Rapoport *
70048a833ccSMike Rapoport * Add new memblock region [@base, @base + @size) to the "memory"
70148a833ccSMike Rapoport * type. See memblock_add_range() description for mode details
70248a833ccSMike Rapoport *
70348a833ccSMike Rapoport * Return:
70448a833ccSMike Rapoport * 0 on success, -errno on failure.
70548a833ccSMike Rapoport */
memblock_add_node(phys_addr_t base,phys_addr_t size,int nid,enum memblock_flags flags)7067fb0bc3fSTejun Heo int __init_memblock memblock_add_node(phys_addr_t base, phys_addr_t size,
707952eea9bSDavid Hildenbrand int nid, enum memblock_flags flags)
7087fb0bc3fSTejun Heo {
70900974b9aSGeert Uytterhoeven phys_addr_t end = base + size - 1;
71000974b9aSGeert Uytterhoeven
711952eea9bSDavid Hildenbrand memblock_dbg("%s: [%pa-%pa] nid=%d flags=%x %pS\n", __func__,
712952eea9bSDavid Hildenbrand &base, &end, nid, flags, (void *)_RET_IP_);
71300974b9aSGeert Uytterhoeven
714952eea9bSDavid Hildenbrand return memblock_add_range(&memblock.memory, base, size, nid, flags);
7157fb0bc3fSTejun Heo }
7167fb0bc3fSTejun Heo
71748a833ccSMike Rapoport /**
71848a833ccSMike Rapoport * memblock_add - add new memblock region
71948a833ccSMike Rapoport * @base: base address of the new region
72048a833ccSMike Rapoport * @size: size of the new region
72148a833ccSMike Rapoport *
72248a833ccSMike Rapoport * Add new memblock region [@base, @base + @size) to the "memory"
72348a833ccSMike Rapoport * type. See memblock_add_range() description for mode details
72448a833ccSMike Rapoport *
72548a833ccSMike Rapoport * Return:
72648a833ccSMike Rapoport * 0 on success, -errno on failure.
72748a833ccSMike Rapoport */
memblock_add(phys_addr_t base,phys_addr_t size)728f705ac4bSAlexander Kuleshov int __init_memblock memblock_add(phys_addr_t base, phys_addr_t size)
7296a4055bcSAlexander Kuleshov {
7305d63f81cSMiles Chen phys_addr_t end = base + size - 1;
7315d63f81cSMiles Chen
732a090d711SAnshuman Khandual memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
7335d63f81cSMiles Chen &base, &end, (void *)_RET_IP_);
7346a4055bcSAlexander Kuleshov
735f705ac4bSAlexander Kuleshov return memblock_add_range(&memblock.memory, base, size, MAX_NUMNODES, 0);
73695f72d1eSYinghai Lu }
73795f72d1eSYinghai Lu
7386a9ceb31STejun Heo /**
7396a9ceb31STejun Heo * memblock_isolate_range - isolate given range into disjoint memblocks
7406a9ceb31STejun Heo * @type: memblock type to isolate range for
7416a9ceb31STejun Heo * @base: base of range to isolate
7426a9ceb31STejun Heo * @size: size of range to isolate
7436a9ceb31STejun Heo * @start_rgn: out parameter for the start of isolated region
7446a9ceb31STejun Heo * @end_rgn: out parameter for the end of isolated region
7456a9ceb31STejun Heo *
7466a9ceb31STejun Heo * Walk @type and ensure that regions don't cross the boundaries defined by
7476a9ceb31STejun Heo * [@base, @base + @size). Crossing regions are split at the boundaries,
7486a9ceb31STejun Heo * which may create at most two more regions. The index of the first
7496a9ceb31STejun Heo * region inside the range is returned in *@start_rgn and end in *@end_rgn.
7506a9ceb31STejun Heo *
75147cec443SMike Rapoport * Return:
7526a9ceb31STejun Heo * 0 on success, -errno on failure.
7536a9ceb31STejun Heo */
memblock_isolate_range(struct memblock_type * type,phys_addr_t base,phys_addr_t size,int * start_rgn,int * end_rgn)7546a9ceb31STejun Heo static int __init_memblock memblock_isolate_range(struct memblock_type *type,
7556a9ceb31STejun Heo phys_addr_t base, phys_addr_t size,
7566a9ceb31STejun Heo int *start_rgn, int *end_rgn)
7576a9ceb31STejun Heo {
758eb18f1b5STejun Heo phys_addr_t end = base + memblock_cap_size(base, &size);
7598c9c1701SAlexander Kuleshov int idx;
7608c9c1701SAlexander Kuleshov struct memblock_region *rgn;
7616a9ceb31STejun Heo
7626a9ceb31STejun Heo *start_rgn = *end_rgn = 0;
7636a9ceb31STejun Heo
764b3dc627cSTejun Heo if (!size)
765b3dc627cSTejun Heo return 0;
766b3dc627cSTejun Heo
7676a9ceb31STejun Heo /* we'll create at most two more regions */
7686a9ceb31STejun Heo while (type->cnt + 2 > type->max)
76948c3b583SGreg Pearson if (memblock_double_array(type, base, size) < 0)
7706a9ceb31STejun Heo return -ENOMEM;
7716a9ceb31STejun Heo
77266e8b438SGioh Kim for_each_memblock_type(idx, type, rgn) {
7736a9ceb31STejun Heo phys_addr_t rbase = rgn->base;
7746a9ceb31STejun Heo phys_addr_t rend = rbase + rgn->size;
7756a9ceb31STejun Heo
7766a9ceb31STejun Heo if (rbase >= end)
7776a9ceb31STejun Heo break;
7786a9ceb31STejun Heo if (rend <= base)
7796a9ceb31STejun Heo continue;
7806a9ceb31STejun Heo
7816a9ceb31STejun Heo if (rbase < base) {
7826a9ceb31STejun Heo /*
7836a9ceb31STejun Heo * @rgn intersects from below. Split and continue
7846a9ceb31STejun Heo * to process the next region - the new top half.
7856a9ceb31STejun Heo */
7866a9ceb31STejun Heo rgn->base = base;
7871440c4e2STejun Heo rgn->size -= base - rbase;
7881440c4e2STejun Heo type->total_size -= base - rbase;
7898c9c1701SAlexander Kuleshov memblock_insert_region(type, idx, rbase, base - rbase,
79066a20757STang Chen memblock_get_region_node(rgn),
79166a20757STang Chen rgn->flags);
7926a9ceb31STejun Heo } else if (rend > end) {
7936a9ceb31STejun Heo /*
7946a9ceb31STejun Heo * @rgn intersects from above. Split and redo the
7956a9ceb31STejun Heo * current region - the new bottom half.
7966a9ceb31STejun Heo */
7976a9ceb31STejun Heo rgn->base = end;
7981440c4e2STejun Heo rgn->size -= end - rbase;
7991440c4e2STejun Heo type->total_size -= end - rbase;
8008c9c1701SAlexander Kuleshov memblock_insert_region(type, idx--, rbase, end - rbase,
80166a20757STang Chen memblock_get_region_node(rgn),
80266a20757STang Chen rgn->flags);
8036a9ceb31STejun Heo } else {
8046a9ceb31STejun Heo /* @rgn is fully contained, record it */
8056a9ceb31STejun Heo if (!*end_rgn)
8068c9c1701SAlexander Kuleshov *start_rgn = idx;
8078c9c1701SAlexander Kuleshov *end_rgn = idx + 1;
8086a9ceb31STejun Heo }
8096a9ceb31STejun Heo }
8106a9ceb31STejun Heo
8116a9ceb31STejun Heo return 0;
8126a9ceb31STejun Heo }
8136a9ceb31STejun Heo
memblock_remove_range(struct memblock_type * type,phys_addr_t base,phys_addr_t size)81435bd16a2SAlexander Kuleshov static int __init_memblock memblock_remove_range(struct memblock_type *type,
8158f7a6605SBenjamin Herrenschmidt phys_addr_t base, phys_addr_t size)
81695f72d1eSYinghai Lu {
81771936180STejun Heo int start_rgn, end_rgn;
81871936180STejun Heo int i, ret;
81995f72d1eSYinghai Lu
82071936180STejun Heo ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
82171936180STejun Heo if (ret)
82271936180STejun Heo return ret;
82395f72d1eSYinghai Lu
82471936180STejun Heo for (i = end_rgn - 1; i >= start_rgn; i--)
82571936180STejun Heo memblock_remove_region(type, i);
82695f72d1eSYinghai Lu return 0;
82795f72d1eSYinghai Lu }
82895f72d1eSYinghai Lu
memblock_remove(phys_addr_t base,phys_addr_t size)829581adcbeSTejun Heo int __init_memblock memblock_remove(phys_addr_t base, phys_addr_t size)
83095f72d1eSYinghai Lu {
83125cf23d7SMinchan Kim phys_addr_t end = base + size - 1;
83225cf23d7SMinchan Kim
833a090d711SAnshuman Khandual memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
83425cf23d7SMinchan Kim &base, &end, (void *)_RET_IP_);
83525cf23d7SMinchan Kim
836f1af9d3aSPhilipp Hachtmann return memblock_remove_range(&memblock.memory, base, size);
83795f72d1eSYinghai Lu }
83895f72d1eSYinghai Lu
8394d72868cSMike Rapoport /**
8404421cca0SMike Rapoport * memblock_free - free boot memory allocation
84177e02cf5SLinus Torvalds * @ptr: starting address of the boot memory allocation
84277e02cf5SLinus Torvalds * @size: size of the boot memory block in bytes
84377e02cf5SLinus Torvalds *
84477e02cf5SLinus Torvalds * Free boot memory block previously allocated by memblock_alloc_xx() API.
84577e02cf5SLinus Torvalds * The freeing memory will not be released to the buddy allocator.
84677e02cf5SLinus Torvalds */
memblock_free(void * ptr,size_t size)8474421cca0SMike Rapoport void __init_memblock memblock_free(void *ptr, size_t size)
84877e02cf5SLinus Torvalds {
84977e02cf5SLinus Torvalds if (ptr)
8503ecc6834SMike Rapoport memblock_phys_free(__pa(ptr), size);
85177e02cf5SLinus Torvalds }
85277e02cf5SLinus Torvalds
85377e02cf5SLinus Torvalds /**
8543ecc6834SMike Rapoport * memblock_phys_free - free boot memory block
8554d72868cSMike Rapoport * @base: phys starting address of the boot memory block
8564d72868cSMike Rapoport * @size: size of the boot memory block in bytes
8574d72868cSMike Rapoport *
858fa81ab49SMiaoqian Lin * Free boot memory block previously allocated by memblock_phys_alloc_xx() API.
8594d72868cSMike Rapoport * The freeing memory will not be released to the buddy allocator.
8604d72868cSMike Rapoport */
memblock_phys_free(phys_addr_t base,phys_addr_t size)8613ecc6834SMike Rapoport int __init_memblock memblock_phys_free(phys_addr_t base, phys_addr_t size)
86295f72d1eSYinghai Lu {
8635d63f81cSMiles Chen phys_addr_t end = base + size - 1;
8645d63f81cSMiles Chen
865a090d711SAnshuman Khandual memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
8665d63f81cSMiles Chen &base, &end, (void *)_RET_IP_);
86724aa0788STejun Heo
8689099daedSCatalin Marinas kmemleak_free_part_phys(base, size);
869f1af9d3aSPhilipp Hachtmann return memblock_remove_range(&memblock.reserved, base, size);
87095f72d1eSYinghai Lu }
87195f72d1eSYinghai Lu
memblock_reserve(phys_addr_t base,phys_addr_t size)872f705ac4bSAlexander Kuleshov int __init_memblock memblock_reserve(phys_addr_t base, phys_addr_t size)
87395f72d1eSYinghai Lu {
8745d63f81cSMiles Chen phys_addr_t end = base + size - 1;
8755d63f81cSMiles Chen
876a090d711SAnshuman Khandual memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
8775d63f81cSMiles Chen &base, &end, (void *)_RET_IP_);
87895f72d1eSYinghai Lu
879f705ac4bSAlexander Kuleshov return memblock_add_range(&memblock.reserved, base, size, MAX_NUMNODES, 0);
88095f72d1eSYinghai Lu }
88195f72d1eSYinghai Lu
88202634a44SAnshuman Khandual #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
memblock_physmem_add(phys_addr_t base,phys_addr_t size)88302634a44SAnshuman Khandual int __init_memblock memblock_physmem_add(phys_addr_t base, phys_addr_t size)
88402634a44SAnshuman Khandual {
88502634a44SAnshuman Khandual phys_addr_t end = base + size - 1;
88602634a44SAnshuman Khandual
88702634a44SAnshuman Khandual memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
88802634a44SAnshuman Khandual &base, &end, (void *)_RET_IP_);
88902634a44SAnshuman Khandual
89077649905SDavid Hildenbrand return memblock_add_range(&physmem, base, size, MAX_NUMNODES, 0);
89102634a44SAnshuman Khandual }
89202634a44SAnshuman Khandual #endif
89302634a44SAnshuman Khandual
89435fd0808STejun Heo /**
89547cec443SMike Rapoport * memblock_setclr_flag - set or clear flag for a memory region
89647cec443SMike Rapoport * @base: base address of the region
89747cec443SMike Rapoport * @size: size of the region
89847cec443SMike Rapoport * @set: set or clear the flag
8998958b249SHaitao Shi * @flag: the flag to update
90066b16edfSTang Chen *
9014308ce17STony Luck * This function isolates region [@base, @base + @size), and sets/clears flag
90266b16edfSTang Chen *
90347cec443SMike Rapoport * Return: 0 on success, -errno on failure.
90466b16edfSTang Chen */
memblock_setclr_flag(phys_addr_t base,phys_addr_t size,int set,int flag)9054308ce17STony Luck static int __init_memblock memblock_setclr_flag(phys_addr_t base,
9064308ce17STony Luck phys_addr_t size, int set, int flag)
90766b16edfSTang Chen {
90866b16edfSTang Chen struct memblock_type *type = &memblock.memory;
90966b16edfSTang Chen int i, ret, start_rgn, end_rgn;
91066b16edfSTang Chen
91166b16edfSTang Chen ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
91266b16edfSTang Chen if (ret)
91366b16edfSTang Chen return ret;
91466b16edfSTang Chen
915fe145124SMike Rapoport for (i = start_rgn; i < end_rgn; i++) {
916fe145124SMike Rapoport struct memblock_region *r = &type->regions[i];
917fe145124SMike Rapoport
9184308ce17STony Luck if (set)
919fe145124SMike Rapoport r->flags |= flag;
9204308ce17STony Luck else
921fe145124SMike Rapoport r->flags &= ~flag;
922fe145124SMike Rapoport }
92366b16edfSTang Chen
9242fe03412SPeng Zhang memblock_merge_regions(type, start_rgn, end_rgn);
92566b16edfSTang Chen return 0;
92666b16edfSTang Chen }
92766b16edfSTang Chen
92866b16edfSTang Chen /**
9294308ce17STony Luck * memblock_mark_hotplug - Mark hotpluggable memory with flag MEMBLOCK_HOTPLUG.
9304308ce17STony Luck * @base: the base phys addr of the region
9314308ce17STony Luck * @size: the size of the region
9324308ce17STony Luck *
93347cec443SMike Rapoport * Return: 0 on success, -errno on failure.
9344308ce17STony Luck */
memblock_mark_hotplug(phys_addr_t base,phys_addr_t size)9354308ce17STony Luck int __init_memblock memblock_mark_hotplug(phys_addr_t base, phys_addr_t size)
9364308ce17STony Luck {
9374308ce17STony Luck return memblock_setclr_flag(base, size, 1, MEMBLOCK_HOTPLUG);
9384308ce17STony Luck }
9394308ce17STony Luck
9404308ce17STony Luck /**
94166b16edfSTang Chen * memblock_clear_hotplug - Clear flag MEMBLOCK_HOTPLUG for a specified region.
94266b16edfSTang Chen * @base: the base phys addr of the region
94366b16edfSTang Chen * @size: the size of the region
94466b16edfSTang Chen *
94547cec443SMike Rapoport * Return: 0 on success, -errno on failure.
94666b16edfSTang Chen */
memblock_clear_hotplug(phys_addr_t base,phys_addr_t size)94766b16edfSTang Chen int __init_memblock memblock_clear_hotplug(phys_addr_t base, phys_addr_t size)
94866b16edfSTang Chen {
9494308ce17STony Luck return memblock_setclr_flag(base, size, 0, MEMBLOCK_HOTPLUG);
95066b16edfSTang Chen }
95166b16edfSTang Chen
95266b16edfSTang Chen /**
953a3f5bafcSTony Luck * memblock_mark_mirror - Mark mirrored memory with flag MEMBLOCK_MIRROR.
954a3f5bafcSTony Luck * @base: the base phys addr of the region
955a3f5bafcSTony Luck * @size: the size of the region
956a3f5bafcSTony Luck *
95747cec443SMike Rapoport * Return: 0 on success, -errno on failure.
958a3f5bafcSTony Luck */
memblock_mark_mirror(phys_addr_t base,phys_addr_t size)959a3f5bafcSTony Luck int __init_memblock memblock_mark_mirror(phys_addr_t base, phys_addr_t size)
960a3f5bafcSTony Luck {
961902c2d91SMa Wupeng if (!mirrored_kernelcore)
962902c2d91SMa Wupeng return 0;
963902c2d91SMa Wupeng
964a3f5bafcSTony Luck system_has_some_mirror = true;
965a3f5bafcSTony Luck
966a3f5bafcSTony Luck return memblock_setclr_flag(base, size, 1, MEMBLOCK_MIRROR);
967a3f5bafcSTony Luck }
968a3f5bafcSTony Luck
969bf3d3cc5SArd Biesheuvel /**
970bf3d3cc5SArd Biesheuvel * memblock_mark_nomap - Mark a memory region with flag MEMBLOCK_NOMAP.
971bf3d3cc5SArd Biesheuvel * @base: the base phys addr of the region
972bf3d3cc5SArd Biesheuvel * @size: the size of the region
973bf3d3cc5SArd Biesheuvel *
9749092d4f7SMike Rapoport * The memory regions marked with %MEMBLOCK_NOMAP will not be added to the
9759092d4f7SMike Rapoport * direct mapping of the physical memory. These regions will still be
9769092d4f7SMike Rapoport * covered by the memory map. The struct page representing NOMAP memory
9779092d4f7SMike Rapoport * frames in the memory map will be PageReserved()
9789092d4f7SMike Rapoport *
979658aafc8SMike Rapoport * Note: if the memory being marked %MEMBLOCK_NOMAP was allocated from
980658aafc8SMike Rapoport * memblock, the caller must inform kmemleak to ignore that memory
981658aafc8SMike Rapoport *
98247cec443SMike Rapoport * Return: 0 on success, -errno on failure.
983bf3d3cc5SArd Biesheuvel */
memblock_mark_nomap(phys_addr_t base,phys_addr_t size)984bf3d3cc5SArd Biesheuvel int __init_memblock memblock_mark_nomap(phys_addr_t base, phys_addr_t size)
985bf3d3cc5SArd Biesheuvel {
9866c9a5455SMike Rapoport return memblock_setclr_flag(base, size, 1, MEMBLOCK_NOMAP);
987bf3d3cc5SArd Biesheuvel }
988a3f5bafcSTony Luck
989a3f5bafcSTony Luck /**
9904c546b8aSAKASHI Takahiro * memblock_clear_nomap - Clear flag MEMBLOCK_NOMAP for a specified region.
9914c546b8aSAKASHI Takahiro * @base: the base phys addr of the region
9924c546b8aSAKASHI Takahiro * @size: the size of the region
9934c546b8aSAKASHI Takahiro *
99447cec443SMike Rapoport * Return: 0 on success, -errno on failure.
9954c546b8aSAKASHI Takahiro */
memblock_clear_nomap(phys_addr_t base,phys_addr_t size)9964c546b8aSAKASHI Takahiro int __init_memblock memblock_clear_nomap(phys_addr_t base, phys_addr_t size)
9974c546b8aSAKASHI Takahiro {
9984c546b8aSAKASHI Takahiro return memblock_setclr_flag(base, size, 0, MEMBLOCK_NOMAP);
9994c546b8aSAKASHI Takahiro }
10004c546b8aSAKASHI Takahiro
should_skip_region(struct memblock_type * type,struct memblock_region * m,int nid,int flags)10019f3d5eaaSMike Rapoport static bool should_skip_region(struct memblock_type *type,
10029f3d5eaaSMike Rapoport struct memblock_region *m,
10039f3d5eaaSMike Rapoport int nid, int flags)
1004c9a688a3SMike Rapoport {
1005c9a688a3SMike Rapoport int m_nid = memblock_get_region_node(m);
1006c9a688a3SMike Rapoport
10079f3d5eaaSMike Rapoport /* we never skip regions when iterating memblock.reserved or physmem */
10089f3d5eaaSMike Rapoport if (type != memblock_memory)
10099f3d5eaaSMike Rapoport return false;
10109f3d5eaaSMike Rapoport
1011c9a688a3SMike Rapoport /* only memory regions are associated with nodes, check it */
1012c9a688a3SMike Rapoport if (nid != NUMA_NO_NODE && nid != m_nid)
1013c9a688a3SMike Rapoport return true;
1014c9a688a3SMike Rapoport
1015c9a688a3SMike Rapoport /* skip hotpluggable memory regions if needed */
101679e482e9SMike Rapoport if (movable_node_is_enabled() && memblock_is_hotpluggable(m) &&
101779e482e9SMike Rapoport !(flags & MEMBLOCK_HOTPLUG))
1018c9a688a3SMike Rapoport return true;
1019c9a688a3SMike Rapoport
1020c9a688a3SMike Rapoport /* if we want mirror memory skip non-mirror memory regions */
1021c9a688a3SMike Rapoport if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m))
1022c9a688a3SMike Rapoport return true;
1023c9a688a3SMike Rapoport
1024c9a688a3SMike Rapoport /* skip nomap memory unless we were asked for it explicitly */
1025c9a688a3SMike Rapoport if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m))
1026c9a688a3SMike Rapoport return true;
1027c9a688a3SMike Rapoport
1028f7892d8eSDavid Hildenbrand /* skip driver-managed memory unless we were asked for it explicitly */
1029f7892d8eSDavid Hildenbrand if (!(flags & MEMBLOCK_DRIVER_MANAGED) && memblock_is_driver_managed(m))
1030f7892d8eSDavid Hildenbrand return true;
1031f7892d8eSDavid Hildenbrand
1032c9a688a3SMike Rapoport return false;
1033c9a688a3SMike Rapoport }
1034c9a688a3SMike Rapoport
10358e7a7f86SRobin Holt /**
1036a2974133SMike Rapoport * __next_mem_range - next function for for_each_free_mem_range() etc.
103735fd0808STejun Heo * @idx: pointer to u64 loop variable
1038b1154233SGrygorii Strashko * @nid: node selector, %NUMA_NO_NODE for all nodes
1039fc6daaf9STony Luck * @flags: pick from blocks based on memory attributes
1040f1af9d3aSPhilipp Hachtmann * @type_a: pointer to memblock_type from where the range is taken
1041f1af9d3aSPhilipp Hachtmann * @type_b: pointer to memblock_type which excludes memory from being taken
1042dad7557eSWanpeng Li * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL
1043dad7557eSWanpeng Li * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL
1044dad7557eSWanpeng Li * @out_nid: ptr to int for nid of the range, can be %NULL
104535fd0808STejun Heo *
1046f1af9d3aSPhilipp Hachtmann * Find the first area from *@idx which matches @nid, fill the out
104735fd0808STejun Heo * parameters, and update *@idx for the next iteration. The lower 32bit of
1048f1af9d3aSPhilipp Hachtmann * *@idx contains index into type_a and the upper 32bit indexes the
1049f1af9d3aSPhilipp Hachtmann * areas before each region in type_b. For example, if type_b regions
105035fd0808STejun Heo * look like the following,
105135fd0808STejun Heo *
105235fd0808STejun Heo * 0:[0-16), 1:[32-48), 2:[128-130)
105335fd0808STejun Heo *
105435fd0808STejun Heo * The upper 32bit indexes the following regions.
105535fd0808STejun Heo *
105635fd0808STejun Heo * 0:[0-0), 1:[16-32), 2:[48-128), 3:[130-MAX)
105735fd0808STejun Heo *
105835fd0808STejun Heo * As both region arrays are sorted, the function advances the two indices
105935fd0808STejun Heo * in lockstep and returns each intersection.
106035fd0808STejun Heo */
__next_mem_range(u64 * idx,int nid,enum memblock_flags flags,struct memblock_type * type_a,struct memblock_type * type_b,phys_addr_t * out_start,phys_addr_t * out_end,int * out_nid)106177649905SDavid Hildenbrand void __next_mem_range(u64 *idx, int nid, enum memblock_flags flags,
1062f1af9d3aSPhilipp Hachtmann struct memblock_type *type_a,
106377649905SDavid Hildenbrand struct memblock_type *type_b, phys_addr_t *out_start,
106435fd0808STejun Heo phys_addr_t *out_end, int *out_nid)
106535fd0808STejun Heo {
1066f1af9d3aSPhilipp Hachtmann int idx_a = *idx & 0xffffffff;
1067f1af9d3aSPhilipp Hachtmann int idx_b = *idx >> 32;
1068b1154233SGrygorii Strashko
1069f1af9d3aSPhilipp Hachtmann if (WARN_ONCE(nid == MAX_NUMNODES,
1070f1af9d3aSPhilipp Hachtmann "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
1071560dca27SGrygorii Strashko nid = NUMA_NO_NODE;
107235fd0808STejun Heo
1073f1af9d3aSPhilipp Hachtmann for (; idx_a < type_a->cnt; idx_a++) {
1074f1af9d3aSPhilipp Hachtmann struct memblock_region *m = &type_a->regions[idx_a];
1075f1af9d3aSPhilipp Hachtmann
107635fd0808STejun Heo phys_addr_t m_start = m->base;
107735fd0808STejun Heo phys_addr_t m_end = m->base + m->size;
1078f1af9d3aSPhilipp Hachtmann int m_nid = memblock_get_region_node(m);
107935fd0808STejun Heo
10809f3d5eaaSMike Rapoport if (should_skip_region(type_a, m, nid, flags))
1081bf3d3cc5SArd Biesheuvel continue;
1082bf3d3cc5SArd Biesheuvel
1083f1af9d3aSPhilipp Hachtmann if (!type_b) {
1084f1af9d3aSPhilipp Hachtmann if (out_start)
1085f1af9d3aSPhilipp Hachtmann *out_start = m_start;
1086f1af9d3aSPhilipp Hachtmann if (out_end)
1087f1af9d3aSPhilipp Hachtmann *out_end = m_end;
1088f1af9d3aSPhilipp Hachtmann if (out_nid)
1089f1af9d3aSPhilipp Hachtmann *out_nid = m_nid;
1090f1af9d3aSPhilipp Hachtmann idx_a++;
1091f1af9d3aSPhilipp Hachtmann *idx = (u32)idx_a | (u64)idx_b << 32;
1092f1af9d3aSPhilipp Hachtmann return;
1093f1af9d3aSPhilipp Hachtmann }
109435fd0808STejun Heo
1095f1af9d3aSPhilipp Hachtmann /* scan areas before each reservation */
1096f1af9d3aSPhilipp Hachtmann for (; idx_b < type_b->cnt + 1; idx_b++) {
1097f1af9d3aSPhilipp Hachtmann struct memblock_region *r;
1098f1af9d3aSPhilipp Hachtmann phys_addr_t r_start;
1099f1af9d3aSPhilipp Hachtmann phys_addr_t r_end;
1100f1af9d3aSPhilipp Hachtmann
1101f1af9d3aSPhilipp Hachtmann r = &type_b->regions[idx_b];
1102f1af9d3aSPhilipp Hachtmann r_start = idx_b ? r[-1].base + r[-1].size : 0;
1103f1af9d3aSPhilipp Hachtmann r_end = idx_b < type_b->cnt ?
11041c4bc43dSStefan Agner r->base : PHYS_ADDR_MAX;
1105f1af9d3aSPhilipp Hachtmann
1106f1af9d3aSPhilipp Hachtmann /*
1107f1af9d3aSPhilipp Hachtmann * if idx_b advanced past idx_a,
1108f1af9d3aSPhilipp Hachtmann * break out to advance idx_a
1109f1af9d3aSPhilipp Hachtmann */
111035fd0808STejun Heo if (r_start >= m_end)
111135fd0808STejun Heo break;
111235fd0808STejun Heo /* if the two regions intersect, we're done */
111335fd0808STejun Heo if (m_start < r_end) {
111435fd0808STejun Heo if (out_start)
1115f1af9d3aSPhilipp Hachtmann *out_start =
1116f1af9d3aSPhilipp Hachtmann max(m_start, r_start);
111735fd0808STejun Heo if (out_end)
111835fd0808STejun Heo *out_end = min(m_end, r_end);
111935fd0808STejun Heo if (out_nid)
1120f1af9d3aSPhilipp Hachtmann *out_nid = m_nid;
112135fd0808STejun Heo /*
1122f1af9d3aSPhilipp Hachtmann * The region which ends first is
1123f1af9d3aSPhilipp Hachtmann * advanced for the next iteration.
112435fd0808STejun Heo */
112535fd0808STejun Heo if (m_end <= r_end)
1126f1af9d3aSPhilipp Hachtmann idx_a++;
112735fd0808STejun Heo else
1128f1af9d3aSPhilipp Hachtmann idx_b++;
1129f1af9d3aSPhilipp Hachtmann *idx = (u32)idx_a | (u64)idx_b << 32;
113035fd0808STejun Heo return;
113135fd0808STejun Heo }
113235fd0808STejun Heo }
113335fd0808STejun Heo }
113435fd0808STejun Heo
113535fd0808STejun Heo /* signal end of iteration */
113635fd0808STejun Heo *idx = ULLONG_MAX;
113735fd0808STejun Heo }
113835fd0808STejun Heo
11397bd0b0f0STejun Heo /**
1140f1af9d3aSPhilipp Hachtmann * __next_mem_range_rev - generic next function for for_each_*_range_rev()
1141f1af9d3aSPhilipp Hachtmann *
11427bd0b0f0STejun Heo * @idx: pointer to u64 loop variable
1143ad5ea8cdSAlexander Kuleshov * @nid: node selector, %NUMA_NO_NODE for all nodes
1144fc6daaf9STony Luck * @flags: pick from blocks based on memory attributes
1145f1af9d3aSPhilipp Hachtmann * @type_a: pointer to memblock_type from where the range is taken
1146f1af9d3aSPhilipp Hachtmann * @type_b: pointer to memblock_type which excludes memory from being taken
1147dad7557eSWanpeng Li * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL
1148dad7557eSWanpeng Li * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL
1149dad7557eSWanpeng Li * @out_nid: ptr to int for nid of the range, can be %NULL
11507bd0b0f0STejun Heo *
115147cec443SMike Rapoport * Finds the next range from type_a which is not marked as unsuitable
115247cec443SMike Rapoport * in type_b.
115347cec443SMike Rapoport *
1154f1af9d3aSPhilipp Hachtmann * Reverse of __next_mem_range().
11557bd0b0f0STejun Heo */
__next_mem_range_rev(u64 * idx,int nid,enum memblock_flags flags,struct memblock_type * type_a,struct memblock_type * type_b,phys_addr_t * out_start,phys_addr_t * out_end,int * out_nid)1156e1720feeSMike Rapoport void __init_memblock __next_mem_range_rev(u64 *idx, int nid,
1157e1720feeSMike Rapoport enum memblock_flags flags,
1158f1af9d3aSPhilipp Hachtmann struct memblock_type *type_a,
1159f1af9d3aSPhilipp Hachtmann struct memblock_type *type_b,
11607bd0b0f0STejun Heo phys_addr_t *out_start,
11617bd0b0f0STejun Heo phys_addr_t *out_end, int *out_nid)
11627bd0b0f0STejun Heo {
1163f1af9d3aSPhilipp Hachtmann int idx_a = *idx & 0xffffffff;
1164f1af9d3aSPhilipp Hachtmann int idx_b = *idx >> 32;
1165b1154233SGrygorii Strashko
1166560dca27SGrygorii Strashko if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
1167560dca27SGrygorii Strashko nid = NUMA_NO_NODE;
11687bd0b0f0STejun Heo
11697bd0b0f0STejun Heo if (*idx == (u64)ULLONG_MAX) {
1170f1af9d3aSPhilipp Hachtmann idx_a = type_a->cnt - 1;
1171e47608abSzijun_hu if (type_b != NULL)
1172f1af9d3aSPhilipp Hachtmann idx_b = type_b->cnt;
1173e47608abSzijun_hu else
1174e47608abSzijun_hu idx_b = 0;
11757bd0b0f0STejun Heo }
11767bd0b0f0STejun Heo
1177f1af9d3aSPhilipp Hachtmann for (; idx_a >= 0; idx_a--) {
1178f1af9d3aSPhilipp Hachtmann struct memblock_region *m = &type_a->regions[idx_a];
1179f1af9d3aSPhilipp Hachtmann
11807bd0b0f0STejun Heo phys_addr_t m_start = m->base;
11817bd0b0f0STejun Heo phys_addr_t m_end = m->base + m->size;
1182f1af9d3aSPhilipp Hachtmann int m_nid = memblock_get_region_node(m);
11837bd0b0f0STejun Heo
11849f3d5eaaSMike Rapoport if (should_skip_region(type_a, m, nid, flags))
1185bf3d3cc5SArd Biesheuvel continue;
1186bf3d3cc5SArd Biesheuvel
1187f1af9d3aSPhilipp Hachtmann if (!type_b) {
1188f1af9d3aSPhilipp Hachtmann if (out_start)
1189f1af9d3aSPhilipp Hachtmann *out_start = m_start;
1190f1af9d3aSPhilipp Hachtmann if (out_end)
1191f1af9d3aSPhilipp Hachtmann *out_end = m_end;
1192f1af9d3aSPhilipp Hachtmann if (out_nid)
1193f1af9d3aSPhilipp Hachtmann *out_nid = m_nid;
1194fb399b48Szijun_hu idx_a--;
1195f1af9d3aSPhilipp Hachtmann *idx = (u32)idx_a | (u64)idx_b << 32;
1196f1af9d3aSPhilipp Hachtmann return;
1197f1af9d3aSPhilipp Hachtmann }
11987bd0b0f0STejun Heo
1199f1af9d3aSPhilipp Hachtmann /* scan areas before each reservation */
1200f1af9d3aSPhilipp Hachtmann for (; idx_b >= 0; idx_b--) {
1201f1af9d3aSPhilipp Hachtmann struct memblock_region *r;
1202f1af9d3aSPhilipp Hachtmann phys_addr_t r_start;
1203f1af9d3aSPhilipp Hachtmann phys_addr_t r_end;
1204f1af9d3aSPhilipp Hachtmann
1205f1af9d3aSPhilipp Hachtmann r = &type_b->regions[idx_b];
1206f1af9d3aSPhilipp Hachtmann r_start = idx_b ? r[-1].base + r[-1].size : 0;
1207f1af9d3aSPhilipp Hachtmann r_end = idx_b < type_b->cnt ?
12081c4bc43dSStefan Agner r->base : PHYS_ADDR_MAX;
1209f1af9d3aSPhilipp Hachtmann /*
1210f1af9d3aSPhilipp Hachtmann * if idx_b advanced past idx_a,
1211f1af9d3aSPhilipp Hachtmann * break out to advance idx_a
1212f1af9d3aSPhilipp Hachtmann */
1213f1af9d3aSPhilipp Hachtmann
12147bd0b0f0STejun Heo if (r_end <= m_start)
12157bd0b0f0STejun Heo break;
12167bd0b0f0STejun Heo /* if the two regions intersect, we're done */
12177bd0b0f0STejun Heo if (m_end > r_start) {
12187bd0b0f0STejun Heo if (out_start)
12197bd0b0f0STejun Heo *out_start = max(m_start, r_start);
12207bd0b0f0STejun Heo if (out_end)
12217bd0b0f0STejun Heo *out_end = min(m_end, r_end);
12227bd0b0f0STejun Heo if (out_nid)
1223f1af9d3aSPhilipp Hachtmann *out_nid = m_nid;
12247bd0b0f0STejun Heo if (m_start >= r_start)
1225f1af9d3aSPhilipp Hachtmann idx_a--;
12267bd0b0f0STejun Heo else
1227f1af9d3aSPhilipp Hachtmann idx_b--;
1228f1af9d3aSPhilipp Hachtmann *idx = (u32)idx_a | (u64)idx_b << 32;
12297bd0b0f0STejun Heo return;
12307bd0b0f0STejun Heo }
12317bd0b0f0STejun Heo }
12327bd0b0f0STejun Heo }
1233f1af9d3aSPhilipp Hachtmann /* signal end of iteration */
12347bd0b0f0STejun Heo *idx = ULLONG_MAX;
12357bd0b0f0STejun Heo }
12367bd0b0f0STejun Heo
12377c0caeb8STejun Heo /*
123845e79815SChen Chang * Common iterator interface used to define for_each_mem_pfn_range().
12397c0caeb8STejun Heo */
__next_mem_pfn_range(int * idx,int nid,unsigned long * out_start_pfn,unsigned long * out_end_pfn,int * out_nid)12407c0caeb8STejun Heo void __init_memblock __next_mem_pfn_range(int *idx, int nid,
12417c0caeb8STejun Heo unsigned long *out_start_pfn,
12427c0caeb8STejun Heo unsigned long *out_end_pfn, int *out_nid)
12437c0caeb8STejun Heo {
12447c0caeb8STejun Heo struct memblock_type *type = &memblock.memory;
12457c0caeb8STejun Heo struct memblock_region *r;
1246d622abf7SMike Rapoport int r_nid;
12477c0caeb8STejun Heo
12487c0caeb8STejun Heo while (++*idx < type->cnt) {
12497c0caeb8STejun Heo r = &type->regions[*idx];
1250d622abf7SMike Rapoport r_nid = memblock_get_region_node(r);
12517c0caeb8STejun Heo
12527c0caeb8STejun Heo if (PFN_UP(r->base) >= PFN_DOWN(r->base + r->size))
12537c0caeb8STejun Heo continue;
1254d622abf7SMike Rapoport if (nid == MAX_NUMNODES || nid == r_nid)
12557c0caeb8STejun Heo break;
12567c0caeb8STejun Heo }
12577c0caeb8STejun Heo if (*idx >= type->cnt) {
12587c0caeb8STejun Heo *idx = -1;
12597c0caeb8STejun Heo return;
12607c0caeb8STejun Heo }
12617c0caeb8STejun Heo
12627c0caeb8STejun Heo if (out_start_pfn)
12637c0caeb8STejun Heo *out_start_pfn = PFN_UP(r->base);
12647c0caeb8STejun Heo if (out_end_pfn)
12657c0caeb8STejun Heo *out_end_pfn = PFN_DOWN(r->base + r->size);
12667c0caeb8STejun Heo if (out_nid)
1267d622abf7SMike Rapoport *out_nid = r_nid;
12687c0caeb8STejun Heo }
12697c0caeb8STejun Heo
12707c0caeb8STejun Heo /**
12717c0caeb8STejun Heo * memblock_set_node - set node ID on memblock regions
12727c0caeb8STejun Heo * @base: base of area to set node ID for
12737c0caeb8STejun Heo * @size: size of area to set node ID for
1274e7e8de59STang Chen * @type: memblock type to set node ID for
12757c0caeb8STejun Heo * @nid: node ID to set
12767c0caeb8STejun Heo *
1277e7e8de59STang Chen * Set the nid of memblock @type regions in [@base, @base + @size) to @nid.
12787c0caeb8STejun Heo * Regions which cross the area boundaries are split as necessary.
12797c0caeb8STejun Heo *
128047cec443SMike Rapoport * Return:
12817c0caeb8STejun Heo * 0 on success, -errno on failure.
12827c0caeb8STejun Heo */
memblock_set_node(phys_addr_t base,phys_addr_t size,struct memblock_type * type,int nid)12837c0caeb8STejun Heo int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size,
1284e7e8de59STang Chen struct memblock_type *type, int nid)
12857c0caeb8STejun Heo {
1286a9ee6cf5SMike Rapoport #ifdef CONFIG_NUMA
12876a9ceb31STejun Heo int start_rgn, end_rgn;
12886a9ceb31STejun Heo int i, ret;
12897c0caeb8STejun Heo
12906a9ceb31STejun Heo ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
12916a9ceb31STejun Heo if (ret)
12926a9ceb31STejun Heo return ret;
12937c0caeb8STejun Heo
12946a9ceb31STejun Heo for (i = start_rgn; i < end_rgn; i++)
1295e9d24ad3SWanpeng Li memblock_set_region_node(&type->regions[i], nid);
12967c0caeb8STejun Heo
12972fe03412SPeng Zhang memblock_merge_regions(type, start_rgn, end_rgn);
12983f08a302SMike Rapoport #endif
12997c0caeb8STejun Heo return 0;
13007c0caeb8STejun Heo }
13013f08a302SMike Rapoport
1302837566e7SAlexander Duyck #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1303837566e7SAlexander Duyck /**
1304837566e7SAlexander Duyck * __next_mem_pfn_range_in_zone - iterator for for_each_*_range_in_zone()
1305837566e7SAlexander Duyck *
1306837566e7SAlexander Duyck * @idx: pointer to u64 loop variable
1307837566e7SAlexander Duyck * @zone: zone in which all of the memory blocks reside
1308837566e7SAlexander Duyck * @out_spfn: ptr to ulong for start pfn of the range, can be %NULL
1309837566e7SAlexander Duyck * @out_epfn: ptr to ulong for end pfn of the range, can be %NULL
1310837566e7SAlexander Duyck *
1311837566e7SAlexander Duyck * This function is meant to be a zone/pfn specific wrapper for the
1312837566e7SAlexander Duyck * for_each_mem_range type iterators. Specifically they are used in the
1313837566e7SAlexander Duyck * deferred memory init routines and as such we were duplicating much of
1314837566e7SAlexander Duyck * this logic throughout the code. So instead of having it in multiple
1315837566e7SAlexander Duyck * locations it seemed like it would make more sense to centralize this to
1316837566e7SAlexander Duyck * one new iterator that does everything they need.
1317837566e7SAlexander Duyck */
1318837566e7SAlexander Duyck void __init_memblock
__next_mem_pfn_range_in_zone(u64 * idx,struct zone * zone,unsigned long * out_spfn,unsigned long * out_epfn)1319837566e7SAlexander Duyck __next_mem_pfn_range_in_zone(u64 *idx, struct zone *zone,
1320837566e7SAlexander Duyck unsigned long *out_spfn, unsigned long *out_epfn)
1321837566e7SAlexander Duyck {
1322837566e7SAlexander Duyck int zone_nid = zone_to_nid(zone);
1323837566e7SAlexander Duyck phys_addr_t spa, epa;
1324837566e7SAlexander Duyck
1325837566e7SAlexander Duyck __next_mem_range(idx, zone_nid, MEMBLOCK_NONE,
1326837566e7SAlexander Duyck &memblock.memory, &memblock.reserved,
1327f30b002cSMiaohe Lin &spa, &epa, NULL);
1328837566e7SAlexander Duyck
1329837566e7SAlexander Duyck while (*idx != U64_MAX) {
1330837566e7SAlexander Duyck unsigned long epfn = PFN_DOWN(epa);
1331837566e7SAlexander Duyck unsigned long spfn = PFN_UP(spa);
1332837566e7SAlexander Duyck
1333837566e7SAlexander Duyck /*
1334837566e7SAlexander Duyck * Verify the end is at least past the start of the zone and
1335837566e7SAlexander Duyck * that we have at least one PFN to initialize.
1336837566e7SAlexander Duyck */
1337837566e7SAlexander Duyck if (zone->zone_start_pfn < epfn && spfn < epfn) {
1338837566e7SAlexander Duyck /* if we went too far just stop searching */
1339837566e7SAlexander Duyck if (zone_end_pfn(zone) <= spfn) {
1340837566e7SAlexander Duyck *idx = U64_MAX;
1341837566e7SAlexander Duyck break;
1342837566e7SAlexander Duyck }
1343837566e7SAlexander Duyck
1344837566e7SAlexander Duyck if (out_spfn)
1345837566e7SAlexander Duyck *out_spfn = max(zone->zone_start_pfn, spfn);
1346837566e7SAlexander Duyck if (out_epfn)
1347837566e7SAlexander Duyck *out_epfn = min(zone_end_pfn(zone), epfn);
1348837566e7SAlexander Duyck
1349837566e7SAlexander Duyck return;
1350837566e7SAlexander Duyck }
1351837566e7SAlexander Duyck
1352837566e7SAlexander Duyck __next_mem_range(idx, zone_nid, MEMBLOCK_NONE,
1353837566e7SAlexander Duyck &memblock.memory, &memblock.reserved,
1354f30b002cSMiaohe Lin &spa, &epa, NULL);
1355837566e7SAlexander Duyck }
1356837566e7SAlexander Duyck
1357837566e7SAlexander Duyck /* signal end of iteration */
1358837566e7SAlexander Duyck if (out_spfn)
1359837566e7SAlexander Duyck *out_spfn = ULONG_MAX;
1360837566e7SAlexander Duyck if (out_epfn)
1361837566e7SAlexander Duyck *out_epfn = 0;
1362837566e7SAlexander Duyck }
1363837566e7SAlexander Duyck
1364837566e7SAlexander Duyck #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
13657c0caeb8STejun Heo
136692d12f95SMike Rapoport /**
136792d12f95SMike Rapoport * memblock_alloc_range_nid - allocate boot memory block
136892d12f95SMike Rapoport * @size: size of memory block to be allocated in bytes
136992d12f95SMike Rapoport * @align: alignment of the region and block's size
137092d12f95SMike Rapoport * @start: the lower bound of the memory region to allocate (phys address)
137192d12f95SMike Rapoport * @end: the upper bound of the memory region to allocate (phys address)
137292d12f95SMike Rapoport * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
13730ac398b1SYunfeng Ye * @exact_nid: control the allocation fall back to other nodes
137492d12f95SMike Rapoport *
137592d12f95SMike Rapoport * The allocation is performed from memory region limited by
137695830666SCao jin * memblock.current_limit if @end == %MEMBLOCK_ALLOC_ACCESSIBLE.
137792d12f95SMike Rapoport *
13780ac398b1SYunfeng Ye * If the specified node can not hold the requested memory and @exact_nid
13790ac398b1SYunfeng Ye * is false, the allocation falls back to any node in the system.
138092d12f95SMike Rapoport *
138192d12f95SMike Rapoport * For systems with memory mirroring, the allocation is attempted first
138292d12f95SMike Rapoport * from the regions with mirroring enabled and then retried from any
138392d12f95SMike Rapoport * memory region.
138492d12f95SMike Rapoport *
1385c200d900SPatrick Wang * In addition, function using kmemleak_alloc_phys for allocated boot
1386c200d900SPatrick Wang * memory block, it is never reported as leaks.
138792d12f95SMike Rapoport *
138892d12f95SMike Rapoport * Return:
138992d12f95SMike Rapoport * Physical address of allocated memory block on success, %0 on failure.
139092d12f95SMike Rapoport */
memblock_alloc_range_nid(phys_addr_t size,phys_addr_t align,phys_addr_t start,phys_addr_t end,int nid,bool exact_nid)13918676af1fSAslan Bakirov phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size,
13922bfc2862SAkinobu Mita phys_addr_t align, phys_addr_t start,
13930ac398b1SYunfeng Ye phys_addr_t end, int nid,
13940ac398b1SYunfeng Ye bool exact_nid)
139595f72d1eSYinghai Lu {
139692d12f95SMike Rapoport enum memblock_flags flags = choose_memblock_flags();
13976ed311b2SBenjamin Herrenschmidt phys_addr_t found;
139895f72d1eSYinghai Lu
139992d12f95SMike Rapoport if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
140092d12f95SMike Rapoport nid = NUMA_NO_NODE;
140192d12f95SMike Rapoport
14022f770806SMike Rapoport if (!align) {
14032f770806SMike Rapoport /* Can't use WARNs this early in boot on powerpc */
14042f770806SMike Rapoport dump_stack();
14052f770806SMike Rapoport align = SMP_CACHE_BYTES;
14062f770806SMike Rapoport }
14072f770806SMike Rapoport
140892d12f95SMike Rapoport again:
1409fc6daaf9STony Luck found = memblock_find_in_range_node(size, align, start, end, nid,
1410fc6daaf9STony Luck flags);
141192d12f95SMike Rapoport if (found && !memblock_reserve(found, size))
141292d12f95SMike Rapoport goto done;
141392d12f95SMike Rapoport
14140ac398b1SYunfeng Ye if (nid != NUMA_NO_NODE && !exact_nid) {
141592d12f95SMike Rapoport found = memblock_find_in_range_node(size, align, start,
141692d12f95SMike Rapoport end, NUMA_NO_NODE,
141792d12f95SMike Rapoport flags);
141892d12f95SMike Rapoport if (found && !memblock_reserve(found, size))
141992d12f95SMike Rapoport goto done;
142092d12f95SMike Rapoport }
142192d12f95SMike Rapoport
142292d12f95SMike Rapoport if (flags & MEMBLOCK_MIRROR) {
142392d12f95SMike Rapoport flags &= ~MEMBLOCK_MIRROR;
142414d9a675SMa Wupeng pr_warn_ratelimited("Could not allocate %pap bytes of mirrored memory\n",
142592d12f95SMike Rapoport &size);
142692d12f95SMike Rapoport goto again;
142792d12f95SMike Rapoport }
142892d12f95SMike Rapoport
142992d12f95SMike Rapoport return 0;
143092d12f95SMike Rapoport
143192d12f95SMike Rapoport done:
1432c6975d7cSQian Cai /*
1433c6975d7cSQian Cai * Skip kmemleak for those places like kasan_init() and
1434c6975d7cSQian Cai * early_pgtable_alloc() due to high volume.
1435c6975d7cSQian Cai */
1436c6975d7cSQian Cai if (end != MEMBLOCK_ALLOC_NOLEAKTRACE)
1437aedf95eaSCatalin Marinas /*
1438c200d900SPatrick Wang * Memblock allocated blocks are never reported as
1439c200d900SPatrick Wang * leaks. This is because many of these blocks are
1440c200d900SPatrick Wang * only referred via the physical address which is
1441c200d900SPatrick Wang * not looked up by kmemleak.
1442aedf95eaSCatalin Marinas */
1443c200d900SPatrick Wang kmemleak_alloc_phys(found, size, 0);
144492d12f95SMike Rapoport
1445dcdfdd40SKirill A. Shutemov /*
1446dcdfdd40SKirill A. Shutemov * Some Virtual Machine platforms, such as Intel TDX or AMD SEV-SNP,
1447dcdfdd40SKirill A. Shutemov * require memory to be accepted before it can be used by the
1448dcdfdd40SKirill A. Shutemov * guest.
1449dcdfdd40SKirill A. Shutemov *
1450dcdfdd40SKirill A. Shutemov * Accept the memory of the allocated buffer.
1451dcdfdd40SKirill A. Shutemov */
1452dcdfdd40SKirill A. Shutemov accept_memory(found, found + size);
1453dcdfdd40SKirill A. Shutemov
14546ed311b2SBenjamin Herrenschmidt return found;
1455aedf95eaSCatalin Marinas }
145695f72d1eSYinghai Lu
1457a2974133SMike Rapoport /**
1458a2974133SMike Rapoport * memblock_phys_alloc_range - allocate a memory block inside specified range
1459a2974133SMike Rapoport * @size: size of memory block to be allocated in bytes
1460a2974133SMike Rapoport * @align: alignment of the region and block's size
1461a2974133SMike Rapoport * @start: the lower bound of the memory region to allocate (physical address)
1462a2974133SMike Rapoport * @end: the upper bound of the memory region to allocate (physical address)
1463a2974133SMike Rapoport *
1464a2974133SMike Rapoport * Allocate @size bytes in the between @start and @end.
1465a2974133SMike Rapoport *
1466a2974133SMike Rapoport * Return: physical address of the allocated memory block on success,
1467a2974133SMike Rapoport * %0 on failure.
1468a2974133SMike Rapoport */
memblock_phys_alloc_range(phys_addr_t size,phys_addr_t align,phys_addr_t start,phys_addr_t end)14698a770c2aSMike Rapoport phys_addr_t __init memblock_phys_alloc_range(phys_addr_t size,
14708a770c2aSMike Rapoport phys_addr_t align,
14718a770c2aSMike Rapoport phys_addr_t start,
14728a770c2aSMike Rapoport phys_addr_t end)
14732bfc2862SAkinobu Mita {
1474b5cf2d6cSFaiyaz Mohammed memblock_dbg("%s: %llu bytes align=0x%llx from=%pa max_addr=%pa %pS\n",
1475b5cf2d6cSFaiyaz Mohammed __func__, (u64)size, (u64)align, &start, &end,
1476b5cf2d6cSFaiyaz Mohammed (void *)_RET_IP_);
14770ac398b1SYunfeng Ye return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE,
14780ac398b1SYunfeng Ye false);
14797bd0b0f0STejun Heo }
14807bd0b0f0STejun Heo
1481a2974133SMike Rapoport /**
148217cbe038SLevi Yun * memblock_phys_alloc_try_nid - allocate a memory block from specified NUMA node
1483a2974133SMike Rapoport * @size: size of memory block to be allocated in bytes
1484a2974133SMike Rapoport * @align: alignment of the region and block's size
1485a2974133SMike Rapoport * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1486a2974133SMike Rapoport *
1487a2974133SMike Rapoport * Allocates memory block from the specified NUMA node. If the node
1488a2974133SMike Rapoport * has no available memory, attempts to allocated from any node in the
1489a2974133SMike Rapoport * system.
1490a2974133SMike Rapoport *
1491a2974133SMike Rapoport * Return: physical address of the allocated memory block on success,
1492a2974133SMike Rapoport * %0 on failure.
1493a2974133SMike Rapoport */
memblock_phys_alloc_try_nid(phys_addr_t size,phys_addr_t align,int nid)14949a8dd708SMike Rapoport phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid)
14959d1e2492SBenjamin Herrenschmidt {
149633755574SMike Rapoport return memblock_alloc_range_nid(size, align, 0,
14970ac398b1SYunfeng Ye MEMBLOCK_ALLOC_ACCESSIBLE, nid, false);
149895f72d1eSYinghai Lu }
149995f72d1eSYinghai Lu
150026f09e9bSSantosh Shilimkar /**
1501eb31d559SMike Rapoport * memblock_alloc_internal - allocate boot memory block
150226f09e9bSSantosh Shilimkar * @size: size of memory block to be allocated in bytes
150326f09e9bSSantosh Shilimkar * @align: alignment of the region and block's size
150426f09e9bSSantosh Shilimkar * @min_addr: the lower bound of the memory region to allocate (phys address)
150526f09e9bSSantosh Shilimkar * @max_addr: the upper bound of the memory region to allocate (phys address)
150626f09e9bSSantosh Shilimkar * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
15070ac398b1SYunfeng Ye * @exact_nid: control the allocation fall back to other nodes
150826f09e9bSSantosh Shilimkar *
150992d12f95SMike Rapoport * Allocates memory block using memblock_alloc_range_nid() and
151092d12f95SMike Rapoport * converts the returned physical address to virtual.
151192d12f95SMike Rapoport *
151226f09e9bSSantosh Shilimkar * The @min_addr limit is dropped if it can not be satisfied and the allocation
151392d12f95SMike Rapoport * will fall back to memory below @min_addr. Other constraints, such
151492d12f95SMike Rapoport * as node and mirrored memory will be handled again in
151592d12f95SMike Rapoport * memblock_alloc_range_nid().
151626f09e9bSSantosh Shilimkar *
151747cec443SMike Rapoport * Return:
151826f09e9bSSantosh Shilimkar * Virtual address of allocated memory block on success, NULL on failure.
151926f09e9bSSantosh Shilimkar */
memblock_alloc_internal(phys_addr_t size,phys_addr_t align,phys_addr_t min_addr,phys_addr_t max_addr,int nid,bool exact_nid)1520eb31d559SMike Rapoport static void * __init memblock_alloc_internal(
152126f09e9bSSantosh Shilimkar phys_addr_t size, phys_addr_t align,
152226f09e9bSSantosh Shilimkar phys_addr_t min_addr, phys_addr_t max_addr,
15230ac398b1SYunfeng Ye int nid, bool exact_nid)
152426f09e9bSSantosh Shilimkar {
152526f09e9bSSantosh Shilimkar phys_addr_t alloc;
152626f09e9bSSantosh Shilimkar
152726f09e9bSSantosh Shilimkar /*
152826f09e9bSSantosh Shilimkar * Detect any accidental use of these APIs after slab is ready, as at
152926f09e9bSSantosh Shilimkar * this moment memblock may be deinitialized already and its
1530c6ffc5caSMike Rapoport * internal data may be destroyed (after execution of memblock_free_all)
153126f09e9bSSantosh Shilimkar */
153226f09e9bSSantosh Shilimkar if (WARN_ON_ONCE(slab_is_available()))
153326f09e9bSSantosh Shilimkar return kzalloc_node(size, GFP_NOWAIT, nid);
153426f09e9bSSantosh Shilimkar
1535f3057ad7SMike Rapoport if (max_addr > memblock.current_limit)
1536f3057ad7SMike Rapoport max_addr = memblock.current_limit;
1537f3057ad7SMike Rapoport
15380ac398b1SYunfeng Ye alloc = memblock_alloc_range_nid(size, align, min_addr, max_addr, nid,
15390ac398b1SYunfeng Ye exact_nid);
15402f770806SMike Rapoport
154192d12f95SMike Rapoport /* retry allocation without lower limit */
154292d12f95SMike Rapoport if (!alloc && min_addr)
15430ac398b1SYunfeng Ye alloc = memblock_alloc_range_nid(size, align, 0, max_addr, nid,
15440ac398b1SYunfeng Ye exact_nid);
154526f09e9bSSantosh Shilimkar
154692d12f95SMike Rapoport if (!alloc)
1547a3f5bafcSTony Luck return NULL;
154826f09e9bSSantosh Shilimkar
154992d12f95SMike Rapoport return phys_to_virt(alloc);
155026f09e9bSSantosh Shilimkar }
155126f09e9bSSantosh Shilimkar
155226f09e9bSSantosh Shilimkar /**
15530ac398b1SYunfeng Ye * memblock_alloc_exact_nid_raw - allocate boot memory block on the exact node
15540ac398b1SYunfeng Ye * without zeroing memory
15550ac398b1SYunfeng Ye * @size: size of memory block to be allocated in bytes
15560ac398b1SYunfeng Ye * @align: alignment of the region and block's size
15570ac398b1SYunfeng Ye * @min_addr: the lower bound of the memory region from where the allocation
15580ac398b1SYunfeng Ye * is preferred (phys address)
15590ac398b1SYunfeng Ye * @max_addr: the upper bound of the memory region from where the allocation
15600ac398b1SYunfeng Ye * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
15610ac398b1SYunfeng Ye * allocate only from memory limited by memblock.current_limit value
15620ac398b1SYunfeng Ye * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
15630ac398b1SYunfeng Ye *
15640ac398b1SYunfeng Ye * Public function, provides additional debug information (including caller
15650ac398b1SYunfeng Ye * info), if enabled. Does not zero allocated memory.
15660ac398b1SYunfeng Ye *
15670ac398b1SYunfeng Ye * Return:
15680ac398b1SYunfeng Ye * Virtual address of allocated memory block on success, NULL on failure.
15690ac398b1SYunfeng Ye */
memblock_alloc_exact_nid_raw(phys_addr_t size,phys_addr_t align,phys_addr_t min_addr,phys_addr_t max_addr,int nid)15700ac398b1SYunfeng Ye void * __init memblock_alloc_exact_nid_raw(
15710ac398b1SYunfeng Ye phys_addr_t size, phys_addr_t align,
15720ac398b1SYunfeng Ye phys_addr_t min_addr, phys_addr_t max_addr,
15730ac398b1SYunfeng Ye int nid)
15740ac398b1SYunfeng Ye {
15750ac398b1SYunfeng Ye memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
15760ac398b1SYunfeng Ye __func__, (u64)size, (u64)align, nid, &min_addr,
15770ac398b1SYunfeng Ye &max_addr, (void *)_RET_IP_);
15780ac398b1SYunfeng Ye
157908678804SMike Rapoport return memblock_alloc_internal(size, align, min_addr, max_addr, nid,
158008678804SMike Rapoport true);
15810ac398b1SYunfeng Ye }
15820ac398b1SYunfeng Ye
15830ac398b1SYunfeng Ye /**
1584eb31d559SMike Rapoport * memblock_alloc_try_nid_raw - allocate boot memory block without zeroing
1585ea1f5f37SPavel Tatashin * memory and without panicking
1586ea1f5f37SPavel Tatashin * @size: size of memory block to be allocated in bytes
1587ea1f5f37SPavel Tatashin * @align: alignment of the region and block's size
1588ea1f5f37SPavel Tatashin * @min_addr: the lower bound of the memory region from where the allocation
1589ea1f5f37SPavel Tatashin * is preferred (phys address)
1590ea1f5f37SPavel Tatashin * @max_addr: the upper bound of the memory region from where the allocation
159197ad1087SMike Rapoport * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
1592ea1f5f37SPavel Tatashin * allocate only from memory limited by memblock.current_limit value
1593ea1f5f37SPavel Tatashin * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1594ea1f5f37SPavel Tatashin *
1595ea1f5f37SPavel Tatashin * Public function, provides additional debug information (including caller
1596ea1f5f37SPavel Tatashin * info), if enabled. Does not zero allocated memory, does not panic if request
1597ea1f5f37SPavel Tatashin * cannot be satisfied.
1598ea1f5f37SPavel Tatashin *
159947cec443SMike Rapoport * Return:
1600ea1f5f37SPavel Tatashin * Virtual address of allocated memory block on success, NULL on failure.
1601ea1f5f37SPavel Tatashin */
memblock_alloc_try_nid_raw(phys_addr_t size,phys_addr_t align,phys_addr_t min_addr,phys_addr_t max_addr,int nid)1602eb31d559SMike Rapoport void * __init memblock_alloc_try_nid_raw(
1603ea1f5f37SPavel Tatashin phys_addr_t size, phys_addr_t align,
1604ea1f5f37SPavel Tatashin phys_addr_t min_addr, phys_addr_t max_addr,
1605ea1f5f37SPavel Tatashin int nid)
1606ea1f5f37SPavel Tatashin {
1607d75f773cSSakari Ailus memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
1608a36aab89SMike Rapoport __func__, (u64)size, (u64)align, nid, &min_addr,
1609a36aab89SMike Rapoport &max_addr, (void *)_RET_IP_);
1610ea1f5f37SPavel Tatashin
161108678804SMike Rapoport return memblock_alloc_internal(size, align, min_addr, max_addr, nid,
161208678804SMike Rapoport false);
1613ea1f5f37SPavel Tatashin }
1614ea1f5f37SPavel Tatashin
1615ea1f5f37SPavel Tatashin /**
1616c0dbe825SMike Rapoport * memblock_alloc_try_nid - allocate boot memory block
161726f09e9bSSantosh Shilimkar * @size: size of memory block to be allocated in bytes
161826f09e9bSSantosh Shilimkar * @align: alignment of the region and block's size
161926f09e9bSSantosh Shilimkar * @min_addr: the lower bound of the memory region from where the allocation
162026f09e9bSSantosh Shilimkar * is preferred (phys address)
162126f09e9bSSantosh Shilimkar * @max_addr: the upper bound of the memory region from where the allocation
162297ad1087SMike Rapoport * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
162326f09e9bSSantosh Shilimkar * allocate only from memory limited by memblock.current_limit value
162426f09e9bSSantosh Shilimkar * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
162526f09e9bSSantosh Shilimkar *
1626c0dbe825SMike Rapoport * Public function, provides additional debug information (including caller
1627c0dbe825SMike Rapoport * info), if enabled. This function zeroes the allocated memory.
162826f09e9bSSantosh Shilimkar *
162947cec443SMike Rapoport * Return:
163026f09e9bSSantosh Shilimkar * Virtual address of allocated memory block on success, NULL on failure.
163126f09e9bSSantosh Shilimkar */
memblock_alloc_try_nid(phys_addr_t size,phys_addr_t align,phys_addr_t min_addr,phys_addr_t max_addr,int nid)1632eb31d559SMike Rapoport void * __init memblock_alloc_try_nid(
163326f09e9bSSantosh Shilimkar phys_addr_t size, phys_addr_t align,
163426f09e9bSSantosh Shilimkar phys_addr_t min_addr, phys_addr_t max_addr,
163526f09e9bSSantosh Shilimkar int nid)
163626f09e9bSSantosh Shilimkar {
163726f09e9bSSantosh Shilimkar void *ptr;
163826f09e9bSSantosh Shilimkar
1639d75f773cSSakari Ailus memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
1640a36aab89SMike Rapoport __func__, (u64)size, (u64)align, nid, &min_addr,
1641a36aab89SMike Rapoport &max_addr, (void *)_RET_IP_);
1642eb31d559SMike Rapoport ptr = memblock_alloc_internal(size, align,
16430ac398b1SYunfeng Ye min_addr, max_addr, nid, false);
1644c0dbe825SMike Rapoport if (ptr)
1645ea1f5f37SPavel Tatashin memset(ptr, 0, size);
164626f09e9bSSantosh Shilimkar
1647c0dbe825SMike Rapoport return ptr;
164826f09e9bSSantosh Shilimkar }
164926f09e9bSSantosh Shilimkar
165026f09e9bSSantosh Shilimkar /**
1651621d9739SMike Rapoport * memblock_free_late - free pages directly to buddy allocator
165248a833ccSMike Rapoport * @base: phys starting address of the boot memory block
165326f09e9bSSantosh Shilimkar * @size: size of the boot memory block in bytes
165426f09e9bSSantosh Shilimkar *
1655a2974133SMike Rapoport * This is only useful when the memblock allocator has already been torn
165626f09e9bSSantosh Shilimkar * down, but we are still initializing the system. Pages are released directly
1657a2974133SMike Rapoport * to the buddy allocator.
165826f09e9bSSantosh Shilimkar */
memblock_free_late(phys_addr_t base,phys_addr_t size)1659621d9739SMike Rapoport void __init memblock_free_late(phys_addr_t base, phys_addr_t size)
166026f09e9bSSantosh Shilimkar {
1661a36aab89SMike Rapoport phys_addr_t cursor, end;
166226f09e9bSSantosh Shilimkar
1663a36aab89SMike Rapoport end = base + size - 1;
1664d75f773cSSakari Ailus memblock_dbg("%s: [%pa-%pa] %pS\n",
1665a36aab89SMike Rapoport __func__, &base, &end, (void *)_RET_IP_);
16669099daedSCatalin Marinas kmemleak_free_part_phys(base, size);
166726f09e9bSSantosh Shilimkar cursor = PFN_UP(base);
166826f09e9bSSantosh Shilimkar end = PFN_DOWN(base + size);
166926f09e9bSSantosh Shilimkar
167026f09e9bSSantosh Shilimkar for (; cursor < end; cursor++) {
1671647037adSAaron Thompson memblock_free_pages(pfn_to_page(cursor), cursor, 0);
1672ca79b0c2SArun KS totalram_pages_inc();
167326f09e9bSSantosh Shilimkar }
167426f09e9bSSantosh Shilimkar }
16759d1e2492SBenjamin Herrenschmidt
16769d1e2492SBenjamin Herrenschmidt /*
16779d1e2492SBenjamin Herrenschmidt * Remaining API functions
16789d1e2492SBenjamin Herrenschmidt */
16799d1e2492SBenjamin Herrenschmidt
memblock_phys_mem_size(void)16801f1ffb8aSDavid Gibson phys_addr_t __init_memblock memblock_phys_mem_size(void)
168195f72d1eSYinghai Lu {
16821440c4e2STejun Heo return memblock.memory.total_size;
168395f72d1eSYinghai Lu }
168495f72d1eSYinghai Lu
memblock_reserved_size(void)16858907de5dSSrikar Dronamraju phys_addr_t __init_memblock memblock_reserved_size(void)
16868907de5dSSrikar Dronamraju {
16878907de5dSSrikar Dronamraju return memblock.reserved.total_size;
16888907de5dSSrikar Dronamraju }
16898907de5dSSrikar Dronamraju
16900a93ebefSSam Ravnborg /* lowest address */
memblock_start_of_DRAM(void)16910a93ebefSSam Ravnborg phys_addr_t __init_memblock memblock_start_of_DRAM(void)
16920a93ebefSSam Ravnborg {
16930a93ebefSSam Ravnborg return memblock.memory.regions[0].base;
16940a93ebefSSam Ravnborg }
16950a93ebefSSam Ravnborg
memblock_end_of_DRAM(void)169610d06439SYinghai Lu phys_addr_t __init_memblock memblock_end_of_DRAM(void)
169795f72d1eSYinghai Lu {
169895f72d1eSYinghai Lu int idx = memblock.memory.cnt - 1;
169995f72d1eSYinghai Lu
1700e3239ff9SBenjamin Herrenschmidt return (memblock.memory.regions[idx].base + memblock.memory.regions[idx].size);
170195f72d1eSYinghai Lu }
170295f72d1eSYinghai Lu
__find_max_addr(phys_addr_t limit)1703a571d4ebSDennis Chen static phys_addr_t __init_memblock __find_max_addr(phys_addr_t limit)
170495f72d1eSYinghai Lu {
17051c4bc43dSStefan Agner phys_addr_t max_addr = PHYS_ADDR_MAX;
1706136199f0SEmil Medve struct memblock_region *r;
170795f72d1eSYinghai Lu
1708a571d4ebSDennis Chen /*
1709a571d4ebSDennis Chen * translate the memory @limit size into the max address within one of
1710a571d4ebSDennis Chen * the memory memblock regions, if the @limit exceeds the total size
17111c4bc43dSStefan Agner * of those regions, max_addr will keep original value PHYS_ADDR_MAX
1712a571d4ebSDennis Chen */
1713cc6de168SMike Rapoport for_each_mem_region(r) {
1714c0ce8fefSTejun Heo if (limit <= r->size) {
1715c0ce8fefSTejun Heo max_addr = r->base + limit;
171695f72d1eSYinghai Lu break;
171795f72d1eSYinghai Lu }
1718c0ce8fefSTejun Heo limit -= r->size;
171995f72d1eSYinghai Lu }
1720c0ce8fefSTejun Heo
1721a571d4ebSDennis Chen return max_addr;
1722a571d4ebSDennis Chen }
1723a571d4ebSDennis Chen
memblock_enforce_memory_limit(phys_addr_t limit)1724a571d4ebSDennis Chen void __init memblock_enforce_memory_limit(phys_addr_t limit)
1725a571d4ebSDennis Chen {
172649aef717SColin Ian King phys_addr_t max_addr;
1727a571d4ebSDennis Chen
1728a571d4ebSDennis Chen if (!limit)
1729a571d4ebSDennis Chen return;
1730a571d4ebSDennis Chen
1731a571d4ebSDennis Chen max_addr = __find_max_addr(limit);
1732a571d4ebSDennis Chen
1733a571d4ebSDennis Chen /* @limit exceeds the total size of the memory, do nothing */
17341c4bc43dSStefan Agner if (max_addr == PHYS_ADDR_MAX)
1735a571d4ebSDennis Chen return;
1736a571d4ebSDennis Chen
1737c0ce8fefSTejun Heo /* truncate both memory and reserved regions */
1738f1af9d3aSPhilipp Hachtmann memblock_remove_range(&memblock.memory, max_addr,
17391c4bc43dSStefan Agner PHYS_ADDR_MAX);
1740f1af9d3aSPhilipp Hachtmann memblock_remove_range(&memblock.reserved, max_addr,
17411c4bc43dSStefan Agner PHYS_ADDR_MAX);
174295f72d1eSYinghai Lu }
174395f72d1eSYinghai Lu
memblock_cap_memory_range(phys_addr_t base,phys_addr_t size)1744c9ca9b4eSAKASHI Takahiro void __init memblock_cap_memory_range(phys_addr_t base, phys_addr_t size)
1745c9ca9b4eSAKASHI Takahiro {
1746c9ca9b4eSAKASHI Takahiro int start_rgn, end_rgn;
1747c9ca9b4eSAKASHI Takahiro int i, ret;
1748c9ca9b4eSAKASHI Takahiro
1749c9ca9b4eSAKASHI Takahiro if (!size)
1750c9ca9b4eSAKASHI Takahiro return;
1751c9ca9b4eSAKASHI Takahiro
17525173ed72SPeng Fan if (!memblock_memory->total_size) {
1753e888fa7bSGeert Uytterhoeven pr_warn("%s: No memory registered yet\n", __func__);
1754e888fa7bSGeert Uytterhoeven return;
1755e888fa7bSGeert Uytterhoeven }
1756e888fa7bSGeert Uytterhoeven
1757c9ca9b4eSAKASHI Takahiro ret = memblock_isolate_range(&memblock.memory, base, size,
1758c9ca9b4eSAKASHI Takahiro &start_rgn, &end_rgn);
1759c9ca9b4eSAKASHI Takahiro if (ret)
1760c9ca9b4eSAKASHI Takahiro return;
1761c9ca9b4eSAKASHI Takahiro
1762c9ca9b4eSAKASHI Takahiro /* remove all the MAP regions */
1763c9ca9b4eSAKASHI Takahiro for (i = memblock.memory.cnt - 1; i >= end_rgn; i--)
1764c9ca9b4eSAKASHI Takahiro if (!memblock_is_nomap(&memblock.memory.regions[i]))
1765c9ca9b4eSAKASHI Takahiro memblock_remove_region(&memblock.memory, i);
1766c9ca9b4eSAKASHI Takahiro
1767c9ca9b4eSAKASHI Takahiro for (i = start_rgn - 1; i >= 0; i--)
1768c9ca9b4eSAKASHI Takahiro if (!memblock_is_nomap(&memblock.memory.regions[i]))
1769c9ca9b4eSAKASHI Takahiro memblock_remove_region(&memblock.memory, i);
1770c9ca9b4eSAKASHI Takahiro
1771c9ca9b4eSAKASHI Takahiro /* truncate the reserved regions */
1772c9ca9b4eSAKASHI Takahiro memblock_remove_range(&memblock.reserved, 0, base);
1773c9ca9b4eSAKASHI Takahiro memblock_remove_range(&memblock.reserved,
17741c4bc43dSStefan Agner base + size, PHYS_ADDR_MAX);
1775c9ca9b4eSAKASHI Takahiro }
1776c9ca9b4eSAKASHI Takahiro
memblock_mem_limit_remove_map(phys_addr_t limit)1777a571d4ebSDennis Chen void __init memblock_mem_limit_remove_map(phys_addr_t limit)
1778a571d4ebSDennis Chen {
1779a571d4ebSDennis Chen phys_addr_t max_addr;
1780a571d4ebSDennis Chen
1781a571d4ebSDennis Chen if (!limit)
1782a571d4ebSDennis Chen return;
1783a571d4ebSDennis Chen
1784a571d4ebSDennis Chen max_addr = __find_max_addr(limit);
1785a571d4ebSDennis Chen
1786a571d4ebSDennis Chen /* @limit exceeds the total size of the memory, do nothing */
17871c4bc43dSStefan Agner if (max_addr == PHYS_ADDR_MAX)
1788a571d4ebSDennis Chen return;
1789a571d4ebSDennis Chen
1790c9ca9b4eSAKASHI Takahiro memblock_cap_memory_range(0, max_addr);
1791a571d4ebSDennis Chen }
1792a571d4ebSDennis Chen
memblock_search(struct memblock_type * type,phys_addr_t addr)1793cd79481dSYinghai Lu static int __init_memblock memblock_search(struct memblock_type *type, phys_addr_t addr)
179472d4b0b4SBenjamin Herrenschmidt {
179572d4b0b4SBenjamin Herrenschmidt unsigned int left = 0, right = type->cnt;
179672d4b0b4SBenjamin Herrenschmidt
179772d4b0b4SBenjamin Herrenschmidt do {
179872d4b0b4SBenjamin Herrenschmidt unsigned int mid = (right + left) / 2;
179972d4b0b4SBenjamin Herrenschmidt
180072d4b0b4SBenjamin Herrenschmidt if (addr < type->regions[mid].base)
180172d4b0b4SBenjamin Herrenschmidt right = mid;
180272d4b0b4SBenjamin Herrenschmidt else if (addr >= (type->regions[mid].base +
180372d4b0b4SBenjamin Herrenschmidt type->regions[mid].size))
180472d4b0b4SBenjamin Herrenschmidt left = mid + 1;
180572d4b0b4SBenjamin Herrenschmidt else
180672d4b0b4SBenjamin Herrenschmidt return mid;
180772d4b0b4SBenjamin Herrenschmidt } while (left < right);
180872d4b0b4SBenjamin Herrenschmidt return -1;
180972d4b0b4SBenjamin Herrenschmidt }
181072d4b0b4SBenjamin Herrenschmidt
memblock_is_reserved(phys_addr_t addr)1811f5a222dcSYueyi Li bool __init_memblock memblock_is_reserved(phys_addr_t addr)
181295f72d1eSYinghai Lu {
181372d4b0b4SBenjamin Herrenschmidt return memblock_search(&memblock.reserved, addr) != -1;
181495f72d1eSYinghai Lu }
181572d4b0b4SBenjamin Herrenschmidt
memblock_is_memory(phys_addr_t addr)1816b4ad0c7eSYaowei Bai bool __init_memblock memblock_is_memory(phys_addr_t addr)
181772d4b0b4SBenjamin Herrenschmidt {
181872d4b0b4SBenjamin Herrenschmidt return memblock_search(&memblock.memory, addr) != -1;
181972d4b0b4SBenjamin Herrenschmidt }
182072d4b0b4SBenjamin Herrenschmidt
memblock_is_map_memory(phys_addr_t addr)1821937f0c26SYaowei Bai bool __init_memblock memblock_is_map_memory(phys_addr_t addr)
1822bf3d3cc5SArd Biesheuvel {
1823bf3d3cc5SArd Biesheuvel int i = memblock_search(&memblock.memory, addr);
1824bf3d3cc5SArd Biesheuvel
1825bf3d3cc5SArd Biesheuvel if (i == -1)
1826bf3d3cc5SArd Biesheuvel return false;
1827bf3d3cc5SArd Biesheuvel return !memblock_is_nomap(&memblock.memory.regions[i]);
1828bf3d3cc5SArd Biesheuvel }
1829bf3d3cc5SArd Biesheuvel
memblock_search_pfn_nid(unsigned long pfn,unsigned long * start_pfn,unsigned long * end_pfn)1830e76b63f8SYinghai Lu int __init_memblock memblock_search_pfn_nid(unsigned long pfn,
1831e76b63f8SYinghai Lu unsigned long *start_pfn, unsigned long *end_pfn)
1832e76b63f8SYinghai Lu {
1833e76b63f8SYinghai Lu struct memblock_type *type = &memblock.memory;
183416763230SFabian Frederick int mid = memblock_search(type, PFN_PHYS(pfn));
1835e76b63f8SYinghai Lu
1836e76b63f8SYinghai Lu if (mid == -1)
1837e76b63f8SYinghai Lu return -1;
1838e76b63f8SYinghai Lu
1839f7e2f7e8SFabian Frederick *start_pfn = PFN_DOWN(type->regions[mid].base);
1840f7e2f7e8SFabian Frederick *end_pfn = PFN_DOWN(type->regions[mid].base + type->regions[mid].size);
1841e76b63f8SYinghai Lu
1842d622abf7SMike Rapoport return memblock_get_region_node(&type->regions[mid]);
1843e76b63f8SYinghai Lu }
1844e76b63f8SYinghai Lu
1845eab30949SStephen Boyd /**
1846eab30949SStephen Boyd * memblock_is_region_memory - check if a region is a subset of memory
1847eab30949SStephen Boyd * @base: base of region to check
1848eab30949SStephen Boyd * @size: size of region to check
1849eab30949SStephen Boyd *
1850eab30949SStephen Boyd * Check if the region [@base, @base + @size) is a subset of a memory block.
1851eab30949SStephen Boyd *
185247cec443SMike Rapoport * Return:
1853eab30949SStephen Boyd * 0 if false, non-zero if true
1854eab30949SStephen Boyd */
memblock_is_region_memory(phys_addr_t base,phys_addr_t size)1855937f0c26SYaowei Bai bool __init_memblock memblock_is_region_memory(phys_addr_t base, phys_addr_t size)
185672d4b0b4SBenjamin Herrenschmidt {
1857abb65272STomi Valkeinen int idx = memblock_search(&memblock.memory, base);
1858eb18f1b5STejun Heo phys_addr_t end = base + memblock_cap_size(base, &size);
185972d4b0b4SBenjamin Herrenschmidt
186072d4b0b4SBenjamin Herrenschmidt if (idx == -1)
1861937f0c26SYaowei Bai return false;
1862ef415ef4SWei Yang return (memblock.memory.regions[idx].base +
1863eb18f1b5STejun Heo memblock.memory.regions[idx].size) >= end;
186495f72d1eSYinghai Lu }
186595f72d1eSYinghai Lu
1866eab30949SStephen Boyd /**
1867eab30949SStephen Boyd * memblock_is_region_reserved - check if a region intersects reserved memory
1868eab30949SStephen Boyd * @base: base of region to check
1869eab30949SStephen Boyd * @size: size of region to check
1870eab30949SStephen Boyd *
187147cec443SMike Rapoport * Check if the region [@base, @base + @size) intersects a reserved
187247cec443SMike Rapoport * memory block.
1873eab30949SStephen Boyd *
187447cec443SMike Rapoport * Return:
1875c5c5c9d1STang Chen * True if they intersect, false if not.
1876eab30949SStephen Boyd */
memblock_is_region_reserved(phys_addr_t base,phys_addr_t size)1877c5c5c9d1STang Chen bool __init_memblock memblock_is_region_reserved(phys_addr_t base, phys_addr_t size)
187895f72d1eSYinghai Lu {
1879c5c5c9d1STang Chen return memblock_overlaps_region(&memblock.reserved, base, size);
188095f72d1eSYinghai Lu }
188195f72d1eSYinghai Lu
memblock_trim_memory(phys_addr_t align)18826ede1fd3SYinghai Lu void __init_memblock memblock_trim_memory(phys_addr_t align)
18836ede1fd3SYinghai Lu {
18846ede1fd3SYinghai Lu phys_addr_t start, end, orig_start, orig_end;
1885136199f0SEmil Medve struct memblock_region *r;
18866ede1fd3SYinghai Lu
1887cc6de168SMike Rapoport for_each_mem_region(r) {
1888136199f0SEmil Medve orig_start = r->base;
1889136199f0SEmil Medve orig_end = r->base + r->size;
18906ede1fd3SYinghai Lu start = round_up(orig_start, align);
18916ede1fd3SYinghai Lu end = round_down(orig_end, align);
18926ede1fd3SYinghai Lu
18936ede1fd3SYinghai Lu if (start == orig_start && end == orig_end)
18946ede1fd3SYinghai Lu continue;
18956ede1fd3SYinghai Lu
18966ede1fd3SYinghai Lu if (start < end) {
1897136199f0SEmil Medve r->base = start;
1898136199f0SEmil Medve r->size = end - start;
18996ede1fd3SYinghai Lu } else {
1900136199f0SEmil Medve memblock_remove_region(&memblock.memory,
1901136199f0SEmil Medve r - memblock.memory.regions);
1902136199f0SEmil Medve r--;
19036ede1fd3SYinghai Lu }
19046ede1fd3SYinghai Lu }
19056ede1fd3SYinghai Lu }
1906e63075a3SBenjamin Herrenschmidt
memblock_set_current_limit(phys_addr_t limit)19073661ca66SYinghai Lu void __init_memblock memblock_set_current_limit(phys_addr_t limit)
1908e63075a3SBenjamin Herrenschmidt {
1909e63075a3SBenjamin Herrenschmidt memblock.current_limit = limit;
1910e63075a3SBenjamin Herrenschmidt }
1911e63075a3SBenjamin Herrenschmidt
memblock_get_current_limit(void)1912fec51014SLaura Abbott phys_addr_t __init_memblock memblock_get_current_limit(void)
1913fec51014SLaura Abbott {
1914fec51014SLaura Abbott return memblock.current_limit;
1915fec51014SLaura Abbott }
1916fec51014SLaura Abbott
memblock_dump(struct memblock_type * type)19170262d9c8SHeiko Carstens static void __init_memblock memblock_dump(struct memblock_type *type)
19186ed311b2SBenjamin Herrenschmidt {
19195d63f81cSMiles Chen phys_addr_t base, end, size;
1920e1720feeSMike Rapoport enum memblock_flags flags;
19218c9c1701SAlexander Kuleshov int idx;
19228c9c1701SAlexander Kuleshov struct memblock_region *rgn;
19236ed311b2SBenjamin Herrenschmidt
19240262d9c8SHeiko Carstens pr_info(" %s.cnt = 0x%lx\n", type->name, type->cnt);
19256ed311b2SBenjamin Herrenschmidt
192666e8b438SGioh Kim for_each_memblock_type(idx, type, rgn) {
19277c0caeb8STejun Heo char nid_buf[32] = "";
19286ed311b2SBenjamin Herrenschmidt
19297c0caeb8STejun Heo base = rgn->base;
19307c0caeb8STejun Heo size = rgn->size;
19315d63f81cSMiles Chen end = base + size - 1;
193266a20757STang Chen flags = rgn->flags;
1933a9ee6cf5SMike Rapoport #ifdef CONFIG_NUMA
19347c0caeb8STejun Heo if (memblock_get_region_node(rgn) != MAX_NUMNODES)
19357c0caeb8STejun Heo snprintf(nid_buf, sizeof(nid_buf), " on node %d",
19367c0caeb8STejun Heo memblock_get_region_node(rgn));
19377c0caeb8STejun Heo #endif
1938e1720feeSMike Rapoport pr_info(" %s[%#x]\t[%pa-%pa], %pa bytes%s flags: %#x\n",
19390262d9c8SHeiko Carstens type->name, idx, &base, &end, &size, nid_buf, flags);
19406ed311b2SBenjamin Herrenschmidt }
19416ed311b2SBenjamin Herrenschmidt }
19426ed311b2SBenjamin Herrenschmidt
__memblock_dump_all(void)194387c55870SMike Rapoport static void __init_memblock __memblock_dump_all(void)
19446ed311b2SBenjamin Herrenschmidt {
19456ed311b2SBenjamin Herrenschmidt pr_info("MEMBLOCK configuration:\n");
19465d63f81cSMiles Chen pr_info(" memory size = %pa reserved size = %pa\n",
19475d63f81cSMiles Chen &memblock.memory.total_size,
19485d63f81cSMiles Chen &memblock.reserved.total_size);
19496ed311b2SBenjamin Herrenschmidt
19500262d9c8SHeiko Carstens memblock_dump(&memblock.memory);
19510262d9c8SHeiko Carstens memblock_dump(&memblock.reserved);
1952409efd4cSHeiko Carstens #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
195377649905SDavid Hildenbrand memblock_dump(&physmem);
1954409efd4cSHeiko Carstens #endif
19556ed311b2SBenjamin Herrenschmidt }
19566ed311b2SBenjamin Herrenschmidt
memblock_dump_all(void)195787c55870SMike Rapoport void __init_memblock memblock_dump_all(void)
195887c55870SMike Rapoport {
195987c55870SMike Rapoport if (memblock_debug)
196087c55870SMike Rapoport __memblock_dump_all();
196187c55870SMike Rapoport }
196287c55870SMike Rapoport
memblock_allow_resize(void)19631aadc056STejun Heo void __init memblock_allow_resize(void)
19646ed311b2SBenjamin Herrenschmidt {
1965142b45a7SBenjamin Herrenschmidt memblock_can_resize = 1;
19666ed311b2SBenjamin Herrenschmidt }
19676ed311b2SBenjamin Herrenschmidt
early_memblock(char * p)19686ed311b2SBenjamin Herrenschmidt static int __init early_memblock(char *p)
19696ed311b2SBenjamin Herrenschmidt {
19706ed311b2SBenjamin Herrenschmidt if (p && strstr(p, "debug"))
19716ed311b2SBenjamin Herrenschmidt memblock_debug = 1;
19726ed311b2SBenjamin Herrenschmidt return 0;
19736ed311b2SBenjamin Herrenschmidt }
19746ed311b2SBenjamin Herrenschmidt early_param("memblock", early_memblock);
19756ed311b2SBenjamin Herrenschmidt
free_memmap(unsigned long start_pfn,unsigned long end_pfn)19764f5b0c17SMike Rapoport static void __init free_memmap(unsigned long start_pfn, unsigned long end_pfn)
19774f5b0c17SMike Rapoport {
19784f5b0c17SMike Rapoport struct page *start_pg, *end_pg;
19794f5b0c17SMike Rapoport phys_addr_t pg, pgend;
19804f5b0c17SMike Rapoport
19814f5b0c17SMike Rapoport /*
19824f5b0c17SMike Rapoport * Convert start_pfn/end_pfn to a struct page pointer.
19834f5b0c17SMike Rapoport */
19844f5b0c17SMike Rapoport start_pg = pfn_to_page(start_pfn - 1) + 1;
19854f5b0c17SMike Rapoport end_pg = pfn_to_page(end_pfn - 1) + 1;
19864f5b0c17SMike Rapoport
19874f5b0c17SMike Rapoport /*
19884f5b0c17SMike Rapoport * Convert to physical addresses, and round start upwards and end
19894f5b0c17SMike Rapoport * downwards.
19904f5b0c17SMike Rapoport */
19914f5b0c17SMike Rapoport pg = PAGE_ALIGN(__pa(start_pg));
19924f5b0c17SMike Rapoport pgend = __pa(end_pg) & PAGE_MASK;
19934f5b0c17SMike Rapoport
19944f5b0c17SMike Rapoport /*
19954f5b0c17SMike Rapoport * If there are free pages between these, free the section of the
19964f5b0c17SMike Rapoport * memmap array.
19974f5b0c17SMike Rapoport */
19984f5b0c17SMike Rapoport if (pg < pgend)
19993ecc6834SMike Rapoport memblock_phys_free(pg, pgend - pg);
20004f5b0c17SMike Rapoport }
20014f5b0c17SMike Rapoport
20024f5b0c17SMike Rapoport /*
20034f5b0c17SMike Rapoport * The mem_map array can get very big. Free the unused area of the memory map.
20044f5b0c17SMike Rapoport */
free_unused_memmap(void)20054f5b0c17SMike Rapoport static void __init free_unused_memmap(void)
20064f5b0c17SMike Rapoport {
20074f5b0c17SMike Rapoport unsigned long start, end, prev_end = 0;
20084f5b0c17SMike Rapoport int i;
20094f5b0c17SMike Rapoport
20104f5b0c17SMike Rapoport if (!IS_ENABLED(CONFIG_HAVE_ARCH_PFN_VALID) ||
20114f5b0c17SMike Rapoport IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP))
20124f5b0c17SMike Rapoport return;
20134f5b0c17SMike Rapoport
20144f5b0c17SMike Rapoport /*
20154f5b0c17SMike Rapoport * This relies on each bank being in address order.
20164f5b0c17SMike Rapoport * The banks are sorted previously in bootmem_init().
20174f5b0c17SMike Rapoport */
20184f5b0c17SMike Rapoport for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, NULL) {
20194f5b0c17SMike Rapoport #ifdef CONFIG_SPARSEMEM
20204f5b0c17SMike Rapoport /*
20214f5b0c17SMike Rapoport * Take care not to free memmap entries that don't exist
20224f5b0c17SMike Rapoport * due to SPARSEMEM sections which aren't present.
20234f5b0c17SMike Rapoport */
20244f5b0c17SMike Rapoport start = min(start, ALIGN(prev_end, PAGES_PER_SECTION));
20254f5b0c17SMike Rapoport #endif
20264f5b0c17SMike Rapoport /*
2027e2a86800SMike Rapoport * Align down here since many operations in VM subsystem
2028e2a86800SMike Rapoport * presume that there are no holes in the memory map inside
2029e2a86800SMike Rapoport * a pageblock
20304f5b0c17SMike Rapoport */
20314f9bc69aSKefeng Wang start = pageblock_start_pfn(start);
20324f5b0c17SMike Rapoport
20334f5b0c17SMike Rapoport /*
20344f5b0c17SMike Rapoport * If we had a previous bank, and there is a space
20354f5b0c17SMike Rapoport * between the current bank and the previous, free it.
20364f5b0c17SMike Rapoport */
20374f5b0c17SMike Rapoport if (prev_end && prev_end < start)
20384f5b0c17SMike Rapoport free_memmap(prev_end, start);
20394f5b0c17SMike Rapoport
20404f5b0c17SMike Rapoport /*
2041e2a86800SMike Rapoport * Align up here since many operations in VM subsystem
2042e2a86800SMike Rapoport * presume that there are no holes in the memory map inside
2043e2a86800SMike Rapoport * a pageblock
20444f5b0c17SMike Rapoport */
20455f7fa13fSKefeng Wang prev_end = pageblock_align(end);
20464f5b0c17SMike Rapoport }
20474f5b0c17SMike Rapoport
20484f5b0c17SMike Rapoport #ifdef CONFIG_SPARSEMEM
2049f921f53eSMike Rapoport if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION)) {
20505f7fa13fSKefeng Wang prev_end = pageblock_align(end);
20514f5b0c17SMike Rapoport free_memmap(prev_end, ALIGN(prev_end, PAGES_PER_SECTION));
2052f921f53eSMike Rapoport }
20534f5b0c17SMike Rapoport #endif
20544f5b0c17SMike Rapoport }
20554f5b0c17SMike Rapoport
__free_pages_memory(unsigned long start,unsigned long end)2056bda49a81SMike Rapoport static void __init __free_pages_memory(unsigned long start, unsigned long end)
2057bda49a81SMike Rapoport {
2058bda49a81SMike Rapoport int order;
2059bda49a81SMike Rapoport
2060bda49a81SMike Rapoport while (start < end) {
206159f876fbSKirill A. Shutemov /*
206259f876fbSKirill A. Shutemov * Free the pages in the largest chunks alignment allows.
206359f876fbSKirill A. Shutemov *
206459f876fbSKirill A. Shutemov * __ffs() behaviour is undefined for 0. start == 0 is
206559f876fbSKirill A. Shutemov * MAX_ORDER-aligned, set order to MAX_ORDER for the case.
206659f876fbSKirill A. Shutemov */
206759f876fbSKirill A. Shutemov if (start)
206823baf831SKirill A. Shutemov order = min_t(int, MAX_ORDER, __ffs(start));
206959f876fbSKirill A. Shutemov else
207059f876fbSKirill A. Shutemov order = MAX_ORDER;
2071bda49a81SMike Rapoport
2072bda49a81SMike Rapoport while (start + (1UL << order) > end)
2073bda49a81SMike Rapoport order--;
2074bda49a81SMike Rapoport
2075bda49a81SMike Rapoport memblock_free_pages(pfn_to_page(start), start, order);
2076bda49a81SMike Rapoport
2077bda49a81SMike Rapoport start += (1UL << order);
2078bda49a81SMike Rapoport }
2079bda49a81SMike Rapoport }
2080bda49a81SMike Rapoport
__free_memory_core(phys_addr_t start,phys_addr_t end)2081bda49a81SMike Rapoport static unsigned long __init __free_memory_core(phys_addr_t start,
2082bda49a81SMike Rapoport phys_addr_t end)
2083bda49a81SMike Rapoport {
2084bda49a81SMike Rapoport unsigned long start_pfn = PFN_UP(start);
2085bda49a81SMike Rapoport unsigned long end_pfn = min_t(unsigned long,
2086bda49a81SMike Rapoport PFN_DOWN(end), max_low_pfn);
2087bda49a81SMike Rapoport
2088bda49a81SMike Rapoport if (start_pfn >= end_pfn)
2089bda49a81SMike Rapoport return 0;
2090bda49a81SMike Rapoport
2091bda49a81SMike Rapoport __free_pages_memory(start_pfn, end_pfn);
2092bda49a81SMike Rapoport
2093bda49a81SMike Rapoport return end_pfn - start_pfn;
2094bda49a81SMike Rapoport }
2095bda49a81SMike Rapoport
memmap_init_reserved_pages(void)20969092d4f7SMike Rapoport static void __init memmap_init_reserved_pages(void)
20979092d4f7SMike Rapoport {
20989092d4f7SMike Rapoport struct memblock_region *region;
20999092d4f7SMike Rapoport phys_addr_t start, end;
210061167ad5SYajun Deng int nid;
21019092d4f7SMike Rapoport
210261167ad5SYajun Deng /*
210361167ad5SYajun Deng * set nid on all reserved pages and also treat struct
210461167ad5SYajun Deng * pages for the NOMAP regions as PageReserved
210561167ad5SYajun Deng */
21069092d4f7SMike Rapoport for_each_mem_region(region) {
210761167ad5SYajun Deng nid = memblock_get_region_node(region);
21089092d4f7SMike Rapoport start = region->base;
21099092d4f7SMike Rapoport end = start + region->size;
211061167ad5SYajun Deng
211161167ad5SYajun Deng if (memblock_is_nomap(region))
211261167ad5SYajun Deng reserve_bootmem_region(start, end, nid);
211361167ad5SYajun Deng
211461167ad5SYajun Deng memblock_set_node(start, end, &memblock.reserved, nid);
21159092d4f7SMike Rapoport }
211661167ad5SYajun Deng
211761167ad5SYajun Deng /* initialize struct pages for the reserved regions */
211861167ad5SYajun Deng for_each_reserved_mem_region(region) {
211961167ad5SYajun Deng nid = memblock_get_region_node(region);
212061167ad5SYajun Deng start = region->base;
212161167ad5SYajun Deng end = start + region->size;
212261167ad5SYajun Deng
2123e791a345SYajun Deng if (nid == NUMA_NO_NODE || nid >= MAX_NUMNODES)
2124e791a345SYajun Deng nid = early_pfn_to_nid(PFN_DOWN(start));
2125e791a345SYajun Deng
212661167ad5SYajun Deng reserve_bootmem_region(start, end, nid);
21279092d4f7SMike Rapoport }
21289092d4f7SMike Rapoport }
21299092d4f7SMike Rapoport
free_low_memory_core_early(void)2130bda49a81SMike Rapoport static unsigned long __init free_low_memory_core_early(void)
2131bda49a81SMike Rapoport {
2132bda49a81SMike Rapoport unsigned long count = 0;
2133bda49a81SMike Rapoport phys_addr_t start, end;
2134bda49a81SMike Rapoport u64 i;
2135bda49a81SMike Rapoport
2136bda49a81SMike Rapoport memblock_clear_hotplug(0, -1);
2137bda49a81SMike Rapoport
21389092d4f7SMike Rapoport memmap_init_reserved_pages();
2139bda49a81SMike Rapoport
2140bda49a81SMike Rapoport /*
2141bda49a81SMike Rapoport * We need to use NUMA_NO_NODE instead of NODE_DATA(0)->node_id
2142bda49a81SMike Rapoport * because in some case like Node0 doesn't have RAM installed
2143bda49a81SMike Rapoport * low ram will be on Node1
2144bda49a81SMike Rapoport */
2145bda49a81SMike Rapoport for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end,
2146bda49a81SMike Rapoport NULL)
2147bda49a81SMike Rapoport count += __free_memory_core(start, end);
2148bda49a81SMike Rapoport
2149bda49a81SMike Rapoport return count;
2150bda49a81SMike Rapoport }
2151bda49a81SMike Rapoport
2152bda49a81SMike Rapoport static int reset_managed_pages_done __initdata;
2153bda49a81SMike Rapoport
reset_node_managed_pages(pg_data_t * pgdat)2154a668968fSHaifeng Xu static void __init reset_node_managed_pages(pg_data_t *pgdat)
2155bda49a81SMike Rapoport {
2156bda49a81SMike Rapoport struct zone *z;
2157bda49a81SMike Rapoport
2158bda49a81SMike Rapoport for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
21599705bea5SArun KS atomic_long_set(&z->managed_pages, 0);
2160bda49a81SMike Rapoport }
2161bda49a81SMike Rapoport
reset_all_zones_managed_pages(void)2162bda49a81SMike Rapoport void __init reset_all_zones_managed_pages(void)
2163bda49a81SMike Rapoport {
2164bda49a81SMike Rapoport struct pglist_data *pgdat;
2165bda49a81SMike Rapoport
2166bda49a81SMike Rapoport if (reset_managed_pages_done)
2167bda49a81SMike Rapoport return;
2168bda49a81SMike Rapoport
2169bda49a81SMike Rapoport for_each_online_pgdat(pgdat)
2170bda49a81SMike Rapoport reset_node_managed_pages(pgdat);
2171bda49a81SMike Rapoport
2172bda49a81SMike Rapoport reset_managed_pages_done = 1;
2173bda49a81SMike Rapoport }
2174bda49a81SMike Rapoport
2175bda49a81SMike Rapoport /**
2176bda49a81SMike Rapoport * memblock_free_all - release free pages to the buddy allocator
2177bda49a81SMike Rapoport */
memblock_free_all(void)2178097d43d8SDaeseok Youn void __init memblock_free_all(void)
2179bda49a81SMike Rapoport {
2180bda49a81SMike Rapoport unsigned long pages;
2181bda49a81SMike Rapoport
21824f5b0c17SMike Rapoport free_unused_memmap();
2183bda49a81SMike Rapoport reset_all_zones_managed_pages();
2184bda49a81SMike Rapoport
2185bda49a81SMike Rapoport pages = free_low_memory_core_early();
2186ca79b0c2SArun KS totalram_pages_add(pages);
2187bda49a81SMike Rapoport }
2188bda49a81SMike Rapoport
2189350e88baSMike Rapoport #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_ARCH_KEEP_MEMBLOCK)
2190493f349eSYuwei Guan static const char * const flagname[] = {
2191493f349eSYuwei Guan [ilog2(MEMBLOCK_HOTPLUG)] = "HOTPLUG",
2192493f349eSYuwei Guan [ilog2(MEMBLOCK_MIRROR)] = "MIRROR",
2193493f349eSYuwei Guan [ilog2(MEMBLOCK_NOMAP)] = "NOMAP",
2194493f349eSYuwei Guan [ilog2(MEMBLOCK_DRIVER_MANAGED)] = "DRV_MNG",
2195493f349eSYuwei Guan };
21966d03b885SBenjamin Herrenschmidt
memblock_debug_show(struct seq_file * m,void * private)21976d03b885SBenjamin Herrenschmidt static int memblock_debug_show(struct seq_file *m, void *private)
21986d03b885SBenjamin Herrenschmidt {
21996d03b885SBenjamin Herrenschmidt struct memblock_type *type = m->private;
22006d03b885SBenjamin Herrenschmidt struct memblock_region *reg;
2201de649e7fSYuwei Guan int i, j, nid;
2202493f349eSYuwei Guan unsigned int count = ARRAY_SIZE(flagname);
22035d63f81cSMiles Chen phys_addr_t end;
22046d03b885SBenjamin Herrenschmidt
22056d03b885SBenjamin Herrenschmidt for (i = 0; i < type->cnt; i++) {
22066d03b885SBenjamin Herrenschmidt reg = &type->regions[i];
22075d63f81cSMiles Chen end = reg->base + reg->size - 1;
2208de649e7fSYuwei Guan nid = memblock_get_region_node(reg);
22096d03b885SBenjamin Herrenschmidt
22105d63f81cSMiles Chen seq_printf(m, "%4d: ", i);
2211493f349eSYuwei Guan seq_printf(m, "%pa..%pa ", ®->base, &end);
2212de649e7fSYuwei Guan if (nid != MAX_NUMNODES)
2213de649e7fSYuwei Guan seq_printf(m, "%4d ", nid);
2214de649e7fSYuwei Guan else
2215de649e7fSYuwei Guan seq_printf(m, "%4c ", 'x');
2216493f349eSYuwei Guan if (reg->flags) {
2217493f349eSYuwei Guan for (j = 0; j < count; j++) {
2218493f349eSYuwei Guan if (reg->flags & (1U << j)) {
2219493f349eSYuwei Guan seq_printf(m, "%s\n", flagname[j]);
2220493f349eSYuwei Guan break;
2221493f349eSYuwei Guan }
2222493f349eSYuwei Guan }
2223493f349eSYuwei Guan if (j == count)
2224493f349eSYuwei Guan seq_printf(m, "%s\n", "UNKNOWN");
2225493f349eSYuwei Guan } else {
2226493f349eSYuwei Guan seq_printf(m, "%s\n", "NONE");
2227493f349eSYuwei Guan }
22286d03b885SBenjamin Herrenschmidt }
22296d03b885SBenjamin Herrenschmidt return 0;
22306d03b885SBenjamin Herrenschmidt }
22315ad35093SAndy Shevchenko DEFINE_SHOW_ATTRIBUTE(memblock_debug);
22326d03b885SBenjamin Herrenschmidt
memblock_init_debugfs(void)22336d03b885SBenjamin Herrenschmidt static int __init memblock_init_debugfs(void)
22346d03b885SBenjamin Herrenschmidt {
22356d03b885SBenjamin Herrenschmidt struct dentry *root = debugfs_create_dir("memblock", NULL);
2236d9f7979cSGreg Kroah-Hartman
22370825a6f9SJoe Perches debugfs_create_file("memory", 0444, root,
22380825a6f9SJoe Perches &memblock.memory, &memblock_debug_fops);
22390825a6f9SJoe Perches debugfs_create_file("reserved", 0444, root,
22400825a6f9SJoe Perches &memblock.reserved, &memblock_debug_fops);
224170210ed9SPhilipp Hachtmann #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
224277649905SDavid Hildenbrand debugfs_create_file("physmem", 0444, root, &physmem,
224377649905SDavid Hildenbrand &memblock_debug_fops);
224470210ed9SPhilipp Hachtmann #endif
22456d03b885SBenjamin Herrenschmidt
22466d03b885SBenjamin Herrenschmidt return 0;
22476d03b885SBenjamin Herrenschmidt }
22486d03b885SBenjamin Herrenschmidt __initcall(memblock_init_debugfs);
22496d03b885SBenjamin Herrenschmidt
22506d03b885SBenjamin Herrenschmidt #endif /* CONFIG_DEBUG_FS */
2251