1 /* 2 * Procedures for maintaining information about logical memory blocks. 3 * 4 * Peter Bergner, IBM Corp. June 2001. 5 * Copyright (C) 2001 Peter Bergner. 6 * 7 * This program is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU General Public License 9 * as published by the Free Software Foundation; either version 10 * 2 of the License, or (at your option) any later version. 11 */ 12 13 #include <linux/kernel.h> 14 #include <linux/slab.h> 15 #include <linux/init.h> 16 #include <linux/bitops.h> 17 #include <linux/poison.h> 18 #include <linux/pfn.h> 19 #include <linux/debugfs.h> 20 #include <linux/seq_file.h> 21 #include <linux/memblock.h> 22 23 #include <asm/sections.h> 24 #include <linux/io.h> 25 26 #include "internal.h" 27 28 static struct memblock_region memblock_memory_init_regions[INIT_MEMBLOCK_REGIONS] __initdata_memblock; 29 static struct memblock_region memblock_reserved_init_regions[INIT_MEMBLOCK_REGIONS] __initdata_memblock; 30 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 31 static struct memblock_region memblock_physmem_init_regions[INIT_PHYSMEM_REGIONS] __initdata_memblock; 32 #endif 33 34 struct memblock memblock __initdata_memblock = { 35 .memory.regions = memblock_memory_init_regions, 36 .memory.cnt = 1, /* empty dummy entry */ 37 .memory.max = INIT_MEMBLOCK_REGIONS, 38 .memory.name = "memory", 39 40 .reserved.regions = memblock_reserved_init_regions, 41 .reserved.cnt = 1, /* empty dummy entry */ 42 .reserved.max = INIT_MEMBLOCK_REGIONS, 43 .reserved.name = "reserved", 44 45 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 46 .physmem.regions = memblock_physmem_init_regions, 47 .physmem.cnt = 1, /* empty dummy entry */ 48 .physmem.max = INIT_PHYSMEM_REGIONS, 49 .physmem.name = "physmem", 50 #endif 51 52 .bottom_up = false, 53 .current_limit = MEMBLOCK_ALLOC_ANYWHERE, 54 }; 55 56 int memblock_debug __initdata_memblock; 57 static bool system_has_some_mirror __initdata_memblock = false; 58 static int memblock_can_resize __initdata_memblock; 59 static int memblock_memory_in_slab __initdata_memblock = 0; 60 static int memblock_reserved_in_slab __initdata_memblock = 0; 61 62 ulong __init_memblock choose_memblock_flags(void) 63 { 64 return system_has_some_mirror ? MEMBLOCK_MIRROR : MEMBLOCK_NONE; 65 } 66 67 /* adjust *@size so that (@base + *@size) doesn't overflow, return new size */ 68 static inline phys_addr_t memblock_cap_size(phys_addr_t base, phys_addr_t *size) 69 { 70 return *size = min(*size, (phys_addr_t)ULLONG_MAX - base); 71 } 72 73 /* 74 * Address comparison utilities 75 */ 76 static unsigned long __init_memblock memblock_addrs_overlap(phys_addr_t base1, phys_addr_t size1, 77 phys_addr_t base2, phys_addr_t size2) 78 { 79 return ((base1 < (base2 + size2)) && (base2 < (base1 + size1))); 80 } 81 82 bool __init_memblock memblock_overlaps_region(struct memblock_type *type, 83 phys_addr_t base, phys_addr_t size) 84 { 85 unsigned long i; 86 87 for (i = 0; i < type->cnt; i++) 88 if (memblock_addrs_overlap(base, size, type->regions[i].base, 89 type->regions[i].size)) 90 break; 91 return i < type->cnt; 92 } 93 94 /* 95 * __memblock_find_range_bottom_up - find free area utility in bottom-up 96 * @start: start of candidate range 97 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_{ANYWHERE|ACCESSIBLE} 98 * @size: size of free area to find 99 * @align: alignment of free area to find 100 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 101 * @flags: pick from blocks based on memory attributes 102 * 103 * Utility called from memblock_find_in_range_node(), find free area bottom-up. 104 * 105 * RETURNS: 106 * Found address on success, 0 on failure. 107 */ 108 static phys_addr_t __init_memblock 109 __memblock_find_range_bottom_up(phys_addr_t start, phys_addr_t end, 110 phys_addr_t size, phys_addr_t align, int nid, 111 ulong flags) 112 { 113 phys_addr_t this_start, this_end, cand; 114 u64 i; 115 116 for_each_free_mem_range(i, nid, flags, &this_start, &this_end, NULL) { 117 this_start = clamp(this_start, start, end); 118 this_end = clamp(this_end, start, end); 119 120 cand = round_up(this_start, align); 121 if (cand < this_end && this_end - cand >= size) 122 return cand; 123 } 124 125 return 0; 126 } 127 128 /** 129 * __memblock_find_range_top_down - find free area utility, in top-down 130 * @start: start of candidate range 131 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_{ANYWHERE|ACCESSIBLE} 132 * @size: size of free area to find 133 * @align: alignment of free area to find 134 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 135 * @flags: pick from blocks based on memory attributes 136 * 137 * Utility called from memblock_find_in_range_node(), find free area top-down. 138 * 139 * RETURNS: 140 * Found address on success, 0 on failure. 141 */ 142 static phys_addr_t __init_memblock 143 __memblock_find_range_top_down(phys_addr_t start, phys_addr_t end, 144 phys_addr_t size, phys_addr_t align, int nid, 145 ulong flags) 146 { 147 phys_addr_t this_start, this_end, cand; 148 u64 i; 149 150 for_each_free_mem_range_reverse(i, nid, flags, &this_start, &this_end, 151 NULL) { 152 this_start = clamp(this_start, start, end); 153 this_end = clamp(this_end, start, end); 154 155 if (this_end < size) 156 continue; 157 158 cand = round_down(this_end - size, align); 159 if (cand >= this_start) 160 return cand; 161 } 162 163 return 0; 164 } 165 166 /** 167 * memblock_find_in_range_node - find free area in given range and node 168 * @size: size of free area to find 169 * @align: alignment of free area to find 170 * @start: start of candidate range 171 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_{ANYWHERE|ACCESSIBLE} 172 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 173 * @flags: pick from blocks based on memory attributes 174 * 175 * Find @size free area aligned to @align in the specified range and node. 176 * 177 * When allocation direction is bottom-up, the @start should be greater 178 * than the end of the kernel image. Otherwise, it will be trimmed. The 179 * reason is that we want the bottom-up allocation just near the kernel 180 * image so it is highly likely that the allocated memory and the kernel 181 * will reside in the same node. 182 * 183 * If bottom-up allocation failed, will try to allocate memory top-down. 184 * 185 * RETURNS: 186 * Found address on success, 0 on failure. 187 */ 188 phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size, 189 phys_addr_t align, phys_addr_t start, 190 phys_addr_t end, int nid, ulong flags) 191 { 192 phys_addr_t kernel_end, ret; 193 194 /* pump up @end */ 195 if (end == MEMBLOCK_ALLOC_ACCESSIBLE) 196 end = memblock.current_limit; 197 198 /* avoid allocating the first page */ 199 start = max_t(phys_addr_t, start, PAGE_SIZE); 200 end = max(start, end); 201 kernel_end = __pa_symbol(_end); 202 203 /* 204 * try bottom-up allocation only when bottom-up mode 205 * is set and @end is above the kernel image. 206 */ 207 if (memblock_bottom_up() && end > kernel_end) { 208 phys_addr_t bottom_up_start; 209 210 /* make sure we will allocate above the kernel */ 211 bottom_up_start = max(start, kernel_end); 212 213 /* ok, try bottom-up allocation first */ 214 ret = __memblock_find_range_bottom_up(bottom_up_start, end, 215 size, align, nid, flags); 216 if (ret) 217 return ret; 218 219 /* 220 * we always limit bottom-up allocation above the kernel, 221 * but top-down allocation doesn't have the limit, so 222 * retrying top-down allocation may succeed when bottom-up 223 * allocation failed. 224 * 225 * bottom-up allocation is expected to be fail very rarely, 226 * so we use WARN_ONCE() here to see the stack trace if 227 * fail happens. 228 */ 229 WARN_ONCE(1, "memblock: bottom-up allocation failed, memory hotunplug may be affected\n"); 230 } 231 232 return __memblock_find_range_top_down(start, end, size, align, nid, 233 flags); 234 } 235 236 /** 237 * memblock_find_in_range - find free area in given range 238 * @start: start of candidate range 239 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_{ANYWHERE|ACCESSIBLE} 240 * @size: size of free area to find 241 * @align: alignment of free area to find 242 * 243 * Find @size free area aligned to @align in the specified range. 244 * 245 * RETURNS: 246 * Found address on success, 0 on failure. 247 */ 248 phys_addr_t __init_memblock memblock_find_in_range(phys_addr_t start, 249 phys_addr_t end, phys_addr_t size, 250 phys_addr_t align) 251 { 252 phys_addr_t ret; 253 ulong flags = choose_memblock_flags(); 254 255 again: 256 ret = memblock_find_in_range_node(size, align, start, end, 257 NUMA_NO_NODE, flags); 258 259 if (!ret && (flags & MEMBLOCK_MIRROR)) { 260 pr_warn("Could not allocate %pap bytes of mirrored memory\n", 261 &size); 262 flags &= ~MEMBLOCK_MIRROR; 263 goto again; 264 } 265 266 return ret; 267 } 268 269 static void __init_memblock memblock_remove_region(struct memblock_type *type, unsigned long r) 270 { 271 type->total_size -= type->regions[r].size; 272 memmove(&type->regions[r], &type->regions[r + 1], 273 (type->cnt - (r + 1)) * sizeof(type->regions[r])); 274 type->cnt--; 275 276 /* Special case for empty arrays */ 277 if (type->cnt == 0) { 278 WARN_ON(type->total_size != 0); 279 type->cnt = 1; 280 type->regions[0].base = 0; 281 type->regions[0].size = 0; 282 type->regions[0].flags = 0; 283 memblock_set_region_node(&type->regions[0], MAX_NUMNODES); 284 } 285 } 286 287 #ifdef CONFIG_ARCH_DISCARD_MEMBLOCK 288 /** 289 * Discard memory and reserved arrays if they were allocated 290 */ 291 void __init memblock_discard(void) 292 { 293 phys_addr_t addr, size; 294 295 if (memblock.reserved.regions != memblock_reserved_init_regions) { 296 addr = __pa(memblock.reserved.regions); 297 size = PAGE_ALIGN(sizeof(struct memblock_region) * 298 memblock.reserved.max); 299 __memblock_free_late(addr, size); 300 } 301 302 if (memblock.memory.regions != memblock_memory_init_regions) { 303 addr = __pa(memblock.memory.regions); 304 size = PAGE_ALIGN(sizeof(struct memblock_region) * 305 memblock.memory.max); 306 __memblock_free_late(addr, size); 307 } 308 } 309 #endif 310 311 /** 312 * memblock_double_array - double the size of the memblock regions array 313 * @type: memblock type of the regions array being doubled 314 * @new_area_start: starting address of memory range to avoid overlap with 315 * @new_area_size: size of memory range to avoid overlap with 316 * 317 * Double the size of the @type regions array. If memblock is being used to 318 * allocate memory for a new reserved regions array and there is a previously 319 * allocated memory range [@new_area_start,@new_area_start+@new_area_size] 320 * waiting to be reserved, ensure the memory used by the new array does 321 * not overlap. 322 * 323 * RETURNS: 324 * 0 on success, -1 on failure. 325 */ 326 static int __init_memblock memblock_double_array(struct memblock_type *type, 327 phys_addr_t new_area_start, 328 phys_addr_t new_area_size) 329 { 330 struct memblock_region *new_array, *old_array; 331 phys_addr_t old_alloc_size, new_alloc_size; 332 phys_addr_t old_size, new_size, addr; 333 int use_slab = slab_is_available(); 334 int *in_slab; 335 336 /* We don't allow resizing until we know about the reserved regions 337 * of memory that aren't suitable for allocation 338 */ 339 if (!memblock_can_resize) 340 return -1; 341 342 /* Calculate new doubled size */ 343 old_size = type->max * sizeof(struct memblock_region); 344 new_size = old_size << 1; 345 /* 346 * We need to allocated new one align to PAGE_SIZE, 347 * so we can free them completely later. 348 */ 349 old_alloc_size = PAGE_ALIGN(old_size); 350 new_alloc_size = PAGE_ALIGN(new_size); 351 352 /* Retrieve the slab flag */ 353 if (type == &memblock.memory) 354 in_slab = &memblock_memory_in_slab; 355 else 356 in_slab = &memblock_reserved_in_slab; 357 358 /* Try to find some space for it. 359 * 360 * WARNING: We assume that either slab_is_available() and we use it or 361 * we use MEMBLOCK for allocations. That means that this is unsafe to 362 * use when bootmem is currently active (unless bootmem itself is 363 * implemented on top of MEMBLOCK which isn't the case yet) 364 * 365 * This should however not be an issue for now, as we currently only 366 * call into MEMBLOCK while it's still active, or much later when slab 367 * is active for memory hotplug operations 368 */ 369 if (use_slab) { 370 new_array = kmalloc(new_size, GFP_KERNEL); 371 addr = new_array ? __pa(new_array) : 0; 372 } else { 373 /* only exclude range when trying to double reserved.regions */ 374 if (type != &memblock.reserved) 375 new_area_start = new_area_size = 0; 376 377 addr = memblock_find_in_range(new_area_start + new_area_size, 378 memblock.current_limit, 379 new_alloc_size, PAGE_SIZE); 380 if (!addr && new_area_size) 381 addr = memblock_find_in_range(0, 382 min(new_area_start, memblock.current_limit), 383 new_alloc_size, PAGE_SIZE); 384 385 new_array = addr ? __va(addr) : NULL; 386 } 387 if (!addr) { 388 pr_err("memblock: Failed to double %s array from %ld to %ld entries !\n", 389 type->name, type->max, type->max * 2); 390 return -1; 391 } 392 393 memblock_dbg("memblock: %s is doubled to %ld at [%#010llx-%#010llx]", 394 type->name, type->max * 2, (u64)addr, 395 (u64)addr + new_size - 1); 396 397 /* 398 * Found space, we now need to move the array over before we add the 399 * reserved region since it may be our reserved array itself that is 400 * full. 401 */ 402 memcpy(new_array, type->regions, old_size); 403 memset(new_array + type->max, 0, old_size); 404 old_array = type->regions; 405 type->regions = new_array; 406 type->max <<= 1; 407 408 /* Free old array. We needn't free it if the array is the static one */ 409 if (*in_slab) 410 kfree(old_array); 411 else if (old_array != memblock_memory_init_regions && 412 old_array != memblock_reserved_init_regions) 413 memblock_free(__pa(old_array), old_alloc_size); 414 415 /* 416 * Reserve the new array if that comes from the memblock. Otherwise, we 417 * needn't do it 418 */ 419 if (!use_slab) 420 BUG_ON(memblock_reserve(addr, new_alloc_size)); 421 422 /* Update slab flag */ 423 *in_slab = use_slab; 424 425 return 0; 426 } 427 428 /** 429 * memblock_merge_regions - merge neighboring compatible regions 430 * @type: memblock type to scan 431 * 432 * Scan @type and merge neighboring compatible regions. 433 */ 434 static void __init_memblock memblock_merge_regions(struct memblock_type *type) 435 { 436 int i = 0; 437 438 /* cnt never goes below 1 */ 439 while (i < type->cnt - 1) { 440 struct memblock_region *this = &type->regions[i]; 441 struct memblock_region *next = &type->regions[i + 1]; 442 443 if (this->base + this->size != next->base || 444 memblock_get_region_node(this) != 445 memblock_get_region_node(next) || 446 this->flags != next->flags) { 447 BUG_ON(this->base + this->size > next->base); 448 i++; 449 continue; 450 } 451 452 this->size += next->size; 453 /* move forward from next + 1, index of which is i + 2 */ 454 memmove(next, next + 1, (type->cnt - (i + 2)) * sizeof(*next)); 455 type->cnt--; 456 } 457 } 458 459 /** 460 * memblock_insert_region - insert new memblock region 461 * @type: memblock type to insert into 462 * @idx: index for the insertion point 463 * @base: base address of the new region 464 * @size: size of the new region 465 * @nid: node id of the new region 466 * @flags: flags of the new region 467 * 468 * Insert new memblock region [@base,@base+@size) into @type at @idx. 469 * @type must already have extra room to accommodate the new region. 470 */ 471 static void __init_memblock memblock_insert_region(struct memblock_type *type, 472 int idx, phys_addr_t base, 473 phys_addr_t size, 474 int nid, unsigned long flags) 475 { 476 struct memblock_region *rgn = &type->regions[idx]; 477 478 BUG_ON(type->cnt >= type->max); 479 memmove(rgn + 1, rgn, (type->cnt - idx) * sizeof(*rgn)); 480 rgn->base = base; 481 rgn->size = size; 482 rgn->flags = flags; 483 memblock_set_region_node(rgn, nid); 484 type->cnt++; 485 type->total_size += size; 486 } 487 488 /** 489 * memblock_add_range - add new memblock region 490 * @type: memblock type to add new region into 491 * @base: base address of the new region 492 * @size: size of the new region 493 * @nid: nid of the new region 494 * @flags: flags of the new region 495 * 496 * Add new memblock region [@base,@base+@size) into @type. The new region 497 * is allowed to overlap with existing ones - overlaps don't affect already 498 * existing regions. @type is guaranteed to be minimal (all neighbouring 499 * compatible regions are merged) after the addition. 500 * 501 * RETURNS: 502 * 0 on success, -errno on failure. 503 */ 504 int __init_memblock memblock_add_range(struct memblock_type *type, 505 phys_addr_t base, phys_addr_t size, 506 int nid, unsigned long flags) 507 { 508 bool insert = false; 509 phys_addr_t obase = base; 510 phys_addr_t end = base + memblock_cap_size(base, &size); 511 int idx, nr_new; 512 struct memblock_region *rgn; 513 514 if (!size) 515 return 0; 516 517 /* special case for empty array */ 518 if (type->regions[0].size == 0) { 519 WARN_ON(type->cnt != 1 || type->total_size); 520 type->regions[0].base = base; 521 type->regions[0].size = size; 522 type->regions[0].flags = flags; 523 memblock_set_region_node(&type->regions[0], nid); 524 type->total_size = size; 525 return 0; 526 } 527 repeat: 528 /* 529 * The following is executed twice. Once with %false @insert and 530 * then with %true. The first counts the number of regions needed 531 * to accommodate the new area. The second actually inserts them. 532 */ 533 base = obase; 534 nr_new = 0; 535 536 for_each_memblock_type(idx, type, rgn) { 537 phys_addr_t rbase = rgn->base; 538 phys_addr_t rend = rbase + rgn->size; 539 540 if (rbase >= end) 541 break; 542 if (rend <= base) 543 continue; 544 /* 545 * @rgn overlaps. If it separates the lower part of new 546 * area, insert that portion. 547 */ 548 if (rbase > base) { 549 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 550 WARN_ON(nid != memblock_get_region_node(rgn)); 551 #endif 552 WARN_ON(flags != rgn->flags); 553 nr_new++; 554 if (insert) 555 memblock_insert_region(type, idx++, base, 556 rbase - base, nid, 557 flags); 558 } 559 /* area below @rend is dealt with, forget about it */ 560 base = min(rend, end); 561 } 562 563 /* insert the remaining portion */ 564 if (base < end) { 565 nr_new++; 566 if (insert) 567 memblock_insert_region(type, idx, base, end - base, 568 nid, flags); 569 } 570 571 if (!nr_new) 572 return 0; 573 574 /* 575 * If this was the first round, resize array and repeat for actual 576 * insertions; otherwise, merge and return. 577 */ 578 if (!insert) { 579 while (type->cnt + nr_new > type->max) 580 if (memblock_double_array(type, obase, size) < 0) 581 return -ENOMEM; 582 insert = true; 583 goto repeat; 584 } else { 585 memblock_merge_regions(type); 586 return 0; 587 } 588 } 589 590 int __init_memblock memblock_add_node(phys_addr_t base, phys_addr_t size, 591 int nid) 592 { 593 return memblock_add_range(&memblock.memory, base, size, nid, 0); 594 } 595 596 int __init_memblock memblock_add(phys_addr_t base, phys_addr_t size) 597 { 598 phys_addr_t end = base + size - 1; 599 600 memblock_dbg("memblock_add: [%pa-%pa] %pF\n", 601 &base, &end, (void *)_RET_IP_); 602 603 return memblock_add_range(&memblock.memory, base, size, MAX_NUMNODES, 0); 604 } 605 606 /** 607 * memblock_isolate_range - isolate given range into disjoint memblocks 608 * @type: memblock type to isolate range for 609 * @base: base of range to isolate 610 * @size: size of range to isolate 611 * @start_rgn: out parameter for the start of isolated region 612 * @end_rgn: out parameter for the end of isolated region 613 * 614 * Walk @type and ensure that regions don't cross the boundaries defined by 615 * [@base,@base+@size). Crossing regions are split at the boundaries, 616 * which may create at most two more regions. The index of the first 617 * region inside the range is returned in *@start_rgn and end in *@end_rgn. 618 * 619 * RETURNS: 620 * 0 on success, -errno on failure. 621 */ 622 static int __init_memblock memblock_isolate_range(struct memblock_type *type, 623 phys_addr_t base, phys_addr_t size, 624 int *start_rgn, int *end_rgn) 625 { 626 phys_addr_t end = base + memblock_cap_size(base, &size); 627 int idx; 628 struct memblock_region *rgn; 629 630 *start_rgn = *end_rgn = 0; 631 632 if (!size) 633 return 0; 634 635 /* we'll create at most two more regions */ 636 while (type->cnt + 2 > type->max) 637 if (memblock_double_array(type, base, size) < 0) 638 return -ENOMEM; 639 640 for_each_memblock_type(idx, type, rgn) { 641 phys_addr_t rbase = rgn->base; 642 phys_addr_t rend = rbase + rgn->size; 643 644 if (rbase >= end) 645 break; 646 if (rend <= base) 647 continue; 648 649 if (rbase < base) { 650 /* 651 * @rgn intersects from below. Split and continue 652 * to process the next region - the new top half. 653 */ 654 rgn->base = base; 655 rgn->size -= base - rbase; 656 type->total_size -= base - rbase; 657 memblock_insert_region(type, idx, rbase, base - rbase, 658 memblock_get_region_node(rgn), 659 rgn->flags); 660 } else if (rend > end) { 661 /* 662 * @rgn intersects from above. Split and redo the 663 * current region - the new bottom half. 664 */ 665 rgn->base = end; 666 rgn->size -= end - rbase; 667 type->total_size -= end - rbase; 668 memblock_insert_region(type, idx--, rbase, end - rbase, 669 memblock_get_region_node(rgn), 670 rgn->flags); 671 } else { 672 /* @rgn is fully contained, record it */ 673 if (!*end_rgn) 674 *start_rgn = idx; 675 *end_rgn = idx + 1; 676 } 677 } 678 679 return 0; 680 } 681 682 static int __init_memblock memblock_remove_range(struct memblock_type *type, 683 phys_addr_t base, phys_addr_t size) 684 { 685 int start_rgn, end_rgn; 686 int i, ret; 687 688 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 689 if (ret) 690 return ret; 691 692 for (i = end_rgn - 1; i >= start_rgn; i--) 693 memblock_remove_region(type, i); 694 return 0; 695 } 696 697 int __init_memblock memblock_remove(phys_addr_t base, phys_addr_t size) 698 { 699 return memblock_remove_range(&memblock.memory, base, size); 700 } 701 702 703 int __init_memblock memblock_free(phys_addr_t base, phys_addr_t size) 704 { 705 phys_addr_t end = base + size - 1; 706 707 memblock_dbg(" memblock_free: [%pa-%pa] %pF\n", 708 &base, &end, (void *)_RET_IP_); 709 710 kmemleak_free_part_phys(base, size); 711 return memblock_remove_range(&memblock.reserved, base, size); 712 } 713 714 int __init_memblock memblock_reserve(phys_addr_t base, phys_addr_t size) 715 { 716 phys_addr_t end = base + size - 1; 717 718 memblock_dbg("memblock_reserve: [%pa-%pa] %pF\n", 719 &base, &end, (void *)_RET_IP_); 720 721 return memblock_add_range(&memblock.reserved, base, size, MAX_NUMNODES, 0); 722 } 723 724 /** 725 * 726 * This function isolates region [@base, @base + @size), and sets/clears flag 727 * 728 * Return 0 on success, -errno on failure. 729 */ 730 static int __init_memblock memblock_setclr_flag(phys_addr_t base, 731 phys_addr_t size, int set, int flag) 732 { 733 struct memblock_type *type = &memblock.memory; 734 int i, ret, start_rgn, end_rgn; 735 736 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 737 if (ret) 738 return ret; 739 740 for (i = start_rgn; i < end_rgn; i++) 741 if (set) 742 memblock_set_region_flags(&type->regions[i], flag); 743 else 744 memblock_clear_region_flags(&type->regions[i], flag); 745 746 memblock_merge_regions(type); 747 return 0; 748 } 749 750 /** 751 * memblock_mark_hotplug - Mark hotpluggable memory with flag MEMBLOCK_HOTPLUG. 752 * @base: the base phys addr of the region 753 * @size: the size of the region 754 * 755 * Return 0 on success, -errno on failure. 756 */ 757 int __init_memblock memblock_mark_hotplug(phys_addr_t base, phys_addr_t size) 758 { 759 return memblock_setclr_flag(base, size, 1, MEMBLOCK_HOTPLUG); 760 } 761 762 /** 763 * memblock_clear_hotplug - Clear flag MEMBLOCK_HOTPLUG for a specified region. 764 * @base: the base phys addr of the region 765 * @size: the size of the region 766 * 767 * Return 0 on success, -errno on failure. 768 */ 769 int __init_memblock memblock_clear_hotplug(phys_addr_t base, phys_addr_t size) 770 { 771 return memblock_setclr_flag(base, size, 0, MEMBLOCK_HOTPLUG); 772 } 773 774 /** 775 * memblock_mark_mirror - Mark mirrored memory with flag MEMBLOCK_MIRROR. 776 * @base: the base phys addr of the region 777 * @size: the size of the region 778 * 779 * Return 0 on success, -errno on failure. 780 */ 781 int __init_memblock memblock_mark_mirror(phys_addr_t base, phys_addr_t size) 782 { 783 system_has_some_mirror = true; 784 785 return memblock_setclr_flag(base, size, 1, MEMBLOCK_MIRROR); 786 } 787 788 /** 789 * memblock_mark_nomap - Mark a memory region with flag MEMBLOCK_NOMAP. 790 * @base: the base phys addr of the region 791 * @size: the size of the region 792 * 793 * Return 0 on success, -errno on failure. 794 */ 795 int __init_memblock memblock_mark_nomap(phys_addr_t base, phys_addr_t size) 796 { 797 return memblock_setclr_flag(base, size, 1, MEMBLOCK_NOMAP); 798 } 799 800 /** 801 * memblock_clear_nomap - Clear flag MEMBLOCK_NOMAP for a specified region. 802 * @base: the base phys addr of the region 803 * @size: the size of the region 804 * 805 * Return 0 on success, -errno on failure. 806 */ 807 int __init_memblock memblock_clear_nomap(phys_addr_t base, phys_addr_t size) 808 { 809 return memblock_setclr_flag(base, size, 0, MEMBLOCK_NOMAP); 810 } 811 812 /** 813 * __next_reserved_mem_region - next function for for_each_reserved_region() 814 * @idx: pointer to u64 loop variable 815 * @out_start: ptr to phys_addr_t for start address of the region, can be %NULL 816 * @out_end: ptr to phys_addr_t for end address of the region, can be %NULL 817 * 818 * Iterate over all reserved memory regions. 819 */ 820 void __init_memblock __next_reserved_mem_region(u64 *idx, 821 phys_addr_t *out_start, 822 phys_addr_t *out_end) 823 { 824 struct memblock_type *type = &memblock.reserved; 825 826 if (*idx < type->cnt) { 827 struct memblock_region *r = &type->regions[*idx]; 828 phys_addr_t base = r->base; 829 phys_addr_t size = r->size; 830 831 if (out_start) 832 *out_start = base; 833 if (out_end) 834 *out_end = base + size - 1; 835 836 *idx += 1; 837 return; 838 } 839 840 /* signal end of iteration */ 841 *idx = ULLONG_MAX; 842 } 843 844 /** 845 * __next__mem_range - next function for for_each_free_mem_range() etc. 846 * @idx: pointer to u64 loop variable 847 * @nid: node selector, %NUMA_NO_NODE for all nodes 848 * @flags: pick from blocks based on memory attributes 849 * @type_a: pointer to memblock_type from where the range is taken 850 * @type_b: pointer to memblock_type which excludes memory from being taken 851 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL 852 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL 853 * @out_nid: ptr to int for nid of the range, can be %NULL 854 * 855 * Find the first area from *@idx which matches @nid, fill the out 856 * parameters, and update *@idx for the next iteration. The lower 32bit of 857 * *@idx contains index into type_a and the upper 32bit indexes the 858 * areas before each region in type_b. For example, if type_b regions 859 * look like the following, 860 * 861 * 0:[0-16), 1:[32-48), 2:[128-130) 862 * 863 * The upper 32bit indexes the following regions. 864 * 865 * 0:[0-0), 1:[16-32), 2:[48-128), 3:[130-MAX) 866 * 867 * As both region arrays are sorted, the function advances the two indices 868 * in lockstep and returns each intersection. 869 */ 870 void __init_memblock __next_mem_range(u64 *idx, int nid, ulong flags, 871 struct memblock_type *type_a, 872 struct memblock_type *type_b, 873 phys_addr_t *out_start, 874 phys_addr_t *out_end, int *out_nid) 875 { 876 int idx_a = *idx & 0xffffffff; 877 int idx_b = *idx >> 32; 878 879 if (WARN_ONCE(nid == MAX_NUMNODES, 880 "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n")) 881 nid = NUMA_NO_NODE; 882 883 for (; idx_a < type_a->cnt; idx_a++) { 884 struct memblock_region *m = &type_a->regions[idx_a]; 885 886 phys_addr_t m_start = m->base; 887 phys_addr_t m_end = m->base + m->size; 888 int m_nid = memblock_get_region_node(m); 889 890 /* only memory regions are associated with nodes, check it */ 891 if (nid != NUMA_NO_NODE && nid != m_nid) 892 continue; 893 894 /* skip hotpluggable memory regions if needed */ 895 if (movable_node_is_enabled() && memblock_is_hotpluggable(m)) 896 continue; 897 898 /* if we want mirror memory skip non-mirror memory regions */ 899 if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m)) 900 continue; 901 902 /* skip nomap memory unless we were asked for it explicitly */ 903 if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m)) 904 continue; 905 906 if (!type_b) { 907 if (out_start) 908 *out_start = m_start; 909 if (out_end) 910 *out_end = m_end; 911 if (out_nid) 912 *out_nid = m_nid; 913 idx_a++; 914 *idx = (u32)idx_a | (u64)idx_b << 32; 915 return; 916 } 917 918 /* scan areas before each reservation */ 919 for (; idx_b < type_b->cnt + 1; idx_b++) { 920 struct memblock_region *r; 921 phys_addr_t r_start; 922 phys_addr_t r_end; 923 924 r = &type_b->regions[idx_b]; 925 r_start = idx_b ? r[-1].base + r[-1].size : 0; 926 r_end = idx_b < type_b->cnt ? 927 r->base : ULLONG_MAX; 928 929 /* 930 * if idx_b advanced past idx_a, 931 * break out to advance idx_a 932 */ 933 if (r_start >= m_end) 934 break; 935 /* if the two regions intersect, we're done */ 936 if (m_start < r_end) { 937 if (out_start) 938 *out_start = 939 max(m_start, r_start); 940 if (out_end) 941 *out_end = min(m_end, r_end); 942 if (out_nid) 943 *out_nid = m_nid; 944 /* 945 * The region which ends first is 946 * advanced for the next iteration. 947 */ 948 if (m_end <= r_end) 949 idx_a++; 950 else 951 idx_b++; 952 *idx = (u32)idx_a | (u64)idx_b << 32; 953 return; 954 } 955 } 956 } 957 958 /* signal end of iteration */ 959 *idx = ULLONG_MAX; 960 } 961 962 /** 963 * __next_mem_range_rev - generic next function for for_each_*_range_rev() 964 * 965 * Finds the next range from type_a which is not marked as unsuitable 966 * in type_b. 967 * 968 * @idx: pointer to u64 loop variable 969 * @nid: node selector, %NUMA_NO_NODE for all nodes 970 * @flags: pick from blocks based on memory attributes 971 * @type_a: pointer to memblock_type from where the range is taken 972 * @type_b: pointer to memblock_type which excludes memory from being taken 973 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL 974 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL 975 * @out_nid: ptr to int for nid of the range, can be %NULL 976 * 977 * Reverse of __next_mem_range(). 978 */ 979 void __init_memblock __next_mem_range_rev(u64 *idx, int nid, ulong flags, 980 struct memblock_type *type_a, 981 struct memblock_type *type_b, 982 phys_addr_t *out_start, 983 phys_addr_t *out_end, int *out_nid) 984 { 985 int idx_a = *idx & 0xffffffff; 986 int idx_b = *idx >> 32; 987 988 if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n")) 989 nid = NUMA_NO_NODE; 990 991 if (*idx == (u64)ULLONG_MAX) { 992 idx_a = type_a->cnt - 1; 993 if (type_b != NULL) 994 idx_b = type_b->cnt; 995 else 996 idx_b = 0; 997 } 998 999 for (; idx_a >= 0; idx_a--) { 1000 struct memblock_region *m = &type_a->regions[idx_a]; 1001 1002 phys_addr_t m_start = m->base; 1003 phys_addr_t m_end = m->base + m->size; 1004 int m_nid = memblock_get_region_node(m); 1005 1006 /* only memory regions are associated with nodes, check it */ 1007 if (nid != NUMA_NO_NODE && nid != m_nid) 1008 continue; 1009 1010 /* skip hotpluggable memory regions if needed */ 1011 if (movable_node_is_enabled() && memblock_is_hotpluggable(m)) 1012 continue; 1013 1014 /* if we want mirror memory skip non-mirror memory regions */ 1015 if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m)) 1016 continue; 1017 1018 /* skip nomap memory unless we were asked for it explicitly */ 1019 if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m)) 1020 continue; 1021 1022 if (!type_b) { 1023 if (out_start) 1024 *out_start = m_start; 1025 if (out_end) 1026 *out_end = m_end; 1027 if (out_nid) 1028 *out_nid = m_nid; 1029 idx_a--; 1030 *idx = (u32)idx_a | (u64)idx_b << 32; 1031 return; 1032 } 1033 1034 /* scan areas before each reservation */ 1035 for (; idx_b >= 0; idx_b--) { 1036 struct memblock_region *r; 1037 phys_addr_t r_start; 1038 phys_addr_t r_end; 1039 1040 r = &type_b->regions[idx_b]; 1041 r_start = idx_b ? r[-1].base + r[-1].size : 0; 1042 r_end = idx_b < type_b->cnt ? 1043 r->base : ULLONG_MAX; 1044 /* 1045 * if idx_b advanced past idx_a, 1046 * break out to advance idx_a 1047 */ 1048 1049 if (r_end <= m_start) 1050 break; 1051 /* if the two regions intersect, we're done */ 1052 if (m_end > r_start) { 1053 if (out_start) 1054 *out_start = max(m_start, r_start); 1055 if (out_end) 1056 *out_end = min(m_end, r_end); 1057 if (out_nid) 1058 *out_nid = m_nid; 1059 if (m_start >= r_start) 1060 idx_a--; 1061 else 1062 idx_b--; 1063 *idx = (u32)idx_a | (u64)idx_b << 32; 1064 return; 1065 } 1066 } 1067 } 1068 /* signal end of iteration */ 1069 *idx = ULLONG_MAX; 1070 } 1071 1072 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 1073 /* 1074 * Common iterator interface used to define for_each_mem_range(). 1075 */ 1076 void __init_memblock __next_mem_pfn_range(int *idx, int nid, 1077 unsigned long *out_start_pfn, 1078 unsigned long *out_end_pfn, int *out_nid) 1079 { 1080 struct memblock_type *type = &memblock.memory; 1081 struct memblock_region *r; 1082 1083 while (++*idx < type->cnt) { 1084 r = &type->regions[*idx]; 1085 1086 if (PFN_UP(r->base) >= PFN_DOWN(r->base + r->size)) 1087 continue; 1088 if (nid == MAX_NUMNODES || nid == r->nid) 1089 break; 1090 } 1091 if (*idx >= type->cnt) { 1092 *idx = -1; 1093 return; 1094 } 1095 1096 if (out_start_pfn) 1097 *out_start_pfn = PFN_UP(r->base); 1098 if (out_end_pfn) 1099 *out_end_pfn = PFN_DOWN(r->base + r->size); 1100 if (out_nid) 1101 *out_nid = r->nid; 1102 } 1103 1104 /** 1105 * memblock_set_node - set node ID on memblock regions 1106 * @base: base of area to set node ID for 1107 * @size: size of area to set node ID for 1108 * @type: memblock type to set node ID for 1109 * @nid: node ID to set 1110 * 1111 * Set the nid of memblock @type regions in [@base,@base+@size) to @nid. 1112 * Regions which cross the area boundaries are split as necessary. 1113 * 1114 * RETURNS: 1115 * 0 on success, -errno on failure. 1116 */ 1117 int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size, 1118 struct memblock_type *type, int nid) 1119 { 1120 int start_rgn, end_rgn; 1121 int i, ret; 1122 1123 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 1124 if (ret) 1125 return ret; 1126 1127 for (i = start_rgn; i < end_rgn; i++) 1128 memblock_set_region_node(&type->regions[i], nid); 1129 1130 memblock_merge_regions(type); 1131 return 0; 1132 } 1133 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 1134 1135 static phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size, 1136 phys_addr_t align, phys_addr_t start, 1137 phys_addr_t end, int nid, ulong flags) 1138 { 1139 phys_addr_t found; 1140 1141 if (!align) 1142 align = SMP_CACHE_BYTES; 1143 1144 found = memblock_find_in_range_node(size, align, start, end, nid, 1145 flags); 1146 if (found && !memblock_reserve(found, size)) { 1147 /* 1148 * The min_count is set to 0 so that memblock allocations are 1149 * never reported as leaks. 1150 */ 1151 kmemleak_alloc_phys(found, size, 0, 0); 1152 return found; 1153 } 1154 return 0; 1155 } 1156 1157 phys_addr_t __init memblock_alloc_range(phys_addr_t size, phys_addr_t align, 1158 phys_addr_t start, phys_addr_t end, 1159 ulong flags) 1160 { 1161 return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE, 1162 flags); 1163 } 1164 1165 static phys_addr_t __init memblock_alloc_base_nid(phys_addr_t size, 1166 phys_addr_t align, phys_addr_t max_addr, 1167 int nid, ulong flags) 1168 { 1169 return memblock_alloc_range_nid(size, align, 0, max_addr, nid, flags); 1170 } 1171 1172 phys_addr_t __init memblock_alloc_nid(phys_addr_t size, phys_addr_t align, int nid) 1173 { 1174 ulong flags = choose_memblock_flags(); 1175 phys_addr_t ret; 1176 1177 again: 1178 ret = memblock_alloc_base_nid(size, align, MEMBLOCK_ALLOC_ACCESSIBLE, 1179 nid, flags); 1180 1181 if (!ret && (flags & MEMBLOCK_MIRROR)) { 1182 flags &= ~MEMBLOCK_MIRROR; 1183 goto again; 1184 } 1185 return ret; 1186 } 1187 1188 phys_addr_t __init __memblock_alloc_base(phys_addr_t size, phys_addr_t align, phys_addr_t max_addr) 1189 { 1190 return memblock_alloc_base_nid(size, align, max_addr, NUMA_NO_NODE, 1191 MEMBLOCK_NONE); 1192 } 1193 1194 phys_addr_t __init memblock_alloc_base(phys_addr_t size, phys_addr_t align, phys_addr_t max_addr) 1195 { 1196 phys_addr_t alloc; 1197 1198 alloc = __memblock_alloc_base(size, align, max_addr); 1199 1200 if (alloc == 0) 1201 panic("ERROR: Failed to allocate %pa bytes below %pa.\n", 1202 &size, &max_addr); 1203 1204 return alloc; 1205 } 1206 1207 phys_addr_t __init memblock_alloc(phys_addr_t size, phys_addr_t align) 1208 { 1209 return memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ACCESSIBLE); 1210 } 1211 1212 phys_addr_t __init memblock_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid) 1213 { 1214 phys_addr_t res = memblock_alloc_nid(size, align, nid); 1215 1216 if (res) 1217 return res; 1218 return memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ACCESSIBLE); 1219 } 1220 1221 /** 1222 * memblock_virt_alloc_internal - allocate boot memory block 1223 * @size: size of memory block to be allocated in bytes 1224 * @align: alignment of the region and block's size 1225 * @min_addr: the lower bound of the memory region to allocate (phys address) 1226 * @max_addr: the upper bound of the memory region to allocate (phys address) 1227 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1228 * 1229 * The @min_addr limit is dropped if it can not be satisfied and the allocation 1230 * will fall back to memory below @min_addr. Also, allocation may fall back 1231 * to any node in the system if the specified node can not 1232 * hold the requested memory. 1233 * 1234 * The allocation is performed from memory region limited by 1235 * memblock.current_limit if @max_addr == %BOOTMEM_ALLOC_ACCESSIBLE. 1236 * 1237 * The memory block is aligned on SMP_CACHE_BYTES if @align == 0. 1238 * 1239 * The phys address of allocated boot memory block is converted to virtual and 1240 * allocated memory is reset to 0. 1241 * 1242 * In addition, function sets the min_count to 0 using kmemleak_alloc for 1243 * allocated boot memory block, so that it is never reported as leaks. 1244 * 1245 * RETURNS: 1246 * Virtual address of allocated memory block on success, NULL on failure. 1247 */ 1248 static void * __init memblock_virt_alloc_internal( 1249 phys_addr_t size, phys_addr_t align, 1250 phys_addr_t min_addr, phys_addr_t max_addr, 1251 int nid) 1252 { 1253 phys_addr_t alloc; 1254 void *ptr; 1255 ulong flags = choose_memblock_flags(); 1256 1257 if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n")) 1258 nid = NUMA_NO_NODE; 1259 1260 /* 1261 * Detect any accidental use of these APIs after slab is ready, as at 1262 * this moment memblock may be deinitialized already and its 1263 * internal data may be destroyed (after execution of free_all_bootmem) 1264 */ 1265 if (WARN_ON_ONCE(slab_is_available())) 1266 return kzalloc_node(size, GFP_NOWAIT, nid); 1267 1268 if (!align) 1269 align = SMP_CACHE_BYTES; 1270 1271 if (max_addr > memblock.current_limit) 1272 max_addr = memblock.current_limit; 1273 again: 1274 alloc = memblock_find_in_range_node(size, align, min_addr, max_addr, 1275 nid, flags); 1276 if (alloc && !memblock_reserve(alloc, size)) 1277 goto done; 1278 1279 if (nid != NUMA_NO_NODE) { 1280 alloc = memblock_find_in_range_node(size, align, min_addr, 1281 max_addr, NUMA_NO_NODE, 1282 flags); 1283 if (alloc && !memblock_reserve(alloc, size)) 1284 goto done; 1285 } 1286 1287 if (min_addr) { 1288 min_addr = 0; 1289 goto again; 1290 } 1291 1292 if (flags & MEMBLOCK_MIRROR) { 1293 flags &= ~MEMBLOCK_MIRROR; 1294 pr_warn("Could not allocate %pap bytes of mirrored memory\n", 1295 &size); 1296 goto again; 1297 } 1298 1299 return NULL; 1300 done: 1301 ptr = phys_to_virt(alloc); 1302 1303 /* 1304 * The min_count is set to 0 so that bootmem allocated blocks 1305 * are never reported as leaks. This is because many of these blocks 1306 * are only referred via the physical address which is not 1307 * looked up by kmemleak. 1308 */ 1309 kmemleak_alloc(ptr, size, 0, 0); 1310 1311 return ptr; 1312 } 1313 1314 /** 1315 * memblock_virt_alloc_try_nid_raw - allocate boot memory block without zeroing 1316 * memory and without panicking 1317 * @size: size of memory block to be allocated in bytes 1318 * @align: alignment of the region and block's size 1319 * @min_addr: the lower bound of the memory region from where the allocation 1320 * is preferred (phys address) 1321 * @max_addr: the upper bound of the memory region from where the allocation 1322 * is preferred (phys address), or %BOOTMEM_ALLOC_ACCESSIBLE to 1323 * allocate only from memory limited by memblock.current_limit value 1324 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1325 * 1326 * Public function, provides additional debug information (including caller 1327 * info), if enabled. Does not zero allocated memory, does not panic if request 1328 * cannot be satisfied. 1329 * 1330 * RETURNS: 1331 * Virtual address of allocated memory block on success, NULL on failure. 1332 */ 1333 void * __init memblock_virt_alloc_try_nid_raw( 1334 phys_addr_t size, phys_addr_t align, 1335 phys_addr_t min_addr, phys_addr_t max_addr, 1336 int nid) 1337 { 1338 void *ptr; 1339 1340 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=0x%llx max_addr=0x%llx %pF\n", 1341 __func__, (u64)size, (u64)align, nid, (u64)min_addr, 1342 (u64)max_addr, (void *)_RET_IP_); 1343 1344 ptr = memblock_virt_alloc_internal(size, align, 1345 min_addr, max_addr, nid); 1346 #ifdef CONFIG_DEBUG_VM 1347 if (ptr && size > 0) 1348 memset(ptr, 0xff, size); 1349 #endif 1350 return ptr; 1351 } 1352 1353 /** 1354 * memblock_virt_alloc_try_nid_nopanic - allocate boot memory block 1355 * @size: size of memory block to be allocated in bytes 1356 * @align: alignment of the region and block's size 1357 * @min_addr: the lower bound of the memory region from where the allocation 1358 * is preferred (phys address) 1359 * @max_addr: the upper bound of the memory region from where the allocation 1360 * is preferred (phys address), or %BOOTMEM_ALLOC_ACCESSIBLE to 1361 * allocate only from memory limited by memblock.current_limit value 1362 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1363 * 1364 * Public function, provides additional debug information (including caller 1365 * info), if enabled. This function zeroes the allocated memory. 1366 * 1367 * RETURNS: 1368 * Virtual address of allocated memory block on success, NULL on failure. 1369 */ 1370 void * __init memblock_virt_alloc_try_nid_nopanic( 1371 phys_addr_t size, phys_addr_t align, 1372 phys_addr_t min_addr, phys_addr_t max_addr, 1373 int nid) 1374 { 1375 void *ptr; 1376 1377 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=0x%llx max_addr=0x%llx %pF\n", 1378 __func__, (u64)size, (u64)align, nid, (u64)min_addr, 1379 (u64)max_addr, (void *)_RET_IP_); 1380 1381 ptr = memblock_virt_alloc_internal(size, align, 1382 min_addr, max_addr, nid); 1383 if (ptr) 1384 memset(ptr, 0, size); 1385 return ptr; 1386 } 1387 1388 /** 1389 * memblock_virt_alloc_try_nid - allocate boot memory block with panicking 1390 * @size: size of memory block to be allocated in bytes 1391 * @align: alignment of the region and block's size 1392 * @min_addr: the lower bound of the memory region from where the allocation 1393 * is preferred (phys address) 1394 * @max_addr: the upper bound of the memory region from where the allocation 1395 * is preferred (phys address), or %BOOTMEM_ALLOC_ACCESSIBLE to 1396 * allocate only from memory limited by memblock.current_limit value 1397 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1398 * 1399 * Public panicking version of memblock_virt_alloc_try_nid_nopanic() 1400 * which provides debug information (including caller info), if enabled, 1401 * and panics if the request can not be satisfied. 1402 * 1403 * RETURNS: 1404 * Virtual address of allocated memory block on success, NULL on failure. 1405 */ 1406 void * __init memblock_virt_alloc_try_nid( 1407 phys_addr_t size, phys_addr_t align, 1408 phys_addr_t min_addr, phys_addr_t max_addr, 1409 int nid) 1410 { 1411 void *ptr; 1412 1413 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=0x%llx max_addr=0x%llx %pF\n", 1414 __func__, (u64)size, (u64)align, nid, (u64)min_addr, 1415 (u64)max_addr, (void *)_RET_IP_); 1416 ptr = memblock_virt_alloc_internal(size, align, 1417 min_addr, max_addr, nid); 1418 if (ptr) { 1419 memset(ptr, 0, size); 1420 return ptr; 1421 } 1422 1423 panic("%s: Failed to allocate %llu bytes align=0x%llx nid=%d from=0x%llx max_addr=0x%llx\n", 1424 __func__, (u64)size, (u64)align, nid, (u64)min_addr, 1425 (u64)max_addr); 1426 return NULL; 1427 } 1428 1429 /** 1430 * __memblock_free_early - free boot memory block 1431 * @base: phys starting address of the boot memory block 1432 * @size: size of the boot memory block in bytes 1433 * 1434 * Free boot memory block previously allocated by memblock_virt_alloc_xx() API. 1435 * The freeing memory will not be released to the buddy allocator. 1436 */ 1437 void __init __memblock_free_early(phys_addr_t base, phys_addr_t size) 1438 { 1439 memblock_dbg("%s: [%#016llx-%#016llx] %pF\n", 1440 __func__, (u64)base, (u64)base + size - 1, 1441 (void *)_RET_IP_); 1442 kmemleak_free_part_phys(base, size); 1443 memblock_remove_range(&memblock.reserved, base, size); 1444 } 1445 1446 /* 1447 * __memblock_free_late - free bootmem block pages directly to buddy allocator 1448 * @addr: phys starting address of the boot memory block 1449 * @size: size of the boot memory block in bytes 1450 * 1451 * This is only useful when the bootmem allocator has already been torn 1452 * down, but we are still initializing the system. Pages are released directly 1453 * to the buddy allocator, no bootmem metadata is updated because it is gone. 1454 */ 1455 void __init __memblock_free_late(phys_addr_t base, phys_addr_t size) 1456 { 1457 u64 cursor, end; 1458 1459 memblock_dbg("%s: [%#016llx-%#016llx] %pF\n", 1460 __func__, (u64)base, (u64)base + size - 1, 1461 (void *)_RET_IP_); 1462 kmemleak_free_part_phys(base, size); 1463 cursor = PFN_UP(base); 1464 end = PFN_DOWN(base + size); 1465 1466 for (; cursor < end; cursor++) { 1467 __free_pages_bootmem(pfn_to_page(cursor), cursor, 0); 1468 totalram_pages++; 1469 } 1470 } 1471 1472 /* 1473 * Remaining API functions 1474 */ 1475 1476 phys_addr_t __init_memblock memblock_phys_mem_size(void) 1477 { 1478 return memblock.memory.total_size; 1479 } 1480 1481 phys_addr_t __init_memblock memblock_reserved_size(void) 1482 { 1483 return memblock.reserved.total_size; 1484 } 1485 1486 phys_addr_t __init memblock_mem_size(unsigned long limit_pfn) 1487 { 1488 unsigned long pages = 0; 1489 struct memblock_region *r; 1490 unsigned long start_pfn, end_pfn; 1491 1492 for_each_memblock(memory, r) { 1493 start_pfn = memblock_region_memory_base_pfn(r); 1494 end_pfn = memblock_region_memory_end_pfn(r); 1495 start_pfn = min_t(unsigned long, start_pfn, limit_pfn); 1496 end_pfn = min_t(unsigned long, end_pfn, limit_pfn); 1497 pages += end_pfn - start_pfn; 1498 } 1499 1500 return PFN_PHYS(pages); 1501 } 1502 1503 /* lowest address */ 1504 phys_addr_t __init_memblock memblock_start_of_DRAM(void) 1505 { 1506 return memblock.memory.regions[0].base; 1507 } 1508 1509 phys_addr_t __init_memblock memblock_end_of_DRAM(void) 1510 { 1511 int idx = memblock.memory.cnt - 1; 1512 1513 return (memblock.memory.regions[idx].base + memblock.memory.regions[idx].size); 1514 } 1515 1516 static phys_addr_t __init_memblock __find_max_addr(phys_addr_t limit) 1517 { 1518 phys_addr_t max_addr = (phys_addr_t)ULLONG_MAX; 1519 struct memblock_region *r; 1520 1521 /* 1522 * translate the memory @limit size into the max address within one of 1523 * the memory memblock regions, if the @limit exceeds the total size 1524 * of those regions, max_addr will keep original value ULLONG_MAX 1525 */ 1526 for_each_memblock(memory, r) { 1527 if (limit <= r->size) { 1528 max_addr = r->base + limit; 1529 break; 1530 } 1531 limit -= r->size; 1532 } 1533 1534 return max_addr; 1535 } 1536 1537 void __init memblock_enforce_memory_limit(phys_addr_t limit) 1538 { 1539 phys_addr_t max_addr = (phys_addr_t)ULLONG_MAX; 1540 1541 if (!limit) 1542 return; 1543 1544 max_addr = __find_max_addr(limit); 1545 1546 /* @limit exceeds the total size of the memory, do nothing */ 1547 if (max_addr == (phys_addr_t)ULLONG_MAX) 1548 return; 1549 1550 /* truncate both memory and reserved regions */ 1551 memblock_remove_range(&memblock.memory, max_addr, 1552 (phys_addr_t)ULLONG_MAX); 1553 memblock_remove_range(&memblock.reserved, max_addr, 1554 (phys_addr_t)ULLONG_MAX); 1555 } 1556 1557 void __init memblock_cap_memory_range(phys_addr_t base, phys_addr_t size) 1558 { 1559 int start_rgn, end_rgn; 1560 int i, ret; 1561 1562 if (!size) 1563 return; 1564 1565 ret = memblock_isolate_range(&memblock.memory, base, size, 1566 &start_rgn, &end_rgn); 1567 if (ret) 1568 return; 1569 1570 /* remove all the MAP regions */ 1571 for (i = memblock.memory.cnt - 1; i >= end_rgn; i--) 1572 if (!memblock_is_nomap(&memblock.memory.regions[i])) 1573 memblock_remove_region(&memblock.memory, i); 1574 1575 for (i = start_rgn - 1; i >= 0; i--) 1576 if (!memblock_is_nomap(&memblock.memory.regions[i])) 1577 memblock_remove_region(&memblock.memory, i); 1578 1579 /* truncate the reserved regions */ 1580 memblock_remove_range(&memblock.reserved, 0, base); 1581 memblock_remove_range(&memblock.reserved, 1582 base + size, (phys_addr_t)ULLONG_MAX); 1583 } 1584 1585 void __init memblock_mem_limit_remove_map(phys_addr_t limit) 1586 { 1587 phys_addr_t max_addr; 1588 1589 if (!limit) 1590 return; 1591 1592 max_addr = __find_max_addr(limit); 1593 1594 /* @limit exceeds the total size of the memory, do nothing */ 1595 if (max_addr == (phys_addr_t)ULLONG_MAX) 1596 return; 1597 1598 memblock_cap_memory_range(0, max_addr); 1599 } 1600 1601 static int __init_memblock memblock_search(struct memblock_type *type, phys_addr_t addr) 1602 { 1603 unsigned int left = 0, right = type->cnt; 1604 1605 do { 1606 unsigned int mid = (right + left) / 2; 1607 1608 if (addr < type->regions[mid].base) 1609 right = mid; 1610 else if (addr >= (type->regions[mid].base + 1611 type->regions[mid].size)) 1612 left = mid + 1; 1613 else 1614 return mid; 1615 } while (left < right); 1616 return -1; 1617 } 1618 1619 bool __init memblock_is_reserved(phys_addr_t addr) 1620 { 1621 return memblock_search(&memblock.reserved, addr) != -1; 1622 } 1623 1624 bool __init_memblock memblock_is_memory(phys_addr_t addr) 1625 { 1626 return memblock_search(&memblock.memory, addr) != -1; 1627 } 1628 1629 bool __init_memblock memblock_is_map_memory(phys_addr_t addr) 1630 { 1631 int i = memblock_search(&memblock.memory, addr); 1632 1633 if (i == -1) 1634 return false; 1635 return !memblock_is_nomap(&memblock.memory.regions[i]); 1636 } 1637 1638 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 1639 int __init_memblock memblock_search_pfn_nid(unsigned long pfn, 1640 unsigned long *start_pfn, unsigned long *end_pfn) 1641 { 1642 struct memblock_type *type = &memblock.memory; 1643 int mid = memblock_search(type, PFN_PHYS(pfn)); 1644 1645 if (mid == -1) 1646 return -1; 1647 1648 *start_pfn = PFN_DOWN(type->regions[mid].base); 1649 *end_pfn = PFN_DOWN(type->regions[mid].base + type->regions[mid].size); 1650 1651 return type->regions[mid].nid; 1652 } 1653 #endif 1654 1655 /** 1656 * memblock_is_region_memory - check if a region is a subset of memory 1657 * @base: base of region to check 1658 * @size: size of region to check 1659 * 1660 * Check if the region [@base, @base+@size) is a subset of a memory block. 1661 * 1662 * RETURNS: 1663 * 0 if false, non-zero if true 1664 */ 1665 bool __init_memblock memblock_is_region_memory(phys_addr_t base, phys_addr_t size) 1666 { 1667 int idx = memblock_search(&memblock.memory, base); 1668 phys_addr_t end = base + memblock_cap_size(base, &size); 1669 1670 if (idx == -1) 1671 return false; 1672 return (memblock.memory.regions[idx].base + 1673 memblock.memory.regions[idx].size) >= end; 1674 } 1675 1676 /** 1677 * memblock_is_region_reserved - check if a region intersects reserved memory 1678 * @base: base of region to check 1679 * @size: size of region to check 1680 * 1681 * Check if the region [@base, @base+@size) intersects a reserved memory block. 1682 * 1683 * RETURNS: 1684 * True if they intersect, false if not. 1685 */ 1686 bool __init_memblock memblock_is_region_reserved(phys_addr_t base, phys_addr_t size) 1687 { 1688 memblock_cap_size(base, &size); 1689 return memblock_overlaps_region(&memblock.reserved, base, size); 1690 } 1691 1692 void __init_memblock memblock_trim_memory(phys_addr_t align) 1693 { 1694 phys_addr_t start, end, orig_start, orig_end; 1695 struct memblock_region *r; 1696 1697 for_each_memblock(memory, r) { 1698 orig_start = r->base; 1699 orig_end = r->base + r->size; 1700 start = round_up(orig_start, align); 1701 end = round_down(orig_end, align); 1702 1703 if (start == orig_start && end == orig_end) 1704 continue; 1705 1706 if (start < end) { 1707 r->base = start; 1708 r->size = end - start; 1709 } else { 1710 memblock_remove_region(&memblock.memory, 1711 r - memblock.memory.regions); 1712 r--; 1713 } 1714 } 1715 } 1716 1717 void __init_memblock memblock_set_current_limit(phys_addr_t limit) 1718 { 1719 memblock.current_limit = limit; 1720 } 1721 1722 phys_addr_t __init_memblock memblock_get_current_limit(void) 1723 { 1724 return memblock.current_limit; 1725 } 1726 1727 static void __init_memblock memblock_dump(struct memblock_type *type) 1728 { 1729 phys_addr_t base, end, size; 1730 unsigned long flags; 1731 int idx; 1732 struct memblock_region *rgn; 1733 1734 pr_info(" %s.cnt = 0x%lx\n", type->name, type->cnt); 1735 1736 for_each_memblock_type(idx, type, rgn) { 1737 char nid_buf[32] = ""; 1738 1739 base = rgn->base; 1740 size = rgn->size; 1741 end = base + size - 1; 1742 flags = rgn->flags; 1743 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 1744 if (memblock_get_region_node(rgn) != MAX_NUMNODES) 1745 snprintf(nid_buf, sizeof(nid_buf), " on node %d", 1746 memblock_get_region_node(rgn)); 1747 #endif 1748 pr_info(" %s[%#x]\t[%pa-%pa], %pa bytes%s flags: %#lx\n", 1749 type->name, idx, &base, &end, &size, nid_buf, flags); 1750 } 1751 } 1752 1753 extern unsigned long __init_memblock 1754 memblock_reserved_memory_within(phys_addr_t start_addr, phys_addr_t end_addr) 1755 { 1756 struct memblock_region *rgn; 1757 unsigned long size = 0; 1758 int idx; 1759 1760 for_each_memblock_type(idx, (&memblock.reserved), rgn) { 1761 phys_addr_t start, end; 1762 1763 if (rgn->base + rgn->size < start_addr) 1764 continue; 1765 if (rgn->base > end_addr) 1766 continue; 1767 1768 start = rgn->base; 1769 end = start + rgn->size; 1770 size += end - start; 1771 } 1772 1773 return size; 1774 } 1775 1776 void __init_memblock __memblock_dump_all(void) 1777 { 1778 pr_info("MEMBLOCK configuration:\n"); 1779 pr_info(" memory size = %pa reserved size = %pa\n", 1780 &memblock.memory.total_size, 1781 &memblock.reserved.total_size); 1782 1783 memblock_dump(&memblock.memory); 1784 memblock_dump(&memblock.reserved); 1785 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 1786 memblock_dump(&memblock.physmem); 1787 #endif 1788 } 1789 1790 void __init memblock_allow_resize(void) 1791 { 1792 memblock_can_resize = 1; 1793 } 1794 1795 static int __init early_memblock(char *p) 1796 { 1797 if (p && strstr(p, "debug")) 1798 memblock_debug = 1; 1799 return 0; 1800 } 1801 early_param("memblock", early_memblock); 1802 1803 #if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_ARCH_DISCARD_MEMBLOCK) 1804 1805 static int memblock_debug_show(struct seq_file *m, void *private) 1806 { 1807 struct memblock_type *type = m->private; 1808 struct memblock_region *reg; 1809 int i; 1810 phys_addr_t end; 1811 1812 for (i = 0; i < type->cnt; i++) { 1813 reg = &type->regions[i]; 1814 end = reg->base + reg->size - 1; 1815 1816 seq_printf(m, "%4d: ", i); 1817 seq_printf(m, "%pa..%pa\n", ®->base, &end); 1818 } 1819 return 0; 1820 } 1821 1822 static int memblock_debug_open(struct inode *inode, struct file *file) 1823 { 1824 return single_open(file, memblock_debug_show, inode->i_private); 1825 } 1826 1827 static const struct file_operations memblock_debug_fops = { 1828 .open = memblock_debug_open, 1829 .read = seq_read, 1830 .llseek = seq_lseek, 1831 .release = single_release, 1832 }; 1833 1834 static int __init memblock_init_debugfs(void) 1835 { 1836 struct dentry *root = debugfs_create_dir("memblock", NULL); 1837 if (!root) 1838 return -ENXIO; 1839 debugfs_create_file("memory", S_IRUGO, root, &memblock.memory, &memblock_debug_fops); 1840 debugfs_create_file("reserved", S_IRUGO, root, &memblock.reserved, &memblock_debug_fops); 1841 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 1842 debugfs_create_file("physmem", S_IRUGO, root, &memblock.physmem, &memblock_debug_fops); 1843 #endif 1844 1845 return 0; 1846 } 1847 __initcall(memblock_init_debugfs); 1848 1849 #endif /* CONFIG_DEBUG_FS */ 1850