1 /* 2 * Procedures for maintaining information about logical memory blocks. 3 * 4 * Peter Bergner, IBM Corp. June 2001. 5 * Copyright (C) 2001 Peter Bergner. 6 * 7 * This program is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU General Public License 9 * as published by the Free Software Foundation; either version 10 * 2 of the License, or (at your option) any later version. 11 */ 12 13 #include <linux/kernel.h> 14 #include <linux/slab.h> 15 #include <linux/init.h> 16 #include <linux/bitops.h> 17 #include <linux/poison.h> 18 #include <linux/pfn.h> 19 #include <linux/debugfs.h> 20 #include <linux/seq_file.h> 21 #include <linux/memblock.h> 22 23 #include <asm-generic/sections.h> 24 #include <linux/io.h> 25 26 #include "internal.h" 27 28 static struct memblock_region memblock_memory_init_regions[INIT_MEMBLOCK_REGIONS] __initdata_memblock; 29 static struct memblock_region memblock_reserved_init_regions[INIT_MEMBLOCK_REGIONS] __initdata_memblock; 30 31 struct memblock memblock __initdata_memblock = { 32 .memory.regions = memblock_memory_init_regions, 33 .memory.cnt = 1, /* empty dummy entry */ 34 .memory.max = INIT_MEMBLOCK_REGIONS, 35 36 .reserved.regions = memblock_reserved_init_regions, 37 .reserved.cnt = 1, /* empty dummy entry */ 38 .reserved.max = INIT_MEMBLOCK_REGIONS, 39 40 .bottom_up = false, 41 .current_limit = MEMBLOCK_ALLOC_ANYWHERE, 42 }; 43 44 int memblock_debug __initdata_memblock; 45 #ifdef CONFIG_MOVABLE_NODE 46 bool movable_node_enabled __initdata_memblock = false; 47 #endif 48 static int memblock_can_resize __initdata_memblock; 49 static int memblock_memory_in_slab __initdata_memblock = 0; 50 static int memblock_reserved_in_slab __initdata_memblock = 0; 51 52 /* inline so we don't get a warning when pr_debug is compiled out */ 53 static __init_memblock const char * 54 memblock_type_name(struct memblock_type *type) 55 { 56 if (type == &memblock.memory) 57 return "memory"; 58 else if (type == &memblock.reserved) 59 return "reserved"; 60 else 61 return "unknown"; 62 } 63 64 /* adjust *@size so that (@base + *@size) doesn't overflow, return new size */ 65 static inline phys_addr_t memblock_cap_size(phys_addr_t base, phys_addr_t *size) 66 { 67 return *size = min(*size, (phys_addr_t)ULLONG_MAX - base); 68 } 69 70 /* 71 * Address comparison utilities 72 */ 73 static unsigned long __init_memblock memblock_addrs_overlap(phys_addr_t base1, phys_addr_t size1, 74 phys_addr_t base2, phys_addr_t size2) 75 { 76 return ((base1 < (base2 + size2)) && (base2 < (base1 + size1))); 77 } 78 79 static long __init_memblock memblock_overlaps_region(struct memblock_type *type, 80 phys_addr_t base, phys_addr_t size) 81 { 82 unsigned long i; 83 84 for (i = 0; i < type->cnt; i++) { 85 phys_addr_t rgnbase = type->regions[i].base; 86 phys_addr_t rgnsize = type->regions[i].size; 87 if (memblock_addrs_overlap(base, size, rgnbase, rgnsize)) 88 break; 89 } 90 91 return (i < type->cnt) ? i : -1; 92 } 93 94 /* 95 * __memblock_find_range_bottom_up - find free area utility in bottom-up 96 * @start: start of candidate range 97 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_{ANYWHERE|ACCESSIBLE} 98 * @size: size of free area to find 99 * @align: alignment of free area to find 100 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 101 * 102 * Utility called from memblock_find_in_range_node(), find free area bottom-up. 103 * 104 * RETURNS: 105 * Found address on success, 0 on failure. 106 */ 107 static phys_addr_t __init_memblock 108 __memblock_find_range_bottom_up(phys_addr_t start, phys_addr_t end, 109 phys_addr_t size, phys_addr_t align, int nid) 110 { 111 phys_addr_t this_start, this_end, cand; 112 u64 i; 113 114 for_each_free_mem_range(i, nid, &this_start, &this_end, NULL) { 115 this_start = clamp(this_start, start, end); 116 this_end = clamp(this_end, start, end); 117 118 cand = round_up(this_start, align); 119 if (cand < this_end && this_end - cand >= size) 120 return cand; 121 } 122 123 return 0; 124 } 125 126 /** 127 * __memblock_find_range_top_down - find free area utility, in top-down 128 * @start: start of candidate range 129 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_{ANYWHERE|ACCESSIBLE} 130 * @size: size of free area to find 131 * @align: alignment of free area to find 132 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 133 * 134 * Utility called from memblock_find_in_range_node(), find free area top-down. 135 * 136 * RETURNS: 137 * Found address on success, 0 on failure. 138 */ 139 static phys_addr_t __init_memblock 140 __memblock_find_range_top_down(phys_addr_t start, phys_addr_t end, 141 phys_addr_t size, phys_addr_t align, int nid) 142 { 143 phys_addr_t this_start, this_end, cand; 144 u64 i; 145 146 for_each_free_mem_range_reverse(i, nid, &this_start, &this_end, NULL) { 147 this_start = clamp(this_start, start, end); 148 this_end = clamp(this_end, start, end); 149 150 if (this_end < size) 151 continue; 152 153 cand = round_down(this_end - size, align); 154 if (cand >= this_start) 155 return cand; 156 } 157 158 return 0; 159 } 160 161 /** 162 * memblock_find_in_range_node - find free area in given range and node 163 * @size: size of free area to find 164 * @align: alignment of free area to find 165 * @start: start of candidate range 166 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_{ANYWHERE|ACCESSIBLE} 167 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 168 * 169 * Find @size free area aligned to @align in the specified range and node. 170 * 171 * When allocation direction is bottom-up, the @start should be greater 172 * than the end of the kernel image. Otherwise, it will be trimmed. The 173 * reason is that we want the bottom-up allocation just near the kernel 174 * image so it is highly likely that the allocated memory and the kernel 175 * will reside in the same node. 176 * 177 * If bottom-up allocation failed, will try to allocate memory top-down. 178 * 179 * RETURNS: 180 * Found address on success, 0 on failure. 181 */ 182 phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size, 183 phys_addr_t align, phys_addr_t start, 184 phys_addr_t end, int nid) 185 { 186 int ret; 187 phys_addr_t kernel_end; 188 189 /* pump up @end */ 190 if (end == MEMBLOCK_ALLOC_ACCESSIBLE) 191 end = memblock.current_limit; 192 193 /* avoid allocating the first page */ 194 start = max_t(phys_addr_t, start, PAGE_SIZE); 195 end = max(start, end); 196 kernel_end = __pa_symbol(_end); 197 198 /* 199 * try bottom-up allocation only when bottom-up mode 200 * is set and @end is above the kernel image. 201 */ 202 if (memblock_bottom_up() && end > kernel_end) { 203 phys_addr_t bottom_up_start; 204 205 /* make sure we will allocate above the kernel */ 206 bottom_up_start = max(start, kernel_end); 207 208 /* ok, try bottom-up allocation first */ 209 ret = __memblock_find_range_bottom_up(bottom_up_start, end, 210 size, align, nid); 211 if (ret) 212 return ret; 213 214 /* 215 * we always limit bottom-up allocation above the kernel, 216 * but top-down allocation doesn't have the limit, so 217 * retrying top-down allocation may succeed when bottom-up 218 * allocation failed. 219 * 220 * bottom-up allocation is expected to be fail very rarely, 221 * so we use WARN_ONCE() here to see the stack trace if 222 * fail happens. 223 */ 224 WARN_ONCE(1, "memblock: bottom-up allocation failed, " 225 "memory hotunplug may be affected\n"); 226 } 227 228 return __memblock_find_range_top_down(start, end, size, align, nid); 229 } 230 231 /** 232 * memblock_find_in_range - find free area in given range 233 * @start: start of candidate range 234 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_{ANYWHERE|ACCESSIBLE} 235 * @size: size of free area to find 236 * @align: alignment of free area to find 237 * 238 * Find @size free area aligned to @align in the specified range. 239 * 240 * RETURNS: 241 * Found address on success, 0 on failure. 242 */ 243 phys_addr_t __init_memblock memblock_find_in_range(phys_addr_t start, 244 phys_addr_t end, phys_addr_t size, 245 phys_addr_t align) 246 { 247 return memblock_find_in_range_node(size, align, start, end, 248 NUMA_NO_NODE); 249 } 250 251 static void __init_memblock memblock_remove_region(struct memblock_type *type, unsigned long r) 252 { 253 type->total_size -= type->regions[r].size; 254 memmove(&type->regions[r], &type->regions[r + 1], 255 (type->cnt - (r + 1)) * sizeof(type->regions[r])); 256 type->cnt--; 257 258 /* Special case for empty arrays */ 259 if (type->cnt == 0) { 260 WARN_ON(type->total_size != 0); 261 type->cnt = 1; 262 type->regions[0].base = 0; 263 type->regions[0].size = 0; 264 type->regions[0].flags = 0; 265 memblock_set_region_node(&type->regions[0], MAX_NUMNODES); 266 } 267 } 268 269 #ifdef CONFIG_ARCH_DISCARD_MEMBLOCK 270 271 phys_addr_t __init_memblock get_allocated_memblock_reserved_regions_info( 272 phys_addr_t *addr) 273 { 274 if (memblock.reserved.regions == memblock_reserved_init_regions) 275 return 0; 276 277 *addr = __pa(memblock.reserved.regions); 278 279 return PAGE_ALIGN(sizeof(struct memblock_region) * 280 memblock.reserved.max); 281 } 282 283 phys_addr_t __init_memblock get_allocated_memblock_memory_regions_info( 284 phys_addr_t *addr) 285 { 286 if (memblock.memory.regions == memblock_memory_init_regions) 287 return 0; 288 289 *addr = __pa(memblock.memory.regions); 290 291 return PAGE_ALIGN(sizeof(struct memblock_region) * 292 memblock.memory.max); 293 } 294 295 #endif 296 297 /** 298 * memblock_double_array - double the size of the memblock regions array 299 * @type: memblock type of the regions array being doubled 300 * @new_area_start: starting address of memory range to avoid overlap with 301 * @new_area_size: size of memory range to avoid overlap with 302 * 303 * Double the size of the @type regions array. If memblock is being used to 304 * allocate memory for a new reserved regions array and there is a previously 305 * allocated memory range [@new_area_start,@new_area_start+@new_area_size] 306 * waiting to be reserved, ensure the memory used by the new array does 307 * not overlap. 308 * 309 * RETURNS: 310 * 0 on success, -1 on failure. 311 */ 312 static int __init_memblock memblock_double_array(struct memblock_type *type, 313 phys_addr_t new_area_start, 314 phys_addr_t new_area_size) 315 { 316 struct memblock_region *new_array, *old_array; 317 phys_addr_t old_alloc_size, new_alloc_size; 318 phys_addr_t old_size, new_size, addr; 319 int use_slab = slab_is_available(); 320 int *in_slab; 321 322 /* We don't allow resizing until we know about the reserved regions 323 * of memory that aren't suitable for allocation 324 */ 325 if (!memblock_can_resize) 326 return -1; 327 328 /* Calculate new doubled size */ 329 old_size = type->max * sizeof(struct memblock_region); 330 new_size = old_size << 1; 331 /* 332 * We need to allocated new one align to PAGE_SIZE, 333 * so we can free them completely later. 334 */ 335 old_alloc_size = PAGE_ALIGN(old_size); 336 new_alloc_size = PAGE_ALIGN(new_size); 337 338 /* Retrieve the slab flag */ 339 if (type == &memblock.memory) 340 in_slab = &memblock_memory_in_slab; 341 else 342 in_slab = &memblock_reserved_in_slab; 343 344 /* Try to find some space for it. 345 * 346 * WARNING: We assume that either slab_is_available() and we use it or 347 * we use MEMBLOCK for allocations. That means that this is unsafe to 348 * use when bootmem is currently active (unless bootmem itself is 349 * implemented on top of MEMBLOCK which isn't the case yet) 350 * 351 * This should however not be an issue for now, as we currently only 352 * call into MEMBLOCK while it's still active, or much later when slab 353 * is active for memory hotplug operations 354 */ 355 if (use_slab) { 356 new_array = kmalloc(new_size, GFP_KERNEL); 357 addr = new_array ? __pa(new_array) : 0; 358 } else { 359 /* only exclude range when trying to double reserved.regions */ 360 if (type != &memblock.reserved) 361 new_area_start = new_area_size = 0; 362 363 addr = memblock_find_in_range(new_area_start + new_area_size, 364 memblock.current_limit, 365 new_alloc_size, PAGE_SIZE); 366 if (!addr && new_area_size) 367 addr = memblock_find_in_range(0, 368 min(new_area_start, memblock.current_limit), 369 new_alloc_size, PAGE_SIZE); 370 371 new_array = addr ? __va(addr) : NULL; 372 } 373 if (!addr) { 374 pr_err("memblock: Failed to double %s array from %ld to %ld entries !\n", 375 memblock_type_name(type), type->max, type->max * 2); 376 return -1; 377 } 378 379 memblock_dbg("memblock: %s is doubled to %ld at [%#010llx-%#010llx]", 380 memblock_type_name(type), type->max * 2, (u64)addr, 381 (u64)addr + new_size - 1); 382 383 /* 384 * Found space, we now need to move the array over before we add the 385 * reserved region since it may be our reserved array itself that is 386 * full. 387 */ 388 memcpy(new_array, type->regions, old_size); 389 memset(new_array + type->max, 0, old_size); 390 old_array = type->regions; 391 type->regions = new_array; 392 type->max <<= 1; 393 394 /* Free old array. We needn't free it if the array is the static one */ 395 if (*in_slab) 396 kfree(old_array); 397 else if (old_array != memblock_memory_init_regions && 398 old_array != memblock_reserved_init_regions) 399 memblock_free(__pa(old_array), old_alloc_size); 400 401 /* 402 * Reserve the new array if that comes from the memblock. Otherwise, we 403 * needn't do it 404 */ 405 if (!use_slab) 406 BUG_ON(memblock_reserve(addr, new_alloc_size)); 407 408 /* Update slab flag */ 409 *in_slab = use_slab; 410 411 return 0; 412 } 413 414 /** 415 * memblock_merge_regions - merge neighboring compatible regions 416 * @type: memblock type to scan 417 * 418 * Scan @type and merge neighboring compatible regions. 419 */ 420 static void __init_memblock memblock_merge_regions(struct memblock_type *type) 421 { 422 int i = 0; 423 424 /* cnt never goes below 1 */ 425 while (i < type->cnt - 1) { 426 struct memblock_region *this = &type->regions[i]; 427 struct memblock_region *next = &type->regions[i + 1]; 428 429 if (this->base + this->size != next->base || 430 memblock_get_region_node(this) != 431 memblock_get_region_node(next) || 432 this->flags != next->flags) { 433 BUG_ON(this->base + this->size > next->base); 434 i++; 435 continue; 436 } 437 438 this->size += next->size; 439 /* move forward from next + 1, index of which is i + 2 */ 440 memmove(next, next + 1, (type->cnt - (i + 2)) * sizeof(*next)); 441 type->cnt--; 442 } 443 } 444 445 /** 446 * memblock_insert_region - insert new memblock region 447 * @type: memblock type to insert into 448 * @idx: index for the insertion point 449 * @base: base address of the new region 450 * @size: size of the new region 451 * @nid: node id of the new region 452 * @flags: flags of the new region 453 * 454 * Insert new memblock region [@base,@base+@size) into @type at @idx. 455 * @type must already have extra room to accomodate the new region. 456 */ 457 static void __init_memblock memblock_insert_region(struct memblock_type *type, 458 int idx, phys_addr_t base, 459 phys_addr_t size, 460 int nid, unsigned long flags) 461 { 462 struct memblock_region *rgn = &type->regions[idx]; 463 464 BUG_ON(type->cnt >= type->max); 465 memmove(rgn + 1, rgn, (type->cnt - idx) * sizeof(*rgn)); 466 rgn->base = base; 467 rgn->size = size; 468 rgn->flags = flags; 469 memblock_set_region_node(rgn, nid); 470 type->cnt++; 471 type->total_size += size; 472 } 473 474 /** 475 * memblock_add_region - add new memblock region 476 * @type: memblock type to add new region into 477 * @base: base address of the new region 478 * @size: size of the new region 479 * @nid: nid of the new region 480 * @flags: flags of the new region 481 * 482 * Add new memblock region [@base,@base+@size) into @type. The new region 483 * is allowed to overlap with existing ones - overlaps don't affect already 484 * existing regions. @type is guaranteed to be minimal (all neighbouring 485 * compatible regions are merged) after the addition. 486 * 487 * RETURNS: 488 * 0 on success, -errno on failure. 489 */ 490 static int __init_memblock memblock_add_region(struct memblock_type *type, 491 phys_addr_t base, phys_addr_t size, 492 int nid, unsigned long flags) 493 { 494 bool insert = false; 495 phys_addr_t obase = base; 496 phys_addr_t end = base + memblock_cap_size(base, &size); 497 int i, nr_new; 498 499 if (!size) 500 return 0; 501 502 /* special case for empty array */ 503 if (type->regions[0].size == 0) { 504 WARN_ON(type->cnt != 1 || type->total_size); 505 type->regions[0].base = base; 506 type->regions[0].size = size; 507 type->regions[0].flags = flags; 508 memblock_set_region_node(&type->regions[0], nid); 509 type->total_size = size; 510 return 0; 511 } 512 repeat: 513 /* 514 * The following is executed twice. Once with %false @insert and 515 * then with %true. The first counts the number of regions needed 516 * to accomodate the new area. The second actually inserts them. 517 */ 518 base = obase; 519 nr_new = 0; 520 521 for (i = 0; i < type->cnt; i++) { 522 struct memblock_region *rgn = &type->regions[i]; 523 phys_addr_t rbase = rgn->base; 524 phys_addr_t rend = rbase + rgn->size; 525 526 if (rbase >= end) 527 break; 528 if (rend <= base) 529 continue; 530 /* 531 * @rgn overlaps. If it separates the lower part of new 532 * area, insert that portion. 533 */ 534 if (rbase > base) { 535 nr_new++; 536 if (insert) 537 memblock_insert_region(type, i++, base, 538 rbase - base, nid, 539 flags); 540 } 541 /* area below @rend is dealt with, forget about it */ 542 base = min(rend, end); 543 } 544 545 /* insert the remaining portion */ 546 if (base < end) { 547 nr_new++; 548 if (insert) 549 memblock_insert_region(type, i, base, end - base, 550 nid, flags); 551 } 552 553 /* 554 * If this was the first round, resize array and repeat for actual 555 * insertions; otherwise, merge and return. 556 */ 557 if (!insert) { 558 while (type->cnt + nr_new > type->max) 559 if (memblock_double_array(type, obase, size) < 0) 560 return -ENOMEM; 561 insert = true; 562 goto repeat; 563 } else { 564 memblock_merge_regions(type); 565 return 0; 566 } 567 } 568 569 int __init_memblock memblock_add_node(phys_addr_t base, phys_addr_t size, 570 int nid) 571 { 572 return memblock_add_region(&memblock.memory, base, size, nid, 0); 573 } 574 575 int __init_memblock memblock_add(phys_addr_t base, phys_addr_t size) 576 { 577 return memblock_add_region(&memblock.memory, base, size, 578 MAX_NUMNODES, 0); 579 } 580 581 /** 582 * memblock_isolate_range - isolate given range into disjoint memblocks 583 * @type: memblock type to isolate range for 584 * @base: base of range to isolate 585 * @size: size of range to isolate 586 * @start_rgn: out parameter for the start of isolated region 587 * @end_rgn: out parameter for the end of isolated region 588 * 589 * Walk @type and ensure that regions don't cross the boundaries defined by 590 * [@base,@base+@size). Crossing regions are split at the boundaries, 591 * which may create at most two more regions. The index of the first 592 * region inside the range is returned in *@start_rgn and end in *@end_rgn. 593 * 594 * RETURNS: 595 * 0 on success, -errno on failure. 596 */ 597 static int __init_memblock memblock_isolate_range(struct memblock_type *type, 598 phys_addr_t base, phys_addr_t size, 599 int *start_rgn, int *end_rgn) 600 { 601 phys_addr_t end = base + memblock_cap_size(base, &size); 602 int i; 603 604 *start_rgn = *end_rgn = 0; 605 606 if (!size) 607 return 0; 608 609 /* we'll create at most two more regions */ 610 while (type->cnt + 2 > type->max) 611 if (memblock_double_array(type, base, size) < 0) 612 return -ENOMEM; 613 614 for (i = 0; i < type->cnt; i++) { 615 struct memblock_region *rgn = &type->regions[i]; 616 phys_addr_t rbase = rgn->base; 617 phys_addr_t rend = rbase + rgn->size; 618 619 if (rbase >= end) 620 break; 621 if (rend <= base) 622 continue; 623 624 if (rbase < base) { 625 /* 626 * @rgn intersects from below. Split and continue 627 * to process the next region - the new top half. 628 */ 629 rgn->base = base; 630 rgn->size -= base - rbase; 631 type->total_size -= base - rbase; 632 memblock_insert_region(type, i, rbase, base - rbase, 633 memblock_get_region_node(rgn), 634 rgn->flags); 635 } else if (rend > end) { 636 /* 637 * @rgn intersects from above. Split and redo the 638 * current region - the new bottom half. 639 */ 640 rgn->base = end; 641 rgn->size -= end - rbase; 642 type->total_size -= end - rbase; 643 memblock_insert_region(type, i--, rbase, end - rbase, 644 memblock_get_region_node(rgn), 645 rgn->flags); 646 } else { 647 /* @rgn is fully contained, record it */ 648 if (!*end_rgn) 649 *start_rgn = i; 650 *end_rgn = i + 1; 651 } 652 } 653 654 return 0; 655 } 656 657 static int __init_memblock __memblock_remove(struct memblock_type *type, 658 phys_addr_t base, phys_addr_t size) 659 { 660 int start_rgn, end_rgn; 661 int i, ret; 662 663 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 664 if (ret) 665 return ret; 666 667 for (i = end_rgn - 1; i >= start_rgn; i--) 668 memblock_remove_region(type, i); 669 return 0; 670 } 671 672 int __init_memblock memblock_remove(phys_addr_t base, phys_addr_t size) 673 { 674 return __memblock_remove(&memblock.memory, base, size); 675 } 676 677 int __init_memblock memblock_free(phys_addr_t base, phys_addr_t size) 678 { 679 memblock_dbg(" memblock_free: [%#016llx-%#016llx] %pF\n", 680 (unsigned long long)base, 681 (unsigned long long)base + size - 1, 682 (void *)_RET_IP_); 683 684 return __memblock_remove(&memblock.reserved, base, size); 685 } 686 687 static int __init_memblock memblock_reserve_region(phys_addr_t base, 688 phys_addr_t size, 689 int nid, 690 unsigned long flags) 691 { 692 struct memblock_type *_rgn = &memblock.reserved; 693 694 memblock_dbg("memblock_reserve: [%#016llx-%#016llx] flags %#02lx %pF\n", 695 (unsigned long long)base, 696 (unsigned long long)base + size - 1, 697 flags, (void *)_RET_IP_); 698 699 return memblock_add_region(_rgn, base, size, nid, flags); 700 } 701 702 int __init_memblock memblock_reserve(phys_addr_t base, phys_addr_t size) 703 { 704 return memblock_reserve_region(base, size, MAX_NUMNODES, 0); 705 } 706 707 /** 708 * memblock_mark_hotplug - Mark hotpluggable memory with flag MEMBLOCK_HOTPLUG. 709 * @base: the base phys addr of the region 710 * @size: the size of the region 711 * 712 * This function isolates region [@base, @base + @size), and mark it with flag 713 * MEMBLOCK_HOTPLUG. 714 * 715 * Return 0 on succees, -errno on failure. 716 */ 717 int __init_memblock memblock_mark_hotplug(phys_addr_t base, phys_addr_t size) 718 { 719 struct memblock_type *type = &memblock.memory; 720 int i, ret, start_rgn, end_rgn; 721 722 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 723 if (ret) 724 return ret; 725 726 for (i = start_rgn; i < end_rgn; i++) 727 memblock_set_region_flags(&type->regions[i], MEMBLOCK_HOTPLUG); 728 729 memblock_merge_regions(type); 730 return 0; 731 } 732 733 /** 734 * memblock_clear_hotplug - Clear flag MEMBLOCK_HOTPLUG for a specified region. 735 * @base: the base phys addr of the region 736 * @size: the size of the region 737 * 738 * This function isolates region [@base, @base + @size), and clear flag 739 * MEMBLOCK_HOTPLUG for the isolated regions. 740 * 741 * Return 0 on succees, -errno on failure. 742 */ 743 int __init_memblock memblock_clear_hotplug(phys_addr_t base, phys_addr_t size) 744 { 745 struct memblock_type *type = &memblock.memory; 746 int i, ret, start_rgn, end_rgn; 747 748 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 749 if (ret) 750 return ret; 751 752 for (i = start_rgn; i < end_rgn; i++) 753 memblock_clear_region_flags(&type->regions[i], 754 MEMBLOCK_HOTPLUG); 755 756 memblock_merge_regions(type); 757 return 0; 758 } 759 760 /** 761 * __next_free_mem_range - next function for for_each_free_mem_range() 762 * @idx: pointer to u64 loop variable 763 * @nid: node selector, %NUMA_NO_NODE for all nodes 764 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL 765 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL 766 * @out_nid: ptr to int for nid of the range, can be %NULL 767 * 768 * Find the first free area from *@idx which matches @nid, fill the out 769 * parameters, and update *@idx for the next iteration. The lower 32bit of 770 * *@idx contains index into memory region and the upper 32bit indexes the 771 * areas before each reserved region. For example, if reserved regions 772 * look like the following, 773 * 774 * 0:[0-16), 1:[32-48), 2:[128-130) 775 * 776 * The upper 32bit indexes the following regions. 777 * 778 * 0:[0-0), 1:[16-32), 2:[48-128), 3:[130-MAX) 779 * 780 * As both region arrays are sorted, the function advances the two indices 781 * in lockstep and returns each intersection. 782 */ 783 void __init_memblock __next_free_mem_range(u64 *idx, int nid, 784 phys_addr_t *out_start, 785 phys_addr_t *out_end, int *out_nid) 786 { 787 struct memblock_type *mem = &memblock.memory; 788 struct memblock_type *rsv = &memblock.reserved; 789 int mi = *idx & 0xffffffff; 790 int ri = *idx >> 32; 791 792 if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n")) 793 nid = NUMA_NO_NODE; 794 795 for ( ; mi < mem->cnt; mi++) { 796 struct memblock_region *m = &mem->regions[mi]; 797 phys_addr_t m_start = m->base; 798 phys_addr_t m_end = m->base + m->size; 799 800 /* only memory regions are associated with nodes, check it */ 801 if (nid != NUMA_NO_NODE && nid != memblock_get_region_node(m)) 802 continue; 803 804 /* scan areas before each reservation for intersection */ 805 for ( ; ri < rsv->cnt + 1; ri++) { 806 struct memblock_region *r = &rsv->regions[ri]; 807 phys_addr_t r_start = ri ? r[-1].base + r[-1].size : 0; 808 phys_addr_t r_end = ri < rsv->cnt ? r->base : ULLONG_MAX; 809 810 /* if ri advanced past mi, break out to advance mi */ 811 if (r_start >= m_end) 812 break; 813 /* if the two regions intersect, we're done */ 814 if (m_start < r_end) { 815 if (out_start) 816 *out_start = max(m_start, r_start); 817 if (out_end) 818 *out_end = min(m_end, r_end); 819 if (out_nid) 820 *out_nid = memblock_get_region_node(m); 821 /* 822 * The region which ends first is advanced 823 * for the next iteration. 824 */ 825 if (m_end <= r_end) 826 mi++; 827 else 828 ri++; 829 *idx = (u32)mi | (u64)ri << 32; 830 return; 831 } 832 } 833 } 834 835 /* signal end of iteration */ 836 *idx = ULLONG_MAX; 837 } 838 839 /** 840 * __next_free_mem_range_rev - next function for for_each_free_mem_range_reverse() 841 * @idx: pointer to u64 loop variable 842 * @nid: nid: node selector, %NUMA_NO_NODE for all nodes 843 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL 844 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL 845 * @out_nid: ptr to int for nid of the range, can be %NULL 846 * 847 * Reverse of __next_free_mem_range(). 848 * 849 * Linux kernel cannot migrate pages used by itself. Memory hotplug users won't 850 * be able to hot-remove hotpluggable memory used by the kernel. So this 851 * function skip hotpluggable regions if needed when allocating memory for the 852 * kernel. 853 */ 854 void __init_memblock __next_free_mem_range_rev(u64 *idx, int nid, 855 phys_addr_t *out_start, 856 phys_addr_t *out_end, int *out_nid) 857 { 858 struct memblock_type *mem = &memblock.memory; 859 struct memblock_type *rsv = &memblock.reserved; 860 int mi = *idx & 0xffffffff; 861 int ri = *idx >> 32; 862 863 if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n")) 864 nid = NUMA_NO_NODE; 865 866 if (*idx == (u64)ULLONG_MAX) { 867 mi = mem->cnt - 1; 868 ri = rsv->cnt; 869 } 870 871 for ( ; mi >= 0; mi--) { 872 struct memblock_region *m = &mem->regions[mi]; 873 phys_addr_t m_start = m->base; 874 phys_addr_t m_end = m->base + m->size; 875 876 /* only memory regions are associated with nodes, check it */ 877 if (nid != NUMA_NO_NODE && nid != memblock_get_region_node(m)) 878 continue; 879 880 /* skip hotpluggable memory regions if needed */ 881 if (movable_node_is_enabled() && memblock_is_hotpluggable(m)) 882 continue; 883 884 /* scan areas before each reservation for intersection */ 885 for ( ; ri >= 0; ri--) { 886 struct memblock_region *r = &rsv->regions[ri]; 887 phys_addr_t r_start = ri ? r[-1].base + r[-1].size : 0; 888 phys_addr_t r_end = ri < rsv->cnt ? r->base : ULLONG_MAX; 889 890 /* if ri advanced past mi, break out to advance mi */ 891 if (r_end <= m_start) 892 break; 893 /* if the two regions intersect, we're done */ 894 if (m_end > r_start) { 895 if (out_start) 896 *out_start = max(m_start, r_start); 897 if (out_end) 898 *out_end = min(m_end, r_end); 899 if (out_nid) 900 *out_nid = memblock_get_region_node(m); 901 902 if (m_start >= r_start) 903 mi--; 904 else 905 ri--; 906 *idx = (u32)mi | (u64)ri << 32; 907 return; 908 } 909 } 910 } 911 912 *idx = ULLONG_MAX; 913 } 914 915 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 916 /* 917 * Common iterator interface used to define for_each_mem_range(). 918 */ 919 void __init_memblock __next_mem_pfn_range(int *idx, int nid, 920 unsigned long *out_start_pfn, 921 unsigned long *out_end_pfn, int *out_nid) 922 { 923 struct memblock_type *type = &memblock.memory; 924 struct memblock_region *r; 925 926 while (++*idx < type->cnt) { 927 r = &type->regions[*idx]; 928 929 if (PFN_UP(r->base) >= PFN_DOWN(r->base + r->size)) 930 continue; 931 if (nid == MAX_NUMNODES || nid == r->nid) 932 break; 933 } 934 if (*idx >= type->cnt) { 935 *idx = -1; 936 return; 937 } 938 939 if (out_start_pfn) 940 *out_start_pfn = PFN_UP(r->base); 941 if (out_end_pfn) 942 *out_end_pfn = PFN_DOWN(r->base + r->size); 943 if (out_nid) 944 *out_nid = r->nid; 945 } 946 947 /** 948 * memblock_set_node - set node ID on memblock regions 949 * @base: base of area to set node ID for 950 * @size: size of area to set node ID for 951 * @type: memblock type to set node ID for 952 * @nid: node ID to set 953 * 954 * Set the nid of memblock @type regions in [@base,@base+@size) to @nid. 955 * Regions which cross the area boundaries are split as necessary. 956 * 957 * RETURNS: 958 * 0 on success, -errno on failure. 959 */ 960 int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size, 961 struct memblock_type *type, int nid) 962 { 963 int start_rgn, end_rgn; 964 int i, ret; 965 966 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 967 if (ret) 968 return ret; 969 970 for (i = start_rgn; i < end_rgn; i++) 971 memblock_set_region_node(&type->regions[i], nid); 972 973 memblock_merge_regions(type); 974 return 0; 975 } 976 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 977 978 static phys_addr_t __init memblock_alloc_base_nid(phys_addr_t size, 979 phys_addr_t align, phys_addr_t max_addr, 980 int nid) 981 { 982 phys_addr_t found; 983 984 if (!align) 985 align = SMP_CACHE_BYTES; 986 987 found = memblock_find_in_range_node(size, align, 0, max_addr, nid); 988 if (found && !memblock_reserve(found, size)) 989 return found; 990 991 return 0; 992 } 993 994 phys_addr_t __init memblock_alloc_nid(phys_addr_t size, phys_addr_t align, int nid) 995 { 996 return memblock_alloc_base_nid(size, align, MEMBLOCK_ALLOC_ACCESSIBLE, nid); 997 } 998 999 phys_addr_t __init __memblock_alloc_base(phys_addr_t size, phys_addr_t align, phys_addr_t max_addr) 1000 { 1001 return memblock_alloc_base_nid(size, align, max_addr, NUMA_NO_NODE); 1002 } 1003 1004 phys_addr_t __init memblock_alloc_base(phys_addr_t size, phys_addr_t align, phys_addr_t max_addr) 1005 { 1006 phys_addr_t alloc; 1007 1008 alloc = __memblock_alloc_base(size, align, max_addr); 1009 1010 if (alloc == 0) 1011 panic("ERROR: Failed to allocate 0x%llx bytes below 0x%llx.\n", 1012 (unsigned long long) size, (unsigned long long) max_addr); 1013 1014 return alloc; 1015 } 1016 1017 phys_addr_t __init memblock_alloc(phys_addr_t size, phys_addr_t align) 1018 { 1019 return memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ACCESSIBLE); 1020 } 1021 1022 phys_addr_t __init memblock_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid) 1023 { 1024 phys_addr_t res = memblock_alloc_nid(size, align, nid); 1025 1026 if (res) 1027 return res; 1028 return memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ACCESSIBLE); 1029 } 1030 1031 /** 1032 * memblock_virt_alloc_internal - allocate boot memory block 1033 * @size: size of memory block to be allocated in bytes 1034 * @align: alignment of the region and block's size 1035 * @min_addr: the lower bound of the memory region to allocate (phys address) 1036 * @max_addr: the upper bound of the memory region to allocate (phys address) 1037 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1038 * 1039 * The @min_addr limit is dropped if it can not be satisfied and the allocation 1040 * will fall back to memory below @min_addr. Also, allocation may fall back 1041 * to any node in the system if the specified node can not 1042 * hold the requested memory. 1043 * 1044 * The allocation is performed from memory region limited by 1045 * memblock.current_limit if @max_addr == %BOOTMEM_ALLOC_ACCESSIBLE. 1046 * 1047 * The memory block is aligned on SMP_CACHE_BYTES if @align == 0. 1048 * 1049 * The phys address of allocated boot memory block is converted to virtual and 1050 * allocated memory is reset to 0. 1051 * 1052 * In addition, function sets the min_count to 0 using kmemleak_alloc for 1053 * allocated boot memory block, so that it is never reported as leaks. 1054 * 1055 * RETURNS: 1056 * Virtual address of allocated memory block on success, NULL on failure. 1057 */ 1058 static void * __init memblock_virt_alloc_internal( 1059 phys_addr_t size, phys_addr_t align, 1060 phys_addr_t min_addr, phys_addr_t max_addr, 1061 int nid) 1062 { 1063 phys_addr_t alloc; 1064 void *ptr; 1065 1066 if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n")) 1067 nid = NUMA_NO_NODE; 1068 1069 /* 1070 * Detect any accidental use of these APIs after slab is ready, as at 1071 * this moment memblock may be deinitialized already and its 1072 * internal data may be destroyed (after execution of free_all_bootmem) 1073 */ 1074 if (WARN_ON_ONCE(slab_is_available())) 1075 return kzalloc_node(size, GFP_NOWAIT, nid); 1076 1077 if (!align) 1078 align = SMP_CACHE_BYTES; 1079 1080 if (max_addr > memblock.current_limit) 1081 max_addr = memblock.current_limit; 1082 1083 again: 1084 alloc = memblock_find_in_range_node(size, align, min_addr, max_addr, 1085 nid); 1086 if (alloc) 1087 goto done; 1088 1089 if (nid != NUMA_NO_NODE) { 1090 alloc = memblock_find_in_range_node(size, align, min_addr, 1091 max_addr, NUMA_NO_NODE); 1092 if (alloc) 1093 goto done; 1094 } 1095 1096 if (min_addr) { 1097 min_addr = 0; 1098 goto again; 1099 } else { 1100 goto error; 1101 } 1102 1103 done: 1104 memblock_reserve(alloc, size); 1105 ptr = phys_to_virt(alloc); 1106 memset(ptr, 0, size); 1107 1108 /* 1109 * The min_count is set to 0 so that bootmem allocated blocks 1110 * are never reported as leaks. This is because many of these blocks 1111 * are only referred via the physical address which is not 1112 * looked up by kmemleak. 1113 */ 1114 kmemleak_alloc(ptr, size, 0, 0); 1115 1116 return ptr; 1117 1118 error: 1119 return NULL; 1120 } 1121 1122 /** 1123 * memblock_virt_alloc_try_nid_nopanic - allocate boot memory block 1124 * @size: size of memory block to be allocated in bytes 1125 * @align: alignment of the region and block's size 1126 * @min_addr: the lower bound of the memory region from where the allocation 1127 * is preferred (phys address) 1128 * @max_addr: the upper bound of the memory region from where the allocation 1129 * is preferred (phys address), or %BOOTMEM_ALLOC_ACCESSIBLE to 1130 * allocate only from memory limited by memblock.current_limit value 1131 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1132 * 1133 * Public version of _memblock_virt_alloc_try_nid_nopanic() which provides 1134 * additional debug information (including caller info), if enabled. 1135 * 1136 * RETURNS: 1137 * Virtual address of allocated memory block on success, NULL on failure. 1138 */ 1139 void * __init memblock_virt_alloc_try_nid_nopanic( 1140 phys_addr_t size, phys_addr_t align, 1141 phys_addr_t min_addr, phys_addr_t max_addr, 1142 int nid) 1143 { 1144 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=0x%llx max_addr=0x%llx %pF\n", 1145 __func__, (u64)size, (u64)align, nid, (u64)min_addr, 1146 (u64)max_addr, (void *)_RET_IP_); 1147 return memblock_virt_alloc_internal(size, align, min_addr, 1148 max_addr, nid); 1149 } 1150 1151 /** 1152 * memblock_virt_alloc_try_nid - allocate boot memory block with panicking 1153 * @size: size of memory block to be allocated in bytes 1154 * @align: alignment of the region and block's size 1155 * @min_addr: the lower bound of the memory region from where the allocation 1156 * is preferred (phys address) 1157 * @max_addr: the upper bound of the memory region from where the allocation 1158 * is preferred (phys address), or %BOOTMEM_ALLOC_ACCESSIBLE to 1159 * allocate only from memory limited by memblock.current_limit value 1160 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1161 * 1162 * Public panicking version of _memblock_virt_alloc_try_nid_nopanic() 1163 * which provides debug information (including caller info), if enabled, 1164 * and panics if the request can not be satisfied. 1165 * 1166 * RETURNS: 1167 * Virtual address of allocated memory block on success, NULL on failure. 1168 */ 1169 void * __init memblock_virt_alloc_try_nid( 1170 phys_addr_t size, phys_addr_t align, 1171 phys_addr_t min_addr, phys_addr_t max_addr, 1172 int nid) 1173 { 1174 void *ptr; 1175 1176 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=0x%llx max_addr=0x%llx %pF\n", 1177 __func__, (u64)size, (u64)align, nid, (u64)min_addr, 1178 (u64)max_addr, (void *)_RET_IP_); 1179 ptr = memblock_virt_alloc_internal(size, align, 1180 min_addr, max_addr, nid); 1181 if (ptr) 1182 return ptr; 1183 1184 panic("%s: Failed to allocate %llu bytes align=0x%llx nid=%d from=0x%llx max_addr=0x%llx\n", 1185 __func__, (u64)size, (u64)align, nid, (u64)min_addr, 1186 (u64)max_addr); 1187 return NULL; 1188 } 1189 1190 /** 1191 * __memblock_free_early - free boot memory block 1192 * @base: phys starting address of the boot memory block 1193 * @size: size of the boot memory block in bytes 1194 * 1195 * Free boot memory block previously allocated by memblock_virt_alloc_xx() API. 1196 * The freeing memory will not be released to the buddy allocator. 1197 */ 1198 void __init __memblock_free_early(phys_addr_t base, phys_addr_t size) 1199 { 1200 memblock_dbg("%s: [%#016llx-%#016llx] %pF\n", 1201 __func__, (u64)base, (u64)base + size - 1, 1202 (void *)_RET_IP_); 1203 kmemleak_free_part(__va(base), size); 1204 __memblock_remove(&memblock.reserved, base, size); 1205 } 1206 1207 /* 1208 * __memblock_free_late - free bootmem block pages directly to buddy allocator 1209 * @addr: phys starting address of the boot memory block 1210 * @size: size of the boot memory block in bytes 1211 * 1212 * This is only useful when the bootmem allocator has already been torn 1213 * down, but we are still initializing the system. Pages are released directly 1214 * to the buddy allocator, no bootmem metadata is updated because it is gone. 1215 */ 1216 void __init __memblock_free_late(phys_addr_t base, phys_addr_t size) 1217 { 1218 u64 cursor, end; 1219 1220 memblock_dbg("%s: [%#016llx-%#016llx] %pF\n", 1221 __func__, (u64)base, (u64)base + size - 1, 1222 (void *)_RET_IP_); 1223 kmemleak_free_part(__va(base), size); 1224 cursor = PFN_UP(base); 1225 end = PFN_DOWN(base + size); 1226 1227 for (; cursor < end; cursor++) { 1228 __free_pages_bootmem(pfn_to_page(cursor), 0); 1229 totalram_pages++; 1230 } 1231 } 1232 1233 /* 1234 * Remaining API functions 1235 */ 1236 1237 phys_addr_t __init memblock_phys_mem_size(void) 1238 { 1239 return memblock.memory.total_size; 1240 } 1241 1242 phys_addr_t __init memblock_mem_size(unsigned long limit_pfn) 1243 { 1244 unsigned long pages = 0; 1245 struct memblock_region *r; 1246 unsigned long start_pfn, end_pfn; 1247 1248 for_each_memblock(memory, r) { 1249 start_pfn = memblock_region_memory_base_pfn(r); 1250 end_pfn = memblock_region_memory_end_pfn(r); 1251 start_pfn = min_t(unsigned long, start_pfn, limit_pfn); 1252 end_pfn = min_t(unsigned long, end_pfn, limit_pfn); 1253 pages += end_pfn - start_pfn; 1254 } 1255 1256 return (phys_addr_t)pages << PAGE_SHIFT; 1257 } 1258 1259 /* lowest address */ 1260 phys_addr_t __init_memblock memblock_start_of_DRAM(void) 1261 { 1262 return memblock.memory.regions[0].base; 1263 } 1264 1265 phys_addr_t __init_memblock memblock_end_of_DRAM(void) 1266 { 1267 int idx = memblock.memory.cnt - 1; 1268 1269 return (memblock.memory.regions[idx].base + memblock.memory.regions[idx].size); 1270 } 1271 1272 void __init memblock_enforce_memory_limit(phys_addr_t limit) 1273 { 1274 unsigned long i; 1275 phys_addr_t max_addr = (phys_addr_t)ULLONG_MAX; 1276 1277 if (!limit) 1278 return; 1279 1280 /* find out max address */ 1281 for (i = 0; i < memblock.memory.cnt; i++) { 1282 struct memblock_region *r = &memblock.memory.regions[i]; 1283 1284 if (limit <= r->size) { 1285 max_addr = r->base + limit; 1286 break; 1287 } 1288 limit -= r->size; 1289 } 1290 1291 /* truncate both memory and reserved regions */ 1292 __memblock_remove(&memblock.memory, max_addr, (phys_addr_t)ULLONG_MAX); 1293 __memblock_remove(&memblock.reserved, max_addr, (phys_addr_t)ULLONG_MAX); 1294 } 1295 1296 static int __init_memblock memblock_search(struct memblock_type *type, phys_addr_t addr) 1297 { 1298 unsigned int left = 0, right = type->cnt; 1299 1300 do { 1301 unsigned int mid = (right + left) / 2; 1302 1303 if (addr < type->regions[mid].base) 1304 right = mid; 1305 else if (addr >= (type->regions[mid].base + 1306 type->regions[mid].size)) 1307 left = mid + 1; 1308 else 1309 return mid; 1310 } while (left < right); 1311 return -1; 1312 } 1313 1314 int __init memblock_is_reserved(phys_addr_t addr) 1315 { 1316 return memblock_search(&memblock.reserved, addr) != -1; 1317 } 1318 1319 int __init_memblock memblock_is_memory(phys_addr_t addr) 1320 { 1321 return memblock_search(&memblock.memory, addr) != -1; 1322 } 1323 1324 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 1325 int __init_memblock memblock_search_pfn_nid(unsigned long pfn, 1326 unsigned long *start_pfn, unsigned long *end_pfn) 1327 { 1328 struct memblock_type *type = &memblock.memory; 1329 int mid = memblock_search(type, (phys_addr_t)pfn << PAGE_SHIFT); 1330 1331 if (mid == -1) 1332 return -1; 1333 1334 *start_pfn = type->regions[mid].base >> PAGE_SHIFT; 1335 *end_pfn = (type->regions[mid].base + type->regions[mid].size) 1336 >> PAGE_SHIFT; 1337 1338 return type->regions[mid].nid; 1339 } 1340 #endif 1341 1342 /** 1343 * memblock_is_region_memory - check if a region is a subset of memory 1344 * @base: base of region to check 1345 * @size: size of region to check 1346 * 1347 * Check if the region [@base, @base+@size) is a subset of a memory block. 1348 * 1349 * RETURNS: 1350 * 0 if false, non-zero if true 1351 */ 1352 int __init_memblock memblock_is_region_memory(phys_addr_t base, phys_addr_t size) 1353 { 1354 int idx = memblock_search(&memblock.memory, base); 1355 phys_addr_t end = base + memblock_cap_size(base, &size); 1356 1357 if (idx == -1) 1358 return 0; 1359 return memblock.memory.regions[idx].base <= base && 1360 (memblock.memory.regions[idx].base + 1361 memblock.memory.regions[idx].size) >= end; 1362 } 1363 1364 /** 1365 * memblock_is_region_reserved - check if a region intersects reserved memory 1366 * @base: base of region to check 1367 * @size: size of region to check 1368 * 1369 * Check if the region [@base, @base+@size) intersects a reserved memory block. 1370 * 1371 * RETURNS: 1372 * 0 if false, non-zero if true 1373 */ 1374 int __init_memblock memblock_is_region_reserved(phys_addr_t base, phys_addr_t size) 1375 { 1376 memblock_cap_size(base, &size); 1377 return memblock_overlaps_region(&memblock.reserved, base, size) >= 0; 1378 } 1379 1380 void __init_memblock memblock_trim_memory(phys_addr_t align) 1381 { 1382 int i; 1383 phys_addr_t start, end, orig_start, orig_end; 1384 struct memblock_type *mem = &memblock.memory; 1385 1386 for (i = 0; i < mem->cnt; i++) { 1387 orig_start = mem->regions[i].base; 1388 orig_end = mem->regions[i].base + mem->regions[i].size; 1389 start = round_up(orig_start, align); 1390 end = round_down(orig_end, align); 1391 1392 if (start == orig_start && end == orig_end) 1393 continue; 1394 1395 if (start < end) { 1396 mem->regions[i].base = start; 1397 mem->regions[i].size = end - start; 1398 } else { 1399 memblock_remove_region(mem, i); 1400 i--; 1401 } 1402 } 1403 } 1404 1405 void __init_memblock memblock_set_current_limit(phys_addr_t limit) 1406 { 1407 memblock.current_limit = limit; 1408 } 1409 1410 static void __init_memblock memblock_dump(struct memblock_type *type, char *name) 1411 { 1412 unsigned long long base, size; 1413 unsigned long flags; 1414 int i; 1415 1416 pr_info(" %s.cnt = 0x%lx\n", name, type->cnt); 1417 1418 for (i = 0; i < type->cnt; i++) { 1419 struct memblock_region *rgn = &type->regions[i]; 1420 char nid_buf[32] = ""; 1421 1422 base = rgn->base; 1423 size = rgn->size; 1424 flags = rgn->flags; 1425 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 1426 if (memblock_get_region_node(rgn) != MAX_NUMNODES) 1427 snprintf(nid_buf, sizeof(nid_buf), " on node %d", 1428 memblock_get_region_node(rgn)); 1429 #endif 1430 pr_info(" %s[%#x]\t[%#016llx-%#016llx], %#llx bytes%s flags: %#lx\n", 1431 name, i, base, base + size - 1, size, nid_buf, flags); 1432 } 1433 } 1434 1435 void __init_memblock __memblock_dump_all(void) 1436 { 1437 pr_info("MEMBLOCK configuration:\n"); 1438 pr_info(" memory size = %#llx reserved size = %#llx\n", 1439 (unsigned long long)memblock.memory.total_size, 1440 (unsigned long long)memblock.reserved.total_size); 1441 1442 memblock_dump(&memblock.memory, "memory"); 1443 memblock_dump(&memblock.reserved, "reserved"); 1444 } 1445 1446 void __init memblock_allow_resize(void) 1447 { 1448 memblock_can_resize = 1; 1449 } 1450 1451 static int __init early_memblock(char *p) 1452 { 1453 if (p && strstr(p, "debug")) 1454 memblock_debug = 1; 1455 return 0; 1456 } 1457 early_param("memblock", early_memblock); 1458 1459 #if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_ARCH_DISCARD_MEMBLOCK) 1460 1461 static int memblock_debug_show(struct seq_file *m, void *private) 1462 { 1463 struct memblock_type *type = m->private; 1464 struct memblock_region *reg; 1465 int i; 1466 1467 for (i = 0; i < type->cnt; i++) { 1468 reg = &type->regions[i]; 1469 seq_printf(m, "%4d: ", i); 1470 if (sizeof(phys_addr_t) == 4) 1471 seq_printf(m, "0x%08lx..0x%08lx\n", 1472 (unsigned long)reg->base, 1473 (unsigned long)(reg->base + reg->size - 1)); 1474 else 1475 seq_printf(m, "0x%016llx..0x%016llx\n", 1476 (unsigned long long)reg->base, 1477 (unsigned long long)(reg->base + reg->size - 1)); 1478 1479 } 1480 return 0; 1481 } 1482 1483 static int memblock_debug_open(struct inode *inode, struct file *file) 1484 { 1485 return single_open(file, memblock_debug_show, inode->i_private); 1486 } 1487 1488 static const struct file_operations memblock_debug_fops = { 1489 .open = memblock_debug_open, 1490 .read = seq_read, 1491 .llseek = seq_lseek, 1492 .release = single_release, 1493 }; 1494 1495 static int __init memblock_init_debugfs(void) 1496 { 1497 struct dentry *root = debugfs_create_dir("memblock", NULL); 1498 if (!root) 1499 return -ENXIO; 1500 debugfs_create_file("memory", S_IRUGO, root, &memblock.memory, &memblock_debug_fops); 1501 debugfs_create_file("reserved", S_IRUGO, root, &memblock.reserved, &memblock_debug_fops); 1502 1503 return 0; 1504 } 1505 __initcall(memblock_init_debugfs); 1506 1507 #endif /* CONFIG_DEBUG_FS */ 1508