18c37cb7dSVasily Gorbik // SPDX-License-Identifier: GPL-2.0 2*f913a660SVasily Gorbik #include <linux/processor.h> 38c37cb7dSVasily Gorbik #include <linux/errno.h> 48c37cb7dSVasily Gorbik #include <linux/init.h> 58c37cb7dSVasily Gorbik #include <asm/physmem_info.h> 6*f913a660SVasily Gorbik #include <asm/stacktrace.h> 7*f913a660SVasily Gorbik #include <asm/boot_data.h> 88c37cb7dSVasily Gorbik #include <asm/sparsemem.h> 9*f913a660SVasily Gorbik #include <asm/sections.h> 10*f913a660SVasily Gorbik #include <asm/setup.h> 11*f913a660SVasily Gorbik #include <asm/sclp.h> 12*f913a660SVasily Gorbik #include <asm/uv.h> 138c37cb7dSVasily Gorbik #include "decompressor.h" 148c37cb7dSVasily Gorbik #include "boot.h" 158c37cb7dSVasily Gorbik 168c37cb7dSVasily Gorbik struct physmem_info __bootdata(physmem_info); 17*f913a660SVasily Gorbik static unsigned int physmem_alloc_ranges; 18*f913a660SVasily Gorbik static unsigned long physmem_alloc_pos; 198c37cb7dSVasily Gorbik 208c37cb7dSVasily Gorbik /* up to 256 storage elements, 1020 subincrements each */ 218c37cb7dSVasily Gorbik #define ENTRIES_EXTENDED_MAX \ 228c37cb7dSVasily Gorbik (256 * (1020 / 2) * sizeof(struct physmem_range)) 238c37cb7dSVasily Gorbik 248c37cb7dSVasily Gorbik static struct physmem_range *__get_physmem_range_ptr(u32 n) 258c37cb7dSVasily Gorbik { 268c37cb7dSVasily Gorbik if (n < MEM_INLINED_ENTRIES) 278c37cb7dSVasily Gorbik return &physmem_info.online[n]; 28*f913a660SVasily Gorbik if (unlikely(!physmem_info.online_extended)) { 29*f913a660SVasily Gorbik physmem_info.online_extended = (struct physmem_range *)physmem_alloc_range( 30*f913a660SVasily Gorbik RR_MEM_DETECT_EXTENDED, ENTRIES_EXTENDED_MAX, sizeof(long), 0, 31*f913a660SVasily Gorbik physmem_alloc_pos, true); 32*f913a660SVasily Gorbik } 338c37cb7dSVasily Gorbik return &physmem_info.online_extended[n - MEM_INLINED_ENTRIES]; 348c37cb7dSVasily Gorbik } 358c37cb7dSVasily Gorbik 368c37cb7dSVasily Gorbik /* 378c37cb7dSVasily Gorbik * sequential calls to add_physmem_online_range with adjacent memory ranges 388c37cb7dSVasily Gorbik * are merged together into single memory range. 398c37cb7dSVasily Gorbik */ 408c37cb7dSVasily Gorbik void add_physmem_online_range(u64 start, u64 end) 418c37cb7dSVasily Gorbik { 428c37cb7dSVasily Gorbik struct physmem_range *range; 438c37cb7dSVasily Gorbik 448c37cb7dSVasily Gorbik if (physmem_info.range_count) { 458c37cb7dSVasily Gorbik range = __get_physmem_range_ptr(physmem_info.range_count - 1); 468c37cb7dSVasily Gorbik if (range->end == start) { 478c37cb7dSVasily Gorbik range->end = end; 488c37cb7dSVasily Gorbik return; 498c37cb7dSVasily Gorbik } 508c37cb7dSVasily Gorbik } 518c37cb7dSVasily Gorbik 528c37cb7dSVasily Gorbik range = __get_physmem_range_ptr(physmem_info.range_count); 538c37cb7dSVasily Gorbik range->start = start; 548c37cb7dSVasily Gorbik range->end = end; 558c37cb7dSVasily Gorbik physmem_info.range_count++; 568c37cb7dSVasily Gorbik } 578c37cb7dSVasily Gorbik 588c37cb7dSVasily Gorbik static int __diag260(unsigned long rx1, unsigned long rx2) 598c37cb7dSVasily Gorbik { 608c37cb7dSVasily Gorbik unsigned long reg1, reg2, ry; 618c37cb7dSVasily Gorbik union register_pair rx; 628c37cb7dSVasily Gorbik psw_t old; 638c37cb7dSVasily Gorbik int rc; 648c37cb7dSVasily Gorbik 658c37cb7dSVasily Gorbik rx.even = rx1; 668c37cb7dSVasily Gorbik rx.odd = rx2; 678c37cb7dSVasily Gorbik ry = 0x10; /* storage configuration */ 688c37cb7dSVasily Gorbik rc = -1; /* fail */ 698c37cb7dSVasily Gorbik asm volatile( 708c37cb7dSVasily Gorbik " mvc 0(16,%[psw_old]),0(%[psw_pgm])\n" 718c37cb7dSVasily Gorbik " epsw %[reg1],%[reg2]\n" 728c37cb7dSVasily Gorbik " st %[reg1],0(%[psw_pgm])\n" 738c37cb7dSVasily Gorbik " st %[reg2],4(%[psw_pgm])\n" 748c37cb7dSVasily Gorbik " larl %[reg1],1f\n" 758c37cb7dSVasily Gorbik " stg %[reg1],8(%[psw_pgm])\n" 768c37cb7dSVasily Gorbik " diag %[rx],%[ry],0x260\n" 778c37cb7dSVasily Gorbik " ipm %[rc]\n" 788c37cb7dSVasily Gorbik " srl %[rc],28\n" 798c37cb7dSVasily Gorbik "1: mvc 0(16,%[psw_pgm]),0(%[psw_old])\n" 808c37cb7dSVasily Gorbik : [reg1] "=&d" (reg1), 818c37cb7dSVasily Gorbik [reg2] "=&a" (reg2), 828c37cb7dSVasily Gorbik [rc] "+&d" (rc), 838c37cb7dSVasily Gorbik [ry] "+&d" (ry), 848c37cb7dSVasily Gorbik "+Q" (S390_lowcore.program_new_psw), 858c37cb7dSVasily Gorbik "=Q" (old) 868c37cb7dSVasily Gorbik : [rx] "d" (rx.pair), 878c37cb7dSVasily Gorbik [psw_old] "a" (&old), 888c37cb7dSVasily Gorbik [psw_pgm] "a" (&S390_lowcore.program_new_psw) 898c37cb7dSVasily Gorbik : "cc", "memory"); 908c37cb7dSVasily Gorbik return rc == 0 ? ry : -1; 918c37cb7dSVasily Gorbik } 928c37cb7dSVasily Gorbik 938c37cb7dSVasily Gorbik static int diag260(void) 948c37cb7dSVasily Gorbik { 958c37cb7dSVasily Gorbik int rc, i; 968c37cb7dSVasily Gorbik 978c37cb7dSVasily Gorbik struct { 988c37cb7dSVasily Gorbik unsigned long start; 998c37cb7dSVasily Gorbik unsigned long end; 1008c37cb7dSVasily Gorbik } storage_extents[8] __aligned(16); /* VM supports up to 8 extends */ 1018c37cb7dSVasily Gorbik 1028c37cb7dSVasily Gorbik memset(storage_extents, 0, sizeof(storage_extents)); 1038c37cb7dSVasily Gorbik rc = __diag260((unsigned long)storage_extents, sizeof(storage_extents)); 1048c37cb7dSVasily Gorbik if (rc == -1) 1058c37cb7dSVasily Gorbik return -1; 1068c37cb7dSVasily Gorbik 1078c37cb7dSVasily Gorbik for (i = 0; i < min_t(int, rc, ARRAY_SIZE(storage_extents)); i++) 1088c37cb7dSVasily Gorbik add_physmem_online_range(storage_extents[i].start, storage_extents[i].end + 1); 1098c37cb7dSVasily Gorbik return 0; 1108c37cb7dSVasily Gorbik } 1118c37cb7dSVasily Gorbik 1128c37cb7dSVasily Gorbik static int tprot(unsigned long addr) 1138c37cb7dSVasily Gorbik { 1148c37cb7dSVasily Gorbik unsigned long reg1, reg2; 1158c37cb7dSVasily Gorbik int rc = -EFAULT; 1168c37cb7dSVasily Gorbik psw_t old; 1178c37cb7dSVasily Gorbik 1188c37cb7dSVasily Gorbik asm volatile( 1198c37cb7dSVasily Gorbik " mvc 0(16,%[psw_old]),0(%[psw_pgm])\n" 1208c37cb7dSVasily Gorbik " epsw %[reg1],%[reg2]\n" 1218c37cb7dSVasily Gorbik " st %[reg1],0(%[psw_pgm])\n" 1228c37cb7dSVasily Gorbik " st %[reg2],4(%[psw_pgm])\n" 1238c37cb7dSVasily Gorbik " larl %[reg1],1f\n" 1248c37cb7dSVasily Gorbik " stg %[reg1],8(%[psw_pgm])\n" 1258c37cb7dSVasily Gorbik " tprot 0(%[addr]),0\n" 1268c37cb7dSVasily Gorbik " ipm %[rc]\n" 1278c37cb7dSVasily Gorbik " srl %[rc],28\n" 1288c37cb7dSVasily Gorbik "1: mvc 0(16,%[psw_pgm]),0(%[psw_old])\n" 1298c37cb7dSVasily Gorbik : [reg1] "=&d" (reg1), 1308c37cb7dSVasily Gorbik [reg2] "=&a" (reg2), 1318c37cb7dSVasily Gorbik [rc] "+&d" (rc), 1328c37cb7dSVasily Gorbik "=Q" (S390_lowcore.program_new_psw.addr), 1338c37cb7dSVasily Gorbik "=Q" (old) 1348c37cb7dSVasily Gorbik : [psw_old] "a" (&old), 1358c37cb7dSVasily Gorbik [psw_pgm] "a" (&S390_lowcore.program_new_psw), 1368c37cb7dSVasily Gorbik [addr] "a" (addr) 1378c37cb7dSVasily Gorbik : "cc", "memory"); 1388c37cb7dSVasily Gorbik return rc; 1398c37cb7dSVasily Gorbik } 1408c37cb7dSVasily Gorbik 1418c37cb7dSVasily Gorbik static unsigned long search_mem_end(void) 1428c37cb7dSVasily Gorbik { 1438c37cb7dSVasily Gorbik unsigned long range = 1 << (MAX_PHYSMEM_BITS - 20); /* in 1MB blocks */ 1448c37cb7dSVasily Gorbik unsigned long offset = 0; 1458c37cb7dSVasily Gorbik unsigned long pivot; 1468c37cb7dSVasily Gorbik 1478c37cb7dSVasily Gorbik while (range > 1) { 1488c37cb7dSVasily Gorbik range >>= 1; 1498c37cb7dSVasily Gorbik pivot = offset + range; 1508c37cb7dSVasily Gorbik if (!tprot(pivot << 20)) 1518c37cb7dSVasily Gorbik offset = pivot; 1528c37cb7dSVasily Gorbik } 1538c37cb7dSVasily Gorbik return (offset + 1) << 20; 1548c37cb7dSVasily Gorbik } 1558c37cb7dSVasily Gorbik 156*f913a660SVasily Gorbik unsigned long detect_max_physmem_end(void) 1578c37cb7dSVasily Gorbik { 1588c37cb7dSVasily Gorbik unsigned long max_physmem_end = 0; 1598c37cb7dSVasily Gorbik 160*f913a660SVasily Gorbik if (!sclp_early_get_memsize(&max_physmem_end)) { 161*f913a660SVasily Gorbik physmem_info.info_source = MEM_DETECT_SCLP_READ_INFO; 162*f913a660SVasily Gorbik } else { 163*f913a660SVasily Gorbik max_physmem_end = search_mem_end(); 164*f913a660SVasily Gorbik physmem_info.info_source = MEM_DETECT_BIN_SEARCH; 165*f913a660SVasily Gorbik } 166*f913a660SVasily Gorbik return max_physmem_end; 167*f913a660SVasily Gorbik } 1688c37cb7dSVasily Gorbik 169*f913a660SVasily Gorbik void detect_physmem_online_ranges(unsigned long max_physmem_end) 170*f913a660SVasily Gorbik { 1718c37cb7dSVasily Gorbik if (!sclp_early_read_storage_info()) { 1728c37cb7dSVasily Gorbik physmem_info.info_source = MEM_DETECT_SCLP_STOR_INFO; 1738c37cb7dSVasily Gorbik } else if (!diag260()) { 1748c37cb7dSVasily Gorbik physmem_info.info_source = MEM_DETECT_DIAG260; 1758c37cb7dSVasily Gorbik } else if (max_physmem_end) { 1768c37cb7dSVasily Gorbik add_physmem_online_range(0, max_physmem_end); 1778c37cb7dSVasily Gorbik } 1788c37cb7dSVasily Gorbik } 1798c37cb7dSVasily Gorbik 1808c37cb7dSVasily Gorbik void physmem_set_usable_limit(unsigned long limit) 1818c37cb7dSVasily Gorbik { 182*f913a660SVasily Gorbik physmem_info.usable = limit; 183*f913a660SVasily Gorbik physmem_alloc_pos = limit; 184*f913a660SVasily Gorbik } 185*f913a660SVasily Gorbik 186*f913a660SVasily Gorbik static void die_oom(unsigned long size, unsigned long align, unsigned long min, unsigned long max) 187*f913a660SVasily Gorbik { 188*f913a660SVasily Gorbik unsigned long start, end, total_mem = 0, total_reserved_mem = 0; 189*f913a660SVasily Gorbik struct reserved_range *range; 190*f913a660SVasily Gorbik enum reserved_range_type t; 1918c37cb7dSVasily Gorbik int i; 1928c37cb7dSVasily Gorbik 193*f913a660SVasily Gorbik decompressor_printk("Linux version %s\n", kernel_version); 194*f913a660SVasily Gorbik if (!is_prot_virt_guest() && early_command_line[0]) 195*f913a660SVasily Gorbik decompressor_printk("Kernel command line: %s\n", early_command_line); 196*f913a660SVasily Gorbik decompressor_printk("Out of memory allocating %lx bytes %lx aligned in range %lx:%lx\n", 197*f913a660SVasily Gorbik size, align, min, max); 198*f913a660SVasily Gorbik decompressor_printk("Reserved memory ranges:\n"); 199*f913a660SVasily Gorbik for_each_physmem_reserved_range(t, range, &start, &end) { 200*f913a660SVasily Gorbik decompressor_printk("%016lx %016lx %s\n", start, end, get_rr_type_name(t)); 201*f913a660SVasily Gorbik total_reserved_mem += end - start; 2028c37cb7dSVasily Gorbik } 203*f913a660SVasily Gorbik decompressor_printk("Usable online memory ranges (info source: %s [%x]):\n", 204*f913a660SVasily Gorbik get_physmem_info_source(), physmem_info.info_source); 205*f913a660SVasily Gorbik for_each_physmem_usable_range(i, &start, &end) { 206*f913a660SVasily Gorbik decompressor_printk("%016lx %016lx\n", start, end); 207*f913a660SVasily Gorbik total_mem += end - start; 2088c37cb7dSVasily Gorbik } 209*f913a660SVasily Gorbik decompressor_printk("Usable online memory total: %lx Reserved: %lx Free: %lx\n", 210*f913a660SVasily Gorbik total_mem, total_reserved_mem, 211*f913a660SVasily Gorbik total_mem > total_reserved_mem ? total_mem - total_reserved_mem : 0); 212*f913a660SVasily Gorbik print_stacktrace(current_frame_address()); 213*f913a660SVasily Gorbik sclp_early_printk("\n\n -- System halted\n"); 214*f913a660SVasily Gorbik disabled_wait(); 215*f913a660SVasily Gorbik } 216*f913a660SVasily Gorbik 217*f913a660SVasily Gorbik void physmem_reserve(enum reserved_range_type type, unsigned long addr, unsigned long size) 218*f913a660SVasily Gorbik { 219*f913a660SVasily Gorbik physmem_info.reserved[type].start = addr; 220*f913a660SVasily Gorbik physmem_info.reserved[type].end = addr + size; 221*f913a660SVasily Gorbik } 222*f913a660SVasily Gorbik 223*f913a660SVasily Gorbik void physmem_free(enum reserved_range_type type) 224*f913a660SVasily Gorbik { 225*f913a660SVasily Gorbik physmem_info.reserved[type].start = 0; 226*f913a660SVasily Gorbik physmem_info.reserved[type].end = 0; 227*f913a660SVasily Gorbik } 228*f913a660SVasily Gorbik 229*f913a660SVasily Gorbik static bool __physmem_alloc_intersects(unsigned long addr, unsigned long size, 230*f913a660SVasily Gorbik unsigned long *intersection_start) 231*f913a660SVasily Gorbik { 232*f913a660SVasily Gorbik unsigned long res_addr, res_size; 233*f913a660SVasily Gorbik int t; 234*f913a660SVasily Gorbik 235*f913a660SVasily Gorbik for (t = 0; t < RR_MAX; t++) { 236*f913a660SVasily Gorbik if (!get_physmem_reserved(t, &res_addr, &res_size)) 237*f913a660SVasily Gorbik continue; 238*f913a660SVasily Gorbik if (intersects(addr, size, res_addr, res_size)) { 239*f913a660SVasily Gorbik *intersection_start = res_addr; 240*f913a660SVasily Gorbik return true; 241*f913a660SVasily Gorbik } 242*f913a660SVasily Gorbik } 243*f913a660SVasily Gorbik return ipl_report_certs_intersects(addr, size, intersection_start); 244*f913a660SVasily Gorbik } 245*f913a660SVasily Gorbik 246*f913a660SVasily Gorbik static unsigned long __physmem_alloc_range(unsigned long size, unsigned long align, 247*f913a660SVasily Gorbik unsigned long min, unsigned long max, 248*f913a660SVasily Gorbik unsigned int from_ranges, unsigned int *ranges_left, 249*f913a660SVasily Gorbik bool die_on_oom) 250*f913a660SVasily Gorbik { 251*f913a660SVasily Gorbik unsigned int nranges = from_ranges ?: physmem_info.range_count; 252*f913a660SVasily Gorbik unsigned long range_start, range_end; 253*f913a660SVasily Gorbik unsigned long intersection_start; 254*f913a660SVasily Gorbik unsigned long addr, pos = max; 255*f913a660SVasily Gorbik 256*f913a660SVasily Gorbik align = max(align, 8UL); 257*f913a660SVasily Gorbik while (nranges) { 258*f913a660SVasily Gorbik __get_physmem_range(nranges - 1, &range_start, &range_end, false); 259*f913a660SVasily Gorbik pos = min(range_end, pos); 260*f913a660SVasily Gorbik 261*f913a660SVasily Gorbik if (round_up(min, align) + size > pos) 262*f913a660SVasily Gorbik break; 263*f913a660SVasily Gorbik addr = round_down(pos - size, align); 264*f913a660SVasily Gorbik if (range_start > addr) { 265*f913a660SVasily Gorbik nranges--; 266*f913a660SVasily Gorbik continue; 267*f913a660SVasily Gorbik } 268*f913a660SVasily Gorbik if (__physmem_alloc_intersects(addr, size, &intersection_start)) { 269*f913a660SVasily Gorbik pos = intersection_start; 270*f913a660SVasily Gorbik continue; 271*f913a660SVasily Gorbik } 272*f913a660SVasily Gorbik 273*f913a660SVasily Gorbik if (ranges_left) 274*f913a660SVasily Gorbik *ranges_left = nranges; 275*f913a660SVasily Gorbik return addr; 276*f913a660SVasily Gorbik } 277*f913a660SVasily Gorbik if (die_on_oom) 278*f913a660SVasily Gorbik die_oom(size, align, min, max); 279*f913a660SVasily Gorbik return 0; 280*f913a660SVasily Gorbik } 281*f913a660SVasily Gorbik 282*f913a660SVasily Gorbik unsigned long physmem_alloc_range(enum reserved_range_type type, unsigned long size, 283*f913a660SVasily Gorbik unsigned long align, unsigned long min, unsigned long max, 284*f913a660SVasily Gorbik bool die_on_oom) 285*f913a660SVasily Gorbik { 286*f913a660SVasily Gorbik unsigned long addr; 287*f913a660SVasily Gorbik 288*f913a660SVasily Gorbik max = min(max, physmem_alloc_pos); 289*f913a660SVasily Gorbik addr = __physmem_alloc_range(size, align, min, max, 0, NULL, die_on_oom); 290*f913a660SVasily Gorbik if (addr) 291*f913a660SVasily Gorbik physmem_reserve(type, addr, size); 292*f913a660SVasily Gorbik return addr; 293*f913a660SVasily Gorbik } 294*f913a660SVasily Gorbik 295*f913a660SVasily Gorbik unsigned long physmem_alloc_top_down(enum reserved_range_type type, unsigned long size, 296*f913a660SVasily Gorbik unsigned long align) 297*f913a660SVasily Gorbik { 298*f913a660SVasily Gorbik struct reserved_range *range = &physmem_info.reserved[type]; 299*f913a660SVasily Gorbik struct reserved_range *new_range; 300*f913a660SVasily Gorbik unsigned int ranges_left; 301*f913a660SVasily Gorbik unsigned long addr; 302*f913a660SVasily Gorbik 303*f913a660SVasily Gorbik addr = __physmem_alloc_range(size, align, 0, physmem_alloc_pos, physmem_alloc_ranges, 304*f913a660SVasily Gorbik &ranges_left, true); 305*f913a660SVasily Gorbik /* if not a consecutive allocation of the same type or first allocation */ 306*f913a660SVasily Gorbik if (range->start != addr + size) { 307*f913a660SVasily Gorbik if (range->end) { 308*f913a660SVasily Gorbik physmem_alloc_pos = __physmem_alloc_range( 309*f913a660SVasily Gorbik sizeof(struct reserved_range), 0, 0, physmem_alloc_pos, 310*f913a660SVasily Gorbik physmem_alloc_ranges, &ranges_left, true); 311*f913a660SVasily Gorbik new_range = (struct reserved_range *)physmem_alloc_pos; 312*f913a660SVasily Gorbik *new_range = *range; 313*f913a660SVasily Gorbik range->chain = new_range; 314*f913a660SVasily Gorbik addr = __physmem_alloc_range(size, align, 0, physmem_alloc_pos, 315*f913a660SVasily Gorbik ranges_left, &ranges_left, true); 316*f913a660SVasily Gorbik } 317*f913a660SVasily Gorbik range->end = addr + size; 318*f913a660SVasily Gorbik } 319*f913a660SVasily Gorbik range->start = addr; 320*f913a660SVasily Gorbik physmem_alloc_pos = addr; 321*f913a660SVasily Gorbik physmem_alloc_ranges = ranges_left; 322*f913a660SVasily Gorbik return addr; 3238c37cb7dSVasily Gorbik } 324