1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Common EFI (Extensible Firmware Interface) support functions 4 * Based on Extensible Firmware Interface Specification version 1.0 5 * 6 * Copyright (C) 1999 VA Linux Systems 7 * Copyright (C) 1999 Walt Drummond <drummond@valinux.com> 8 * Copyright (C) 1999-2002 Hewlett-Packard Co. 9 * David Mosberger-Tang <davidm@hpl.hp.com> 10 * Stephane Eranian <eranian@hpl.hp.com> 11 * Copyright (C) 2005-2008 Intel Co. 12 * Fenghua Yu <fenghua.yu@intel.com> 13 * Bibo Mao <bibo.mao@intel.com> 14 * Chandramouli Narayanan <mouli@linux.intel.com> 15 * Huang Ying <ying.huang@intel.com> 16 * Copyright (C) 2013 SuSE Labs 17 * Borislav Petkov <bp@suse.de> - runtime services VA mapping 18 * 19 * Copied from efi_32.c to eliminate the duplicated code between EFI 20 * 32/64 support code. --ying 2007-10-26 21 * 22 * All EFI Runtime Services are not implemented yet as EFI only 23 * supports physical mode addressing on SoftSDV. This is to be fixed 24 * in a future version. --drummond 1999-07-20 25 * 26 * Implemented EFI runtime services and virtual mode calls. --davidm 27 * 28 * Goutham Rao: <goutham.rao@intel.com> 29 * Skip non-WB memory and ignore empty memory ranges. 30 */ 31 32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 33 34 #include <linux/kernel.h> 35 #include <linux/init.h> 36 #include <linux/efi.h> 37 #include <linux/efi-bgrt.h> 38 #include <linux/export.h> 39 #include <linux/memblock.h> 40 #include <linux/slab.h> 41 #include <linux/spinlock.h> 42 #include <linux/uaccess.h> 43 #include <linux/time.h> 44 #include <linux/io.h> 45 #include <linux/reboot.h> 46 #include <linux/bcd.h> 47 48 #include <asm/setup.h> 49 #include <asm/efi.h> 50 #include <asm/e820/api.h> 51 #include <asm/time.h> 52 #include <asm/set_memory.h> 53 #include <asm/tlbflush.h> 54 #include <asm/x86_init.h> 55 #include <asm/uv/uv.h> 56 57 static struct efi efi_phys __initdata; 58 static efi_system_table_t efi_systab __initdata; 59 60 static efi_config_table_type_t arch_tables[] __initdata = { 61 #ifdef CONFIG_X86_UV 62 {UV_SYSTEM_TABLE_GUID, "UVsystab", &efi.uv_systab}, 63 #endif 64 {NULL_GUID, NULL, NULL}, 65 }; 66 67 u64 efi_setup; /* efi setup_data physical address */ 68 69 static int add_efi_memmap __initdata; 70 static int __init setup_add_efi_memmap(char *arg) 71 { 72 add_efi_memmap = 1; 73 return 0; 74 } 75 early_param("add_efi_memmap", setup_add_efi_memmap); 76 77 static efi_status_t __init phys_efi_set_virtual_address_map( 78 unsigned long memory_map_size, 79 unsigned long descriptor_size, 80 u32 descriptor_version, 81 efi_memory_desc_t *virtual_map) 82 { 83 efi_status_t status; 84 unsigned long flags; 85 pgd_t *save_pgd; 86 87 save_pgd = efi_call_phys_prolog(); 88 if (!save_pgd) 89 return EFI_ABORTED; 90 91 /* Disable interrupts around EFI calls: */ 92 local_irq_save(flags); 93 status = efi_call_phys(efi_phys.set_virtual_address_map, 94 memory_map_size, descriptor_size, 95 descriptor_version, virtual_map); 96 local_irq_restore(flags); 97 98 efi_call_phys_epilog(save_pgd); 99 100 return status; 101 } 102 103 void __init efi_find_mirror(void) 104 { 105 efi_memory_desc_t *md; 106 u64 mirror_size = 0, total_size = 0; 107 108 for_each_efi_memory_desc(md) { 109 unsigned long long start = md->phys_addr; 110 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT; 111 112 total_size += size; 113 if (md->attribute & EFI_MEMORY_MORE_RELIABLE) { 114 memblock_mark_mirror(start, size); 115 mirror_size += size; 116 } 117 } 118 if (mirror_size) 119 pr_info("Memory: %lldM/%lldM mirrored memory\n", 120 mirror_size>>20, total_size>>20); 121 } 122 123 /* 124 * Tell the kernel about the EFI memory map. This might include 125 * more than the max 128 entries that can fit in the e820 legacy 126 * (zeropage) memory map. 127 */ 128 129 static void __init do_add_efi_memmap(void) 130 { 131 efi_memory_desc_t *md; 132 133 for_each_efi_memory_desc(md) { 134 unsigned long long start = md->phys_addr; 135 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT; 136 int e820_type; 137 138 switch (md->type) { 139 case EFI_LOADER_CODE: 140 case EFI_LOADER_DATA: 141 case EFI_BOOT_SERVICES_CODE: 142 case EFI_BOOT_SERVICES_DATA: 143 case EFI_CONVENTIONAL_MEMORY: 144 if (md->attribute & EFI_MEMORY_WB) 145 e820_type = E820_TYPE_RAM; 146 else 147 e820_type = E820_TYPE_RESERVED; 148 break; 149 case EFI_ACPI_RECLAIM_MEMORY: 150 e820_type = E820_TYPE_ACPI; 151 break; 152 case EFI_ACPI_MEMORY_NVS: 153 e820_type = E820_TYPE_NVS; 154 break; 155 case EFI_UNUSABLE_MEMORY: 156 e820_type = E820_TYPE_UNUSABLE; 157 break; 158 case EFI_PERSISTENT_MEMORY: 159 e820_type = E820_TYPE_PMEM; 160 break; 161 default: 162 /* 163 * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE 164 * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO 165 * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE 166 */ 167 e820_type = E820_TYPE_RESERVED; 168 break; 169 } 170 e820__range_add(start, size, e820_type); 171 } 172 e820__update_table(e820_table); 173 } 174 175 int __init efi_memblock_x86_reserve_range(void) 176 { 177 struct efi_info *e = &boot_params.efi_info; 178 struct efi_memory_map_data data; 179 phys_addr_t pmap; 180 int rv; 181 182 if (efi_enabled(EFI_PARAVIRT)) 183 return 0; 184 185 #ifdef CONFIG_X86_32 186 /* Can't handle data above 4GB at this time */ 187 if (e->efi_memmap_hi) { 188 pr_err("Memory map is above 4GB, disabling EFI.\n"); 189 return -EINVAL; 190 } 191 pmap = e->efi_memmap; 192 #else 193 pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32)); 194 #endif 195 data.phys_map = pmap; 196 data.size = e->efi_memmap_size; 197 data.desc_size = e->efi_memdesc_size; 198 data.desc_version = e->efi_memdesc_version; 199 200 rv = efi_memmap_init_early(&data); 201 if (rv) 202 return rv; 203 204 if (add_efi_memmap) 205 do_add_efi_memmap(); 206 207 WARN(efi.memmap.desc_version != 1, 208 "Unexpected EFI_MEMORY_DESCRIPTOR version %ld", 209 efi.memmap.desc_version); 210 211 memblock_reserve(pmap, efi.memmap.nr_map * efi.memmap.desc_size); 212 213 return 0; 214 } 215 216 #define OVERFLOW_ADDR_SHIFT (64 - EFI_PAGE_SHIFT) 217 #define OVERFLOW_ADDR_MASK (U64_MAX << OVERFLOW_ADDR_SHIFT) 218 #define U64_HIGH_BIT (~(U64_MAX >> 1)) 219 220 static bool __init efi_memmap_entry_valid(const efi_memory_desc_t *md, int i) 221 { 222 u64 end = (md->num_pages << EFI_PAGE_SHIFT) + md->phys_addr - 1; 223 u64 end_hi = 0; 224 char buf[64]; 225 226 if (md->num_pages == 0) { 227 end = 0; 228 } else if (md->num_pages > EFI_PAGES_MAX || 229 EFI_PAGES_MAX - md->num_pages < 230 (md->phys_addr >> EFI_PAGE_SHIFT)) { 231 end_hi = (md->num_pages & OVERFLOW_ADDR_MASK) 232 >> OVERFLOW_ADDR_SHIFT; 233 234 if ((md->phys_addr & U64_HIGH_BIT) && !(end & U64_HIGH_BIT)) 235 end_hi += 1; 236 } else { 237 return true; 238 } 239 240 pr_warn_once(FW_BUG "Invalid EFI memory map entries:\n"); 241 242 if (end_hi) { 243 pr_warn("mem%02u: %s range=[0x%016llx-0x%llx%016llx] (invalid)\n", 244 i, efi_md_typeattr_format(buf, sizeof(buf), md), 245 md->phys_addr, end_hi, end); 246 } else { 247 pr_warn("mem%02u: %s range=[0x%016llx-0x%016llx] (invalid)\n", 248 i, efi_md_typeattr_format(buf, sizeof(buf), md), 249 md->phys_addr, end); 250 } 251 return false; 252 } 253 254 static void __init efi_clean_memmap(void) 255 { 256 efi_memory_desc_t *out = efi.memmap.map; 257 const efi_memory_desc_t *in = out; 258 const efi_memory_desc_t *end = efi.memmap.map_end; 259 int i, n_removal; 260 261 for (i = n_removal = 0; in < end; i++) { 262 if (efi_memmap_entry_valid(in, i)) { 263 if (out != in) 264 memcpy(out, in, efi.memmap.desc_size); 265 out = (void *)out + efi.memmap.desc_size; 266 } else { 267 n_removal++; 268 } 269 in = (void *)in + efi.memmap.desc_size; 270 } 271 272 if (n_removal > 0) { 273 u64 size = efi.memmap.nr_map - n_removal; 274 275 pr_warn("Removing %d invalid memory map entries.\n", n_removal); 276 efi_memmap_install(efi.memmap.phys_map, size); 277 } 278 } 279 280 void __init efi_print_memmap(void) 281 { 282 efi_memory_desc_t *md; 283 int i = 0; 284 285 for_each_efi_memory_desc(md) { 286 char buf[64]; 287 288 pr_info("mem%02u: %s range=[0x%016llx-0x%016llx] (%lluMB)\n", 289 i++, efi_md_typeattr_format(buf, sizeof(buf), md), 290 md->phys_addr, 291 md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT) - 1, 292 (md->num_pages >> (20 - EFI_PAGE_SHIFT))); 293 } 294 } 295 296 static int __init efi_systab_init(void *phys) 297 { 298 if (efi_enabled(EFI_64BIT)) { 299 efi_system_table_64_t *systab64; 300 struct efi_setup_data *data = NULL; 301 u64 tmp = 0; 302 303 if (efi_setup) { 304 data = early_memremap(efi_setup, sizeof(*data)); 305 if (!data) 306 return -ENOMEM; 307 } 308 systab64 = early_memremap((unsigned long)phys, 309 sizeof(*systab64)); 310 if (systab64 == NULL) { 311 pr_err("Couldn't map the system table!\n"); 312 if (data) 313 early_memunmap(data, sizeof(*data)); 314 return -ENOMEM; 315 } 316 317 efi_systab.hdr = systab64->hdr; 318 efi_systab.fw_vendor = data ? (unsigned long)data->fw_vendor : 319 systab64->fw_vendor; 320 tmp |= data ? data->fw_vendor : systab64->fw_vendor; 321 efi_systab.fw_revision = systab64->fw_revision; 322 efi_systab.con_in_handle = systab64->con_in_handle; 323 tmp |= systab64->con_in_handle; 324 efi_systab.con_in = systab64->con_in; 325 tmp |= systab64->con_in; 326 efi_systab.con_out_handle = systab64->con_out_handle; 327 tmp |= systab64->con_out_handle; 328 efi_systab.con_out = systab64->con_out; 329 tmp |= systab64->con_out; 330 efi_systab.stderr_handle = systab64->stderr_handle; 331 tmp |= systab64->stderr_handle; 332 efi_systab.stderr = systab64->stderr; 333 tmp |= systab64->stderr; 334 efi_systab.runtime = data ? 335 (void *)(unsigned long)data->runtime : 336 (void *)(unsigned long)systab64->runtime; 337 tmp |= data ? data->runtime : systab64->runtime; 338 efi_systab.boottime = (void *)(unsigned long)systab64->boottime; 339 tmp |= systab64->boottime; 340 efi_systab.nr_tables = systab64->nr_tables; 341 efi_systab.tables = data ? (unsigned long)data->tables : 342 systab64->tables; 343 tmp |= data ? data->tables : systab64->tables; 344 345 early_memunmap(systab64, sizeof(*systab64)); 346 if (data) 347 early_memunmap(data, sizeof(*data)); 348 #ifdef CONFIG_X86_32 349 if (tmp >> 32) { 350 pr_err("EFI data located above 4GB, disabling EFI.\n"); 351 return -EINVAL; 352 } 353 #endif 354 } else { 355 efi_system_table_32_t *systab32; 356 357 systab32 = early_memremap((unsigned long)phys, 358 sizeof(*systab32)); 359 if (systab32 == NULL) { 360 pr_err("Couldn't map the system table!\n"); 361 return -ENOMEM; 362 } 363 364 efi_systab.hdr = systab32->hdr; 365 efi_systab.fw_vendor = systab32->fw_vendor; 366 efi_systab.fw_revision = systab32->fw_revision; 367 efi_systab.con_in_handle = systab32->con_in_handle; 368 efi_systab.con_in = systab32->con_in; 369 efi_systab.con_out_handle = systab32->con_out_handle; 370 efi_systab.con_out = systab32->con_out; 371 efi_systab.stderr_handle = systab32->stderr_handle; 372 efi_systab.stderr = systab32->stderr; 373 efi_systab.runtime = (void *)(unsigned long)systab32->runtime; 374 efi_systab.boottime = (void *)(unsigned long)systab32->boottime; 375 efi_systab.nr_tables = systab32->nr_tables; 376 efi_systab.tables = systab32->tables; 377 378 early_memunmap(systab32, sizeof(*systab32)); 379 } 380 381 efi.systab = &efi_systab; 382 383 /* 384 * Verify the EFI Table 385 */ 386 if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) { 387 pr_err("System table signature incorrect!\n"); 388 return -EINVAL; 389 } 390 if ((efi.systab->hdr.revision >> 16) == 0) 391 pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n", 392 efi.systab->hdr.revision >> 16, 393 efi.systab->hdr.revision & 0xffff); 394 395 return 0; 396 } 397 398 static int __init efi_runtime_init32(void) 399 { 400 efi_runtime_services_32_t *runtime; 401 402 runtime = early_memremap((unsigned long)efi.systab->runtime, 403 sizeof(efi_runtime_services_32_t)); 404 if (!runtime) { 405 pr_err("Could not map the runtime service table!\n"); 406 return -ENOMEM; 407 } 408 409 /* 410 * We will only need *early* access to the SetVirtualAddressMap 411 * EFI runtime service. All other runtime services will be called 412 * via the virtual mapping. 413 */ 414 efi_phys.set_virtual_address_map = 415 (efi_set_virtual_address_map_t *) 416 (unsigned long)runtime->set_virtual_address_map; 417 early_memunmap(runtime, sizeof(efi_runtime_services_32_t)); 418 419 return 0; 420 } 421 422 static int __init efi_runtime_init64(void) 423 { 424 efi_runtime_services_64_t *runtime; 425 426 runtime = early_memremap((unsigned long)efi.systab->runtime, 427 sizeof(efi_runtime_services_64_t)); 428 if (!runtime) { 429 pr_err("Could not map the runtime service table!\n"); 430 return -ENOMEM; 431 } 432 433 /* 434 * We will only need *early* access to the SetVirtualAddressMap 435 * EFI runtime service. All other runtime services will be called 436 * via the virtual mapping. 437 */ 438 efi_phys.set_virtual_address_map = 439 (efi_set_virtual_address_map_t *) 440 (unsigned long)runtime->set_virtual_address_map; 441 early_memunmap(runtime, sizeof(efi_runtime_services_64_t)); 442 443 return 0; 444 } 445 446 static int __init efi_runtime_init(void) 447 { 448 int rv; 449 450 /* 451 * Check out the runtime services table. We need to map 452 * the runtime services table so that we can grab the physical 453 * address of several of the EFI runtime functions, needed to 454 * set the firmware into virtual mode. 455 * 456 * When EFI_PARAVIRT is in force then we could not map runtime 457 * service memory region because we do not have direct access to it. 458 * However, runtime services are available through proxy functions 459 * (e.g. in case of Xen dom0 EFI implementation they call special 460 * hypercall which executes relevant EFI functions) and that is why 461 * they are always enabled. 462 */ 463 464 if (!efi_enabled(EFI_PARAVIRT)) { 465 if (efi_enabled(EFI_64BIT)) 466 rv = efi_runtime_init64(); 467 else 468 rv = efi_runtime_init32(); 469 470 if (rv) 471 return rv; 472 } 473 474 set_bit(EFI_RUNTIME_SERVICES, &efi.flags); 475 476 return 0; 477 } 478 479 void __init efi_init(void) 480 { 481 efi_char16_t *c16; 482 char vendor[100] = "unknown"; 483 int i = 0; 484 void *tmp; 485 486 #ifdef CONFIG_X86_32 487 if (boot_params.efi_info.efi_systab_hi || 488 boot_params.efi_info.efi_memmap_hi) { 489 pr_info("Table located above 4GB, disabling EFI.\n"); 490 return; 491 } 492 efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab; 493 #else 494 efi_phys.systab = (efi_system_table_t *) 495 (boot_params.efi_info.efi_systab | 496 ((__u64)boot_params.efi_info.efi_systab_hi<<32)); 497 #endif 498 499 if (efi_systab_init(efi_phys.systab)) 500 return; 501 502 efi.config_table = (unsigned long)efi.systab->tables; 503 efi.fw_vendor = (unsigned long)efi.systab->fw_vendor; 504 efi.runtime = (unsigned long)efi.systab->runtime; 505 506 /* 507 * Show what we know for posterity 508 */ 509 c16 = tmp = early_memremap(efi.systab->fw_vendor, 2); 510 if (c16) { 511 for (i = 0; i < sizeof(vendor) - 1 && *c16; ++i) 512 vendor[i] = *c16++; 513 vendor[i] = '\0'; 514 } else 515 pr_err("Could not map the firmware vendor!\n"); 516 early_memunmap(tmp, 2); 517 518 pr_info("EFI v%u.%.02u by %s\n", 519 efi.systab->hdr.revision >> 16, 520 efi.systab->hdr.revision & 0xffff, vendor); 521 522 if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables)) 523 return; 524 525 if (efi_config_init(arch_tables)) 526 return; 527 528 /* 529 * Note: We currently don't support runtime services on an EFI 530 * that doesn't match the kernel 32/64-bit mode. 531 */ 532 533 if (!efi_runtime_supported()) 534 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n"); 535 else { 536 if (efi_runtime_disabled() || efi_runtime_init()) { 537 efi_memmap_unmap(); 538 return; 539 } 540 } 541 542 efi_clean_memmap(); 543 544 if (efi_enabled(EFI_DBG)) 545 efi_print_memmap(); 546 } 547 548 void __init efi_set_executable(efi_memory_desc_t *md, bool executable) 549 { 550 u64 addr, npages; 551 552 addr = md->virt_addr; 553 npages = md->num_pages; 554 555 memrange_efi_to_native(&addr, &npages); 556 557 if (executable) 558 set_memory_x(addr, npages); 559 else 560 set_memory_nx(addr, npages); 561 } 562 563 void __init runtime_code_page_mkexec(void) 564 { 565 efi_memory_desc_t *md; 566 567 /* Make EFI runtime service code area executable */ 568 for_each_efi_memory_desc(md) { 569 if (md->type != EFI_RUNTIME_SERVICES_CODE) 570 continue; 571 572 efi_set_executable(md, true); 573 } 574 } 575 576 void __init efi_memory_uc(u64 addr, unsigned long size) 577 { 578 unsigned long page_shift = 1UL << EFI_PAGE_SHIFT; 579 u64 npages; 580 581 npages = round_up(size, page_shift) / page_shift; 582 memrange_efi_to_native(&addr, &npages); 583 set_memory_uc(addr, npages); 584 } 585 586 void __init old_map_region(efi_memory_desc_t *md) 587 { 588 u64 start_pfn, end_pfn, end; 589 unsigned long size; 590 void *va; 591 592 start_pfn = PFN_DOWN(md->phys_addr); 593 size = md->num_pages << PAGE_SHIFT; 594 end = md->phys_addr + size; 595 end_pfn = PFN_UP(end); 596 597 if (pfn_range_is_mapped(start_pfn, end_pfn)) { 598 va = __va(md->phys_addr); 599 600 if (!(md->attribute & EFI_MEMORY_WB)) 601 efi_memory_uc((u64)(unsigned long)va, size); 602 } else 603 va = efi_ioremap(md->phys_addr, size, 604 md->type, md->attribute); 605 606 md->virt_addr = (u64) (unsigned long) va; 607 if (!va) 608 pr_err("ioremap of 0x%llX failed!\n", 609 (unsigned long long)md->phys_addr); 610 } 611 612 /* Merge contiguous regions of the same type and attribute */ 613 static void __init efi_merge_regions(void) 614 { 615 efi_memory_desc_t *md, *prev_md = NULL; 616 617 for_each_efi_memory_desc(md) { 618 u64 prev_size; 619 620 if (!prev_md) { 621 prev_md = md; 622 continue; 623 } 624 625 if (prev_md->type != md->type || 626 prev_md->attribute != md->attribute) { 627 prev_md = md; 628 continue; 629 } 630 631 prev_size = prev_md->num_pages << EFI_PAGE_SHIFT; 632 633 if (md->phys_addr == (prev_md->phys_addr + prev_size)) { 634 prev_md->num_pages += md->num_pages; 635 md->type = EFI_RESERVED_TYPE; 636 md->attribute = 0; 637 continue; 638 } 639 prev_md = md; 640 } 641 } 642 643 static void __init get_systab_virt_addr(efi_memory_desc_t *md) 644 { 645 unsigned long size; 646 u64 end, systab; 647 648 size = md->num_pages << EFI_PAGE_SHIFT; 649 end = md->phys_addr + size; 650 systab = (u64)(unsigned long)efi_phys.systab; 651 if (md->phys_addr <= systab && systab < end) { 652 systab += md->virt_addr - md->phys_addr; 653 efi.systab = (efi_system_table_t *)(unsigned long)systab; 654 } 655 } 656 657 static void *realloc_pages(void *old_memmap, int old_shift) 658 { 659 void *ret; 660 661 ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1); 662 if (!ret) 663 goto out; 664 665 /* 666 * A first-time allocation doesn't have anything to copy. 667 */ 668 if (!old_memmap) 669 return ret; 670 671 memcpy(ret, old_memmap, PAGE_SIZE << old_shift); 672 673 out: 674 free_pages((unsigned long)old_memmap, old_shift); 675 return ret; 676 } 677 678 /* 679 * Iterate the EFI memory map in reverse order because the regions 680 * will be mapped top-down. The end result is the same as if we had 681 * mapped things forward, but doesn't require us to change the 682 * existing implementation of efi_map_region(). 683 */ 684 static inline void *efi_map_next_entry_reverse(void *entry) 685 { 686 /* Initial call */ 687 if (!entry) 688 return efi.memmap.map_end - efi.memmap.desc_size; 689 690 entry -= efi.memmap.desc_size; 691 if (entry < efi.memmap.map) 692 return NULL; 693 694 return entry; 695 } 696 697 /* 698 * efi_map_next_entry - Return the next EFI memory map descriptor 699 * @entry: Previous EFI memory map descriptor 700 * 701 * This is a helper function to iterate over the EFI memory map, which 702 * we do in different orders depending on the current configuration. 703 * 704 * To begin traversing the memory map @entry must be %NULL. 705 * 706 * Returns %NULL when we reach the end of the memory map. 707 */ 708 static void *efi_map_next_entry(void *entry) 709 { 710 if (!efi_enabled(EFI_OLD_MEMMAP) && efi_enabled(EFI_64BIT)) { 711 /* 712 * Starting in UEFI v2.5 the EFI_PROPERTIES_TABLE 713 * config table feature requires us to map all entries 714 * in the same order as they appear in the EFI memory 715 * map. That is to say, entry N must have a lower 716 * virtual address than entry N+1. This is because the 717 * firmware toolchain leaves relative references in 718 * the code/data sections, which are split and become 719 * separate EFI memory regions. Mapping things 720 * out-of-order leads to the firmware accessing 721 * unmapped addresses. 722 * 723 * Since we need to map things this way whether or not 724 * the kernel actually makes use of 725 * EFI_PROPERTIES_TABLE, let's just switch to this 726 * scheme by default for 64-bit. 727 */ 728 return efi_map_next_entry_reverse(entry); 729 } 730 731 /* Initial call */ 732 if (!entry) 733 return efi.memmap.map; 734 735 entry += efi.memmap.desc_size; 736 if (entry >= efi.memmap.map_end) 737 return NULL; 738 739 return entry; 740 } 741 742 static bool should_map_region(efi_memory_desc_t *md) 743 { 744 /* 745 * Runtime regions always require runtime mappings (obviously). 746 */ 747 if (md->attribute & EFI_MEMORY_RUNTIME) 748 return true; 749 750 /* 751 * 32-bit EFI doesn't suffer from the bug that requires us to 752 * reserve boot services regions, and mixed mode support 753 * doesn't exist for 32-bit kernels. 754 */ 755 if (IS_ENABLED(CONFIG_X86_32)) 756 return false; 757 758 /* 759 * Map all of RAM so that we can access arguments in the 1:1 760 * mapping when making EFI runtime calls. 761 */ 762 if (IS_ENABLED(CONFIG_EFI_MIXED) && !efi_is_native()) { 763 if (md->type == EFI_CONVENTIONAL_MEMORY || 764 md->type == EFI_LOADER_DATA || 765 md->type == EFI_LOADER_CODE) 766 return true; 767 } 768 769 /* 770 * Map boot services regions as a workaround for buggy 771 * firmware that accesses them even when they shouldn't. 772 * 773 * See efi_{reserve,free}_boot_services(). 774 */ 775 if (md->type == EFI_BOOT_SERVICES_CODE || 776 md->type == EFI_BOOT_SERVICES_DATA) 777 return true; 778 779 return false; 780 } 781 782 /* 783 * Map the efi memory ranges of the runtime services and update new_mmap with 784 * virtual addresses. 785 */ 786 static void * __init efi_map_regions(int *count, int *pg_shift) 787 { 788 void *p, *new_memmap = NULL; 789 unsigned long left = 0; 790 unsigned long desc_size; 791 efi_memory_desc_t *md; 792 793 desc_size = efi.memmap.desc_size; 794 795 p = NULL; 796 while ((p = efi_map_next_entry(p))) { 797 md = p; 798 799 if (!should_map_region(md)) 800 continue; 801 802 efi_map_region(md); 803 get_systab_virt_addr(md); 804 805 if (left < desc_size) { 806 new_memmap = realloc_pages(new_memmap, *pg_shift); 807 if (!new_memmap) 808 return NULL; 809 810 left += PAGE_SIZE << *pg_shift; 811 (*pg_shift)++; 812 } 813 814 memcpy(new_memmap + (*count * desc_size), md, desc_size); 815 816 left -= desc_size; 817 (*count)++; 818 } 819 820 return new_memmap; 821 } 822 823 static void __init kexec_enter_virtual_mode(void) 824 { 825 #ifdef CONFIG_KEXEC_CORE 826 efi_memory_desc_t *md; 827 unsigned int num_pages; 828 829 efi.systab = NULL; 830 831 /* 832 * We don't do virtual mode, since we don't do runtime services, on 833 * non-native EFI. With efi=old_map, we don't do runtime services in 834 * kexec kernel because in the initial boot something else might 835 * have been mapped at these virtual addresses. 836 */ 837 if (!efi_is_native() || efi_enabled(EFI_OLD_MEMMAP)) { 838 efi_memmap_unmap(); 839 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 840 return; 841 } 842 843 if (efi_alloc_page_tables()) { 844 pr_err("Failed to allocate EFI page tables\n"); 845 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 846 return; 847 } 848 849 /* 850 * Map efi regions which were passed via setup_data. The virt_addr is a 851 * fixed addr which was used in first kernel of a kexec boot. 852 */ 853 for_each_efi_memory_desc(md) { 854 efi_map_region_fixed(md); /* FIXME: add error handling */ 855 get_systab_virt_addr(md); 856 } 857 858 /* 859 * Unregister the early EFI memmap from efi_init() and install 860 * the new EFI memory map. 861 */ 862 efi_memmap_unmap(); 863 864 if (efi_memmap_init_late(efi.memmap.phys_map, 865 efi.memmap.desc_size * efi.memmap.nr_map)) { 866 pr_err("Failed to remap late EFI memory map\n"); 867 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 868 return; 869 } 870 871 BUG_ON(!efi.systab); 872 873 num_pages = ALIGN(efi.memmap.nr_map * efi.memmap.desc_size, PAGE_SIZE); 874 num_pages >>= PAGE_SHIFT; 875 876 if (efi_setup_page_tables(efi.memmap.phys_map, num_pages)) { 877 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 878 return; 879 } 880 881 efi_sync_low_kernel_mappings(); 882 883 /* 884 * Now that EFI is in virtual mode, update the function 885 * pointers in the runtime service table to the new virtual addresses. 886 * 887 * Call EFI services through wrapper functions. 888 */ 889 efi.runtime_version = efi_systab.hdr.revision; 890 891 efi_native_runtime_setup(); 892 893 efi.set_virtual_address_map = NULL; 894 895 if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX)) 896 runtime_code_page_mkexec(); 897 898 /* clean DUMMY object */ 899 efi_delete_dummy_variable(); 900 #endif 901 } 902 903 /* 904 * This function will switch the EFI runtime services to virtual mode. 905 * Essentially, we look through the EFI memmap and map every region that 906 * has the runtime attribute bit set in its memory descriptor into the 907 * efi_pgd page table. 908 * 909 * The old method which used to update that memory descriptor with the 910 * virtual address obtained from ioremap() is still supported when the 911 * kernel is booted with efi=old_map on its command line. Same old 912 * method enabled the runtime services to be called without having to 913 * thunk back into physical mode for every invocation. 914 * 915 * The new method does a pagetable switch in a preemption-safe manner 916 * so that we're in a different address space when calling a runtime 917 * function. For function arguments passing we do copy the PUDs of the 918 * kernel page table into efi_pgd prior to each call. 919 * 920 * Specially for kexec boot, efi runtime maps in previous kernel should 921 * be passed in via setup_data. In that case runtime ranges will be mapped 922 * to the same virtual addresses as the first kernel, see 923 * kexec_enter_virtual_mode(). 924 */ 925 static void __init __efi_enter_virtual_mode(void) 926 { 927 int count = 0, pg_shift = 0; 928 void *new_memmap = NULL; 929 efi_status_t status; 930 unsigned long pa; 931 932 efi.systab = NULL; 933 934 if (efi_alloc_page_tables()) { 935 pr_err("Failed to allocate EFI page tables\n"); 936 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 937 return; 938 } 939 940 efi_merge_regions(); 941 new_memmap = efi_map_regions(&count, &pg_shift); 942 if (!new_memmap) { 943 pr_err("Error reallocating memory, EFI runtime non-functional!\n"); 944 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 945 return; 946 } 947 948 pa = __pa(new_memmap); 949 950 /* 951 * Unregister the early EFI memmap from efi_init() and install 952 * the new EFI memory map that we are about to pass to the 953 * firmware via SetVirtualAddressMap(). 954 */ 955 efi_memmap_unmap(); 956 957 if (efi_memmap_init_late(pa, efi.memmap.desc_size * count)) { 958 pr_err("Failed to remap late EFI memory map\n"); 959 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 960 return; 961 } 962 963 if (efi_enabled(EFI_DBG)) { 964 pr_info("EFI runtime memory map:\n"); 965 efi_print_memmap(); 966 } 967 968 BUG_ON(!efi.systab); 969 970 if (efi_setup_page_tables(pa, 1 << pg_shift)) { 971 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 972 return; 973 } 974 975 efi_sync_low_kernel_mappings(); 976 977 if (efi_is_native()) { 978 status = phys_efi_set_virtual_address_map( 979 efi.memmap.desc_size * count, 980 efi.memmap.desc_size, 981 efi.memmap.desc_version, 982 (efi_memory_desc_t *)pa); 983 } else { 984 status = efi_thunk_set_virtual_address_map( 985 efi_phys.set_virtual_address_map, 986 efi.memmap.desc_size * count, 987 efi.memmap.desc_size, 988 efi.memmap.desc_version, 989 (efi_memory_desc_t *)pa); 990 } 991 992 if (status != EFI_SUCCESS) { 993 pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n", 994 status); 995 panic("EFI call to SetVirtualAddressMap() failed!"); 996 } 997 998 efi_free_boot_services(); 999 1000 /* 1001 * Now that EFI is in virtual mode, update the function 1002 * pointers in the runtime service table to the new virtual addresses. 1003 * 1004 * Call EFI services through wrapper functions. 1005 */ 1006 efi.runtime_version = efi_systab.hdr.revision; 1007 1008 if (efi_is_native()) 1009 efi_native_runtime_setup(); 1010 else 1011 efi_thunk_runtime_setup(); 1012 1013 efi.set_virtual_address_map = NULL; 1014 1015 /* 1016 * Apply more restrictive page table mapping attributes now that 1017 * SVAM() has been called and the firmware has performed all 1018 * necessary relocation fixups for the new virtual addresses. 1019 */ 1020 efi_runtime_update_mappings(); 1021 1022 /* clean DUMMY object */ 1023 efi_delete_dummy_variable(); 1024 } 1025 1026 void __init efi_enter_virtual_mode(void) 1027 { 1028 if (efi_enabled(EFI_PARAVIRT)) 1029 return; 1030 1031 if (efi_setup) 1032 kexec_enter_virtual_mode(); 1033 else 1034 __efi_enter_virtual_mode(); 1035 1036 efi_dump_pagetable(); 1037 } 1038 1039 static int __init arch_parse_efi_cmdline(char *str) 1040 { 1041 if (!str) { 1042 pr_warn("need at least one option\n"); 1043 return -EINVAL; 1044 } 1045 1046 if (parse_option_str(str, "old_map")) 1047 set_bit(EFI_OLD_MEMMAP, &efi.flags); 1048 1049 return 0; 1050 } 1051 early_param("efi", arch_parse_efi_cmdline); 1052