1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Common EFI (Extensible Firmware Interface) support functions 4 * Based on Extensible Firmware Interface Specification version 1.0 5 * 6 * Copyright (C) 1999 VA Linux Systems 7 * Copyright (C) 1999 Walt Drummond <drummond@valinux.com> 8 * Copyright (C) 1999-2002 Hewlett-Packard Co. 9 * David Mosberger-Tang <davidm@hpl.hp.com> 10 * Stephane Eranian <eranian@hpl.hp.com> 11 * Copyright (C) 2005-2008 Intel Co. 12 * Fenghua Yu <fenghua.yu@intel.com> 13 * Bibo Mao <bibo.mao@intel.com> 14 * Chandramouli Narayanan <mouli@linux.intel.com> 15 * Huang Ying <ying.huang@intel.com> 16 * Copyright (C) 2013 SuSE Labs 17 * Borislav Petkov <bp@suse.de> - runtime services VA mapping 18 * 19 * Copied from efi_32.c to eliminate the duplicated code between EFI 20 * 32/64 support code. --ying 2007-10-26 21 * 22 * All EFI Runtime Services are not implemented yet as EFI only 23 * supports physical mode addressing on SoftSDV. This is to be fixed 24 * in a future version. --drummond 1999-07-20 25 * 26 * Implemented EFI runtime services and virtual mode calls. --davidm 27 * 28 * Goutham Rao: <goutham.rao@intel.com> 29 * Skip non-WB memory and ignore empty memory ranges. 30 */ 31 32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 33 34 #include <linux/kernel.h> 35 #include <linux/init.h> 36 #include <linux/efi.h> 37 #include <linux/efi-bgrt.h> 38 #include <linux/export.h> 39 #include <linux/memblock.h> 40 #include <linux/slab.h> 41 #include <linux/spinlock.h> 42 #include <linux/uaccess.h> 43 #include <linux/time.h> 44 #include <linux/io.h> 45 #include <linux/reboot.h> 46 #include <linux/bcd.h> 47 48 #include <asm/setup.h> 49 #include <asm/efi.h> 50 #include <asm/e820/api.h> 51 #include <asm/time.h> 52 #include <asm/set_memory.h> 53 #include <asm/tlbflush.h> 54 #include <asm/x86_init.h> 55 #include <asm/uv/uv.h> 56 57 static struct efi efi_phys __initdata; 58 static efi_system_table_t efi_systab __initdata; 59 60 static efi_config_table_type_t arch_tables[] __initdata = { 61 #ifdef CONFIG_X86_UV 62 {UV_SYSTEM_TABLE_GUID, "UVsystab", &uv_systab_phys}, 63 #endif 64 {NULL_GUID, NULL, NULL}, 65 }; 66 67 static const unsigned long * const efi_tables[] = { 68 &efi.mps, 69 &efi.acpi, 70 &efi.acpi20, 71 &efi.smbios, 72 &efi.smbios3, 73 &efi.boot_info, 74 &efi.hcdp, 75 &efi.uga, 76 #ifdef CONFIG_X86_UV 77 &uv_systab_phys, 78 #endif 79 &efi.fw_vendor, 80 &efi.runtime, 81 &efi.config_table, 82 &efi.esrt, 83 &efi.properties_table, 84 &efi.mem_attr_table, 85 #ifdef CONFIG_EFI_RCI2_TABLE 86 &rci2_table_phys, 87 #endif 88 }; 89 90 u64 efi_setup; /* efi setup_data physical address */ 91 92 static int add_efi_memmap __initdata; 93 static int __init setup_add_efi_memmap(char *arg) 94 { 95 add_efi_memmap = 1; 96 return 0; 97 } 98 early_param("add_efi_memmap", setup_add_efi_memmap); 99 100 static efi_status_t __init phys_efi_set_virtual_address_map( 101 unsigned long memory_map_size, 102 unsigned long descriptor_size, 103 u32 descriptor_version, 104 efi_memory_desc_t *virtual_map) 105 { 106 efi_status_t status; 107 unsigned long flags; 108 pgd_t *save_pgd; 109 110 save_pgd = efi_call_phys_prolog(); 111 if (!save_pgd) 112 return EFI_ABORTED; 113 114 /* Disable interrupts around EFI calls: */ 115 local_irq_save(flags); 116 status = efi_call_phys(efi_phys.set_virtual_address_map, 117 memory_map_size, descriptor_size, 118 descriptor_version, virtual_map); 119 local_irq_restore(flags); 120 121 efi_call_phys_epilog(save_pgd); 122 123 return status; 124 } 125 126 void __init efi_find_mirror(void) 127 { 128 efi_memory_desc_t *md; 129 u64 mirror_size = 0, total_size = 0; 130 131 for_each_efi_memory_desc(md) { 132 unsigned long long start = md->phys_addr; 133 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT; 134 135 total_size += size; 136 if (md->attribute & EFI_MEMORY_MORE_RELIABLE) { 137 memblock_mark_mirror(start, size); 138 mirror_size += size; 139 } 140 } 141 if (mirror_size) 142 pr_info("Memory: %lldM/%lldM mirrored memory\n", 143 mirror_size>>20, total_size>>20); 144 } 145 146 /* 147 * Tell the kernel about the EFI memory map. This might include 148 * more than the max 128 entries that can fit in the e820 legacy 149 * (zeropage) memory map. 150 */ 151 152 static void __init do_add_efi_memmap(void) 153 { 154 efi_memory_desc_t *md; 155 156 for_each_efi_memory_desc(md) { 157 unsigned long long start = md->phys_addr; 158 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT; 159 int e820_type; 160 161 switch (md->type) { 162 case EFI_LOADER_CODE: 163 case EFI_LOADER_DATA: 164 case EFI_BOOT_SERVICES_CODE: 165 case EFI_BOOT_SERVICES_DATA: 166 case EFI_CONVENTIONAL_MEMORY: 167 if (md->attribute & EFI_MEMORY_WB) 168 e820_type = E820_TYPE_RAM; 169 else 170 e820_type = E820_TYPE_RESERVED; 171 break; 172 case EFI_ACPI_RECLAIM_MEMORY: 173 e820_type = E820_TYPE_ACPI; 174 break; 175 case EFI_ACPI_MEMORY_NVS: 176 e820_type = E820_TYPE_NVS; 177 break; 178 case EFI_UNUSABLE_MEMORY: 179 e820_type = E820_TYPE_UNUSABLE; 180 break; 181 case EFI_PERSISTENT_MEMORY: 182 e820_type = E820_TYPE_PMEM; 183 break; 184 default: 185 /* 186 * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE 187 * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO 188 * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE 189 */ 190 e820_type = E820_TYPE_RESERVED; 191 break; 192 } 193 e820__range_add(start, size, e820_type); 194 } 195 e820__update_table(e820_table); 196 } 197 198 int __init efi_memblock_x86_reserve_range(void) 199 { 200 struct efi_info *e = &boot_params.efi_info; 201 struct efi_memory_map_data data; 202 phys_addr_t pmap; 203 int rv; 204 205 if (efi_enabled(EFI_PARAVIRT)) 206 return 0; 207 208 #ifdef CONFIG_X86_32 209 /* Can't handle data above 4GB at this time */ 210 if (e->efi_memmap_hi) { 211 pr_err("Memory map is above 4GB, disabling EFI.\n"); 212 return -EINVAL; 213 } 214 pmap = e->efi_memmap; 215 #else 216 pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32)); 217 #endif 218 data.phys_map = pmap; 219 data.size = e->efi_memmap_size; 220 data.desc_size = e->efi_memdesc_size; 221 data.desc_version = e->efi_memdesc_version; 222 223 rv = efi_memmap_init_early(&data); 224 if (rv) 225 return rv; 226 227 if (add_efi_memmap) 228 do_add_efi_memmap(); 229 230 WARN(efi.memmap.desc_version != 1, 231 "Unexpected EFI_MEMORY_DESCRIPTOR version %ld", 232 efi.memmap.desc_version); 233 234 memblock_reserve(pmap, efi.memmap.nr_map * efi.memmap.desc_size); 235 236 return 0; 237 } 238 239 #define OVERFLOW_ADDR_SHIFT (64 - EFI_PAGE_SHIFT) 240 #define OVERFLOW_ADDR_MASK (U64_MAX << OVERFLOW_ADDR_SHIFT) 241 #define U64_HIGH_BIT (~(U64_MAX >> 1)) 242 243 static bool __init efi_memmap_entry_valid(const efi_memory_desc_t *md, int i) 244 { 245 u64 end = (md->num_pages << EFI_PAGE_SHIFT) + md->phys_addr - 1; 246 u64 end_hi = 0; 247 char buf[64]; 248 249 if (md->num_pages == 0) { 250 end = 0; 251 } else if (md->num_pages > EFI_PAGES_MAX || 252 EFI_PAGES_MAX - md->num_pages < 253 (md->phys_addr >> EFI_PAGE_SHIFT)) { 254 end_hi = (md->num_pages & OVERFLOW_ADDR_MASK) 255 >> OVERFLOW_ADDR_SHIFT; 256 257 if ((md->phys_addr & U64_HIGH_BIT) && !(end & U64_HIGH_BIT)) 258 end_hi += 1; 259 } else { 260 return true; 261 } 262 263 pr_warn_once(FW_BUG "Invalid EFI memory map entries:\n"); 264 265 if (end_hi) { 266 pr_warn("mem%02u: %s range=[0x%016llx-0x%llx%016llx] (invalid)\n", 267 i, efi_md_typeattr_format(buf, sizeof(buf), md), 268 md->phys_addr, end_hi, end); 269 } else { 270 pr_warn("mem%02u: %s range=[0x%016llx-0x%016llx] (invalid)\n", 271 i, efi_md_typeattr_format(buf, sizeof(buf), md), 272 md->phys_addr, end); 273 } 274 return false; 275 } 276 277 static void __init efi_clean_memmap(void) 278 { 279 efi_memory_desc_t *out = efi.memmap.map; 280 const efi_memory_desc_t *in = out; 281 const efi_memory_desc_t *end = efi.memmap.map_end; 282 int i, n_removal; 283 284 for (i = n_removal = 0; in < end; i++) { 285 if (efi_memmap_entry_valid(in, i)) { 286 if (out != in) 287 memcpy(out, in, efi.memmap.desc_size); 288 out = (void *)out + efi.memmap.desc_size; 289 } else { 290 n_removal++; 291 } 292 in = (void *)in + efi.memmap.desc_size; 293 } 294 295 if (n_removal > 0) { 296 u64 size = efi.memmap.nr_map - n_removal; 297 298 pr_warn("Removing %d invalid memory map entries.\n", n_removal); 299 efi_memmap_install(efi.memmap.phys_map, size); 300 } 301 } 302 303 void __init efi_print_memmap(void) 304 { 305 efi_memory_desc_t *md; 306 int i = 0; 307 308 for_each_efi_memory_desc(md) { 309 char buf[64]; 310 311 pr_info("mem%02u: %s range=[0x%016llx-0x%016llx] (%lluMB)\n", 312 i++, efi_md_typeattr_format(buf, sizeof(buf), md), 313 md->phys_addr, 314 md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT) - 1, 315 (md->num_pages >> (20 - EFI_PAGE_SHIFT))); 316 } 317 } 318 319 static int __init efi_systab_init(void *phys) 320 { 321 if (efi_enabled(EFI_64BIT)) { 322 efi_system_table_64_t *systab64; 323 struct efi_setup_data *data = NULL; 324 u64 tmp = 0; 325 326 if (efi_setup) { 327 data = early_memremap(efi_setup, sizeof(*data)); 328 if (!data) 329 return -ENOMEM; 330 } 331 systab64 = early_memremap((unsigned long)phys, 332 sizeof(*systab64)); 333 if (systab64 == NULL) { 334 pr_err("Couldn't map the system table!\n"); 335 if (data) 336 early_memunmap(data, sizeof(*data)); 337 return -ENOMEM; 338 } 339 340 efi_systab.hdr = systab64->hdr; 341 efi_systab.fw_vendor = data ? (unsigned long)data->fw_vendor : 342 systab64->fw_vendor; 343 tmp |= data ? data->fw_vendor : systab64->fw_vendor; 344 efi_systab.fw_revision = systab64->fw_revision; 345 efi_systab.con_in_handle = systab64->con_in_handle; 346 tmp |= systab64->con_in_handle; 347 efi_systab.con_in = systab64->con_in; 348 tmp |= systab64->con_in; 349 efi_systab.con_out_handle = systab64->con_out_handle; 350 tmp |= systab64->con_out_handle; 351 efi_systab.con_out = systab64->con_out; 352 tmp |= systab64->con_out; 353 efi_systab.stderr_handle = systab64->stderr_handle; 354 tmp |= systab64->stderr_handle; 355 efi_systab.stderr = systab64->stderr; 356 tmp |= systab64->stderr; 357 efi_systab.runtime = data ? 358 (void *)(unsigned long)data->runtime : 359 (void *)(unsigned long)systab64->runtime; 360 tmp |= data ? data->runtime : systab64->runtime; 361 efi_systab.boottime = (void *)(unsigned long)systab64->boottime; 362 tmp |= systab64->boottime; 363 efi_systab.nr_tables = systab64->nr_tables; 364 efi_systab.tables = data ? (unsigned long)data->tables : 365 systab64->tables; 366 tmp |= data ? data->tables : systab64->tables; 367 368 early_memunmap(systab64, sizeof(*systab64)); 369 if (data) 370 early_memunmap(data, sizeof(*data)); 371 #ifdef CONFIG_X86_32 372 if (tmp >> 32) { 373 pr_err("EFI data located above 4GB, disabling EFI.\n"); 374 return -EINVAL; 375 } 376 #endif 377 } else { 378 efi_system_table_32_t *systab32; 379 380 systab32 = early_memremap((unsigned long)phys, 381 sizeof(*systab32)); 382 if (systab32 == NULL) { 383 pr_err("Couldn't map the system table!\n"); 384 return -ENOMEM; 385 } 386 387 efi_systab.hdr = systab32->hdr; 388 efi_systab.fw_vendor = systab32->fw_vendor; 389 efi_systab.fw_revision = systab32->fw_revision; 390 efi_systab.con_in_handle = systab32->con_in_handle; 391 efi_systab.con_in = systab32->con_in; 392 efi_systab.con_out_handle = systab32->con_out_handle; 393 efi_systab.con_out = systab32->con_out; 394 efi_systab.stderr_handle = systab32->stderr_handle; 395 efi_systab.stderr = systab32->stderr; 396 efi_systab.runtime = (void *)(unsigned long)systab32->runtime; 397 efi_systab.boottime = (void *)(unsigned long)systab32->boottime; 398 efi_systab.nr_tables = systab32->nr_tables; 399 efi_systab.tables = systab32->tables; 400 401 early_memunmap(systab32, sizeof(*systab32)); 402 } 403 404 efi.systab = &efi_systab; 405 406 /* 407 * Verify the EFI Table 408 */ 409 if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) { 410 pr_err("System table signature incorrect!\n"); 411 return -EINVAL; 412 } 413 if ((efi.systab->hdr.revision >> 16) == 0) 414 pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n", 415 efi.systab->hdr.revision >> 16, 416 efi.systab->hdr.revision & 0xffff); 417 418 return 0; 419 } 420 421 static int __init efi_runtime_init32(void) 422 { 423 efi_runtime_services_32_t *runtime; 424 425 runtime = early_memremap((unsigned long)efi.systab->runtime, 426 sizeof(efi_runtime_services_32_t)); 427 if (!runtime) { 428 pr_err("Could not map the runtime service table!\n"); 429 return -ENOMEM; 430 } 431 432 /* 433 * We will only need *early* access to the SetVirtualAddressMap 434 * EFI runtime service. All other runtime services will be called 435 * via the virtual mapping. 436 */ 437 efi_phys.set_virtual_address_map = 438 (efi_set_virtual_address_map_t *) 439 (unsigned long)runtime->set_virtual_address_map; 440 early_memunmap(runtime, sizeof(efi_runtime_services_32_t)); 441 442 return 0; 443 } 444 445 static int __init efi_runtime_init64(void) 446 { 447 efi_runtime_services_64_t *runtime; 448 449 runtime = early_memremap((unsigned long)efi.systab->runtime, 450 sizeof(efi_runtime_services_64_t)); 451 if (!runtime) { 452 pr_err("Could not map the runtime service table!\n"); 453 return -ENOMEM; 454 } 455 456 /* 457 * We will only need *early* access to the SetVirtualAddressMap 458 * EFI runtime service. All other runtime services will be called 459 * via the virtual mapping. 460 */ 461 efi_phys.set_virtual_address_map = 462 (efi_set_virtual_address_map_t *) 463 (unsigned long)runtime->set_virtual_address_map; 464 early_memunmap(runtime, sizeof(efi_runtime_services_64_t)); 465 466 return 0; 467 } 468 469 static int __init efi_runtime_init(void) 470 { 471 int rv; 472 473 /* 474 * Check out the runtime services table. We need to map 475 * the runtime services table so that we can grab the physical 476 * address of several of the EFI runtime functions, needed to 477 * set the firmware into virtual mode. 478 * 479 * When EFI_PARAVIRT is in force then we could not map runtime 480 * service memory region because we do not have direct access to it. 481 * However, runtime services are available through proxy functions 482 * (e.g. in case of Xen dom0 EFI implementation they call special 483 * hypercall which executes relevant EFI functions) and that is why 484 * they are always enabled. 485 */ 486 487 if (!efi_enabled(EFI_PARAVIRT)) { 488 if (efi_enabled(EFI_64BIT)) 489 rv = efi_runtime_init64(); 490 else 491 rv = efi_runtime_init32(); 492 493 if (rv) 494 return rv; 495 } 496 497 set_bit(EFI_RUNTIME_SERVICES, &efi.flags); 498 499 return 0; 500 } 501 502 void __init efi_init(void) 503 { 504 efi_char16_t *c16; 505 char vendor[100] = "unknown"; 506 int i = 0; 507 void *tmp; 508 509 #ifdef CONFIG_X86_32 510 if (boot_params.efi_info.efi_systab_hi || 511 boot_params.efi_info.efi_memmap_hi) { 512 pr_info("Table located above 4GB, disabling EFI.\n"); 513 return; 514 } 515 efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab; 516 #else 517 efi_phys.systab = (efi_system_table_t *) 518 (boot_params.efi_info.efi_systab | 519 ((__u64)boot_params.efi_info.efi_systab_hi<<32)); 520 #endif 521 522 if (efi_systab_init(efi_phys.systab)) 523 return; 524 525 efi.config_table = (unsigned long)efi.systab->tables; 526 efi.fw_vendor = (unsigned long)efi.systab->fw_vendor; 527 efi.runtime = (unsigned long)efi.systab->runtime; 528 529 /* 530 * Show what we know for posterity 531 */ 532 c16 = tmp = early_memremap(efi.systab->fw_vendor, 2); 533 if (c16) { 534 for (i = 0; i < sizeof(vendor) - 1 && *c16; ++i) 535 vendor[i] = *c16++; 536 vendor[i] = '\0'; 537 } else 538 pr_err("Could not map the firmware vendor!\n"); 539 early_memunmap(tmp, 2); 540 541 pr_info("EFI v%u.%.02u by %s\n", 542 efi.systab->hdr.revision >> 16, 543 efi.systab->hdr.revision & 0xffff, vendor); 544 545 if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables)) 546 return; 547 548 if (efi_config_init(arch_tables)) 549 return; 550 551 /* 552 * Note: We currently don't support runtime services on an EFI 553 * that doesn't match the kernel 32/64-bit mode. 554 */ 555 556 if (!efi_runtime_supported()) 557 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n"); 558 else { 559 if (efi_runtime_disabled() || efi_runtime_init()) { 560 efi_memmap_unmap(); 561 return; 562 } 563 } 564 565 efi_clean_memmap(); 566 567 if (efi_enabled(EFI_DBG)) 568 efi_print_memmap(); 569 } 570 571 void __init efi_set_executable(efi_memory_desc_t *md, bool executable) 572 { 573 u64 addr, npages; 574 575 addr = md->virt_addr; 576 npages = md->num_pages; 577 578 memrange_efi_to_native(&addr, &npages); 579 580 if (executable) 581 set_memory_x(addr, npages); 582 else 583 set_memory_nx(addr, npages); 584 } 585 586 void __init runtime_code_page_mkexec(void) 587 { 588 efi_memory_desc_t *md; 589 590 /* Make EFI runtime service code area executable */ 591 for_each_efi_memory_desc(md) { 592 if (md->type != EFI_RUNTIME_SERVICES_CODE) 593 continue; 594 595 efi_set_executable(md, true); 596 } 597 } 598 599 void __init efi_memory_uc(u64 addr, unsigned long size) 600 { 601 unsigned long page_shift = 1UL << EFI_PAGE_SHIFT; 602 u64 npages; 603 604 npages = round_up(size, page_shift) / page_shift; 605 memrange_efi_to_native(&addr, &npages); 606 set_memory_uc(addr, npages); 607 } 608 609 void __init old_map_region(efi_memory_desc_t *md) 610 { 611 u64 start_pfn, end_pfn, end; 612 unsigned long size; 613 void *va; 614 615 start_pfn = PFN_DOWN(md->phys_addr); 616 size = md->num_pages << PAGE_SHIFT; 617 end = md->phys_addr + size; 618 end_pfn = PFN_UP(end); 619 620 if (pfn_range_is_mapped(start_pfn, end_pfn)) { 621 va = __va(md->phys_addr); 622 623 if (!(md->attribute & EFI_MEMORY_WB)) 624 efi_memory_uc((u64)(unsigned long)va, size); 625 } else 626 va = efi_ioremap(md->phys_addr, size, 627 md->type, md->attribute); 628 629 md->virt_addr = (u64) (unsigned long) va; 630 if (!va) 631 pr_err("ioremap of 0x%llX failed!\n", 632 (unsigned long long)md->phys_addr); 633 } 634 635 /* Merge contiguous regions of the same type and attribute */ 636 static void __init efi_merge_regions(void) 637 { 638 efi_memory_desc_t *md, *prev_md = NULL; 639 640 for_each_efi_memory_desc(md) { 641 u64 prev_size; 642 643 if (!prev_md) { 644 prev_md = md; 645 continue; 646 } 647 648 if (prev_md->type != md->type || 649 prev_md->attribute != md->attribute) { 650 prev_md = md; 651 continue; 652 } 653 654 prev_size = prev_md->num_pages << EFI_PAGE_SHIFT; 655 656 if (md->phys_addr == (prev_md->phys_addr + prev_size)) { 657 prev_md->num_pages += md->num_pages; 658 md->type = EFI_RESERVED_TYPE; 659 md->attribute = 0; 660 continue; 661 } 662 prev_md = md; 663 } 664 } 665 666 static void __init get_systab_virt_addr(efi_memory_desc_t *md) 667 { 668 unsigned long size; 669 u64 end, systab; 670 671 size = md->num_pages << EFI_PAGE_SHIFT; 672 end = md->phys_addr + size; 673 systab = (u64)(unsigned long)efi_phys.systab; 674 if (md->phys_addr <= systab && systab < end) { 675 systab += md->virt_addr - md->phys_addr; 676 efi.systab = (efi_system_table_t *)(unsigned long)systab; 677 } 678 } 679 680 static void *realloc_pages(void *old_memmap, int old_shift) 681 { 682 void *ret; 683 684 ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1); 685 if (!ret) 686 goto out; 687 688 /* 689 * A first-time allocation doesn't have anything to copy. 690 */ 691 if (!old_memmap) 692 return ret; 693 694 memcpy(ret, old_memmap, PAGE_SIZE << old_shift); 695 696 out: 697 free_pages((unsigned long)old_memmap, old_shift); 698 return ret; 699 } 700 701 /* 702 * Iterate the EFI memory map in reverse order because the regions 703 * will be mapped top-down. The end result is the same as if we had 704 * mapped things forward, but doesn't require us to change the 705 * existing implementation of efi_map_region(). 706 */ 707 static inline void *efi_map_next_entry_reverse(void *entry) 708 { 709 /* Initial call */ 710 if (!entry) 711 return efi.memmap.map_end - efi.memmap.desc_size; 712 713 entry -= efi.memmap.desc_size; 714 if (entry < efi.memmap.map) 715 return NULL; 716 717 return entry; 718 } 719 720 /* 721 * efi_map_next_entry - Return the next EFI memory map descriptor 722 * @entry: Previous EFI memory map descriptor 723 * 724 * This is a helper function to iterate over the EFI memory map, which 725 * we do in different orders depending on the current configuration. 726 * 727 * To begin traversing the memory map @entry must be %NULL. 728 * 729 * Returns %NULL when we reach the end of the memory map. 730 */ 731 static void *efi_map_next_entry(void *entry) 732 { 733 if (!efi_enabled(EFI_OLD_MEMMAP) && efi_enabled(EFI_64BIT)) { 734 /* 735 * Starting in UEFI v2.5 the EFI_PROPERTIES_TABLE 736 * config table feature requires us to map all entries 737 * in the same order as they appear in the EFI memory 738 * map. That is to say, entry N must have a lower 739 * virtual address than entry N+1. This is because the 740 * firmware toolchain leaves relative references in 741 * the code/data sections, which are split and become 742 * separate EFI memory regions. Mapping things 743 * out-of-order leads to the firmware accessing 744 * unmapped addresses. 745 * 746 * Since we need to map things this way whether or not 747 * the kernel actually makes use of 748 * EFI_PROPERTIES_TABLE, let's just switch to this 749 * scheme by default for 64-bit. 750 */ 751 return efi_map_next_entry_reverse(entry); 752 } 753 754 /* Initial call */ 755 if (!entry) 756 return efi.memmap.map; 757 758 entry += efi.memmap.desc_size; 759 if (entry >= efi.memmap.map_end) 760 return NULL; 761 762 return entry; 763 } 764 765 static bool should_map_region(efi_memory_desc_t *md) 766 { 767 /* 768 * Runtime regions always require runtime mappings (obviously). 769 */ 770 if (md->attribute & EFI_MEMORY_RUNTIME) 771 return true; 772 773 /* 774 * 32-bit EFI doesn't suffer from the bug that requires us to 775 * reserve boot services regions, and mixed mode support 776 * doesn't exist for 32-bit kernels. 777 */ 778 if (IS_ENABLED(CONFIG_X86_32)) 779 return false; 780 781 /* 782 * Map all of RAM so that we can access arguments in the 1:1 783 * mapping when making EFI runtime calls. 784 */ 785 if (IS_ENABLED(CONFIG_EFI_MIXED) && !efi_is_native()) { 786 if (md->type == EFI_CONVENTIONAL_MEMORY || 787 md->type == EFI_LOADER_DATA || 788 md->type == EFI_LOADER_CODE) 789 return true; 790 } 791 792 /* 793 * Map boot services regions as a workaround for buggy 794 * firmware that accesses them even when they shouldn't. 795 * 796 * See efi_{reserve,free}_boot_services(). 797 */ 798 if (md->type == EFI_BOOT_SERVICES_CODE || 799 md->type == EFI_BOOT_SERVICES_DATA) 800 return true; 801 802 return false; 803 } 804 805 /* 806 * Map the efi memory ranges of the runtime services and update new_mmap with 807 * virtual addresses. 808 */ 809 static void * __init efi_map_regions(int *count, int *pg_shift) 810 { 811 void *p, *new_memmap = NULL; 812 unsigned long left = 0; 813 unsigned long desc_size; 814 efi_memory_desc_t *md; 815 816 desc_size = efi.memmap.desc_size; 817 818 p = NULL; 819 while ((p = efi_map_next_entry(p))) { 820 md = p; 821 822 if (!should_map_region(md)) 823 continue; 824 825 efi_map_region(md); 826 get_systab_virt_addr(md); 827 828 if (left < desc_size) { 829 new_memmap = realloc_pages(new_memmap, *pg_shift); 830 if (!new_memmap) 831 return NULL; 832 833 left += PAGE_SIZE << *pg_shift; 834 (*pg_shift)++; 835 } 836 837 memcpy(new_memmap + (*count * desc_size), md, desc_size); 838 839 left -= desc_size; 840 (*count)++; 841 } 842 843 return new_memmap; 844 } 845 846 static void __init kexec_enter_virtual_mode(void) 847 { 848 #ifdef CONFIG_KEXEC_CORE 849 efi_memory_desc_t *md; 850 unsigned int num_pages; 851 852 efi.systab = NULL; 853 854 /* 855 * We don't do virtual mode, since we don't do runtime services, on 856 * non-native EFI. With efi=old_map, we don't do runtime services in 857 * kexec kernel because in the initial boot something else might 858 * have been mapped at these virtual addresses. 859 */ 860 if (!efi_is_native() || efi_enabled(EFI_OLD_MEMMAP)) { 861 efi_memmap_unmap(); 862 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 863 return; 864 } 865 866 if (efi_alloc_page_tables()) { 867 pr_err("Failed to allocate EFI page tables\n"); 868 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 869 return; 870 } 871 872 /* 873 * Map efi regions which were passed via setup_data. The virt_addr is a 874 * fixed addr which was used in first kernel of a kexec boot. 875 */ 876 for_each_efi_memory_desc(md) { 877 efi_map_region_fixed(md); /* FIXME: add error handling */ 878 get_systab_virt_addr(md); 879 } 880 881 /* 882 * Unregister the early EFI memmap from efi_init() and install 883 * the new EFI memory map. 884 */ 885 efi_memmap_unmap(); 886 887 if (efi_memmap_init_late(efi.memmap.phys_map, 888 efi.memmap.desc_size * efi.memmap.nr_map)) { 889 pr_err("Failed to remap late EFI memory map\n"); 890 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 891 return; 892 } 893 894 BUG_ON(!efi.systab); 895 896 num_pages = ALIGN(efi.memmap.nr_map * efi.memmap.desc_size, PAGE_SIZE); 897 num_pages >>= PAGE_SHIFT; 898 899 if (efi_setup_page_tables(efi.memmap.phys_map, num_pages)) { 900 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 901 return; 902 } 903 904 efi_sync_low_kernel_mappings(); 905 906 /* 907 * Now that EFI is in virtual mode, update the function 908 * pointers in the runtime service table to the new virtual addresses. 909 * 910 * Call EFI services through wrapper functions. 911 */ 912 efi.runtime_version = efi_systab.hdr.revision; 913 914 efi_native_runtime_setup(); 915 916 efi.set_virtual_address_map = NULL; 917 918 if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX)) 919 runtime_code_page_mkexec(); 920 #endif 921 } 922 923 /* 924 * This function will switch the EFI runtime services to virtual mode. 925 * Essentially, we look through the EFI memmap and map every region that 926 * has the runtime attribute bit set in its memory descriptor into the 927 * efi_pgd page table. 928 * 929 * The old method which used to update that memory descriptor with the 930 * virtual address obtained from ioremap() is still supported when the 931 * kernel is booted with efi=old_map on its command line. Same old 932 * method enabled the runtime services to be called without having to 933 * thunk back into physical mode for every invocation. 934 * 935 * The new method does a pagetable switch in a preemption-safe manner 936 * so that we're in a different address space when calling a runtime 937 * function. For function arguments passing we do copy the PUDs of the 938 * kernel page table into efi_pgd prior to each call. 939 * 940 * Specially for kexec boot, efi runtime maps in previous kernel should 941 * be passed in via setup_data. In that case runtime ranges will be mapped 942 * to the same virtual addresses as the first kernel, see 943 * kexec_enter_virtual_mode(). 944 */ 945 static void __init __efi_enter_virtual_mode(void) 946 { 947 int count = 0, pg_shift = 0; 948 void *new_memmap = NULL; 949 efi_status_t status; 950 unsigned long pa; 951 952 efi.systab = NULL; 953 954 if (efi_alloc_page_tables()) { 955 pr_err("Failed to allocate EFI page tables\n"); 956 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 957 return; 958 } 959 960 efi_merge_regions(); 961 new_memmap = efi_map_regions(&count, &pg_shift); 962 if (!new_memmap) { 963 pr_err("Error reallocating memory, EFI runtime non-functional!\n"); 964 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 965 return; 966 } 967 968 pa = __pa(new_memmap); 969 970 /* 971 * Unregister the early EFI memmap from efi_init() and install 972 * the new EFI memory map that we are about to pass to the 973 * firmware via SetVirtualAddressMap(). 974 */ 975 efi_memmap_unmap(); 976 977 if (efi_memmap_init_late(pa, efi.memmap.desc_size * count)) { 978 pr_err("Failed to remap late EFI memory map\n"); 979 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 980 return; 981 } 982 983 if (efi_enabled(EFI_DBG)) { 984 pr_info("EFI runtime memory map:\n"); 985 efi_print_memmap(); 986 } 987 988 BUG_ON(!efi.systab); 989 990 if (efi_setup_page_tables(pa, 1 << pg_shift)) { 991 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 992 return; 993 } 994 995 efi_sync_low_kernel_mappings(); 996 997 if (efi_is_native()) { 998 status = phys_efi_set_virtual_address_map( 999 efi.memmap.desc_size * count, 1000 efi.memmap.desc_size, 1001 efi.memmap.desc_version, 1002 (efi_memory_desc_t *)pa); 1003 } else { 1004 status = efi_thunk_set_virtual_address_map( 1005 efi_phys.set_virtual_address_map, 1006 efi.memmap.desc_size * count, 1007 efi.memmap.desc_size, 1008 efi.memmap.desc_version, 1009 (efi_memory_desc_t *)pa); 1010 } 1011 1012 if (status != EFI_SUCCESS) { 1013 pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n", 1014 status); 1015 panic("EFI call to SetVirtualAddressMap() failed!"); 1016 } 1017 1018 efi_free_boot_services(); 1019 1020 /* 1021 * Now that EFI is in virtual mode, update the function 1022 * pointers in the runtime service table to the new virtual addresses. 1023 * 1024 * Call EFI services through wrapper functions. 1025 */ 1026 efi.runtime_version = efi_systab.hdr.revision; 1027 1028 if (efi_is_native()) 1029 efi_native_runtime_setup(); 1030 else 1031 efi_thunk_runtime_setup(); 1032 1033 efi.set_virtual_address_map = NULL; 1034 1035 /* 1036 * Apply more restrictive page table mapping attributes now that 1037 * SVAM() has been called and the firmware has performed all 1038 * necessary relocation fixups for the new virtual addresses. 1039 */ 1040 efi_runtime_update_mappings(); 1041 1042 /* clean DUMMY object */ 1043 efi_delete_dummy_variable(); 1044 } 1045 1046 void __init efi_enter_virtual_mode(void) 1047 { 1048 if (efi_enabled(EFI_PARAVIRT)) 1049 return; 1050 1051 if (efi_setup) 1052 kexec_enter_virtual_mode(); 1053 else 1054 __efi_enter_virtual_mode(); 1055 1056 efi_dump_pagetable(); 1057 } 1058 1059 static int __init arch_parse_efi_cmdline(char *str) 1060 { 1061 if (!str) { 1062 pr_warn("need at least one option\n"); 1063 return -EINVAL; 1064 } 1065 1066 if (parse_option_str(str, "old_map")) 1067 set_bit(EFI_OLD_MEMMAP, &efi.flags); 1068 1069 return 0; 1070 } 1071 early_param("efi", arch_parse_efi_cmdline); 1072 1073 bool efi_is_table_address(unsigned long phys_addr) 1074 { 1075 unsigned int i; 1076 1077 if (phys_addr == EFI_INVALID_TABLE_ADDR) 1078 return false; 1079 1080 for (i = 0; i < ARRAY_SIZE(efi_tables); i++) 1081 if (*(efi_tables[i]) == phys_addr) 1082 return true; 1083 1084 return false; 1085 } 1086