1 /* 2 * Common EFI (Extensible Firmware Interface) support functions 3 * Based on Extensible Firmware Interface Specification version 1.0 4 * 5 * Copyright (C) 1999 VA Linux Systems 6 * Copyright (C) 1999 Walt Drummond <drummond@valinux.com> 7 * Copyright (C) 1999-2002 Hewlett-Packard Co. 8 * David Mosberger-Tang <davidm@hpl.hp.com> 9 * Stephane Eranian <eranian@hpl.hp.com> 10 * Copyright (C) 2005-2008 Intel Co. 11 * Fenghua Yu <fenghua.yu@intel.com> 12 * Bibo Mao <bibo.mao@intel.com> 13 * Chandramouli Narayanan <mouli@linux.intel.com> 14 * Huang Ying <ying.huang@intel.com> 15 * Copyright (C) 2013 SuSE Labs 16 * Borislav Petkov <bp@suse.de> - runtime services VA mapping 17 * 18 * Copied from efi_32.c to eliminate the duplicated code between EFI 19 * 32/64 support code. --ying 2007-10-26 20 * 21 * All EFI Runtime Services are not implemented yet as EFI only 22 * supports physical mode addressing on SoftSDV. This is to be fixed 23 * in a future version. --drummond 1999-07-20 24 * 25 * Implemented EFI runtime services and virtual mode calls. --davidm 26 * 27 * Goutham Rao: <goutham.rao@intel.com> 28 * Skip non-WB memory and ignore empty memory ranges. 29 */ 30 31 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 32 33 #include <linux/kernel.h> 34 #include <linux/init.h> 35 #include <linux/efi.h> 36 #include <linux/efi-bgrt.h> 37 #include <linux/export.h> 38 #include <linux/bootmem.h> 39 #include <linux/slab.h> 40 #include <linux/memblock.h> 41 #include <linux/spinlock.h> 42 #include <linux/uaccess.h> 43 #include <linux/time.h> 44 #include <linux/io.h> 45 #include <linux/reboot.h> 46 #include <linux/bcd.h> 47 48 #include <asm/setup.h> 49 #include <asm/efi.h> 50 #include <asm/time.h> 51 #include <asm/cacheflush.h> 52 #include <asm/tlbflush.h> 53 #include <asm/x86_init.h> 54 #include <asm/rtc.h> 55 #include <asm/uv/uv.h> 56 57 #define EFI_DEBUG 58 59 struct efi_memory_map memmap; 60 61 static struct efi efi_phys __initdata; 62 static efi_system_table_t efi_systab __initdata; 63 64 static efi_config_table_type_t arch_tables[] __initdata = { 65 #ifdef CONFIG_X86_UV 66 {UV_SYSTEM_TABLE_GUID, "UVsystab", &efi.uv_systab}, 67 #endif 68 {NULL_GUID, NULL, NULL}, 69 }; 70 71 u64 efi_setup; /* efi setup_data physical address */ 72 73 static int add_efi_memmap __initdata; 74 static int __init setup_add_efi_memmap(char *arg) 75 { 76 add_efi_memmap = 1; 77 return 0; 78 } 79 early_param("add_efi_memmap", setup_add_efi_memmap); 80 81 static efi_status_t __init phys_efi_set_virtual_address_map( 82 unsigned long memory_map_size, 83 unsigned long descriptor_size, 84 u32 descriptor_version, 85 efi_memory_desc_t *virtual_map) 86 { 87 efi_status_t status; 88 unsigned long flags; 89 pgd_t *save_pgd; 90 91 save_pgd = efi_call_phys_prolog(); 92 93 /* Disable interrupts around EFI calls: */ 94 local_irq_save(flags); 95 status = efi_call_phys(efi_phys.set_virtual_address_map, 96 memory_map_size, descriptor_size, 97 descriptor_version, virtual_map); 98 local_irq_restore(flags); 99 100 efi_call_phys_epilog(save_pgd); 101 102 return status; 103 } 104 105 void efi_get_time(struct timespec *now) 106 { 107 efi_status_t status; 108 efi_time_t eft; 109 efi_time_cap_t cap; 110 111 status = efi.get_time(&eft, &cap); 112 if (status != EFI_SUCCESS) 113 pr_err("Oops: efitime: can't read time!\n"); 114 115 now->tv_sec = mktime(eft.year, eft.month, eft.day, eft.hour, 116 eft.minute, eft.second); 117 now->tv_nsec = 0; 118 } 119 120 void __init efi_find_mirror(void) 121 { 122 void *p; 123 u64 mirror_size = 0, total_size = 0; 124 125 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { 126 efi_memory_desc_t *md = p; 127 unsigned long long start = md->phys_addr; 128 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT; 129 130 total_size += size; 131 if (md->attribute & EFI_MEMORY_MORE_RELIABLE) { 132 memblock_mark_mirror(start, size); 133 mirror_size += size; 134 } 135 } 136 if (mirror_size) 137 pr_info("Memory: %lldM/%lldM mirrored memory\n", 138 mirror_size>>20, total_size>>20); 139 } 140 141 /* 142 * Tell the kernel about the EFI memory map. This might include 143 * more than the max 128 entries that can fit in the e820 legacy 144 * (zeropage) memory map. 145 */ 146 147 static void __init do_add_efi_memmap(void) 148 { 149 void *p; 150 151 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { 152 efi_memory_desc_t *md = p; 153 unsigned long long start = md->phys_addr; 154 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT; 155 int e820_type; 156 157 switch (md->type) { 158 case EFI_LOADER_CODE: 159 case EFI_LOADER_DATA: 160 case EFI_BOOT_SERVICES_CODE: 161 case EFI_BOOT_SERVICES_DATA: 162 case EFI_CONVENTIONAL_MEMORY: 163 if (md->attribute & EFI_MEMORY_WB) 164 e820_type = E820_RAM; 165 else 166 e820_type = E820_RESERVED; 167 break; 168 case EFI_ACPI_RECLAIM_MEMORY: 169 e820_type = E820_ACPI; 170 break; 171 case EFI_ACPI_MEMORY_NVS: 172 e820_type = E820_NVS; 173 break; 174 case EFI_UNUSABLE_MEMORY: 175 e820_type = E820_UNUSABLE; 176 break; 177 case EFI_PERSISTENT_MEMORY: 178 e820_type = E820_PMEM; 179 break; 180 default: 181 /* 182 * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE 183 * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO 184 * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE 185 */ 186 e820_type = E820_RESERVED; 187 break; 188 } 189 e820_add_region(start, size, e820_type); 190 } 191 sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map); 192 } 193 194 int __init efi_memblock_x86_reserve_range(void) 195 { 196 struct efi_info *e = &boot_params.efi_info; 197 phys_addr_t pmap; 198 199 if (efi_enabled(EFI_PARAVIRT)) 200 return 0; 201 202 #ifdef CONFIG_X86_32 203 /* Can't handle data above 4GB at this time */ 204 if (e->efi_memmap_hi) { 205 pr_err("Memory map is above 4GB, disabling EFI.\n"); 206 return -EINVAL; 207 } 208 pmap = e->efi_memmap; 209 #else 210 pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32)); 211 #endif 212 memmap.phys_map = pmap; 213 memmap.nr_map = e->efi_memmap_size / 214 e->efi_memdesc_size; 215 memmap.desc_size = e->efi_memdesc_size; 216 memmap.desc_version = e->efi_memdesc_version; 217 218 memblock_reserve(pmap, memmap.nr_map * memmap.desc_size); 219 220 efi.memmap = &memmap; 221 222 return 0; 223 } 224 225 void __init efi_print_memmap(void) 226 { 227 #ifdef EFI_DEBUG 228 efi_memory_desc_t *md; 229 void *p; 230 int i; 231 232 for (p = memmap.map, i = 0; 233 p < memmap.map_end; 234 p += memmap.desc_size, i++) { 235 char buf[64]; 236 237 md = p; 238 pr_info("mem%02u: %s range=[0x%016llx-0x%016llx] (%lluMB)\n", 239 i, efi_md_typeattr_format(buf, sizeof(buf), md), 240 md->phys_addr, 241 md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT) - 1, 242 (md->num_pages >> (20 - EFI_PAGE_SHIFT))); 243 } 244 #endif /* EFI_DEBUG */ 245 } 246 247 void __init efi_unmap_memmap(void) 248 { 249 clear_bit(EFI_MEMMAP, &efi.flags); 250 if (memmap.map) { 251 early_memunmap(memmap.map, memmap.nr_map * memmap.desc_size); 252 memmap.map = NULL; 253 } 254 } 255 256 static int __init efi_systab_init(void *phys) 257 { 258 if (efi_enabled(EFI_64BIT)) { 259 efi_system_table_64_t *systab64; 260 struct efi_setup_data *data = NULL; 261 u64 tmp = 0; 262 263 if (efi_setup) { 264 data = early_memremap(efi_setup, sizeof(*data)); 265 if (!data) 266 return -ENOMEM; 267 } 268 systab64 = early_memremap((unsigned long)phys, 269 sizeof(*systab64)); 270 if (systab64 == NULL) { 271 pr_err("Couldn't map the system table!\n"); 272 if (data) 273 early_memunmap(data, sizeof(*data)); 274 return -ENOMEM; 275 } 276 277 efi_systab.hdr = systab64->hdr; 278 efi_systab.fw_vendor = data ? (unsigned long)data->fw_vendor : 279 systab64->fw_vendor; 280 tmp |= data ? data->fw_vendor : systab64->fw_vendor; 281 efi_systab.fw_revision = systab64->fw_revision; 282 efi_systab.con_in_handle = systab64->con_in_handle; 283 tmp |= systab64->con_in_handle; 284 efi_systab.con_in = systab64->con_in; 285 tmp |= systab64->con_in; 286 efi_systab.con_out_handle = systab64->con_out_handle; 287 tmp |= systab64->con_out_handle; 288 efi_systab.con_out = systab64->con_out; 289 tmp |= systab64->con_out; 290 efi_systab.stderr_handle = systab64->stderr_handle; 291 tmp |= systab64->stderr_handle; 292 efi_systab.stderr = systab64->stderr; 293 tmp |= systab64->stderr; 294 efi_systab.runtime = data ? 295 (void *)(unsigned long)data->runtime : 296 (void *)(unsigned long)systab64->runtime; 297 tmp |= data ? data->runtime : systab64->runtime; 298 efi_systab.boottime = (void *)(unsigned long)systab64->boottime; 299 tmp |= systab64->boottime; 300 efi_systab.nr_tables = systab64->nr_tables; 301 efi_systab.tables = data ? (unsigned long)data->tables : 302 systab64->tables; 303 tmp |= data ? data->tables : systab64->tables; 304 305 early_memunmap(systab64, sizeof(*systab64)); 306 if (data) 307 early_memunmap(data, sizeof(*data)); 308 #ifdef CONFIG_X86_32 309 if (tmp >> 32) { 310 pr_err("EFI data located above 4GB, disabling EFI.\n"); 311 return -EINVAL; 312 } 313 #endif 314 } else { 315 efi_system_table_32_t *systab32; 316 317 systab32 = early_memremap((unsigned long)phys, 318 sizeof(*systab32)); 319 if (systab32 == NULL) { 320 pr_err("Couldn't map the system table!\n"); 321 return -ENOMEM; 322 } 323 324 efi_systab.hdr = systab32->hdr; 325 efi_systab.fw_vendor = systab32->fw_vendor; 326 efi_systab.fw_revision = systab32->fw_revision; 327 efi_systab.con_in_handle = systab32->con_in_handle; 328 efi_systab.con_in = systab32->con_in; 329 efi_systab.con_out_handle = systab32->con_out_handle; 330 efi_systab.con_out = systab32->con_out; 331 efi_systab.stderr_handle = systab32->stderr_handle; 332 efi_systab.stderr = systab32->stderr; 333 efi_systab.runtime = (void *)(unsigned long)systab32->runtime; 334 efi_systab.boottime = (void *)(unsigned long)systab32->boottime; 335 efi_systab.nr_tables = systab32->nr_tables; 336 efi_systab.tables = systab32->tables; 337 338 early_memunmap(systab32, sizeof(*systab32)); 339 } 340 341 efi.systab = &efi_systab; 342 343 /* 344 * Verify the EFI Table 345 */ 346 if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) { 347 pr_err("System table signature incorrect!\n"); 348 return -EINVAL; 349 } 350 if ((efi.systab->hdr.revision >> 16) == 0) 351 pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n", 352 efi.systab->hdr.revision >> 16, 353 efi.systab->hdr.revision & 0xffff); 354 355 return 0; 356 } 357 358 static int __init efi_runtime_init32(void) 359 { 360 efi_runtime_services_32_t *runtime; 361 362 runtime = early_memremap((unsigned long)efi.systab->runtime, 363 sizeof(efi_runtime_services_32_t)); 364 if (!runtime) { 365 pr_err("Could not map the runtime service table!\n"); 366 return -ENOMEM; 367 } 368 369 /* 370 * We will only need *early* access to the SetVirtualAddressMap 371 * EFI runtime service. All other runtime services will be called 372 * via the virtual mapping. 373 */ 374 efi_phys.set_virtual_address_map = 375 (efi_set_virtual_address_map_t *) 376 (unsigned long)runtime->set_virtual_address_map; 377 early_memunmap(runtime, sizeof(efi_runtime_services_32_t)); 378 379 return 0; 380 } 381 382 static int __init efi_runtime_init64(void) 383 { 384 efi_runtime_services_64_t *runtime; 385 386 runtime = early_memremap((unsigned long)efi.systab->runtime, 387 sizeof(efi_runtime_services_64_t)); 388 if (!runtime) { 389 pr_err("Could not map the runtime service table!\n"); 390 return -ENOMEM; 391 } 392 393 /* 394 * We will only need *early* access to the SetVirtualAddressMap 395 * EFI runtime service. All other runtime services will be called 396 * via the virtual mapping. 397 */ 398 efi_phys.set_virtual_address_map = 399 (efi_set_virtual_address_map_t *) 400 (unsigned long)runtime->set_virtual_address_map; 401 early_memunmap(runtime, sizeof(efi_runtime_services_64_t)); 402 403 return 0; 404 } 405 406 static int __init efi_runtime_init(void) 407 { 408 int rv; 409 410 /* 411 * Check out the runtime services table. We need to map 412 * the runtime services table so that we can grab the physical 413 * address of several of the EFI runtime functions, needed to 414 * set the firmware into virtual mode. 415 * 416 * When EFI_PARAVIRT is in force then we could not map runtime 417 * service memory region because we do not have direct access to it. 418 * However, runtime services are available through proxy functions 419 * (e.g. in case of Xen dom0 EFI implementation they call special 420 * hypercall which executes relevant EFI functions) and that is why 421 * they are always enabled. 422 */ 423 424 if (!efi_enabled(EFI_PARAVIRT)) { 425 if (efi_enabled(EFI_64BIT)) 426 rv = efi_runtime_init64(); 427 else 428 rv = efi_runtime_init32(); 429 430 if (rv) 431 return rv; 432 } 433 434 set_bit(EFI_RUNTIME_SERVICES, &efi.flags); 435 436 return 0; 437 } 438 439 static int __init efi_memmap_init(void) 440 { 441 if (efi_enabled(EFI_PARAVIRT)) 442 return 0; 443 444 /* Map the EFI memory map */ 445 memmap.map = early_memremap((unsigned long)memmap.phys_map, 446 memmap.nr_map * memmap.desc_size); 447 if (memmap.map == NULL) { 448 pr_err("Could not map the memory map!\n"); 449 return -ENOMEM; 450 } 451 memmap.map_end = memmap.map + (memmap.nr_map * memmap.desc_size); 452 453 if (add_efi_memmap) 454 do_add_efi_memmap(); 455 456 set_bit(EFI_MEMMAP, &efi.flags); 457 458 return 0; 459 } 460 461 void __init efi_init(void) 462 { 463 efi_char16_t *c16; 464 char vendor[100] = "unknown"; 465 int i = 0; 466 void *tmp; 467 468 #ifdef CONFIG_X86_32 469 if (boot_params.efi_info.efi_systab_hi || 470 boot_params.efi_info.efi_memmap_hi) { 471 pr_info("Table located above 4GB, disabling EFI.\n"); 472 return; 473 } 474 efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab; 475 #else 476 efi_phys.systab = (efi_system_table_t *) 477 (boot_params.efi_info.efi_systab | 478 ((__u64)boot_params.efi_info.efi_systab_hi<<32)); 479 #endif 480 481 if (efi_systab_init(efi_phys.systab)) 482 return; 483 484 efi.config_table = (unsigned long)efi.systab->tables; 485 efi.fw_vendor = (unsigned long)efi.systab->fw_vendor; 486 efi.runtime = (unsigned long)efi.systab->runtime; 487 488 /* 489 * Show what we know for posterity 490 */ 491 c16 = tmp = early_memremap(efi.systab->fw_vendor, 2); 492 if (c16) { 493 for (i = 0; i < sizeof(vendor) - 1 && *c16; ++i) 494 vendor[i] = *c16++; 495 vendor[i] = '\0'; 496 } else 497 pr_err("Could not map the firmware vendor!\n"); 498 early_memunmap(tmp, 2); 499 500 pr_info("EFI v%u.%.02u by %s\n", 501 efi.systab->hdr.revision >> 16, 502 efi.systab->hdr.revision & 0xffff, vendor); 503 504 if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables)) 505 return; 506 507 if (efi_config_init(arch_tables)) 508 return; 509 510 /* 511 * Note: We currently don't support runtime services on an EFI 512 * that doesn't match the kernel 32/64-bit mode. 513 */ 514 515 if (!efi_runtime_supported()) 516 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n"); 517 else { 518 if (efi_runtime_disabled() || efi_runtime_init()) 519 return; 520 } 521 if (efi_memmap_init()) 522 return; 523 524 if (efi_enabled(EFI_DBG)) 525 efi_print_memmap(); 526 527 efi_esrt_init(); 528 } 529 530 void __init efi_late_init(void) 531 { 532 efi_bgrt_init(); 533 } 534 535 void __init efi_set_executable(efi_memory_desc_t *md, bool executable) 536 { 537 u64 addr, npages; 538 539 addr = md->virt_addr; 540 npages = md->num_pages; 541 542 memrange_efi_to_native(&addr, &npages); 543 544 if (executable) 545 set_memory_x(addr, npages); 546 else 547 set_memory_nx(addr, npages); 548 } 549 550 void __init runtime_code_page_mkexec(void) 551 { 552 efi_memory_desc_t *md; 553 void *p; 554 555 /* Make EFI runtime service code area executable */ 556 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { 557 md = p; 558 559 if (md->type != EFI_RUNTIME_SERVICES_CODE) 560 continue; 561 562 efi_set_executable(md, true); 563 } 564 } 565 566 void __init efi_memory_uc(u64 addr, unsigned long size) 567 { 568 unsigned long page_shift = 1UL << EFI_PAGE_SHIFT; 569 u64 npages; 570 571 npages = round_up(size, page_shift) / page_shift; 572 memrange_efi_to_native(&addr, &npages); 573 set_memory_uc(addr, npages); 574 } 575 576 void __init old_map_region(efi_memory_desc_t *md) 577 { 578 u64 start_pfn, end_pfn, end; 579 unsigned long size; 580 void *va; 581 582 start_pfn = PFN_DOWN(md->phys_addr); 583 size = md->num_pages << PAGE_SHIFT; 584 end = md->phys_addr + size; 585 end_pfn = PFN_UP(end); 586 587 if (pfn_range_is_mapped(start_pfn, end_pfn)) { 588 va = __va(md->phys_addr); 589 590 if (!(md->attribute & EFI_MEMORY_WB)) 591 efi_memory_uc((u64)(unsigned long)va, size); 592 } else 593 va = efi_ioremap(md->phys_addr, size, 594 md->type, md->attribute); 595 596 md->virt_addr = (u64) (unsigned long) va; 597 if (!va) 598 pr_err("ioremap of 0x%llX failed!\n", 599 (unsigned long long)md->phys_addr); 600 } 601 602 /* Merge contiguous regions of the same type and attribute */ 603 static void __init efi_merge_regions(void) 604 { 605 void *p; 606 efi_memory_desc_t *md, *prev_md = NULL; 607 608 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { 609 u64 prev_size; 610 md = p; 611 612 if (!prev_md) { 613 prev_md = md; 614 continue; 615 } 616 617 if (prev_md->type != md->type || 618 prev_md->attribute != md->attribute) { 619 prev_md = md; 620 continue; 621 } 622 623 prev_size = prev_md->num_pages << EFI_PAGE_SHIFT; 624 625 if (md->phys_addr == (prev_md->phys_addr + prev_size)) { 626 prev_md->num_pages += md->num_pages; 627 md->type = EFI_RESERVED_TYPE; 628 md->attribute = 0; 629 continue; 630 } 631 prev_md = md; 632 } 633 } 634 635 static void __init get_systab_virt_addr(efi_memory_desc_t *md) 636 { 637 unsigned long size; 638 u64 end, systab; 639 640 size = md->num_pages << EFI_PAGE_SHIFT; 641 end = md->phys_addr + size; 642 systab = (u64)(unsigned long)efi_phys.systab; 643 if (md->phys_addr <= systab && systab < end) { 644 systab += md->virt_addr - md->phys_addr; 645 efi.systab = (efi_system_table_t *)(unsigned long)systab; 646 } 647 } 648 649 static void __init save_runtime_map(void) 650 { 651 #ifdef CONFIG_KEXEC_CORE 652 efi_memory_desc_t *md; 653 void *tmp, *p, *q = NULL; 654 int count = 0; 655 656 if (efi_enabled(EFI_OLD_MEMMAP)) 657 return; 658 659 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { 660 md = p; 661 662 if (!(md->attribute & EFI_MEMORY_RUNTIME) || 663 (md->type == EFI_BOOT_SERVICES_CODE) || 664 (md->type == EFI_BOOT_SERVICES_DATA)) 665 continue; 666 tmp = krealloc(q, (count + 1) * memmap.desc_size, GFP_KERNEL); 667 if (!tmp) 668 goto out; 669 q = tmp; 670 671 memcpy(q + count * memmap.desc_size, md, memmap.desc_size); 672 count++; 673 } 674 675 efi_runtime_map_setup(q, count, memmap.desc_size); 676 return; 677 678 out: 679 kfree(q); 680 pr_err("Error saving runtime map, efi runtime on kexec non-functional!!\n"); 681 #endif 682 } 683 684 static void *realloc_pages(void *old_memmap, int old_shift) 685 { 686 void *ret; 687 688 ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1); 689 if (!ret) 690 goto out; 691 692 /* 693 * A first-time allocation doesn't have anything to copy. 694 */ 695 if (!old_memmap) 696 return ret; 697 698 memcpy(ret, old_memmap, PAGE_SIZE << old_shift); 699 700 out: 701 free_pages((unsigned long)old_memmap, old_shift); 702 return ret; 703 } 704 705 /* 706 * Iterate the EFI memory map in reverse order because the regions 707 * will be mapped top-down. The end result is the same as if we had 708 * mapped things forward, but doesn't require us to change the 709 * existing implementation of efi_map_region(). 710 */ 711 static inline void *efi_map_next_entry_reverse(void *entry) 712 { 713 /* Initial call */ 714 if (!entry) 715 return memmap.map_end - memmap.desc_size; 716 717 entry -= memmap.desc_size; 718 if (entry < memmap.map) 719 return NULL; 720 721 return entry; 722 } 723 724 /* 725 * efi_map_next_entry - Return the next EFI memory map descriptor 726 * @entry: Previous EFI memory map descriptor 727 * 728 * This is a helper function to iterate over the EFI memory map, which 729 * we do in different orders depending on the current configuration. 730 * 731 * To begin traversing the memory map @entry must be %NULL. 732 * 733 * Returns %NULL when we reach the end of the memory map. 734 */ 735 static void *efi_map_next_entry(void *entry) 736 { 737 if (!efi_enabled(EFI_OLD_MEMMAP) && efi_enabled(EFI_64BIT)) { 738 /* 739 * Starting in UEFI v2.5 the EFI_PROPERTIES_TABLE 740 * config table feature requires us to map all entries 741 * in the same order as they appear in the EFI memory 742 * map. That is to say, entry N must have a lower 743 * virtual address than entry N+1. This is because the 744 * firmware toolchain leaves relative references in 745 * the code/data sections, which are split and become 746 * separate EFI memory regions. Mapping things 747 * out-of-order leads to the firmware accessing 748 * unmapped addresses. 749 * 750 * Since we need to map things this way whether or not 751 * the kernel actually makes use of 752 * EFI_PROPERTIES_TABLE, let's just switch to this 753 * scheme by default for 64-bit. 754 */ 755 return efi_map_next_entry_reverse(entry); 756 } 757 758 /* Initial call */ 759 if (!entry) 760 return memmap.map; 761 762 entry += memmap.desc_size; 763 if (entry >= memmap.map_end) 764 return NULL; 765 766 return entry; 767 } 768 769 /* 770 * Map the efi memory ranges of the runtime services and update new_mmap with 771 * virtual addresses. 772 */ 773 static void * __init efi_map_regions(int *count, int *pg_shift) 774 { 775 void *p, *new_memmap = NULL; 776 unsigned long left = 0; 777 efi_memory_desc_t *md; 778 779 p = NULL; 780 while ((p = efi_map_next_entry(p))) { 781 md = p; 782 if (!(md->attribute & EFI_MEMORY_RUNTIME)) { 783 #ifdef CONFIG_X86_64 784 if (md->type != EFI_BOOT_SERVICES_CODE && 785 md->type != EFI_BOOT_SERVICES_DATA) 786 #endif 787 continue; 788 } 789 790 efi_map_region(md); 791 get_systab_virt_addr(md); 792 793 if (left < memmap.desc_size) { 794 new_memmap = realloc_pages(new_memmap, *pg_shift); 795 if (!new_memmap) 796 return NULL; 797 798 left += PAGE_SIZE << *pg_shift; 799 (*pg_shift)++; 800 } 801 802 memcpy(new_memmap + (*count * memmap.desc_size), md, 803 memmap.desc_size); 804 805 left -= memmap.desc_size; 806 (*count)++; 807 } 808 809 return new_memmap; 810 } 811 812 static void __init kexec_enter_virtual_mode(void) 813 { 814 #ifdef CONFIG_KEXEC_CORE 815 efi_memory_desc_t *md; 816 unsigned int num_pages; 817 void *p; 818 819 efi.systab = NULL; 820 821 /* 822 * We don't do virtual mode, since we don't do runtime services, on 823 * non-native EFI 824 */ 825 if (!efi_is_native()) { 826 efi_unmap_memmap(); 827 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 828 return; 829 } 830 831 if (efi_alloc_page_tables()) { 832 pr_err("Failed to allocate EFI page tables\n"); 833 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 834 return; 835 } 836 837 /* 838 * Map efi regions which were passed via setup_data. The virt_addr is a 839 * fixed addr which was used in first kernel of a kexec boot. 840 */ 841 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { 842 md = p; 843 efi_map_region_fixed(md); /* FIXME: add error handling */ 844 get_systab_virt_addr(md); 845 } 846 847 save_runtime_map(); 848 849 BUG_ON(!efi.systab); 850 851 num_pages = ALIGN(memmap.nr_map * memmap.desc_size, PAGE_SIZE); 852 num_pages >>= PAGE_SHIFT; 853 854 if (efi_setup_page_tables(memmap.phys_map, num_pages)) { 855 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 856 return; 857 } 858 859 efi_sync_low_kernel_mappings(); 860 861 /* 862 * Now that EFI is in virtual mode, update the function 863 * pointers in the runtime service table to the new virtual addresses. 864 * 865 * Call EFI services through wrapper functions. 866 */ 867 efi.runtime_version = efi_systab.hdr.revision; 868 869 efi_native_runtime_setup(); 870 871 efi.set_virtual_address_map = NULL; 872 873 if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX)) 874 runtime_code_page_mkexec(); 875 876 /* clean DUMMY object */ 877 efi_delete_dummy_variable(); 878 #endif 879 } 880 881 /* 882 * This function will switch the EFI runtime services to virtual mode. 883 * Essentially, we look through the EFI memmap and map every region that 884 * has the runtime attribute bit set in its memory descriptor into the 885 * efi_pgd page table. 886 * 887 * The old method which used to update that memory descriptor with the 888 * virtual address obtained from ioremap() is still supported when the 889 * kernel is booted with efi=old_map on its command line. Same old 890 * method enabled the runtime services to be called without having to 891 * thunk back into physical mode for every invocation. 892 * 893 * The new method does a pagetable switch in a preemption-safe manner 894 * so that we're in a different address space when calling a runtime 895 * function. For function arguments passing we do copy the PUDs of the 896 * kernel page table into efi_pgd prior to each call. 897 * 898 * Specially for kexec boot, efi runtime maps in previous kernel should 899 * be passed in via setup_data. In that case runtime ranges will be mapped 900 * to the same virtual addresses as the first kernel, see 901 * kexec_enter_virtual_mode(). 902 */ 903 static void __init __efi_enter_virtual_mode(void) 904 { 905 int count = 0, pg_shift = 0; 906 void *new_memmap = NULL; 907 efi_status_t status; 908 909 efi.systab = NULL; 910 911 if (efi_alloc_page_tables()) { 912 pr_err("Failed to allocate EFI page tables\n"); 913 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 914 return; 915 } 916 917 efi_merge_regions(); 918 new_memmap = efi_map_regions(&count, &pg_shift); 919 if (!new_memmap) { 920 pr_err("Error reallocating memory, EFI runtime non-functional!\n"); 921 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 922 return; 923 } 924 925 save_runtime_map(); 926 927 BUG_ON(!efi.systab); 928 929 if (efi_setup_page_tables(__pa(new_memmap), 1 << pg_shift)) { 930 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); 931 return; 932 } 933 934 efi_sync_low_kernel_mappings(); 935 936 if (efi_is_native()) { 937 status = phys_efi_set_virtual_address_map( 938 memmap.desc_size * count, 939 memmap.desc_size, 940 memmap.desc_version, 941 (efi_memory_desc_t *)__pa(new_memmap)); 942 } else { 943 status = efi_thunk_set_virtual_address_map( 944 efi_phys.set_virtual_address_map, 945 memmap.desc_size * count, 946 memmap.desc_size, 947 memmap.desc_version, 948 (efi_memory_desc_t *)__pa(new_memmap)); 949 } 950 951 if (status != EFI_SUCCESS) { 952 pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n", 953 status); 954 panic("EFI call to SetVirtualAddressMap() failed!"); 955 } 956 957 /* 958 * Now that EFI is in virtual mode, update the function 959 * pointers in the runtime service table to the new virtual addresses. 960 * 961 * Call EFI services through wrapper functions. 962 */ 963 efi.runtime_version = efi_systab.hdr.revision; 964 965 if (efi_is_native()) 966 efi_native_runtime_setup(); 967 else 968 efi_thunk_runtime_setup(); 969 970 efi.set_virtual_address_map = NULL; 971 972 /* 973 * Apply more restrictive page table mapping attributes now that 974 * SVAM() has been called and the firmware has performed all 975 * necessary relocation fixups for the new virtual addresses. 976 */ 977 efi_runtime_update_mappings(); 978 efi_dump_pagetable(); 979 980 /* 981 * We mapped the descriptor array into the EFI pagetable above 982 * but we're not unmapping it here because if we're running in 983 * EFI mixed mode we need all of memory to be accessible when 984 * we pass parameters to the EFI runtime services in the 985 * thunking code. 986 * 987 * efi_cleanup_page_tables(__pa(new_memmap), 1 << pg_shift); 988 */ 989 free_pages((unsigned long)new_memmap, pg_shift); 990 991 /* clean DUMMY object */ 992 efi_delete_dummy_variable(); 993 } 994 995 void __init efi_enter_virtual_mode(void) 996 { 997 if (efi_enabled(EFI_PARAVIRT)) 998 return; 999 1000 if (efi_setup) 1001 kexec_enter_virtual_mode(); 1002 else 1003 __efi_enter_virtual_mode(); 1004 } 1005 1006 /* 1007 * Convenience functions to obtain memory types and attributes 1008 */ 1009 u32 efi_mem_type(unsigned long phys_addr) 1010 { 1011 efi_memory_desc_t *md; 1012 void *p; 1013 1014 if (!efi_enabled(EFI_MEMMAP)) 1015 return 0; 1016 1017 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { 1018 md = p; 1019 if ((md->phys_addr <= phys_addr) && 1020 (phys_addr < (md->phys_addr + 1021 (md->num_pages << EFI_PAGE_SHIFT)))) 1022 return md->type; 1023 } 1024 return 0; 1025 } 1026 1027 static int __init arch_parse_efi_cmdline(char *str) 1028 { 1029 if (!str) { 1030 pr_warn("need at least one option\n"); 1031 return -EINVAL; 1032 } 1033 1034 if (parse_option_str(str, "old_map")) 1035 set_bit(EFI_OLD_MEMMAP, &efi.flags); 1036 1037 return 0; 1038 } 1039 early_param("efi", arch_parse_efi_cmdline); 1040