1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * x86_64 specific EFI support functions 4 * Based on Extensible Firmware Interface Specification version 1.0 5 * 6 * Copyright (C) 2005-2008 Intel Co. 7 * Fenghua Yu <fenghua.yu@intel.com> 8 * Bibo Mao <bibo.mao@intel.com> 9 * Chandramouli Narayanan <mouli@linux.intel.com> 10 * Huang Ying <ying.huang@intel.com> 11 * 12 * Code to convert EFI to E820 map has been implemented in elilo bootloader 13 * based on a EFI patch by Edgar Hucek. Based on the E820 map, the page table 14 * is setup appropriately for EFI runtime code. 15 * - mouli 06/14/2007. 16 * 17 */ 18 19 #define pr_fmt(fmt) "efi: " fmt 20 21 #include <linux/kernel.h> 22 #include <linux/init.h> 23 #include <linux/mm.h> 24 #include <linux/types.h> 25 #include <linux/spinlock.h> 26 #include <linux/memblock.h> 27 #include <linux/ioport.h> 28 #include <linux/mc146818rtc.h> 29 #include <linux/efi.h> 30 #include <linux/export.h> 31 #include <linux/uaccess.h> 32 #include <linux/io.h> 33 #include <linux/reboot.h> 34 #include <linux/slab.h> 35 #include <linux/ucs2_string.h> 36 #include <linux/mem_encrypt.h> 37 #include <linux/sched/task.h> 38 39 #include <asm/setup.h> 40 #include <asm/page.h> 41 #include <asm/e820/api.h> 42 #include <asm/pgtable.h> 43 #include <asm/tlbflush.h> 44 #include <asm/proto.h> 45 #include <asm/efi.h> 46 #include <asm/cacheflush.h> 47 #include <asm/fixmap.h> 48 #include <asm/realmode.h> 49 #include <asm/time.h> 50 #include <asm/pgalloc.h> 51 52 /* 53 * We allocate runtime services regions top-down, starting from -4G, i.e. 54 * 0xffff_ffff_0000_0000 and limit EFI VA mapping space to 64G. 55 */ 56 static u64 efi_va = EFI_VA_START; 57 58 struct efi_scratch efi_scratch; 59 60 static void __init early_code_mapping_set_exec(int executable) 61 { 62 efi_memory_desc_t *md; 63 64 if (!(__supported_pte_mask & _PAGE_NX)) 65 return; 66 67 /* Make EFI service code area executable */ 68 for_each_efi_memory_desc(md) { 69 if (md->type == EFI_RUNTIME_SERVICES_CODE || 70 md->type == EFI_BOOT_SERVICES_CODE) 71 efi_set_executable(md, executable); 72 } 73 } 74 75 pgd_t * __init efi_call_phys_prolog(void) 76 { 77 unsigned long vaddr, addr_pgd, addr_p4d, addr_pud; 78 pgd_t *save_pgd, *pgd_k, *pgd_efi; 79 p4d_t *p4d, *p4d_k, *p4d_efi; 80 pud_t *pud; 81 82 int pgd; 83 int n_pgds, i, j; 84 85 if (!efi_enabled(EFI_OLD_MEMMAP)) { 86 efi_switch_mm(&efi_mm); 87 return efi_mm.pgd; 88 } 89 90 early_code_mapping_set_exec(1); 91 92 n_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT), PGDIR_SIZE); 93 save_pgd = kmalloc_array(n_pgds, sizeof(*save_pgd), GFP_KERNEL); 94 if (!save_pgd) 95 return NULL; 96 97 /* 98 * Build 1:1 identity mapping for efi=old_map usage. Note that 99 * PAGE_OFFSET is PGDIR_SIZE aligned when KASLR is disabled, while 100 * it is PUD_SIZE ALIGNED with KASLR enabled. So for a given physical 101 * address X, the pud_index(X) != pud_index(__va(X)), we can only copy 102 * PUD entry of __va(X) to fill in pud entry of X to build 1:1 mapping. 103 * This means here we can only reuse the PMD tables of the direct mapping. 104 */ 105 for (pgd = 0; pgd < n_pgds; pgd++) { 106 addr_pgd = (unsigned long)(pgd * PGDIR_SIZE); 107 vaddr = (unsigned long)__va(pgd * PGDIR_SIZE); 108 pgd_efi = pgd_offset_k(addr_pgd); 109 save_pgd[pgd] = *pgd_efi; 110 111 p4d = p4d_alloc(&init_mm, pgd_efi, addr_pgd); 112 if (!p4d) { 113 pr_err("Failed to allocate p4d table!\n"); 114 goto out; 115 } 116 117 for (i = 0; i < PTRS_PER_P4D; i++) { 118 addr_p4d = addr_pgd + i * P4D_SIZE; 119 p4d_efi = p4d + p4d_index(addr_p4d); 120 121 pud = pud_alloc(&init_mm, p4d_efi, addr_p4d); 122 if (!pud) { 123 pr_err("Failed to allocate pud table!\n"); 124 goto out; 125 } 126 127 for (j = 0; j < PTRS_PER_PUD; j++) { 128 addr_pud = addr_p4d + j * PUD_SIZE; 129 130 if (addr_pud > (max_pfn << PAGE_SHIFT)) 131 break; 132 133 vaddr = (unsigned long)__va(addr_pud); 134 135 pgd_k = pgd_offset_k(vaddr); 136 p4d_k = p4d_offset(pgd_k, vaddr); 137 pud[j] = *pud_offset(p4d_k, vaddr); 138 } 139 } 140 pgd_offset_k(pgd * PGDIR_SIZE)->pgd &= ~_PAGE_NX; 141 } 142 143 __flush_tlb_all(); 144 return save_pgd; 145 out: 146 efi_call_phys_epilog(save_pgd); 147 return NULL; 148 } 149 150 void __init efi_call_phys_epilog(pgd_t *save_pgd) 151 { 152 /* 153 * After the lock is released, the original page table is restored. 154 */ 155 int pgd_idx, i; 156 int nr_pgds; 157 pgd_t *pgd; 158 p4d_t *p4d; 159 pud_t *pud; 160 161 if (!efi_enabled(EFI_OLD_MEMMAP)) { 162 efi_switch_mm(efi_scratch.prev_mm); 163 return; 164 } 165 166 nr_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT) , PGDIR_SIZE); 167 168 for (pgd_idx = 0; pgd_idx < nr_pgds; pgd_idx++) { 169 pgd = pgd_offset_k(pgd_idx * PGDIR_SIZE); 170 set_pgd(pgd_offset_k(pgd_idx * PGDIR_SIZE), save_pgd[pgd_idx]); 171 172 if (!pgd_present(*pgd)) 173 continue; 174 175 for (i = 0; i < PTRS_PER_P4D; i++) { 176 p4d = p4d_offset(pgd, 177 pgd_idx * PGDIR_SIZE + i * P4D_SIZE); 178 179 if (!p4d_present(*p4d)) 180 continue; 181 182 pud = (pud_t *)p4d_page_vaddr(*p4d); 183 pud_free(&init_mm, pud); 184 } 185 186 p4d = (p4d_t *)pgd_page_vaddr(*pgd); 187 p4d_free(&init_mm, p4d); 188 } 189 190 kfree(save_pgd); 191 192 __flush_tlb_all(); 193 early_code_mapping_set_exec(0); 194 } 195 196 EXPORT_SYMBOL_GPL(efi_mm); 197 198 /* 199 * We need our own copy of the higher levels of the page tables 200 * because we want to avoid inserting EFI region mappings (EFI_VA_END 201 * to EFI_VA_START) into the standard kernel page tables. Everything 202 * else can be shared, see efi_sync_low_kernel_mappings(). 203 * 204 * We don't want the pgd on the pgd_list and cannot use pgd_alloc() for the 205 * allocation. 206 */ 207 int __init efi_alloc_page_tables(void) 208 { 209 pgd_t *pgd, *efi_pgd; 210 p4d_t *p4d; 211 pud_t *pud; 212 gfp_t gfp_mask; 213 214 if (efi_enabled(EFI_OLD_MEMMAP)) 215 return 0; 216 217 gfp_mask = GFP_KERNEL | __GFP_ZERO; 218 efi_pgd = (pgd_t *)__get_free_pages(gfp_mask, PGD_ALLOCATION_ORDER); 219 if (!efi_pgd) 220 return -ENOMEM; 221 222 pgd = efi_pgd + pgd_index(EFI_VA_END); 223 p4d = p4d_alloc(&init_mm, pgd, EFI_VA_END); 224 if (!p4d) { 225 free_page((unsigned long)efi_pgd); 226 return -ENOMEM; 227 } 228 229 pud = pud_alloc(&init_mm, p4d, EFI_VA_END); 230 if (!pud) { 231 if (pgtable_l5_enabled()) 232 free_page((unsigned long) pgd_page_vaddr(*pgd)); 233 free_pages((unsigned long)efi_pgd, PGD_ALLOCATION_ORDER); 234 return -ENOMEM; 235 } 236 237 efi_mm.pgd = efi_pgd; 238 mm_init_cpumask(&efi_mm); 239 init_new_context(NULL, &efi_mm); 240 241 return 0; 242 } 243 244 /* 245 * Add low kernel mappings for passing arguments to EFI functions. 246 */ 247 void efi_sync_low_kernel_mappings(void) 248 { 249 unsigned num_entries; 250 pgd_t *pgd_k, *pgd_efi; 251 p4d_t *p4d_k, *p4d_efi; 252 pud_t *pud_k, *pud_efi; 253 pgd_t *efi_pgd = efi_mm.pgd; 254 255 if (efi_enabled(EFI_OLD_MEMMAP)) 256 return; 257 258 /* 259 * We can share all PGD entries apart from the one entry that 260 * covers the EFI runtime mapping space. 261 * 262 * Make sure the EFI runtime region mappings are guaranteed to 263 * only span a single PGD entry and that the entry also maps 264 * other important kernel regions. 265 */ 266 MAYBE_BUILD_BUG_ON(pgd_index(EFI_VA_END) != pgd_index(MODULES_END)); 267 MAYBE_BUILD_BUG_ON((EFI_VA_START & PGDIR_MASK) != 268 (EFI_VA_END & PGDIR_MASK)); 269 270 pgd_efi = efi_pgd + pgd_index(PAGE_OFFSET); 271 pgd_k = pgd_offset_k(PAGE_OFFSET); 272 273 num_entries = pgd_index(EFI_VA_END) - pgd_index(PAGE_OFFSET); 274 memcpy(pgd_efi, pgd_k, sizeof(pgd_t) * num_entries); 275 276 /* 277 * As with PGDs, we share all P4D entries apart from the one entry 278 * that covers the EFI runtime mapping space. 279 */ 280 BUILD_BUG_ON(p4d_index(EFI_VA_END) != p4d_index(MODULES_END)); 281 BUILD_BUG_ON((EFI_VA_START & P4D_MASK) != (EFI_VA_END & P4D_MASK)); 282 283 pgd_efi = efi_pgd + pgd_index(EFI_VA_END); 284 pgd_k = pgd_offset_k(EFI_VA_END); 285 p4d_efi = p4d_offset(pgd_efi, 0); 286 p4d_k = p4d_offset(pgd_k, 0); 287 288 num_entries = p4d_index(EFI_VA_END); 289 memcpy(p4d_efi, p4d_k, sizeof(p4d_t) * num_entries); 290 291 /* 292 * We share all the PUD entries apart from those that map the 293 * EFI regions. Copy around them. 294 */ 295 BUILD_BUG_ON((EFI_VA_START & ~PUD_MASK) != 0); 296 BUILD_BUG_ON((EFI_VA_END & ~PUD_MASK) != 0); 297 298 p4d_efi = p4d_offset(pgd_efi, EFI_VA_END); 299 p4d_k = p4d_offset(pgd_k, EFI_VA_END); 300 pud_efi = pud_offset(p4d_efi, 0); 301 pud_k = pud_offset(p4d_k, 0); 302 303 num_entries = pud_index(EFI_VA_END); 304 memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries); 305 306 pud_efi = pud_offset(p4d_efi, EFI_VA_START); 307 pud_k = pud_offset(p4d_k, EFI_VA_START); 308 309 num_entries = PTRS_PER_PUD - pud_index(EFI_VA_START); 310 memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries); 311 } 312 313 /* 314 * Wrapper for slow_virt_to_phys() that handles NULL addresses. 315 */ 316 static inline phys_addr_t 317 virt_to_phys_or_null_size(void *va, unsigned long size) 318 { 319 bool bad_size; 320 321 if (!va) 322 return 0; 323 324 if (virt_addr_valid(va)) 325 return virt_to_phys(va); 326 327 /* 328 * A fully aligned variable on the stack is guaranteed not to 329 * cross a page bounary. Try to catch strings on the stack by 330 * checking that 'size' is a power of two. 331 */ 332 bad_size = size > PAGE_SIZE || !is_power_of_2(size); 333 334 WARN_ON(!IS_ALIGNED((unsigned long)va, size) || bad_size); 335 336 return slow_virt_to_phys(va); 337 } 338 339 #define virt_to_phys_or_null(addr) \ 340 virt_to_phys_or_null_size((addr), sizeof(*(addr))) 341 342 int __init efi_setup_page_tables(unsigned long pa_memmap, unsigned num_pages) 343 { 344 unsigned long pfn, text, pf; 345 struct page *page; 346 unsigned npages; 347 pgd_t *pgd = efi_mm.pgd; 348 349 if (efi_enabled(EFI_OLD_MEMMAP)) 350 return 0; 351 352 /* 353 * It can happen that the physical address of new_memmap lands in memory 354 * which is not mapped in the EFI page table. Therefore we need to go 355 * and ident-map those pages containing the map before calling 356 * phys_efi_set_virtual_address_map(). 357 */ 358 pfn = pa_memmap >> PAGE_SHIFT; 359 pf = _PAGE_NX | _PAGE_RW | _PAGE_ENC; 360 if (kernel_map_pages_in_pgd(pgd, pfn, pa_memmap, num_pages, pf)) { 361 pr_err("Error ident-mapping new memmap (0x%lx)!\n", pa_memmap); 362 return 1; 363 } 364 365 /* 366 * Certain firmware versions are way too sentimential and still believe 367 * they are exclusive and unquestionable owners of the first physical page, 368 * even though they explicitly mark it as EFI_CONVENTIONAL_MEMORY 369 * (but then write-access it later during SetVirtualAddressMap()). 370 * 371 * Create a 1:1 mapping for this page, to avoid triple faults during early 372 * boot with such firmware. We are free to hand this page to the BIOS, 373 * as trim_bios_range() will reserve the first page and isolate it away 374 * from memory allocators anyway. 375 */ 376 pf = _PAGE_RW; 377 if (sev_active()) 378 pf |= _PAGE_ENC; 379 380 if (kernel_map_pages_in_pgd(pgd, 0x0, 0x0, 1, pf)) { 381 pr_err("Failed to create 1:1 mapping for the first page!\n"); 382 return 1; 383 } 384 385 /* 386 * When making calls to the firmware everything needs to be 1:1 387 * mapped and addressable with 32-bit pointers. Map the kernel 388 * text and allocate a new stack because we can't rely on the 389 * stack pointer being < 4GB. 390 */ 391 if (!IS_ENABLED(CONFIG_EFI_MIXED) || efi_is_native()) 392 return 0; 393 394 page = alloc_page(GFP_KERNEL|__GFP_DMA32); 395 if (!page) 396 panic("Unable to allocate EFI runtime stack < 4GB\n"); 397 398 efi_scratch.phys_stack = virt_to_phys(page_address(page)); 399 efi_scratch.phys_stack += PAGE_SIZE; /* stack grows down */ 400 401 npages = (_etext - _text) >> PAGE_SHIFT; 402 text = __pa(_text); 403 pfn = text >> PAGE_SHIFT; 404 405 pf = _PAGE_RW | _PAGE_ENC; 406 if (kernel_map_pages_in_pgd(pgd, pfn, text, npages, pf)) { 407 pr_err("Failed to map kernel text 1:1\n"); 408 return 1; 409 } 410 411 return 0; 412 } 413 414 static void __init __map_region(efi_memory_desc_t *md, u64 va) 415 { 416 unsigned long flags = _PAGE_RW; 417 unsigned long pfn; 418 pgd_t *pgd = efi_mm.pgd; 419 420 if (!(md->attribute & EFI_MEMORY_WB)) 421 flags |= _PAGE_PCD; 422 423 if (sev_active() && md->type != EFI_MEMORY_MAPPED_IO) 424 flags |= _PAGE_ENC; 425 426 pfn = md->phys_addr >> PAGE_SHIFT; 427 if (kernel_map_pages_in_pgd(pgd, pfn, va, md->num_pages, flags)) 428 pr_warn("Error mapping PA 0x%llx -> VA 0x%llx!\n", 429 md->phys_addr, va); 430 } 431 432 void __init efi_map_region(efi_memory_desc_t *md) 433 { 434 unsigned long size = md->num_pages << PAGE_SHIFT; 435 u64 pa = md->phys_addr; 436 437 if (efi_enabled(EFI_OLD_MEMMAP)) 438 return old_map_region(md); 439 440 /* 441 * Make sure the 1:1 mappings are present as a catch-all for b0rked 442 * firmware which doesn't update all internal pointers after switching 443 * to virtual mode and would otherwise crap on us. 444 */ 445 __map_region(md, md->phys_addr); 446 447 /* 448 * Enforce the 1:1 mapping as the default virtual address when 449 * booting in EFI mixed mode, because even though we may be 450 * running a 64-bit kernel, the firmware may only be 32-bit. 451 */ 452 if (!efi_is_native () && IS_ENABLED(CONFIG_EFI_MIXED)) { 453 md->virt_addr = md->phys_addr; 454 return; 455 } 456 457 efi_va -= size; 458 459 /* Is PA 2M-aligned? */ 460 if (!(pa & (PMD_SIZE - 1))) { 461 efi_va &= PMD_MASK; 462 } else { 463 u64 pa_offset = pa & (PMD_SIZE - 1); 464 u64 prev_va = efi_va; 465 466 /* get us the same offset within this 2M page */ 467 efi_va = (efi_va & PMD_MASK) + pa_offset; 468 469 if (efi_va > prev_va) 470 efi_va -= PMD_SIZE; 471 } 472 473 if (efi_va < EFI_VA_END) { 474 pr_warn(FW_WARN "VA address range overflow!\n"); 475 return; 476 } 477 478 /* Do the VA map */ 479 __map_region(md, efi_va); 480 md->virt_addr = efi_va; 481 } 482 483 /* 484 * kexec kernel will use efi_map_region_fixed to map efi runtime memory ranges. 485 * md->virt_addr is the original virtual address which had been mapped in kexec 486 * 1st kernel. 487 */ 488 void __init efi_map_region_fixed(efi_memory_desc_t *md) 489 { 490 __map_region(md, md->phys_addr); 491 __map_region(md, md->virt_addr); 492 } 493 494 void __iomem *__init efi_ioremap(unsigned long phys_addr, unsigned long size, 495 u32 type, u64 attribute) 496 { 497 unsigned long last_map_pfn; 498 499 if (type == EFI_MEMORY_MAPPED_IO) 500 return ioremap(phys_addr, size); 501 502 last_map_pfn = init_memory_mapping(phys_addr, phys_addr + size); 503 if ((last_map_pfn << PAGE_SHIFT) < phys_addr + size) { 504 unsigned long top = last_map_pfn << PAGE_SHIFT; 505 efi_ioremap(top, size - (top - phys_addr), type, attribute); 506 } 507 508 if (!(attribute & EFI_MEMORY_WB)) 509 efi_memory_uc((u64)(unsigned long)__va(phys_addr), size); 510 511 return (void __iomem *)__va(phys_addr); 512 } 513 514 void __init parse_efi_setup(u64 phys_addr, u32 data_len) 515 { 516 efi_setup = phys_addr + sizeof(struct setup_data); 517 } 518 519 static int __init efi_update_mappings(efi_memory_desc_t *md, unsigned long pf) 520 { 521 unsigned long pfn; 522 pgd_t *pgd = efi_mm.pgd; 523 int err1, err2; 524 525 /* Update the 1:1 mapping */ 526 pfn = md->phys_addr >> PAGE_SHIFT; 527 err1 = kernel_map_pages_in_pgd(pgd, pfn, md->phys_addr, md->num_pages, pf); 528 if (err1) { 529 pr_err("Error while updating 1:1 mapping PA 0x%llx -> VA 0x%llx!\n", 530 md->phys_addr, md->virt_addr); 531 } 532 533 err2 = kernel_map_pages_in_pgd(pgd, pfn, md->virt_addr, md->num_pages, pf); 534 if (err2) { 535 pr_err("Error while updating VA mapping PA 0x%llx -> VA 0x%llx!\n", 536 md->phys_addr, md->virt_addr); 537 } 538 539 return err1 || err2; 540 } 541 542 static int __init efi_update_mem_attr(struct mm_struct *mm, efi_memory_desc_t *md) 543 { 544 unsigned long pf = 0; 545 546 if (md->attribute & EFI_MEMORY_XP) 547 pf |= _PAGE_NX; 548 549 if (!(md->attribute & EFI_MEMORY_RO)) 550 pf |= _PAGE_RW; 551 552 if (sev_active()) 553 pf |= _PAGE_ENC; 554 555 return efi_update_mappings(md, pf); 556 } 557 558 void __init efi_runtime_update_mappings(void) 559 { 560 efi_memory_desc_t *md; 561 562 if (efi_enabled(EFI_OLD_MEMMAP)) { 563 if (__supported_pte_mask & _PAGE_NX) 564 runtime_code_page_mkexec(); 565 return; 566 } 567 568 /* 569 * Use the EFI Memory Attribute Table for mapping permissions if it 570 * exists, since it is intended to supersede EFI_PROPERTIES_TABLE. 571 */ 572 if (efi_enabled(EFI_MEM_ATTR)) { 573 efi_memattr_apply_permissions(NULL, efi_update_mem_attr); 574 return; 575 } 576 577 /* 578 * EFI_MEMORY_ATTRIBUTES_TABLE is intended to replace 579 * EFI_PROPERTIES_TABLE. So, use EFI_PROPERTIES_TABLE to update 580 * permissions only if EFI_MEMORY_ATTRIBUTES_TABLE is not 581 * published by the firmware. Even if we find a buggy implementation of 582 * EFI_MEMORY_ATTRIBUTES_TABLE, don't fall back to 583 * EFI_PROPERTIES_TABLE, because of the same reason. 584 */ 585 586 if (!efi_enabled(EFI_NX_PE_DATA)) 587 return; 588 589 for_each_efi_memory_desc(md) { 590 unsigned long pf = 0; 591 592 if (!(md->attribute & EFI_MEMORY_RUNTIME)) 593 continue; 594 595 if (!(md->attribute & EFI_MEMORY_WB)) 596 pf |= _PAGE_PCD; 597 598 if ((md->attribute & EFI_MEMORY_XP) || 599 (md->type == EFI_RUNTIME_SERVICES_DATA)) 600 pf |= _PAGE_NX; 601 602 if (!(md->attribute & EFI_MEMORY_RO) && 603 (md->type != EFI_RUNTIME_SERVICES_CODE)) 604 pf |= _PAGE_RW; 605 606 if (sev_active()) 607 pf |= _PAGE_ENC; 608 609 efi_update_mappings(md, pf); 610 } 611 } 612 613 void __init efi_dump_pagetable(void) 614 { 615 #ifdef CONFIG_EFI_PGT_DUMP 616 if (efi_enabled(EFI_OLD_MEMMAP)) 617 ptdump_walk_pgd_level(NULL, swapper_pg_dir); 618 else 619 ptdump_walk_pgd_level(NULL, efi_mm.pgd); 620 #endif 621 } 622 623 /* 624 * Makes the calling thread switch to/from efi_mm context. Can be used 625 * in a kernel thread and user context. Preemption needs to remain disabled 626 * while the EFI-mm is borrowed. mmgrab()/mmdrop() is not used because the mm 627 * can not change under us. 628 * It should be ensured that there are no concurent calls to this function. 629 */ 630 void efi_switch_mm(struct mm_struct *mm) 631 { 632 efi_scratch.prev_mm = current->active_mm; 633 current->active_mm = mm; 634 switch_mm(efi_scratch.prev_mm, mm, NULL); 635 } 636 637 #ifdef CONFIG_EFI_MIXED 638 extern efi_status_t efi64_thunk(u32, ...); 639 640 static DEFINE_SPINLOCK(efi_runtime_lock); 641 642 #define runtime_service32(func) \ 643 ({ \ 644 u32 table = (u32)(unsigned long)efi.systab; \ 645 u32 *rt, *___f; \ 646 \ 647 rt = (u32 *)(table + offsetof(efi_system_table_32_t, runtime)); \ 648 ___f = (u32 *)(*rt + offsetof(efi_runtime_services_32_t, func)); \ 649 *___f; \ 650 }) 651 652 /* 653 * Switch to the EFI page tables early so that we can access the 1:1 654 * runtime services mappings which are not mapped in any other page 655 * tables. This function must be called before runtime_service32(). 656 * 657 * Also, disable interrupts because the IDT points to 64-bit handlers, 658 * which aren't going to function correctly when we switch to 32-bit. 659 */ 660 #define efi_thunk(f, ...) \ 661 ({ \ 662 efi_status_t __s; \ 663 u32 __func; \ 664 \ 665 arch_efi_call_virt_setup(); \ 666 \ 667 __func = runtime_service32(f); \ 668 __s = efi64_thunk(__func, __VA_ARGS__); \ 669 \ 670 arch_efi_call_virt_teardown(); \ 671 \ 672 __s; \ 673 }) 674 675 efi_status_t efi_thunk_set_virtual_address_map( 676 void *phys_set_virtual_address_map, 677 unsigned long memory_map_size, 678 unsigned long descriptor_size, 679 u32 descriptor_version, 680 efi_memory_desc_t *virtual_map) 681 { 682 efi_status_t status; 683 unsigned long flags; 684 u32 func; 685 686 efi_sync_low_kernel_mappings(); 687 local_irq_save(flags); 688 689 efi_switch_mm(&efi_mm); 690 691 func = (u32)(unsigned long)phys_set_virtual_address_map; 692 status = efi64_thunk(func, memory_map_size, descriptor_size, 693 descriptor_version, virtual_map); 694 695 efi_switch_mm(efi_scratch.prev_mm); 696 local_irq_restore(flags); 697 698 return status; 699 } 700 701 static efi_status_t efi_thunk_get_time(efi_time_t *tm, efi_time_cap_t *tc) 702 { 703 efi_status_t status; 704 u32 phys_tm, phys_tc; 705 unsigned long flags; 706 707 spin_lock(&rtc_lock); 708 spin_lock_irqsave(&efi_runtime_lock, flags); 709 710 phys_tm = virt_to_phys_or_null(tm); 711 phys_tc = virt_to_phys_or_null(tc); 712 713 status = efi_thunk(get_time, phys_tm, phys_tc); 714 715 spin_unlock_irqrestore(&efi_runtime_lock, flags); 716 spin_unlock(&rtc_lock); 717 718 return status; 719 } 720 721 static efi_status_t efi_thunk_set_time(efi_time_t *tm) 722 { 723 efi_status_t status; 724 u32 phys_tm; 725 unsigned long flags; 726 727 spin_lock(&rtc_lock); 728 spin_lock_irqsave(&efi_runtime_lock, flags); 729 730 phys_tm = virt_to_phys_or_null(tm); 731 732 status = efi_thunk(set_time, phys_tm); 733 734 spin_unlock_irqrestore(&efi_runtime_lock, flags); 735 spin_unlock(&rtc_lock); 736 737 return status; 738 } 739 740 static efi_status_t 741 efi_thunk_get_wakeup_time(efi_bool_t *enabled, efi_bool_t *pending, 742 efi_time_t *tm) 743 { 744 efi_status_t status; 745 u32 phys_enabled, phys_pending, phys_tm; 746 unsigned long flags; 747 748 spin_lock(&rtc_lock); 749 spin_lock_irqsave(&efi_runtime_lock, flags); 750 751 phys_enabled = virt_to_phys_or_null(enabled); 752 phys_pending = virt_to_phys_or_null(pending); 753 phys_tm = virt_to_phys_or_null(tm); 754 755 status = efi_thunk(get_wakeup_time, phys_enabled, 756 phys_pending, phys_tm); 757 758 spin_unlock_irqrestore(&efi_runtime_lock, flags); 759 spin_unlock(&rtc_lock); 760 761 return status; 762 } 763 764 static efi_status_t 765 efi_thunk_set_wakeup_time(efi_bool_t enabled, efi_time_t *tm) 766 { 767 efi_status_t status; 768 u32 phys_tm; 769 unsigned long flags; 770 771 spin_lock(&rtc_lock); 772 spin_lock_irqsave(&efi_runtime_lock, flags); 773 774 phys_tm = virt_to_phys_or_null(tm); 775 776 status = efi_thunk(set_wakeup_time, enabled, phys_tm); 777 778 spin_unlock_irqrestore(&efi_runtime_lock, flags); 779 spin_unlock(&rtc_lock); 780 781 return status; 782 } 783 784 static unsigned long efi_name_size(efi_char16_t *name) 785 { 786 return ucs2_strsize(name, EFI_VAR_NAME_LEN) + 1; 787 } 788 789 static efi_status_t 790 efi_thunk_get_variable(efi_char16_t *name, efi_guid_t *vendor, 791 u32 *attr, unsigned long *data_size, void *data) 792 { 793 efi_status_t status; 794 u32 phys_name, phys_vendor, phys_attr; 795 u32 phys_data_size, phys_data; 796 unsigned long flags; 797 798 spin_lock_irqsave(&efi_runtime_lock, flags); 799 800 phys_data_size = virt_to_phys_or_null(data_size); 801 phys_vendor = virt_to_phys_or_null(vendor); 802 phys_name = virt_to_phys_or_null_size(name, efi_name_size(name)); 803 phys_attr = virt_to_phys_or_null(attr); 804 phys_data = virt_to_phys_or_null_size(data, *data_size); 805 806 status = efi_thunk(get_variable, phys_name, phys_vendor, 807 phys_attr, phys_data_size, phys_data); 808 809 spin_unlock_irqrestore(&efi_runtime_lock, flags); 810 811 return status; 812 } 813 814 static efi_status_t 815 efi_thunk_set_variable(efi_char16_t *name, efi_guid_t *vendor, 816 u32 attr, unsigned long data_size, void *data) 817 { 818 u32 phys_name, phys_vendor, phys_data; 819 efi_status_t status; 820 unsigned long flags; 821 822 spin_lock_irqsave(&efi_runtime_lock, flags); 823 824 phys_name = virt_to_phys_or_null_size(name, efi_name_size(name)); 825 phys_vendor = virt_to_phys_or_null(vendor); 826 phys_data = virt_to_phys_or_null_size(data, data_size); 827 828 /* If data_size is > sizeof(u32) we've got problems */ 829 status = efi_thunk(set_variable, phys_name, phys_vendor, 830 attr, data_size, phys_data); 831 832 spin_unlock_irqrestore(&efi_runtime_lock, flags); 833 834 return status; 835 } 836 837 static efi_status_t 838 efi_thunk_set_variable_nonblocking(efi_char16_t *name, efi_guid_t *vendor, 839 u32 attr, unsigned long data_size, 840 void *data) 841 { 842 u32 phys_name, phys_vendor, phys_data; 843 efi_status_t status; 844 unsigned long flags; 845 846 if (!spin_trylock_irqsave(&efi_runtime_lock, flags)) 847 return EFI_NOT_READY; 848 849 phys_name = virt_to_phys_or_null_size(name, efi_name_size(name)); 850 phys_vendor = virt_to_phys_or_null(vendor); 851 phys_data = virt_to_phys_or_null_size(data, data_size); 852 853 /* If data_size is > sizeof(u32) we've got problems */ 854 status = efi_thunk(set_variable, phys_name, phys_vendor, 855 attr, data_size, phys_data); 856 857 spin_unlock_irqrestore(&efi_runtime_lock, flags); 858 859 return status; 860 } 861 862 static efi_status_t 863 efi_thunk_get_next_variable(unsigned long *name_size, 864 efi_char16_t *name, 865 efi_guid_t *vendor) 866 { 867 efi_status_t status; 868 u32 phys_name_size, phys_name, phys_vendor; 869 unsigned long flags; 870 871 spin_lock_irqsave(&efi_runtime_lock, flags); 872 873 phys_name_size = virt_to_phys_or_null(name_size); 874 phys_vendor = virt_to_phys_or_null(vendor); 875 phys_name = virt_to_phys_or_null_size(name, *name_size); 876 877 status = efi_thunk(get_next_variable, phys_name_size, 878 phys_name, phys_vendor); 879 880 spin_unlock_irqrestore(&efi_runtime_lock, flags); 881 882 return status; 883 } 884 885 static efi_status_t 886 efi_thunk_get_next_high_mono_count(u32 *count) 887 { 888 efi_status_t status; 889 u32 phys_count; 890 unsigned long flags; 891 892 spin_lock_irqsave(&efi_runtime_lock, flags); 893 894 phys_count = virt_to_phys_or_null(count); 895 status = efi_thunk(get_next_high_mono_count, phys_count); 896 897 spin_unlock_irqrestore(&efi_runtime_lock, flags); 898 899 return status; 900 } 901 902 static void 903 efi_thunk_reset_system(int reset_type, efi_status_t status, 904 unsigned long data_size, efi_char16_t *data) 905 { 906 u32 phys_data; 907 unsigned long flags; 908 909 spin_lock_irqsave(&efi_runtime_lock, flags); 910 911 phys_data = virt_to_phys_or_null_size(data, data_size); 912 913 efi_thunk(reset_system, reset_type, status, data_size, phys_data); 914 915 spin_unlock_irqrestore(&efi_runtime_lock, flags); 916 } 917 918 static efi_status_t 919 efi_thunk_update_capsule(efi_capsule_header_t **capsules, 920 unsigned long count, unsigned long sg_list) 921 { 922 /* 923 * To properly support this function we would need to repackage 924 * 'capsules' because the firmware doesn't understand 64-bit 925 * pointers. 926 */ 927 return EFI_UNSUPPORTED; 928 } 929 930 static efi_status_t 931 efi_thunk_query_variable_info(u32 attr, u64 *storage_space, 932 u64 *remaining_space, 933 u64 *max_variable_size) 934 { 935 efi_status_t status; 936 u32 phys_storage, phys_remaining, phys_max; 937 unsigned long flags; 938 939 if (efi.runtime_version < EFI_2_00_SYSTEM_TABLE_REVISION) 940 return EFI_UNSUPPORTED; 941 942 spin_lock_irqsave(&efi_runtime_lock, flags); 943 944 phys_storage = virt_to_phys_or_null(storage_space); 945 phys_remaining = virt_to_phys_or_null(remaining_space); 946 phys_max = virt_to_phys_or_null(max_variable_size); 947 948 status = efi_thunk(query_variable_info, attr, phys_storage, 949 phys_remaining, phys_max); 950 951 spin_unlock_irqrestore(&efi_runtime_lock, flags); 952 953 return status; 954 } 955 956 static efi_status_t 957 efi_thunk_query_variable_info_nonblocking(u32 attr, u64 *storage_space, 958 u64 *remaining_space, 959 u64 *max_variable_size) 960 { 961 efi_status_t status; 962 u32 phys_storage, phys_remaining, phys_max; 963 unsigned long flags; 964 965 if (efi.runtime_version < EFI_2_00_SYSTEM_TABLE_REVISION) 966 return EFI_UNSUPPORTED; 967 968 if (!spin_trylock_irqsave(&efi_runtime_lock, flags)) 969 return EFI_NOT_READY; 970 971 phys_storage = virt_to_phys_or_null(storage_space); 972 phys_remaining = virt_to_phys_or_null(remaining_space); 973 phys_max = virt_to_phys_or_null(max_variable_size); 974 975 status = efi_thunk(query_variable_info, attr, phys_storage, 976 phys_remaining, phys_max); 977 978 spin_unlock_irqrestore(&efi_runtime_lock, flags); 979 980 return status; 981 } 982 983 static efi_status_t 984 efi_thunk_query_capsule_caps(efi_capsule_header_t **capsules, 985 unsigned long count, u64 *max_size, 986 int *reset_type) 987 { 988 /* 989 * To properly support this function we would need to repackage 990 * 'capsules' because the firmware doesn't understand 64-bit 991 * pointers. 992 */ 993 return EFI_UNSUPPORTED; 994 } 995 996 void efi_thunk_runtime_setup(void) 997 { 998 efi.get_time = efi_thunk_get_time; 999 efi.set_time = efi_thunk_set_time; 1000 efi.get_wakeup_time = efi_thunk_get_wakeup_time; 1001 efi.set_wakeup_time = efi_thunk_set_wakeup_time; 1002 efi.get_variable = efi_thunk_get_variable; 1003 efi.get_next_variable = efi_thunk_get_next_variable; 1004 efi.set_variable = efi_thunk_set_variable; 1005 efi.set_variable_nonblocking = efi_thunk_set_variable_nonblocking; 1006 efi.get_next_high_mono_count = efi_thunk_get_next_high_mono_count; 1007 efi.reset_system = efi_thunk_reset_system; 1008 efi.query_variable_info = efi_thunk_query_variable_info; 1009 efi.query_variable_info_nonblocking = efi_thunk_query_variable_info_nonblocking; 1010 efi.update_capsule = efi_thunk_update_capsule; 1011 efi.query_capsule_caps = efi_thunk_query_capsule_caps; 1012 } 1013 #endif /* CONFIG_EFI_MIXED */ 1014