12053bc57SKirill A. Shutemov // SPDX-License-Identifier: GPL-2.0-only 22053bc57SKirill A. Shutemov 32053bc57SKirill A. Shutemov #include <linux/efi.h> 42053bc57SKirill A. Shutemov #include <linux/memblock.h> 52053bc57SKirill A. Shutemov #include <linux/spinlock.h> 62053bc57SKirill A. Shutemov #include <asm/unaccepted_memory.h> 72053bc57SKirill A. Shutemov 8*50e782a8SKirill A. Shutemov /* Protects unaccepted memory bitmap and accepting_list */ 92053bc57SKirill A. Shutemov static DEFINE_SPINLOCK(unaccepted_memory_lock); 102053bc57SKirill A. Shutemov 11*50e782a8SKirill A. Shutemov struct accept_range { 12*50e782a8SKirill A. Shutemov struct list_head list; 13*50e782a8SKirill A. Shutemov unsigned long start; 14*50e782a8SKirill A. Shutemov unsigned long end; 15*50e782a8SKirill A. Shutemov }; 16*50e782a8SKirill A. Shutemov 17*50e782a8SKirill A. Shutemov static LIST_HEAD(accepting_list); 18*50e782a8SKirill A. Shutemov 192053bc57SKirill A. Shutemov /* 202053bc57SKirill A. Shutemov * accept_memory() -- Consult bitmap and accept the memory if needed. 212053bc57SKirill A. Shutemov * 222053bc57SKirill A. Shutemov * Only memory that is explicitly marked as unaccepted in the bitmap requires 232053bc57SKirill A. Shutemov * an action. All the remaining memory is implicitly accepted and doesn't need 242053bc57SKirill A. Shutemov * acceptance. 252053bc57SKirill A. Shutemov * 262053bc57SKirill A. Shutemov * No need to accept: 272053bc57SKirill A. Shutemov * - anything if the system has no unaccepted table; 282053bc57SKirill A. Shutemov * - memory that is below phys_base; 292053bc57SKirill A. Shutemov * - memory that is above the memory that addressable by the bitmap; 302053bc57SKirill A. Shutemov */ 312053bc57SKirill A. Shutemov void accept_memory(phys_addr_t start, phys_addr_t end) 322053bc57SKirill A. Shutemov { 332053bc57SKirill A. Shutemov struct efi_unaccepted_memory *unaccepted; 342053bc57SKirill A. Shutemov unsigned long range_start, range_end; 35*50e782a8SKirill A. Shutemov struct accept_range range, *entry; 362053bc57SKirill A. Shutemov unsigned long flags; 372053bc57SKirill A. Shutemov u64 unit_size; 382053bc57SKirill A. Shutemov 392053bc57SKirill A. Shutemov unaccepted = efi_get_unaccepted_table(); 402053bc57SKirill A. Shutemov if (!unaccepted) 412053bc57SKirill A. Shutemov return; 422053bc57SKirill A. Shutemov 432053bc57SKirill A. Shutemov unit_size = unaccepted->unit_size; 442053bc57SKirill A. Shutemov 452053bc57SKirill A. Shutemov /* 462053bc57SKirill A. Shutemov * Only care for the part of the range that is represented 472053bc57SKirill A. Shutemov * in the bitmap. 482053bc57SKirill A. Shutemov */ 492053bc57SKirill A. Shutemov if (start < unaccepted->phys_base) 502053bc57SKirill A. Shutemov start = unaccepted->phys_base; 512053bc57SKirill A. Shutemov if (end < unaccepted->phys_base) 522053bc57SKirill A. Shutemov return; 532053bc57SKirill A. Shutemov 542053bc57SKirill A. Shutemov /* Translate to offsets from the beginning of the bitmap */ 552053bc57SKirill A. Shutemov start -= unaccepted->phys_base; 562053bc57SKirill A. Shutemov end -= unaccepted->phys_base; 572053bc57SKirill A. Shutemov 58c211c19eSKirill A. Shutemov /* 59c211c19eSKirill A. Shutemov * load_unaligned_zeropad() can lead to unwanted loads across page 60c211c19eSKirill A. Shutemov * boundaries. The unwanted loads are typically harmless. But, they 61c211c19eSKirill A. Shutemov * might be made to totally unrelated or even unmapped memory. 62c211c19eSKirill A. Shutemov * load_unaligned_zeropad() relies on exception fixup (#PF, #GP and now 63c211c19eSKirill A. Shutemov * #VE) to recover from these unwanted loads. 64c211c19eSKirill A. Shutemov * 65c211c19eSKirill A. Shutemov * But, this approach does not work for unaccepted memory. For TDX, a 66c211c19eSKirill A. Shutemov * load from unaccepted memory will not lead to a recoverable exception 67c211c19eSKirill A. Shutemov * within the guest. The guest will exit to the VMM where the only 68c211c19eSKirill A. Shutemov * recourse is to terminate the guest. 69c211c19eSKirill A. Shutemov * 70c211c19eSKirill A. Shutemov * There are two parts to fix this issue and comprehensively avoid 71c211c19eSKirill A. Shutemov * access to unaccepted memory. Together these ensure that an extra 72c211c19eSKirill A. Shutemov * "guard" page is accepted in addition to the memory that needs to be 73c211c19eSKirill A. Shutemov * used: 74c211c19eSKirill A. Shutemov * 75c211c19eSKirill A. Shutemov * 1. Implicitly extend the range_contains_unaccepted_memory(start, end) 76c211c19eSKirill A. Shutemov * checks up to end+unit_size if 'end' is aligned on a unit_size 77c211c19eSKirill A. Shutemov * boundary. 78c211c19eSKirill A. Shutemov * 79c211c19eSKirill A. Shutemov * 2. Implicitly extend accept_memory(start, end) to end+unit_size if 80c211c19eSKirill A. Shutemov * 'end' is aligned on a unit_size boundary. (immediately following 81c211c19eSKirill A. Shutemov * this comment) 82c211c19eSKirill A. Shutemov */ 83c211c19eSKirill A. Shutemov if (!(end % unit_size)) 84c211c19eSKirill A. Shutemov end += unit_size; 85c211c19eSKirill A. Shutemov 862053bc57SKirill A. Shutemov /* Make sure not to overrun the bitmap */ 872053bc57SKirill A. Shutemov if (end > unaccepted->size * unit_size * BITS_PER_BYTE) 882053bc57SKirill A. Shutemov end = unaccepted->size * unit_size * BITS_PER_BYTE; 892053bc57SKirill A. Shutemov 90*50e782a8SKirill A. Shutemov range.start = start / unit_size; 91*50e782a8SKirill A. Shutemov range.end = DIV_ROUND_UP(end, unit_size); 92*50e782a8SKirill A. Shutemov retry: 932053bc57SKirill A. Shutemov spin_lock_irqsave(&unaccepted_memory_lock, flags); 94*50e782a8SKirill A. Shutemov 95*50e782a8SKirill A. Shutemov /* 96*50e782a8SKirill A. Shutemov * Check if anybody works on accepting the same range of the memory. 97*50e782a8SKirill A. Shutemov * 98*50e782a8SKirill A. Shutemov * The check is done with unit_size granularity. It is crucial to catch 99*50e782a8SKirill A. Shutemov * all accept requests to the same unit_size block, even if they don't 100*50e782a8SKirill A. Shutemov * overlap on physical address level. 101*50e782a8SKirill A. Shutemov */ 102*50e782a8SKirill A. Shutemov list_for_each_entry(entry, &accepting_list, list) { 103*50e782a8SKirill A. Shutemov if (entry->end < range.start) 104*50e782a8SKirill A. Shutemov continue; 105*50e782a8SKirill A. Shutemov if (entry->start >= range.end) 106*50e782a8SKirill A. Shutemov continue; 107*50e782a8SKirill A. Shutemov 108*50e782a8SKirill A. Shutemov /* 109*50e782a8SKirill A. Shutemov * Somebody else accepting the range. Or at least part of it. 110*50e782a8SKirill A. Shutemov * 111*50e782a8SKirill A. Shutemov * Drop the lock and retry until it is complete. 112*50e782a8SKirill A. Shutemov */ 113*50e782a8SKirill A. Shutemov spin_unlock_irqrestore(&unaccepted_memory_lock, flags); 114*50e782a8SKirill A. Shutemov goto retry; 115*50e782a8SKirill A. Shutemov } 116*50e782a8SKirill A. Shutemov 117*50e782a8SKirill A. Shutemov /* 118*50e782a8SKirill A. Shutemov * Register that the range is about to be accepted. 119*50e782a8SKirill A. Shutemov * Make sure nobody else will accept it. 120*50e782a8SKirill A. Shutemov */ 121*50e782a8SKirill A. Shutemov list_add(&range.list, &accepting_list); 122*50e782a8SKirill A. Shutemov 123*50e782a8SKirill A. Shutemov range_start = range.start; 1242053bc57SKirill A. Shutemov for_each_set_bitrange_from(range_start, range_end, unaccepted->bitmap, 125*50e782a8SKirill A. Shutemov range.end) { 1262053bc57SKirill A. Shutemov unsigned long phys_start, phys_end; 1272053bc57SKirill A. Shutemov unsigned long len = range_end - range_start; 1282053bc57SKirill A. Shutemov 1292053bc57SKirill A. Shutemov phys_start = range_start * unit_size + unaccepted->phys_base; 1302053bc57SKirill A. Shutemov phys_end = range_end * unit_size + unaccepted->phys_base; 1312053bc57SKirill A. Shutemov 132*50e782a8SKirill A. Shutemov /* 133*50e782a8SKirill A. Shutemov * Keep interrupts disabled until the accept operation is 134*50e782a8SKirill A. Shutemov * complete in order to prevent deadlocks. 135*50e782a8SKirill A. Shutemov * 136*50e782a8SKirill A. Shutemov * Enabling interrupts before calling arch_accept_memory() 137*50e782a8SKirill A. Shutemov * creates an opportunity for an interrupt handler to request 138*50e782a8SKirill A. Shutemov * acceptance for the same memory. The handler will continuously 139*50e782a8SKirill A. Shutemov * spin with interrupts disabled, preventing other task from 140*50e782a8SKirill A. Shutemov * making progress with the acceptance process. 141*50e782a8SKirill A. Shutemov */ 142*50e782a8SKirill A. Shutemov spin_unlock(&unaccepted_memory_lock); 143*50e782a8SKirill A. Shutemov 1442053bc57SKirill A. Shutemov arch_accept_memory(phys_start, phys_end); 145*50e782a8SKirill A. Shutemov 146*50e782a8SKirill A. Shutemov spin_lock(&unaccepted_memory_lock); 1472053bc57SKirill A. Shutemov bitmap_clear(unaccepted->bitmap, range_start, len); 1482053bc57SKirill A. Shutemov } 149*50e782a8SKirill A. Shutemov 150*50e782a8SKirill A. Shutemov list_del(&range.list); 1512053bc57SKirill A. Shutemov spin_unlock_irqrestore(&unaccepted_memory_lock, flags); 1522053bc57SKirill A. Shutemov } 1532053bc57SKirill A. Shutemov 1542053bc57SKirill A. Shutemov bool range_contains_unaccepted_memory(phys_addr_t start, phys_addr_t end) 1552053bc57SKirill A. Shutemov { 1562053bc57SKirill A. Shutemov struct efi_unaccepted_memory *unaccepted; 1572053bc57SKirill A. Shutemov unsigned long flags; 1582053bc57SKirill A. Shutemov bool ret = false; 1592053bc57SKirill A. Shutemov u64 unit_size; 1602053bc57SKirill A. Shutemov 1612053bc57SKirill A. Shutemov unaccepted = efi_get_unaccepted_table(); 1622053bc57SKirill A. Shutemov if (!unaccepted) 1632053bc57SKirill A. Shutemov return false; 1642053bc57SKirill A. Shutemov 1652053bc57SKirill A. Shutemov unit_size = unaccepted->unit_size; 1662053bc57SKirill A. Shutemov 1672053bc57SKirill A. Shutemov /* 1682053bc57SKirill A. Shutemov * Only care for the part of the range that is represented 1692053bc57SKirill A. Shutemov * in the bitmap. 1702053bc57SKirill A. Shutemov */ 1712053bc57SKirill A. Shutemov if (start < unaccepted->phys_base) 1722053bc57SKirill A. Shutemov start = unaccepted->phys_base; 1732053bc57SKirill A. Shutemov if (end < unaccepted->phys_base) 1742053bc57SKirill A. Shutemov return false; 1752053bc57SKirill A. Shutemov 1762053bc57SKirill A. Shutemov /* Translate to offsets from the beginning of the bitmap */ 1772053bc57SKirill A. Shutemov start -= unaccepted->phys_base; 1782053bc57SKirill A. Shutemov end -= unaccepted->phys_base; 1792053bc57SKirill A. Shutemov 180c211c19eSKirill A. Shutemov /* 181c211c19eSKirill A. Shutemov * Also consider the unaccepted state of the *next* page. See fix #1 in 182c211c19eSKirill A. Shutemov * the comment on load_unaligned_zeropad() in accept_memory(). 183c211c19eSKirill A. Shutemov */ 184c211c19eSKirill A. Shutemov if (!(end % unit_size)) 185c211c19eSKirill A. Shutemov end += unit_size; 186c211c19eSKirill A. Shutemov 1872053bc57SKirill A. Shutemov /* Make sure not to overrun the bitmap */ 1882053bc57SKirill A. Shutemov if (end > unaccepted->size * unit_size * BITS_PER_BYTE) 1892053bc57SKirill A. Shutemov end = unaccepted->size * unit_size * BITS_PER_BYTE; 1902053bc57SKirill A. Shutemov 1912053bc57SKirill A. Shutemov spin_lock_irqsave(&unaccepted_memory_lock, flags); 1922053bc57SKirill A. Shutemov while (start < end) { 1932053bc57SKirill A. Shutemov if (test_bit(start / unit_size, unaccepted->bitmap)) { 1942053bc57SKirill A. Shutemov ret = true; 1952053bc57SKirill A. Shutemov break; 1962053bc57SKirill A. Shutemov } 1972053bc57SKirill A. Shutemov 1982053bc57SKirill A. Shutemov start += unit_size; 1992053bc57SKirill A. Shutemov } 2002053bc57SKirill A. Shutemov spin_unlock_irqrestore(&unaccepted_memory_lock, flags); 2012053bc57SKirill A. Shutemov 2022053bc57SKirill A. Shutemov return ret; 2032053bc57SKirill A. Shutemov } 204