12053bc57SKirill A. Shutemov // SPDX-License-Identifier: GPL-2.0-only
22053bc57SKirill A. Shutemov
32053bc57SKirill A. Shutemov #include <linux/efi.h>
42053bc57SKirill A. Shutemov #include <linux/memblock.h>
52053bc57SKirill A. Shutemov #include <linux/spinlock.h>
62053bc57SKirill A. Shutemov #include <asm/unaccepted_memory.h>
72053bc57SKirill A. Shutemov
850e782a8SKirill A. Shutemov /* Protects unaccepted memory bitmap and accepting_list */
92053bc57SKirill A. Shutemov static DEFINE_SPINLOCK(unaccepted_memory_lock);
102053bc57SKirill A. Shutemov
1150e782a8SKirill A. Shutemov struct accept_range {
1250e782a8SKirill A. Shutemov struct list_head list;
1350e782a8SKirill A. Shutemov unsigned long start;
1450e782a8SKirill A. Shutemov unsigned long end;
1550e782a8SKirill A. Shutemov };
1650e782a8SKirill A. Shutemov
1750e782a8SKirill A. Shutemov static LIST_HEAD(accepting_list);
1850e782a8SKirill A. Shutemov
192053bc57SKirill A. Shutemov /*
202053bc57SKirill A. Shutemov * accept_memory() -- Consult bitmap and accept the memory if needed.
212053bc57SKirill A. Shutemov *
222053bc57SKirill A. Shutemov * Only memory that is explicitly marked as unaccepted in the bitmap requires
232053bc57SKirill A. Shutemov * an action. All the remaining memory is implicitly accepted and doesn't need
242053bc57SKirill A. Shutemov * acceptance.
252053bc57SKirill A. Shutemov *
262053bc57SKirill A. Shutemov * No need to accept:
272053bc57SKirill A. Shutemov * - anything if the system has no unaccepted table;
282053bc57SKirill A. Shutemov * - memory that is below phys_base;
292053bc57SKirill A. Shutemov * - memory that is above the memory that addressable by the bitmap;
302053bc57SKirill A. Shutemov */
accept_memory(phys_addr_t start,phys_addr_t end)312053bc57SKirill A. Shutemov void accept_memory(phys_addr_t start, phys_addr_t end)
322053bc57SKirill A. Shutemov {
332053bc57SKirill A. Shutemov struct efi_unaccepted_memory *unaccepted;
342053bc57SKirill A. Shutemov unsigned long range_start, range_end;
3550e782a8SKirill A. Shutemov struct accept_range range, *entry;
362053bc57SKirill A. Shutemov unsigned long flags;
372053bc57SKirill A. Shutemov u64 unit_size;
382053bc57SKirill A. Shutemov
392053bc57SKirill A. Shutemov unaccepted = efi_get_unaccepted_table();
402053bc57SKirill A. Shutemov if (!unaccepted)
412053bc57SKirill A. Shutemov return;
422053bc57SKirill A. Shutemov
432053bc57SKirill A. Shutemov unit_size = unaccepted->unit_size;
442053bc57SKirill A. Shutemov
452053bc57SKirill A. Shutemov /*
462053bc57SKirill A. Shutemov * Only care for the part of the range that is represented
472053bc57SKirill A. Shutemov * in the bitmap.
482053bc57SKirill A. Shutemov */
492053bc57SKirill A. Shutemov if (start < unaccepted->phys_base)
502053bc57SKirill A. Shutemov start = unaccepted->phys_base;
512053bc57SKirill A. Shutemov if (end < unaccepted->phys_base)
522053bc57SKirill A. Shutemov return;
532053bc57SKirill A. Shutemov
542053bc57SKirill A. Shutemov /* Translate to offsets from the beginning of the bitmap */
552053bc57SKirill A. Shutemov start -= unaccepted->phys_base;
562053bc57SKirill A. Shutemov end -= unaccepted->phys_base;
572053bc57SKirill A. Shutemov
58c211c19eSKirill A. Shutemov /*
59c211c19eSKirill A. Shutemov * load_unaligned_zeropad() can lead to unwanted loads across page
60c211c19eSKirill A. Shutemov * boundaries. The unwanted loads are typically harmless. But, they
61c211c19eSKirill A. Shutemov * might be made to totally unrelated or even unmapped memory.
62c211c19eSKirill A. Shutemov * load_unaligned_zeropad() relies on exception fixup (#PF, #GP and now
63c211c19eSKirill A. Shutemov * #VE) to recover from these unwanted loads.
64c211c19eSKirill A. Shutemov *
65c211c19eSKirill A. Shutemov * But, this approach does not work for unaccepted memory. For TDX, a
66c211c19eSKirill A. Shutemov * load from unaccepted memory will not lead to a recoverable exception
67c211c19eSKirill A. Shutemov * within the guest. The guest will exit to the VMM where the only
68c211c19eSKirill A. Shutemov * recourse is to terminate the guest.
69c211c19eSKirill A. Shutemov *
70c211c19eSKirill A. Shutemov * There are two parts to fix this issue and comprehensively avoid
71c211c19eSKirill A. Shutemov * access to unaccepted memory. Together these ensure that an extra
72c211c19eSKirill A. Shutemov * "guard" page is accepted in addition to the memory that needs to be
73c211c19eSKirill A. Shutemov * used:
74c211c19eSKirill A. Shutemov *
75c211c19eSKirill A. Shutemov * 1. Implicitly extend the range_contains_unaccepted_memory(start, end)
76c211c19eSKirill A. Shutemov * checks up to end+unit_size if 'end' is aligned on a unit_size
77c211c19eSKirill A. Shutemov * boundary.
78c211c19eSKirill A. Shutemov *
79c211c19eSKirill A. Shutemov * 2. Implicitly extend accept_memory(start, end) to end+unit_size if
80c211c19eSKirill A. Shutemov * 'end' is aligned on a unit_size boundary. (immediately following
81c211c19eSKirill A. Shutemov * this comment)
82c211c19eSKirill A. Shutemov */
83c211c19eSKirill A. Shutemov if (!(end % unit_size))
84c211c19eSKirill A. Shutemov end += unit_size;
85c211c19eSKirill A. Shutemov
862053bc57SKirill A. Shutemov /* Make sure not to overrun the bitmap */
872053bc57SKirill A. Shutemov if (end > unaccepted->size * unit_size * BITS_PER_BYTE)
882053bc57SKirill A. Shutemov end = unaccepted->size * unit_size * BITS_PER_BYTE;
892053bc57SKirill A. Shutemov
9050e782a8SKirill A. Shutemov range.start = start / unit_size;
9150e782a8SKirill A. Shutemov range.end = DIV_ROUND_UP(end, unit_size);
9250e782a8SKirill A. Shutemov retry:
932053bc57SKirill A. Shutemov spin_lock_irqsave(&unaccepted_memory_lock, flags);
9450e782a8SKirill A. Shutemov
9550e782a8SKirill A. Shutemov /*
9650e782a8SKirill A. Shutemov * Check if anybody works on accepting the same range of the memory.
9750e782a8SKirill A. Shutemov *
9850e782a8SKirill A. Shutemov * The check is done with unit_size granularity. It is crucial to catch
9950e782a8SKirill A. Shutemov * all accept requests to the same unit_size block, even if they don't
10050e782a8SKirill A. Shutemov * overlap on physical address level.
10150e782a8SKirill A. Shutemov */
10250e782a8SKirill A. Shutemov list_for_each_entry(entry, &accepting_list, list) {
103*589959bfSMichael Roth if (entry->end <= range.start)
10450e782a8SKirill A. Shutemov continue;
10550e782a8SKirill A. Shutemov if (entry->start >= range.end)
10650e782a8SKirill A. Shutemov continue;
10750e782a8SKirill A. Shutemov
10850e782a8SKirill A. Shutemov /*
10950e782a8SKirill A. Shutemov * Somebody else accepting the range. Or at least part of it.
11050e782a8SKirill A. Shutemov *
11150e782a8SKirill A. Shutemov * Drop the lock and retry until it is complete.
11250e782a8SKirill A. Shutemov */
11350e782a8SKirill A. Shutemov spin_unlock_irqrestore(&unaccepted_memory_lock, flags);
11450e782a8SKirill A. Shutemov goto retry;
11550e782a8SKirill A. Shutemov }
11650e782a8SKirill A. Shutemov
11750e782a8SKirill A. Shutemov /*
11850e782a8SKirill A. Shutemov * Register that the range is about to be accepted.
11950e782a8SKirill A. Shutemov * Make sure nobody else will accept it.
12050e782a8SKirill A. Shutemov */
12150e782a8SKirill A. Shutemov list_add(&range.list, &accepting_list);
12250e782a8SKirill A. Shutemov
12350e782a8SKirill A. Shutemov range_start = range.start;
1242053bc57SKirill A. Shutemov for_each_set_bitrange_from(range_start, range_end, unaccepted->bitmap,
12550e782a8SKirill A. Shutemov range.end) {
1262053bc57SKirill A. Shutemov unsigned long phys_start, phys_end;
1272053bc57SKirill A. Shutemov unsigned long len = range_end - range_start;
1282053bc57SKirill A. Shutemov
1292053bc57SKirill A. Shutemov phys_start = range_start * unit_size + unaccepted->phys_base;
1302053bc57SKirill A. Shutemov phys_end = range_end * unit_size + unaccepted->phys_base;
1312053bc57SKirill A. Shutemov
13250e782a8SKirill A. Shutemov /*
13350e782a8SKirill A. Shutemov * Keep interrupts disabled until the accept operation is
13450e782a8SKirill A. Shutemov * complete in order to prevent deadlocks.
13550e782a8SKirill A. Shutemov *
13650e782a8SKirill A. Shutemov * Enabling interrupts before calling arch_accept_memory()
13750e782a8SKirill A. Shutemov * creates an opportunity for an interrupt handler to request
13850e782a8SKirill A. Shutemov * acceptance for the same memory. The handler will continuously
13950e782a8SKirill A. Shutemov * spin with interrupts disabled, preventing other task from
14050e782a8SKirill A. Shutemov * making progress with the acceptance process.
14150e782a8SKirill A. Shutemov */
14250e782a8SKirill A. Shutemov spin_unlock(&unaccepted_memory_lock);
14350e782a8SKirill A. Shutemov
1442053bc57SKirill A. Shutemov arch_accept_memory(phys_start, phys_end);
14550e782a8SKirill A. Shutemov
14650e782a8SKirill A. Shutemov spin_lock(&unaccepted_memory_lock);
1472053bc57SKirill A. Shutemov bitmap_clear(unaccepted->bitmap, range_start, len);
1482053bc57SKirill A. Shutemov }
14950e782a8SKirill A. Shutemov
15050e782a8SKirill A. Shutemov list_del(&range.list);
1512053bc57SKirill A. Shutemov spin_unlock_irqrestore(&unaccepted_memory_lock, flags);
1522053bc57SKirill A. Shutemov }
1532053bc57SKirill A. Shutemov
range_contains_unaccepted_memory(phys_addr_t start,phys_addr_t end)1542053bc57SKirill A. Shutemov bool range_contains_unaccepted_memory(phys_addr_t start, phys_addr_t end)
1552053bc57SKirill A. Shutemov {
1562053bc57SKirill A. Shutemov struct efi_unaccepted_memory *unaccepted;
1572053bc57SKirill A. Shutemov unsigned long flags;
1582053bc57SKirill A. Shutemov bool ret = false;
1592053bc57SKirill A. Shutemov u64 unit_size;
1602053bc57SKirill A. Shutemov
1612053bc57SKirill A. Shutemov unaccepted = efi_get_unaccepted_table();
1622053bc57SKirill A. Shutemov if (!unaccepted)
1632053bc57SKirill A. Shutemov return false;
1642053bc57SKirill A. Shutemov
1652053bc57SKirill A. Shutemov unit_size = unaccepted->unit_size;
1662053bc57SKirill A. Shutemov
1672053bc57SKirill A. Shutemov /*
1682053bc57SKirill A. Shutemov * Only care for the part of the range that is represented
1692053bc57SKirill A. Shutemov * in the bitmap.
1702053bc57SKirill A. Shutemov */
1712053bc57SKirill A. Shutemov if (start < unaccepted->phys_base)
1722053bc57SKirill A. Shutemov start = unaccepted->phys_base;
1732053bc57SKirill A. Shutemov if (end < unaccepted->phys_base)
1742053bc57SKirill A. Shutemov return false;
1752053bc57SKirill A. Shutemov
1762053bc57SKirill A. Shutemov /* Translate to offsets from the beginning of the bitmap */
1772053bc57SKirill A. Shutemov start -= unaccepted->phys_base;
1782053bc57SKirill A. Shutemov end -= unaccepted->phys_base;
1792053bc57SKirill A. Shutemov
180c211c19eSKirill A. Shutemov /*
181c211c19eSKirill A. Shutemov * Also consider the unaccepted state of the *next* page. See fix #1 in
182c211c19eSKirill A. Shutemov * the comment on load_unaligned_zeropad() in accept_memory().
183c211c19eSKirill A. Shutemov */
184c211c19eSKirill A. Shutemov if (!(end % unit_size))
185c211c19eSKirill A. Shutemov end += unit_size;
186c211c19eSKirill A. Shutemov
1872053bc57SKirill A. Shutemov /* Make sure not to overrun the bitmap */
1882053bc57SKirill A. Shutemov if (end > unaccepted->size * unit_size * BITS_PER_BYTE)
1892053bc57SKirill A. Shutemov end = unaccepted->size * unit_size * BITS_PER_BYTE;
1902053bc57SKirill A. Shutemov
1912053bc57SKirill A. Shutemov spin_lock_irqsave(&unaccepted_memory_lock, flags);
1922053bc57SKirill A. Shutemov while (start < end) {
1932053bc57SKirill A. Shutemov if (test_bit(start / unit_size, unaccepted->bitmap)) {
1942053bc57SKirill A. Shutemov ret = true;
1952053bc57SKirill A. Shutemov break;
1962053bc57SKirill A. Shutemov }
1972053bc57SKirill A. Shutemov
1982053bc57SKirill A. Shutemov start += unit_size;
1992053bc57SKirill A. Shutemov }
2002053bc57SKirill A. Shutemov spin_unlock_irqrestore(&unaccepted_memory_lock, flags);
2012053bc57SKirill A. Shutemov
2022053bc57SKirill A. Shutemov return ret;
2032053bc57SKirill A. Shutemov }
204