1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2017 ARM Ltd. 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #include <linux/kvm_host.h> 8 #include <linux/random.h> 9 #include <linux/memblock.h> 10 #include <asm/alternative.h> 11 #include <asm/debug-monitors.h> 12 #include <asm/insn.h> 13 #include <asm/kvm_mmu.h> 14 15 /* 16 * The LSB of the HYP VA tag 17 */ 18 static u8 tag_lsb; 19 /* 20 * The HYP VA tag value with the region bit 21 */ 22 static u64 tag_val; 23 static u64 va_mask; 24 25 /* 26 * We want to generate a hyp VA with the following format (with V == 27 * vabits_actual): 28 * 29 * 63 ... V | V-1 | V-2 .. tag_lsb | tag_lsb - 1 .. 0 30 * --------------------------------------------------------- 31 * | 0000000 | hyp_va_msb | random tag | kern linear VA | 32 * |--------- tag_val -----------|----- va_mask ---| 33 * 34 * which does not conflict with the idmap regions. 35 */ 36 __init void kvm_compute_layout(void) 37 { 38 phys_addr_t idmap_addr = __pa_symbol(__hyp_idmap_text_start); 39 u64 hyp_va_msb; 40 41 /* Where is my RAM region? */ 42 hyp_va_msb = idmap_addr & BIT(vabits_actual - 1); 43 hyp_va_msb ^= BIT(vabits_actual - 1); 44 45 tag_lsb = fls64((u64)phys_to_virt(memblock_start_of_DRAM()) ^ 46 (u64)(high_memory - 1)); 47 48 va_mask = GENMASK_ULL(tag_lsb - 1, 0); 49 tag_val = hyp_va_msb; 50 51 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && tag_lsb != (vabits_actual - 1)) { 52 /* We have some free bits to insert a random tag. */ 53 tag_val |= get_random_long() & GENMASK_ULL(vabits_actual - 2, tag_lsb); 54 } 55 tag_val >>= tag_lsb; 56 } 57 58 static u32 compute_instruction(int n, u32 rd, u32 rn) 59 { 60 u32 insn = AARCH64_BREAK_FAULT; 61 62 switch (n) { 63 case 0: 64 insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_AND, 65 AARCH64_INSN_VARIANT_64BIT, 66 rn, rd, va_mask); 67 break; 68 69 case 1: 70 /* ROR is a variant of EXTR with Rm = Rn */ 71 insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT, 72 rn, rn, rd, 73 tag_lsb); 74 break; 75 76 case 2: 77 insn = aarch64_insn_gen_add_sub_imm(rd, rn, 78 tag_val & GENMASK(11, 0), 79 AARCH64_INSN_VARIANT_64BIT, 80 AARCH64_INSN_ADSB_ADD); 81 break; 82 83 case 3: 84 insn = aarch64_insn_gen_add_sub_imm(rd, rn, 85 tag_val & GENMASK(23, 12), 86 AARCH64_INSN_VARIANT_64BIT, 87 AARCH64_INSN_ADSB_ADD); 88 break; 89 90 case 4: 91 /* ROR is a variant of EXTR with Rm = Rn */ 92 insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT, 93 rn, rn, rd, 64 - tag_lsb); 94 break; 95 } 96 97 return insn; 98 } 99 100 void __init kvm_update_va_mask(struct alt_instr *alt, 101 __le32 *origptr, __le32 *updptr, int nr_inst) 102 { 103 int i; 104 105 BUG_ON(nr_inst != 5); 106 107 for (i = 0; i < nr_inst; i++) { 108 u32 rd, rn, insn, oinsn; 109 110 /* 111 * VHE doesn't need any address translation, let's NOP 112 * everything. 113 * 114 * Alternatively, if the tag is zero (because the layout 115 * dictates it and we don't have any spare bits in the 116 * address), NOP everything after masking the kernel VA. 117 */ 118 if (has_vhe() || (!tag_val && i > 0)) { 119 updptr[i] = cpu_to_le32(aarch64_insn_gen_nop()); 120 continue; 121 } 122 123 oinsn = le32_to_cpu(origptr[i]); 124 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn); 125 rn = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RN, oinsn); 126 127 insn = compute_instruction(i, rd, rn); 128 BUG_ON(insn == AARCH64_BREAK_FAULT); 129 130 updptr[i] = cpu_to_le32(insn); 131 } 132 } 133 134 void *__kvm_bp_vect_base; 135 int __kvm_harden_el2_vector_slot; 136 137 void kvm_patch_vector_branch(struct alt_instr *alt, 138 __le32 *origptr, __le32 *updptr, int nr_inst) 139 { 140 u64 addr; 141 u32 insn; 142 143 BUG_ON(nr_inst != 5); 144 145 if (has_vhe() || !cpus_have_const_cap(ARM64_HARDEN_EL2_VECTORS)) { 146 WARN_ON_ONCE(cpus_have_const_cap(ARM64_HARDEN_EL2_VECTORS)); 147 return; 148 } 149 150 /* 151 * Compute HYP VA by using the same computation as kern_hyp_va() 152 */ 153 addr = (uintptr_t)kvm_ksym_ref(__kvm_hyp_vector); 154 addr &= va_mask; 155 addr |= tag_val << tag_lsb; 156 157 /* Use PC[10:7] to branch to the same vector in KVM */ 158 addr |= ((u64)origptr & GENMASK_ULL(10, 7)); 159 160 /* 161 * Branch over the preamble in order to avoid the initial store on 162 * the stack (which we already perform in the hardening vectors). 163 */ 164 addr += KVM_VECTOR_PREAMBLE; 165 166 /* stp x0, x1, [sp, #-16]! */ 167 insn = aarch64_insn_gen_load_store_pair(AARCH64_INSN_REG_0, 168 AARCH64_INSN_REG_1, 169 AARCH64_INSN_REG_SP, 170 -16, 171 AARCH64_INSN_VARIANT_64BIT, 172 AARCH64_INSN_LDST_STORE_PAIR_PRE_INDEX); 173 *updptr++ = cpu_to_le32(insn); 174 175 /* movz x0, #(addr & 0xffff) */ 176 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0, 177 (u16)addr, 178 0, 179 AARCH64_INSN_VARIANT_64BIT, 180 AARCH64_INSN_MOVEWIDE_ZERO); 181 *updptr++ = cpu_to_le32(insn); 182 183 /* movk x0, #((addr >> 16) & 0xffff), lsl #16 */ 184 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0, 185 (u16)(addr >> 16), 186 16, 187 AARCH64_INSN_VARIANT_64BIT, 188 AARCH64_INSN_MOVEWIDE_KEEP); 189 *updptr++ = cpu_to_le32(insn); 190 191 /* movk x0, #((addr >> 32) & 0xffff), lsl #32 */ 192 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0, 193 (u16)(addr >> 32), 194 32, 195 AARCH64_INSN_VARIANT_64BIT, 196 AARCH64_INSN_MOVEWIDE_KEEP); 197 *updptr++ = cpu_to_le32(insn); 198 199 /* br x0 */ 200 insn = aarch64_insn_gen_branch_reg(AARCH64_INSN_REG_0, 201 AARCH64_INSN_BRANCH_NOLINK); 202 *updptr++ = cpu_to_le32(insn); 203 } 204