1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2017 ARM Ltd. 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #include <linux/kvm_host.h> 8 #include <linux/random.h> 9 #include <linux/memblock.h> 10 #include <asm/alternative.h> 11 #include <asm/debug-monitors.h> 12 #include <asm/insn.h> 13 #include <asm/kvm_mmu.h> 14 #include <asm/memory.h> 15 16 /* 17 * The LSB of the HYP VA tag 18 */ 19 static u8 tag_lsb; 20 /* 21 * The HYP VA tag value with the region bit 22 */ 23 static u64 tag_val; 24 static u64 va_mask; 25 26 /* 27 * Compute HYP VA by using the same computation as kern_hyp_va(). 28 */ 29 static u64 __early_kern_hyp_va(u64 addr) 30 { 31 addr &= va_mask; 32 addr |= tag_val << tag_lsb; 33 return addr; 34 } 35 36 /* 37 * Store a hyp VA <-> PA offset into a hyp-owned variable. 38 */ 39 static void init_hyp_physvirt_offset(void) 40 { 41 extern s64 kvm_nvhe_sym(hyp_physvirt_offset); 42 u64 kern_va, hyp_va; 43 44 /* Compute the offset from the hyp VA and PA of a random symbol. */ 45 kern_va = (u64)kvm_ksym_ref(__hyp_text_start); 46 hyp_va = __early_kern_hyp_va(kern_va); 47 CHOOSE_NVHE_SYM(hyp_physvirt_offset) = (s64)__pa(kern_va) - (s64)hyp_va; 48 } 49 50 /* 51 * We want to generate a hyp VA with the following format (with V == 52 * vabits_actual): 53 * 54 * 63 ... V | V-1 | V-2 .. tag_lsb | tag_lsb - 1 .. 0 55 * --------------------------------------------------------- 56 * | 0000000 | hyp_va_msb | random tag | kern linear VA | 57 * |--------- tag_val -----------|----- va_mask ---| 58 * 59 * which does not conflict with the idmap regions. 60 */ 61 __init void kvm_compute_layout(void) 62 { 63 phys_addr_t idmap_addr = __pa_symbol(__hyp_idmap_text_start); 64 u64 hyp_va_msb; 65 66 /* Where is my RAM region? */ 67 hyp_va_msb = idmap_addr & BIT(vabits_actual - 1); 68 hyp_va_msb ^= BIT(vabits_actual - 1); 69 70 tag_lsb = fls64((u64)phys_to_virt(memblock_start_of_DRAM()) ^ 71 (u64)(high_memory - 1)); 72 73 va_mask = GENMASK_ULL(tag_lsb - 1, 0); 74 tag_val = hyp_va_msb; 75 76 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && tag_lsb != (vabits_actual - 1)) { 77 /* We have some free bits to insert a random tag. */ 78 tag_val |= get_random_long() & GENMASK_ULL(vabits_actual - 2, tag_lsb); 79 } 80 tag_val >>= tag_lsb; 81 82 init_hyp_physvirt_offset(); 83 } 84 85 static u32 compute_instruction(int n, u32 rd, u32 rn) 86 { 87 u32 insn = AARCH64_BREAK_FAULT; 88 89 switch (n) { 90 case 0: 91 insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_AND, 92 AARCH64_INSN_VARIANT_64BIT, 93 rn, rd, va_mask); 94 break; 95 96 case 1: 97 /* ROR is a variant of EXTR with Rm = Rn */ 98 insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT, 99 rn, rn, rd, 100 tag_lsb); 101 break; 102 103 case 2: 104 insn = aarch64_insn_gen_add_sub_imm(rd, rn, 105 tag_val & GENMASK(11, 0), 106 AARCH64_INSN_VARIANT_64BIT, 107 AARCH64_INSN_ADSB_ADD); 108 break; 109 110 case 3: 111 insn = aarch64_insn_gen_add_sub_imm(rd, rn, 112 tag_val & GENMASK(23, 12), 113 AARCH64_INSN_VARIANT_64BIT, 114 AARCH64_INSN_ADSB_ADD); 115 break; 116 117 case 4: 118 /* ROR is a variant of EXTR with Rm = Rn */ 119 insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT, 120 rn, rn, rd, 64 - tag_lsb); 121 break; 122 } 123 124 return insn; 125 } 126 127 void __init kvm_update_va_mask(struct alt_instr *alt, 128 __le32 *origptr, __le32 *updptr, int nr_inst) 129 { 130 int i; 131 132 BUG_ON(nr_inst != 5); 133 134 for (i = 0; i < nr_inst; i++) { 135 u32 rd, rn, insn, oinsn; 136 137 /* 138 * VHE doesn't need any address translation, let's NOP 139 * everything. 140 * 141 * Alternatively, if the tag is zero (because the layout 142 * dictates it and we don't have any spare bits in the 143 * address), NOP everything after masking the kernel VA. 144 */ 145 if (has_vhe() || (!tag_val && i > 0)) { 146 updptr[i] = cpu_to_le32(aarch64_insn_gen_nop()); 147 continue; 148 } 149 150 oinsn = le32_to_cpu(origptr[i]); 151 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn); 152 rn = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RN, oinsn); 153 154 insn = compute_instruction(i, rd, rn); 155 BUG_ON(insn == AARCH64_BREAK_FAULT); 156 157 updptr[i] = cpu_to_le32(insn); 158 } 159 } 160 161 void kvm_patch_vector_branch(struct alt_instr *alt, 162 __le32 *origptr, __le32 *updptr, int nr_inst) 163 { 164 u64 addr; 165 u32 insn; 166 167 BUG_ON(nr_inst != 4); 168 169 if (!cpus_have_const_cap(ARM64_SPECTRE_V3A) || WARN_ON_ONCE(has_vhe())) 170 return; 171 172 /* 173 * Compute HYP VA by using the same computation as kern_hyp_va() 174 */ 175 addr = __early_kern_hyp_va((u64)kvm_ksym_ref(__kvm_hyp_vector)); 176 177 /* Use PC[10:7] to branch to the same vector in KVM */ 178 addr |= ((u64)origptr & GENMASK_ULL(10, 7)); 179 180 /* 181 * Branch over the preamble in order to avoid the initial store on 182 * the stack (which we already perform in the hardening vectors). 183 */ 184 addr += KVM_VECTOR_PREAMBLE; 185 186 /* movz x0, #(addr & 0xffff) */ 187 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0, 188 (u16)addr, 189 0, 190 AARCH64_INSN_VARIANT_64BIT, 191 AARCH64_INSN_MOVEWIDE_ZERO); 192 *updptr++ = cpu_to_le32(insn); 193 194 /* movk x0, #((addr >> 16) & 0xffff), lsl #16 */ 195 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0, 196 (u16)(addr >> 16), 197 16, 198 AARCH64_INSN_VARIANT_64BIT, 199 AARCH64_INSN_MOVEWIDE_KEEP); 200 *updptr++ = cpu_to_le32(insn); 201 202 /* movk x0, #((addr >> 32) & 0xffff), lsl #32 */ 203 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0, 204 (u16)(addr >> 32), 205 32, 206 AARCH64_INSN_VARIANT_64BIT, 207 AARCH64_INSN_MOVEWIDE_KEEP); 208 *updptr++ = cpu_to_le32(insn); 209 210 /* br x0 */ 211 insn = aarch64_insn_gen_branch_reg(AARCH64_INSN_REG_0, 212 AARCH64_INSN_BRANCH_NOLINK); 213 *updptr++ = cpu_to_le32(insn); 214 } 215 216 static void generate_mov_q(u64 val, __le32 *origptr, __le32 *updptr, int nr_inst) 217 { 218 u32 insn, oinsn, rd; 219 220 BUG_ON(nr_inst != 4); 221 222 /* Compute target register */ 223 oinsn = le32_to_cpu(*origptr); 224 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn); 225 226 /* movz rd, #(val & 0xffff) */ 227 insn = aarch64_insn_gen_movewide(rd, 228 (u16)val, 229 0, 230 AARCH64_INSN_VARIANT_64BIT, 231 AARCH64_INSN_MOVEWIDE_ZERO); 232 *updptr++ = cpu_to_le32(insn); 233 234 /* movk rd, #((val >> 16) & 0xffff), lsl #16 */ 235 insn = aarch64_insn_gen_movewide(rd, 236 (u16)(val >> 16), 237 16, 238 AARCH64_INSN_VARIANT_64BIT, 239 AARCH64_INSN_MOVEWIDE_KEEP); 240 *updptr++ = cpu_to_le32(insn); 241 242 /* movk rd, #((val >> 32) & 0xffff), lsl #32 */ 243 insn = aarch64_insn_gen_movewide(rd, 244 (u16)(val >> 32), 245 32, 246 AARCH64_INSN_VARIANT_64BIT, 247 AARCH64_INSN_MOVEWIDE_KEEP); 248 *updptr++ = cpu_to_le32(insn); 249 250 /* movk rd, #((val >> 48) & 0xffff), lsl #48 */ 251 insn = aarch64_insn_gen_movewide(rd, 252 (u16)(val >> 48), 253 48, 254 AARCH64_INSN_VARIANT_64BIT, 255 AARCH64_INSN_MOVEWIDE_KEEP); 256 *updptr++ = cpu_to_le32(insn); 257 } 258 259 void kvm_update_kimg_phys_offset(struct alt_instr *alt, 260 __le32 *origptr, __le32 *updptr, int nr_inst) 261 { 262 generate_mov_q(kimage_voffset + PHYS_OFFSET, origptr, updptr, nr_inst); 263 } 264 265 void kvm_get_kimage_voffset(struct alt_instr *alt, 266 __le32 *origptr, __le32 *updptr, int nr_inst) 267 { 268 generate_mov_q(kimage_voffset, origptr, updptr, nr_inst); 269 } 270