1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * alternative runtime patching 4 * inspired by the x86 version 5 * 6 * Copyright (C) 2014 ARM Ltd. 7 */ 8 9 #define pr_fmt(fmt) "alternatives: " fmt 10 11 #include <linux/init.h> 12 #include <linux/cpu.h> 13 #include <asm/cacheflush.h> 14 #include <asm/alternative.h> 15 #include <asm/cpufeature.h> 16 #include <asm/insn.h> 17 #include <asm/sections.h> 18 #include <linux/stop_machine.h> 19 20 #define __ALT_PTR(a,f) ((void *)&(a)->f + (a)->f) 21 #define ALT_ORIG_PTR(a) __ALT_PTR(a, orig_offset) 22 #define ALT_REPL_PTR(a) __ALT_PTR(a, alt_offset) 23 24 static int all_alternatives_applied; 25 26 static DECLARE_BITMAP(applied_alternatives, ARM64_NCAPS); 27 28 struct alt_region { 29 struct alt_instr *begin; 30 struct alt_instr *end; 31 }; 32 33 bool alternative_is_applied(u16 cpufeature) 34 { 35 if (WARN_ON(cpufeature >= ARM64_NCAPS)) 36 return false; 37 38 return test_bit(cpufeature, applied_alternatives); 39 } 40 41 /* 42 * Check if the target PC is within an alternative block. 43 */ 44 static bool branch_insn_requires_update(struct alt_instr *alt, unsigned long pc) 45 { 46 unsigned long replptr; 47 48 if (kernel_text_address(pc)) 49 return true; 50 51 replptr = (unsigned long)ALT_REPL_PTR(alt); 52 if (pc >= replptr && pc <= (replptr + alt->alt_len)) 53 return false; 54 55 /* 56 * Branching into *another* alternate sequence is doomed, and 57 * we're not even trying to fix it up. 58 */ 59 BUG(); 60 } 61 62 #define align_down(x, a) ((unsigned long)(x) & ~(((unsigned long)(a)) - 1)) 63 64 static u32 get_alt_insn(struct alt_instr *alt, __le32 *insnptr, __le32 *altinsnptr) 65 { 66 u32 insn; 67 68 insn = le32_to_cpu(*altinsnptr); 69 70 if (aarch64_insn_is_branch_imm(insn)) { 71 s32 offset = aarch64_get_branch_offset(insn); 72 unsigned long target; 73 74 target = (unsigned long)altinsnptr + offset; 75 76 /* 77 * If we're branching inside the alternate sequence, 78 * do not rewrite the instruction, as it is already 79 * correct. Otherwise, generate the new instruction. 80 */ 81 if (branch_insn_requires_update(alt, target)) { 82 offset = target - (unsigned long)insnptr; 83 insn = aarch64_set_branch_offset(insn, offset); 84 } 85 } else if (aarch64_insn_is_adrp(insn)) { 86 s32 orig_offset, new_offset; 87 unsigned long target; 88 89 /* 90 * If we're replacing an adrp instruction, which uses PC-relative 91 * immediate addressing, adjust the offset to reflect the new 92 * PC. adrp operates on 4K aligned addresses. 93 */ 94 orig_offset = aarch64_insn_adrp_get_offset(insn); 95 target = align_down(altinsnptr, SZ_4K) + orig_offset; 96 new_offset = target - align_down(insnptr, SZ_4K); 97 insn = aarch64_insn_adrp_set_offset(insn, new_offset); 98 } else if (aarch64_insn_uses_literal(insn)) { 99 /* 100 * Disallow patching unhandled instructions using PC relative 101 * literal addresses 102 */ 103 BUG(); 104 } 105 106 return insn; 107 } 108 109 static void patch_alternative(struct alt_instr *alt, 110 __le32 *origptr, __le32 *updptr, int nr_inst) 111 { 112 __le32 *replptr; 113 int i; 114 115 replptr = ALT_REPL_PTR(alt); 116 for (i = 0; i < nr_inst; i++) { 117 u32 insn; 118 119 insn = get_alt_insn(alt, origptr + i, replptr + i); 120 updptr[i] = cpu_to_le32(insn); 121 } 122 } 123 124 /* 125 * We provide our own, private D-cache cleaning function so that we don't 126 * accidentally call into the cache.S code, which is patched by us at 127 * runtime. 128 */ 129 static void clean_dcache_range_nopatch(u64 start, u64 end) 130 { 131 u64 cur, d_size, ctr_el0; 132 133 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0); 134 d_size = 4 << cpuid_feature_extract_unsigned_field(ctr_el0, 135 CTR_DMINLINE_SHIFT); 136 cur = start & ~(d_size - 1); 137 do { 138 /* 139 * We must clean+invalidate to the PoC in order to avoid 140 * Cortex-A53 errata 826319, 827319, 824069 and 819472 141 * (this corresponds to ARM64_WORKAROUND_CLEAN_CACHE) 142 */ 143 asm volatile("dc civac, %0" : : "r" (cur) : "memory"); 144 } while (cur += d_size, cur < end); 145 } 146 147 static void __apply_alternatives(void *alt_region, bool is_module, 148 unsigned long *feature_mask) 149 { 150 struct alt_instr *alt; 151 struct alt_region *region = alt_region; 152 __le32 *origptr, *updptr; 153 alternative_cb_t alt_cb; 154 155 for (alt = region->begin; alt < region->end; alt++) { 156 int nr_inst; 157 158 if (!test_bit(alt->cpufeature, feature_mask)) 159 continue; 160 161 /* Use ARM64_CB_PATCH as an unconditional patch */ 162 if (alt->cpufeature < ARM64_CB_PATCH && 163 !cpus_have_cap(alt->cpufeature)) 164 continue; 165 166 if (alt->cpufeature == ARM64_CB_PATCH) 167 BUG_ON(alt->alt_len != 0); 168 else 169 BUG_ON(alt->alt_len != alt->orig_len); 170 171 pr_info_once("patching kernel code\n"); 172 173 origptr = ALT_ORIG_PTR(alt); 174 updptr = is_module ? origptr : lm_alias(origptr); 175 nr_inst = alt->orig_len / AARCH64_INSN_SIZE; 176 177 if (alt->cpufeature < ARM64_CB_PATCH) 178 alt_cb = patch_alternative; 179 else 180 alt_cb = ALT_REPL_PTR(alt); 181 182 alt_cb(alt, origptr, updptr, nr_inst); 183 184 if (!is_module) { 185 clean_dcache_range_nopatch((u64)origptr, 186 (u64)(origptr + nr_inst)); 187 } 188 } 189 190 /* 191 * The core module code takes care of cache maintenance in 192 * flush_module_icache(). 193 */ 194 if (!is_module) { 195 dsb(ish); 196 __flush_icache_all(); 197 isb(); 198 199 /* Ignore ARM64_CB bit from feature mask */ 200 bitmap_or(applied_alternatives, applied_alternatives, 201 feature_mask, ARM64_NCAPS); 202 bitmap_and(applied_alternatives, applied_alternatives, 203 cpu_hwcaps, ARM64_NCAPS); 204 } 205 } 206 207 /* 208 * We might be patching the stop_machine state machine, so implement a 209 * really simple polling protocol here. 210 */ 211 static int __apply_alternatives_multi_stop(void *unused) 212 { 213 struct alt_region region = { 214 .begin = (struct alt_instr *)__alt_instructions, 215 .end = (struct alt_instr *)__alt_instructions_end, 216 }; 217 218 /* We always have a CPU 0 at this point (__init) */ 219 if (smp_processor_id()) { 220 while (!READ_ONCE(all_alternatives_applied)) 221 cpu_relax(); 222 isb(); 223 } else { 224 DECLARE_BITMAP(remaining_capabilities, ARM64_NPATCHABLE); 225 226 bitmap_complement(remaining_capabilities, boot_capabilities, 227 ARM64_NPATCHABLE); 228 229 BUG_ON(all_alternatives_applied); 230 __apply_alternatives(®ion, false, remaining_capabilities); 231 /* Barriers provided by the cache flushing */ 232 WRITE_ONCE(all_alternatives_applied, 1); 233 } 234 235 return 0; 236 } 237 238 void __init apply_alternatives_all(void) 239 { 240 /* better not try code patching on a live SMP system */ 241 stop_machine(__apply_alternatives_multi_stop, NULL, cpu_online_mask); 242 } 243 244 /* 245 * This is called very early in the boot process (directly after we run 246 * a feature detect on the boot CPU). No need to worry about other CPUs 247 * here. 248 */ 249 void __init apply_boot_alternatives(void) 250 { 251 struct alt_region region = { 252 .begin = (struct alt_instr *)__alt_instructions, 253 .end = (struct alt_instr *)__alt_instructions_end, 254 }; 255 256 /* If called on non-boot cpu things could go wrong */ 257 WARN_ON(smp_processor_id() != 0); 258 259 __apply_alternatives(®ion, false, &boot_capabilities[0]); 260 } 261 262 #ifdef CONFIG_MODULES 263 void apply_alternatives_module(void *start, size_t length) 264 { 265 struct alt_region region = { 266 .begin = start, 267 .end = start + length, 268 }; 269 DECLARE_BITMAP(all_capabilities, ARM64_NPATCHABLE); 270 271 bitmap_fill(all_capabilities, ARM64_NPATCHABLE); 272 273 __apply_alternatives(®ion, true, &all_capabilities[0]); 274 } 275 #endif 276