1caab277bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2e039ee4eSAndre Przywara /*
3e039ee4eSAndre Przywara * alternative runtime patching
4e039ee4eSAndre Przywara * inspired by the x86 version
5e039ee4eSAndre Przywara *
6e039ee4eSAndre Przywara * Copyright (C) 2014 ARM Ltd.
7e039ee4eSAndre Przywara */
8e039ee4eSAndre Przywara
9e039ee4eSAndre Przywara #define pr_fmt(fmt) "alternatives: " fmt
10e039ee4eSAndre Przywara
11e039ee4eSAndre Przywara #include <linux/init.h>
12e039ee4eSAndre Przywara #include <linux/cpu.h>
134e3bca8fSJoey Gouly #include <linux/elf.h>
14e039ee4eSAndre Przywara #include <asm/cacheflush.h>
15e039ee4eSAndre Przywara #include <asm/alternative.h>
16e039ee4eSAndre Przywara #include <asm/cpufeature.h>
177616fc8bSMarc Zyngier #include <asm/insn.h>
184e3bca8fSJoey Gouly #include <asm/module.h>
19ee78fdc7SJames Morse #include <asm/sections.h>
204e3bca8fSJoey Gouly #include <asm/vdso.h>
21e039ee4eSAndre Przywara #include <linux/stop_machine.h>
22e039ee4eSAndre Przywara
2315ad6aceSLuc Van Oostenryck #define __ALT_PTR(a, f) ((void *)&(a)->f + (a)->f)
247616fc8bSMarc Zyngier #define ALT_ORIG_PTR(a) __ALT_PTR(a, orig_offset)
257616fc8bSMarc Zyngier #define ALT_REPL_PTR(a) __ALT_PTR(a, alt_offset)
267616fc8bSMarc Zyngier
274c0bd995SMark Rutland #define ALT_CAP(a) ((a)->cpucap & ~ARM64_CB_BIT)
284c0bd995SMark Rutland #define ALT_HAS_CB(a) ((a)->cpucap & ARM64_CB_BIT)
294c0bd995SMark Rutland
305af76fb4SWill Deacon /* Volatile, as we may be patching the guts of READ_ONCE() */
315af76fb4SWill Deacon static volatile int all_alternatives_applied;
32e9ab7a2eSJulien Thierry
33e9ab7a2eSJulien Thierry static DECLARE_BITMAP(applied_alternatives, ARM64_NCAPS);
346d99b689SJames Morse
35932ded4bSAndre Przywara struct alt_region {
36932ded4bSAndre Przywara struct alt_instr *begin;
37932ded4bSAndre Przywara struct alt_instr *end;
38932ded4bSAndre Przywara };
39932ded4bSAndre Przywara
alternative_is_applied(u16 cpucap)40e9ab7a2eSJulien Thierry bool alternative_is_applied(u16 cpucap)
41e9ab7a2eSJulien Thierry {
42e9ab7a2eSJulien Thierry if (WARN_ON(cpucap >= ARM64_NCAPS))
43e9ab7a2eSJulien Thierry return false;
44e9ab7a2eSJulien Thierry
45e9ab7a2eSJulien Thierry return test_bit(cpucap, applied_alternatives);
46e9ab7a2eSJulien Thierry }
47e9ab7a2eSJulien Thierry
487616fc8bSMarc Zyngier /*
497616fc8bSMarc Zyngier * Check if the target PC is within an alternative block.
507616fc8bSMarc Zyngier */
branch_insn_requires_update(struct alt_instr * alt,unsigned long pc)51a2c0b0fbSJoey Gouly static __always_inline bool branch_insn_requires_update(struct alt_instr *alt, unsigned long pc)
527616fc8bSMarc Zyngier {
535679b281SArd Biesheuvel unsigned long replptr = (unsigned long)ALT_REPL_PTR(alt);
545679b281SArd Biesheuvel return !(pc >= replptr && pc <= (replptr + alt->alt_len));
557616fc8bSMarc Zyngier }
567616fc8bSMarc Zyngier
57c831b2aeSSuzuki K Poulose #define align_down(x, a) ((unsigned long)(x) & ~(((unsigned long)(a)) - 1))
58c831b2aeSSuzuki K Poulose
get_alt_insn(struct alt_instr * alt,__le32 * insnptr,__le32 * altinsnptr)59a2c0b0fbSJoey Gouly static __always_inline u32 get_alt_insn(struct alt_instr *alt, __le32 *insnptr, __le32 *altinsnptr)
607616fc8bSMarc Zyngier {
617616fc8bSMarc Zyngier u32 insn;
627616fc8bSMarc Zyngier
637616fc8bSMarc Zyngier insn = le32_to_cpu(*altinsnptr);
647616fc8bSMarc Zyngier
657616fc8bSMarc Zyngier if (aarch64_insn_is_branch_imm(insn)) {
667616fc8bSMarc Zyngier s32 offset = aarch64_get_branch_offset(insn);
677616fc8bSMarc Zyngier unsigned long target;
687616fc8bSMarc Zyngier
697616fc8bSMarc Zyngier target = (unsigned long)altinsnptr + offset;
707616fc8bSMarc Zyngier
717616fc8bSMarc Zyngier /*
727616fc8bSMarc Zyngier * If we're branching inside the alternate sequence,
737616fc8bSMarc Zyngier * do not rewrite the instruction, as it is already
747616fc8bSMarc Zyngier * correct. Otherwise, generate the new instruction.
757616fc8bSMarc Zyngier */
767616fc8bSMarc Zyngier if (branch_insn_requires_update(alt, target)) {
777616fc8bSMarc Zyngier offset = target - (unsigned long)insnptr;
787616fc8bSMarc Zyngier insn = aarch64_set_branch_offset(insn, offset);
797616fc8bSMarc Zyngier }
80c831b2aeSSuzuki K Poulose } else if (aarch64_insn_is_adrp(insn)) {
81c831b2aeSSuzuki K Poulose s32 orig_offset, new_offset;
82c831b2aeSSuzuki K Poulose unsigned long target;
83c831b2aeSSuzuki K Poulose
84c831b2aeSSuzuki K Poulose /*
85c831b2aeSSuzuki K Poulose * If we're replacing an adrp instruction, which uses PC-relative
86c831b2aeSSuzuki K Poulose * immediate addressing, adjust the offset to reflect the new
87c831b2aeSSuzuki K Poulose * PC. adrp operates on 4K aligned addresses.
88c831b2aeSSuzuki K Poulose */
89c831b2aeSSuzuki K Poulose orig_offset = aarch64_insn_adrp_get_offset(insn);
90c831b2aeSSuzuki K Poulose target = align_down(altinsnptr, SZ_4K) + orig_offset;
91c831b2aeSSuzuki K Poulose new_offset = target - align_down(insnptr, SZ_4K);
92c831b2aeSSuzuki K Poulose insn = aarch64_insn_adrp_set_offset(insn, new_offset);
93baa763b5SSuzuki K Poulose } else if (aarch64_insn_uses_literal(insn)) {
94baa763b5SSuzuki K Poulose /*
95baa763b5SSuzuki K Poulose * Disallow patching unhandled instructions using PC relative
96baa763b5SSuzuki K Poulose * literal addresses
97baa763b5SSuzuki K Poulose */
98baa763b5SSuzuki K Poulose BUG();
997616fc8bSMarc Zyngier }
1007616fc8bSMarc Zyngier
1017616fc8bSMarc Zyngier return insn;
1027616fc8bSMarc Zyngier }
1037616fc8bSMarc Zyngier
patch_alternative(struct alt_instr * alt,__le32 * origptr,__le32 * updptr,int nr_inst)104a2c0b0fbSJoey Gouly static noinstr void patch_alternative(struct alt_instr *alt,
105dea5e2a4SMarc Zyngier __le32 *origptr, __le32 *updptr, int nr_inst)
106dea5e2a4SMarc Zyngier {
107dea5e2a4SMarc Zyngier __le32 *replptr;
108dea5e2a4SMarc Zyngier int i;
109dea5e2a4SMarc Zyngier
110dea5e2a4SMarc Zyngier replptr = ALT_REPL_PTR(alt);
111dea5e2a4SMarc Zyngier for (i = 0; i < nr_inst; i++) {
112dea5e2a4SMarc Zyngier u32 insn;
113dea5e2a4SMarc Zyngier
114dea5e2a4SMarc Zyngier insn = get_alt_insn(alt, origptr + i, replptr + i);
115dea5e2a4SMarc Zyngier updptr[i] = cpu_to_le32(insn);
116dea5e2a4SMarc Zyngier }
117dea5e2a4SMarc Zyngier }
118dea5e2a4SMarc Zyngier
11942938868SWill Deacon /*
12042938868SWill Deacon * We provide our own, private D-cache cleaning function so that we don't
12142938868SWill Deacon * accidentally call into the cache.S code, which is patched by us at
12242938868SWill Deacon * runtime.
12342938868SWill Deacon */
clean_dcache_range_nopatch(u64 start,u64 end)124*39138093SMark Rutland static noinstr void clean_dcache_range_nopatch(u64 start, u64 end)
12542938868SWill Deacon {
12642938868SWill Deacon u64 cur, d_size, ctr_el0;
12742938868SWill Deacon
128*39138093SMark Rutland ctr_el0 = arm64_ftr_reg_ctrel0.sys_val;
12942938868SWill Deacon d_size = 4 << cpuid_feature_extract_unsigned_field(ctr_el0,
1305b345e39SMark Brown CTR_EL0_DminLine_SHIFT);
13142938868SWill Deacon cur = start & ~(d_size - 1);
13242938868SWill Deacon do {
13342938868SWill Deacon /*
13442938868SWill Deacon * We must clean+invalidate to the PoC in order to avoid
13542938868SWill Deacon * Cortex-A53 errata 826319, 827319, 824069 and 819472
13642938868SWill Deacon * (this corresponds to ARM64_WORKAROUND_CLEAN_CACHE)
13742938868SWill Deacon */
13842938868SWill Deacon asm volatile("dc civac, %0" : : "r" (cur) : "memory");
13942938868SWill Deacon } while (cur += d_size, cur < end);
14042938868SWill Deacon }
14142938868SWill Deacon
__apply_alternatives(const struct alt_region * region,bool is_module,unsigned long * cpucap_mask)14218fd0497SLinus Torvalds static void __apply_alternatives(const struct alt_region *region,
143b723edf3SMark Rutland bool is_module,
1440ceb0d56SDaniel Thompson unsigned long *cpucap_mask)
145e039ee4eSAndre Przywara {
146e039ee4eSAndre Przywara struct alt_instr *alt;
147dea5e2a4SMarc Zyngier __le32 *origptr, *updptr;
148dea5e2a4SMarc Zyngier alternative_cb_t alt_cb;
149e039ee4eSAndre Przywara
150932ded4bSAndre Przywara for (alt = region->begin; alt < region->end; alt++) {
151dea5e2a4SMarc Zyngier int nr_inst;
1524c0bd995SMark Rutland int cap = ALT_CAP(alt);
1537616fc8bSMarc Zyngier
1544c0bd995SMark Rutland if (!test_bit(cap, cpucap_mask))
1550ceb0d56SDaniel Thompson continue;
1560ceb0d56SDaniel Thompson
1574c0bd995SMark Rutland if (!cpus_have_cap(cap))
158e039ee4eSAndre Przywara continue;
159e039ee4eSAndre Przywara
1604c0bd995SMark Rutland if (ALT_HAS_CB(alt))
161dea5e2a4SMarc Zyngier BUG_ON(alt->alt_len != 0);
162dea5e2a4SMarc Zyngier else
163fef7f2b2SMarc Zyngier BUG_ON(alt->alt_len != alt->orig_len);
164e039ee4eSAndre Przywara
1657616fc8bSMarc Zyngier origptr = ALT_ORIG_PTR(alt);
16642938868SWill Deacon updptr = is_module ? origptr : lm_alias(origptr);
167dea5e2a4SMarc Zyngier nr_inst = alt->orig_len / AARCH64_INSN_SIZE;
1687616fc8bSMarc Zyngier
1694c0bd995SMark Rutland if (ALT_HAS_CB(alt))
170dea5e2a4SMarc Zyngier alt_cb = ALT_REPL_PTR(alt);
1714c0bd995SMark Rutland else
1724c0bd995SMark Rutland alt_cb = patch_alternative;
173dea5e2a4SMarc Zyngier
174dea5e2a4SMarc Zyngier alt_cb(alt, origptr, updptr, nr_inst);
1757616fc8bSMarc Zyngier
17642938868SWill Deacon if (!is_module) {
17742938868SWill Deacon clean_dcache_range_nopatch((u64)origptr,
17842938868SWill Deacon (u64)(origptr + nr_inst));
17942938868SWill Deacon }
18042938868SWill Deacon }
18142938868SWill Deacon
18242938868SWill Deacon /*
18342938868SWill Deacon * The core module code takes care of cache maintenance in
18442938868SWill Deacon * flush_module_icache().
18542938868SWill Deacon */
18642938868SWill Deacon if (!is_module) {
18742938868SWill Deacon dsb(ish);
188fade9c2cSFuad Tabba icache_inval_all_pou();
18942938868SWill Deacon isb();
190e9ab7a2eSJulien Thierry
1910ceb0d56SDaniel Thompson bitmap_or(applied_alternatives, applied_alternatives,
1920ceb0d56SDaniel Thompson cpucap_mask, ARM64_NCAPS);
1930ceb0d56SDaniel Thompson bitmap_and(applied_alternatives, applied_alternatives,
1940ceb0d56SDaniel Thompson system_cpucaps, ARM64_NCAPS);
1950ceb0d56SDaniel Thompson }
196e039ee4eSAndre Przywara }
197e039ee4eSAndre Przywara
apply_alternatives_vdso(void)198e039ee4eSAndre Przywara static void __init apply_alternatives_vdso(void)
19967bc5b2dSJisheng Zhang {
2004e3bca8fSJoey Gouly struct alt_region region;
2014e3bca8fSJoey Gouly const struct elf64_hdr *hdr;
2024e3bca8fSJoey Gouly const struct elf64_shdr *shdr;
2034e3bca8fSJoey Gouly const struct elf64_shdr *alt;
2044e3bca8fSJoey Gouly DECLARE_BITMAP(all_capabilities, ARM64_NCAPS);
205c704cf27SCatalin Marinas
2064e3bca8fSJoey Gouly bitmap_fill(all_capabilities, ARM64_NCAPS);
207c704cf27SCatalin Marinas
2084e3bca8fSJoey Gouly hdr = (struct elf64_hdr *)vdso_start;
2094e3bca8fSJoey Gouly shdr = (void *)hdr + hdr->e_shoff;
2104e3bca8fSJoey Gouly alt = find_section(hdr, shdr, ".altinstructions");
2114e3bca8fSJoey Gouly if (!alt)
2124e3bca8fSJoey Gouly return;
2134e3bca8fSJoey Gouly
2144e3bca8fSJoey Gouly region = (struct alt_region){
2154e3bca8fSJoey Gouly .begin = (void *)hdr + alt->sh_offset,
2164e3bca8fSJoey Gouly .end = (void *)hdr + alt->sh_offset + alt->sh_size,
2174e3bca8fSJoey Gouly };
2184e3bca8fSJoey Gouly
2194e3bca8fSJoey Gouly __apply_alternatives(®ion, false, &all_capabilities[0]);
2204e3bca8fSJoey Gouly }
2214e3bca8fSJoey Gouly
2224e3bca8fSJoey Gouly static const struct alt_region kernel_alternatives __initconst = {
22367bc5b2dSJisheng Zhang .begin = (struct alt_instr *)__alt_instructions,
224b723edf3SMark Rutland .end = (struct alt_instr *)__alt_instructions_end,
225b723edf3SMark Rutland };
226b723edf3SMark Rutland
227b723edf3SMark Rutland /*
228ef5e724bSWill Deacon * We might be patching the stop_machine state machine, so implement a
229ef5e724bSWill Deacon * really simple polling protocol here.
230ef5e724bSWill Deacon */
__apply_alternatives_multi_stop(void * unused)231ef5e724bSWill Deacon static int __init __apply_alternatives_multi_stop(void *unused)
23267bc5b2dSJisheng Zhang {
233e039ee4eSAndre Przywara /* We always have a CPU 0 at this point (__init) */
234ef5e724bSWill Deacon if (smp_processor_id()) {
235ef5e724bSWill Deacon while (!all_alternatives_applied)
2365af76fb4SWill Deacon cpu_relax();
237ef5e724bSWill Deacon isb();
23804b8637bSWill Deacon } else {
239ef5e724bSWill Deacon DECLARE_BITMAP(remaining_capabilities, ARM64_NCAPS);
2404c0bd995SMark Rutland
2410ceb0d56SDaniel Thompson bitmap_complement(remaining_capabilities, boot_cpucaps,
2420ceb0d56SDaniel Thompson ARM64_NCAPS);
2434c0bd995SMark Rutland
2440ceb0d56SDaniel Thompson BUG_ON(all_alternatives_applied);
245e9ab7a2eSJulien Thierry __apply_alternatives(&kernel_alternatives, false,
246b723edf3SMark Rutland remaining_capabilities);
247b723edf3SMark Rutland /* Barriers provided by the cache flushing */
248ef5e724bSWill Deacon all_alternatives_applied = 1;
2495af76fb4SWill Deacon }
250ef5e724bSWill Deacon
251ef5e724bSWill Deacon return 0;
252ef5e724bSWill Deacon }
253ef5e724bSWill Deacon
apply_alternatives_all(void)254ef5e724bSWill Deacon void __init apply_alternatives_all(void)
255ef5e724bSWill Deacon {
256ef5e724bSWill Deacon pr_info("applying system-wide alternatives\n");
257c5ba0326SMark Rutland
258c5ba0326SMark Rutland apply_alternatives_vdso();
2594e3bca8fSJoey Gouly /* better not try code patching on a live SMP system */
260e039ee4eSAndre Przywara stop_machine(__apply_alternatives_multi_stop, NULL, cpu_online_mask);
261ef5e724bSWill Deacon }
262932ded4bSAndre Przywara
263932ded4bSAndre Przywara /*
2640ceb0d56SDaniel Thompson * This is called very early in the boot process (directly after we run
2650ceb0d56SDaniel Thompson * a feature detect on the boot CPU). No need to worry about other CPUs
2660ceb0d56SDaniel Thompson * here.
2670ceb0d56SDaniel Thompson */
apply_boot_alternatives(void)2680ceb0d56SDaniel Thompson void __init apply_boot_alternatives(void)
2690ceb0d56SDaniel Thompson {
2700ceb0d56SDaniel Thompson /* If called on non-boot cpu things could go wrong */
2710ceb0d56SDaniel Thompson WARN_ON(smp_processor_id() != 0);
2720ceb0d56SDaniel Thompson
2730ceb0d56SDaniel Thompson pr_info("applying boot alternatives\n");
274c5ba0326SMark Rutland
275c5ba0326SMark Rutland __apply_alternatives(&kernel_alternatives, false,
276b723edf3SMark Rutland &boot_cpucaps[0]);
277b723edf3SMark Rutland }
2780ceb0d56SDaniel Thompson
2790ceb0d56SDaniel Thompson #ifdef CONFIG_MODULES
apply_alternatives_module(void * start,size_t length)28042938868SWill Deacon void apply_alternatives_module(void *start, size_t length)
28142938868SWill Deacon {
282932ded4bSAndre Przywara struct alt_region region = {
283932ded4bSAndre Przywara .begin = start,
284932ded4bSAndre Przywara .end = start + length,
285932ded4bSAndre Przywara };
286932ded4bSAndre Przywara DECLARE_BITMAP(all_capabilities, ARM64_NCAPS);
2874c0bd995SMark Rutland
288932ded4bSAndre Przywara bitmap_fill(all_capabilities, ARM64_NCAPS);
2894c0bd995SMark Rutland
2900ceb0d56SDaniel Thompson __apply_alternatives(®ion, true, &all_capabilities[0]);
2910ceb0d56SDaniel Thompson }
292e039ee4eSAndre Przywara #endif
29342938868SWill Deacon
alt_cb_patch_nops(struct alt_instr * alt,__le32 * origptr,__le32 * updptr,int nr_inst)294d926079fSMark Rutland noinstr void alt_cb_patch_nops(struct alt_instr *alt, __le32 *origptr,
295d926079fSMark Rutland __le32 *updptr, int nr_inst)
296d926079fSMark Rutland {
297d926079fSMark Rutland for (int i = 0; i < nr_inst; i++)
298d926079fSMark Rutland updptr[i] = cpu_to_le32(aarch64_insn_gen_nop());
299d926079fSMark Rutland }
300d926079fSMark Rutland EXPORT_SYMBOL(alt_cb_patch_nops);
301d926079fSMark Rutland