1e039ee4eSAndre Przywara /* 2e039ee4eSAndre Przywara * alternative runtime patching 3e039ee4eSAndre Przywara * inspired by the x86 version 4e039ee4eSAndre Przywara * 5e039ee4eSAndre Przywara * Copyright (C) 2014 ARM Ltd. 6e039ee4eSAndre Przywara * 7e039ee4eSAndre Przywara * This program is free software; you can redistribute it and/or modify 8e039ee4eSAndre Przywara * it under the terms of the GNU General Public License version 2 as 9e039ee4eSAndre Przywara * published by the Free Software Foundation. 10e039ee4eSAndre Przywara * 11e039ee4eSAndre Przywara * This program is distributed in the hope that it will be useful, 12e039ee4eSAndre Przywara * but WITHOUT ANY WARRANTY; without even the implied warranty of 13e039ee4eSAndre Przywara * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14e039ee4eSAndre Przywara * GNU General Public License for more details. 15e039ee4eSAndre Przywara * 16e039ee4eSAndre Przywara * You should have received a copy of the GNU General Public License 17e039ee4eSAndre Przywara * along with this program. If not, see <http://www.gnu.org/licenses/>. 18e039ee4eSAndre Przywara */ 19e039ee4eSAndre Przywara 20e039ee4eSAndre Przywara #define pr_fmt(fmt) "alternatives: " fmt 21e039ee4eSAndre Przywara 22e039ee4eSAndre Przywara #include <linux/init.h> 23e039ee4eSAndre Przywara #include <linux/cpu.h> 24e039ee4eSAndre Przywara #include <asm/cacheflush.h> 25e039ee4eSAndre Przywara #include <asm/alternative.h> 26e039ee4eSAndre Przywara #include <asm/cpufeature.h> 277616fc8bSMarc Zyngier #include <asm/insn.h> 28ee78fdc7SJames Morse #include <asm/sections.h> 29e039ee4eSAndre Przywara #include <linux/stop_machine.h> 30e039ee4eSAndre Przywara 317616fc8bSMarc Zyngier #define __ALT_PTR(a,f) (u32 *)((void *)&(a)->f + (a)->f) 327616fc8bSMarc Zyngier #define ALT_ORIG_PTR(a) __ALT_PTR(a, orig_offset) 337616fc8bSMarc Zyngier #define ALT_REPL_PTR(a) __ALT_PTR(a, alt_offset) 347616fc8bSMarc Zyngier 35932ded4bSAndre Przywara struct alt_region { 36932ded4bSAndre Przywara struct alt_instr *begin; 37932ded4bSAndre Przywara struct alt_instr *end; 38932ded4bSAndre Przywara }; 39932ded4bSAndre Przywara 407616fc8bSMarc Zyngier /* 417616fc8bSMarc Zyngier * Check if the target PC is within an alternative block. 427616fc8bSMarc Zyngier */ 437616fc8bSMarc Zyngier static bool branch_insn_requires_update(struct alt_instr *alt, unsigned long pc) 447616fc8bSMarc Zyngier { 457616fc8bSMarc Zyngier unsigned long replptr; 467616fc8bSMarc Zyngier 477616fc8bSMarc Zyngier if (kernel_text_address(pc)) 487616fc8bSMarc Zyngier return 1; 497616fc8bSMarc Zyngier 507616fc8bSMarc Zyngier replptr = (unsigned long)ALT_REPL_PTR(alt); 517616fc8bSMarc Zyngier if (pc >= replptr && pc <= (replptr + alt->alt_len)) 527616fc8bSMarc Zyngier return 0; 537616fc8bSMarc Zyngier 547616fc8bSMarc Zyngier /* 557616fc8bSMarc Zyngier * Branching into *another* alternate sequence is doomed, and 567616fc8bSMarc Zyngier * we're not even trying to fix it up. 577616fc8bSMarc Zyngier */ 587616fc8bSMarc Zyngier BUG(); 597616fc8bSMarc Zyngier } 607616fc8bSMarc Zyngier 61c831b2aeSSuzuki K Poulose #define align_down(x, a) ((unsigned long)(x) & ~(((unsigned long)(a)) - 1)) 62c831b2aeSSuzuki K Poulose 637616fc8bSMarc Zyngier static u32 get_alt_insn(struct alt_instr *alt, u32 *insnptr, u32 *altinsnptr) 647616fc8bSMarc Zyngier { 657616fc8bSMarc Zyngier u32 insn; 667616fc8bSMarc Zyngier 677616fc8bSMarc Zyngier insn = le32_to_cpu(*altinsnptr); 687616fc8bSMarc Zyngier 697616fc8bSMarc Zyngier if (aarch64_insn_is_branch_imm(insn)) { 707616fc8bSMarc Zyngier s32 offset = aarch64_get_branch_offset(insn); 717616fc8bSMarc Zyngier unsigned long target; 727616fc8bSMarc Zyngier 737616fc8bSMarc Zyngier target = (unsigned long)altinsnptr + offset; 747616fc8bSMarc Zyngier 757616fc8bSMarc Zyngier /* 767616fc8bSMarc Zyngier * If we're branching inside the alternate sequence, 777616fc8bSMarc Zyngier * do not rewrite the instruction, as it is already 787616fc8bSMarc Zyngier * correct. Otherwise, generate the new instruction. 797616fc8bSMarc Zyngier */ 807616fc8bSMarc Zyngier if (branch_insn_requires_update(alt, target)) { 817616fc8bSMarc Zyngier offset = target - (unsigned long)insnptr; 827616fc8bSMarc Zyngier insn = aarch64_set_branch_offset(insn, offset); 837616fc8bSMarc Zyngier } 84c831b2aeSSuzuki K Poulose } else if (aarch64_insn_is_adrp(insn)) { 85c831b2aeSSuzuki K Poulose s32 orig_offset, new_offset; 86c831b2aeSSuzuki K Poulose unsigned long target; 87c831b2aeSSuzuki K Poulose 88c831b2aeSSuzuki K Poulose /* 89c831b2aeSSuzuki K Poulose * If we're replacing an adrp instruction, which uses PC-relative 90c831b2aeSSuzuki K Poulose * immediate addressing, adjust the offset to reflect the new 91c831b2aeSSuzuki K Poulose * PC. adrp operates on 4K aligned addresses. 92c831b2aeSSuzuki K Poulose */ 93c831b2aeSSuzuki K Poulose orig_offset = aarch64_insn_adrp_get_offset(insn); 94c831b2aeSSuzuki K Poulose target = align_down(altinsnptr, SZ_4K) + orig_offset; 95c831b2aeSSuzuki K Poulose new_offset = target - align_down(insnptr, SZ_4K); 96c831b2aeSSuzuki K Poulose insn = aarch64_insn_adrp_set_offset(insn, new_offset); 97baa763b5SSuzuki K Poulose } else if (aarch64_insn_uses_literal(insn)) { 98baa763b5SSuzuki K Poulose /* 99baa763b5SSuzuki K Poulose * Disallow patching unhandled instructions using PC relative 100baa763b5SSuzuki K Poulose * literal addresses 101baa763b5SSuzuki K Poulose */ 102baa763b5SSuzuki K Poulose BUG(); 1037616fc8bSMarc Zyngier } 1047616fc8bSMarc Zyngier 1057616fc8bSMarc Zyngier return insn; 1067616fc8bSMarc Zyngier } 1077616fc8bSMarc Zyngier 108ef5e724bSWill Deacon static void __apply_alternatives(void *alt_region) 109e039ee4eSAndre Przywara { 110e039ee4eSAndre Przywara struct alt_instr *alt; 111932ded4bSAndre Przywara struct alt_region *region = alt_region; 1127616fc8bSMarc Zyngier u32 *origptr, *replptr; 113e039ee4eSAndre Przywara 114932ded4bSAndre Przywara for (alt = region->begin; alt < region->end; alt++) { 1157616fc8bSMarc Zyngier u32 insn; 1167616fc8bSMarc Zyngier int i, nr_inst; 1177616fc8bSMarc Zyngier 118e039ee4eSAndre Przywara if (!cpus_have_cap(alt->cpufeature)) 119e039ee4eSAndre Przywara continue; 120e039ee4eSAndre Przywara 121fef7f2b2SMarc Zyngier BUG_ON(alt->alt_len != alt->orig_len); 122e039ee4eSAndre Przywara 123e039ee4eSAndre Przywara pr_info_once("patching kernel code\n"); 124e039ee4eSAndre Przywara 1257616fc8bSMarc Zyngier origptr = ALT_ORIG_PTR(alt); 1267616fc8bSMarc Zyngier replptr = ALT_REPL_PTR(alt); 1277616fc8bSMarc Zyngier nr_inst = alt->alt_len / sizeof(insn); 1287616fc8bSMarc Zyngier 1297616fc8bSMarc Zyngier for (i = 0; i < nr_inst; i++) { 1307616fc8bSMarc Zyngier insn = get_alt_insn(alt, origptr + i, replptr + i); 1317616fc8bSMarc Zyngier *(origptr + i) = cpu_to_le32(insn); 1327616fc8bSMarc Zyngier } 1337616fc8bSMarc Zyngier 134e039ee4eSAndre Przywara flush_icache_range((uintptr_t)origptr, 1357616fc8bSMarc Zyngier (uintptr_t)(origptr + nr_inst)); 136e039ee4eSAndre Przywara } 137e039ee4eSAndre Przywara } 138e039ee4eSAndre Przywara 139ef5e724bSWill Deacon /* 140ef5e724bSWill Deacon * We might be patching the stop_machine state machine, so implement a 141ef5e724bSWill Deacon * really simple polling protocol here. 142ef5e724bSWill Deacon */ 143ef5e724bSWill Deacon static int __apply_alternatives_multi_stop(void *unused) 144e039ee4eSAndre Przywara { 145ef5e724bSWill Deacon static int patched = 0; 146932ded4bSAndre Przywara struct alt_region region = { 147ee78fdc7SJames Morse .begin = (struct alt_instr *)__alt_instructions, 148ee78fdc7SJames Morse .end = (struct alt_instr *)__alt_instructions_end, 149932ded4bSAndre Przywara }; 150932ded4bSAndre Przywara 151ef5e724bSWill Deacon /* We always have a CPU 0 at this point (__init) */ 152ef5e724bSWill Deacon if (smp_processor_id()) { 153ef5e724bSWill Deacon while (!READ_ONCE(patched)) 154ef5e724bSWill Deacon cpu_relax(); 15504b8637bSWill Deacon isb(); 156ef5e724bSWill Deacon } else { 157ef5e724bSWill Deacon BUG_ON(patched); 158ef5e724bSWill Deacon __apply_alternatives(®ion); 159ef5e724bSWill Deacon /* Barriers provided by the cache flushing */ 160ef5e724bSWill Deacon WRITE_ONCE(patched, 1); 161ef5e724bSWill Deacon } 162ef5e724bSWill Deacon 163ef5e724bSWill Deacon return 0; 164ef5e724bSWill Deacon } 165ef5e724bSWill Deacon 166ef5e724bSWill Deacon void __init apply_alternatives_all(void) 167ef5e724bSWill Deacon { 168e039ee4eSAndre Przywara /* better not try code patching on a live SMP system */ 169ef5e724bSWill Deacon stop_machine(__apply_alternatives_multi_stop, NULL, cpu_online_mask); 170932ded4bSAndre Przywara } 171932ded4bSAndre Przywara 172932ded4bSAndre Przywara void apply_alternatives(void *start, size_t length) 173932ded4bSAndre Przywara { 174932ded4bSAndre Przywara struct alt_region region = { 175932ded4bSAndre Przywara .begin = start, 176932ded4bSAndre Przywara .end = start + length, 177932ded4bSAndre Przywara }; 178932ded4bSAndre Przywara 179932ded4bSAndre Przywara __apply_alternatives(®ion); 180e039ee4eSAndre Przywara } 181