1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/module.h> 3 #include <linux/cpu.h> 4 #include <linux/smp.h> 5 #include <asm/text-patching.h> 6 #include <asm/alternative.h> 7 #include <asm/facility.h> 8 #include <asm/nospec-branch.h> 9 10 #define MAX_PATCH_LEN (255 - 1) 11 12 static int __initdata_or_module alt_instr_disabled; 13 14 static int __init disable_alternative_instructions(char *str) 15 { 16 alt_instr_disabled = 1; 17 return 0; 18 } 19 20 early_param("noaltinstr", disable_alternative_instructions); 21 22 struct brcl_insn { 23 u16 opc; 24 s32 disp; 25 } __packed; 26 27 static u16 __initdata_or_module nop16 = 0x0700; 28 static u32 __initdata_or_module nop32 = 0x47000000; 29 static struct brcl_insn __initdata_or_module nop48 = { 30 0xc004, 0 31 }; 32 33 static const void *nops[] __initdata_or_module = { 34 &nop16, 35 &nop32, 36 &nop48 37 }; 38 39 static void __init_or_module add_jump_padding(void *insns, unsigned int len) 40 { 41 struct brcl_insn brcl = { 42 0xc0f4, 43 len / 2 44 }; 45 46 memcpy(insns, &brcl, sizeof(brcl)); 47 insns += sizeof(brcl); 48 len -= sizeof(brcl); 49 50 while (len > 0) { 51 memcpy(insns, &nop16, 2); 52 insns += 2; 53 len -= 2; 54 } 55 } 56 57 static void __init_or_module add_padding(void *insns, unsigned int len) 58 { 59 if (len > 6) 60 add_jump_padding(insns, len); 61 else if (len >= 2) 62 memcpy(insns, nops[len / 2 - 1], len); 63 } 64 65 static void __init_or_module __apply_alternatives(struct alt_instr *start, 66 struct alt_instr *end) 67 { 68 struct alt_instr *a; 69 u8 *instr, *replacement; 70 u8 insnbuf[MAX_PATCH_LEN]; 71 72 /* 73 * The scan order should be from start to end. A later scanned 74 * alternative code can overwrite previously scanned alternative code. 75 */ 76 for (a = start; a < end; a++) { 77 int insnbuf_sz = 0; 78 79 instr = (u8 *)&a->instr_offset + a->instr_offset; 80 replacement = (u8 *)&a->repl_offset + a->repl_offset; 81 82 if (!__test_facility(a->facility, alt_stfle_fac_list)) 83 continue; 84 85 if (unlikely(a->instrlen % 2 || a->replacementlen % 2)) { 86 WARN_ONCE(1, "cpu alternatives instructions length is " 87 "odd, skipping patching\n"); 88 continue; 89 } 90 91 memcpy(insnbuf, replacement, a->replacementlen); 92 insnbuf_sz = a->replacementlen; 93 94 if (a->instrlen > a->replacementlen) { 95 add_padding(insnbuf + a->replacementlen, 96 a->instrlen - a->replacementlen); 97 insnbuf_sz += a->instrlen - a->replacementlen; 98 } 99 100 s390_kernel_write(instr, insnbuf, insnbuf_sz); 101 } 102 } 103 104 void __init_or_module apply_alternatives(struct alt_instr *start, 105 struct alt_instr *end) 106 { 107 if (!alt_instr_disabled) 108 __apply_alternatives(start, end); 109 } 110 111 extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; 112 void __init apply_alternative_instructions(void) 113 { 114 apply_alternatives(__alt_instructions, __alt_instructions_end); 115 } 116 117 static void do_sync_core(void *info) 118 { 119 sync_core(); 120 } 121 122 void text_poke_sync(void) 123 { 124 on_each_cpu(do_sync_core, NULL, 1); 125 } 126 127 void text_poke_sync_lock(void) 128 { 129 cpus_read_lock(); 130 text_poke_sync(); 131 cpus_read_unlock(); 132 } 133