1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * jump label x86 support 4 * 5 * Copyright (C) 2009 Jason Baron <jbaron@redhat.com> 6 * 7 */ 8 #include <linux/jump_label.h> 9 #include <linux/memory.h> 10 #include <linux/uaccess.h> 11 #include <linux/module.h> 12 #include <linux/list.h> 13 #include <linux/jhash.h> 14 #include <linux/cpu.h> 15 #include <asm/kprobes.h> 16 #include <asm/alternative.h> 17 #include <asm/text-patching.h> 18 19 #ifdef HAVE_JUMP_LABEL 20 21 union jump_code_union { 22 char code[JUMP_LABEL_NOP_SIZE]; 23 struct { 24 char jump; 25 int offset; 26 } __attribute__((packed)); 27 }; 28 29 static void bug_at(unsigned char *ip, int line) 30 { 31 /* 32 * The location is not an op that we were expecting. 33 * Something went wrong. Crash the box, as something could be 34 * corrupting the kernel. 35 */ 36 pr_crit("jump_label: Fatal kernel bug, unexpected op at %pS [%p] (%5ph) %d\n", ip, ip, ip, line); 37 BUG(); 38 } 39 40 static void __ref __jump_label_transform(struct jump_entry *entry, 41 enum jump_label_type type, 42 void *(*poker)(void *, const void *, size_t), 43 int init) 44 { 45 union jump_code_union jmp; 46 const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP }; 47 const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5]; 48 const void *expect, *code; 49 int line; 50 51 jmp.jump = 0xe9; 52 jmp.offset = jump_entry_target(entry) - 53 (jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE); 54 55 if (early_boot_irqs_disabled) 56 poker = text_poke_early; 57 58 if (type == JUMP_LABEL_JMP) { 59 if (init) { 60 expect = default_nop; line = __LINE__; 61 } else { 62 expect = ideal_nop; line = __LINE__; 63 } 64 65 code = &jmp.code; 66 } else { 67 if (init) { 68 expect = default_nop; line = __LINE__; 69 } else { 70 expect = &jmp.code; line = __LINE__; 71 } 72 73 code = ideal_nop; 74 } 75 76 if (memcmp((void *)jump_entry_code(entry), expect, JUMP_LABEL_NOP_SIZE)) 77 bug_at((void *)jump_entry_code(entry), line); 78 79 /* 80 * Make text_poke_bp() a default fallback poker. 81 * 82 * At the time the change is being done, just ignore whether we 83 * are doing nop -> jump or jump -> nop transition, and assume 84 * always nop being the 'currently valid' instruction 85 * 86 */ 87 if (poker) { 88 (*poker)((void *)jump_entry_code(entry), code, 89 JUMP_LABEL_NOP_SIZE); 90 return; 91 } 92 93 text_poke_bp((void *)jump_entry_code(entry), code, JUMP_LABEL_NOP_SIZE, 94 (void *)jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE); 95 } 96 97 void arch_jump_label_transform(struct jump_entry *entry, 98 enum jump_label_type type) 99 { 100 mutex_lock(&text_mutex); 101 __jump_label_transform(entry, type, NULL, 0); 102 mutex_unlock(&text_mutex); 103 } 104 105 static enum { 106 JL_STATE_START, 107 JL_STATE_NO_UPDATE, 108 JL_STATE_UPDATE, 109 } jlstate __initdata_or_module = JL_STATE_START; 110 111 __init_or_module void arch_jump_label_transform_static(struct jump_entry *entry, 112 enum jump_label_type type) 113 { 114 /* 115 * This function is called at boot up and when modules are 116 * first loaded. Check if the default nop, the one that is 117 * inserted at compile time, is the ideal nop. If it is, then 118 * we do not need to update the nop, and we can leave it as is. 119 * If it is not, then we need to update the nop to the ideal nop. 120 */ 121 if (jlstate == JL_STATE_START) { 122 const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP }; 123 const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5]; 124 125 if (memcmp(ideal_nop, default_nop, 5) != 0) 126 jlstate = JL_STATE_UPDATE; 127 else 128 jlstate = JL_STATE_NO_UPDATE; 129 } 130 if (jlstate == JL_STATE_UPDATE) 131 __jump_label_transform(entry, type, text_poke_early, 1); 132 } 133 134 #endif 135