1 #ifndef _ASM_X86_JUMP_LABEL_H 2 #define _ASM_X86_JUMP_LABEL_H 3 4 #ifndef HAVE_JUMP_LABEL 5 /* 6 * For better or for worse, if jump labels (the gcc extension) are missing, 7 * then the entire static branch patching infrastructure is compiled out. 8 * If that happens, the code in here will malfunction. Raise a compiler 9 * error instead. 10 * 11 * In theory, jump labels and the static branch patching infrastructure 12 * could be decoupled to fix this. 13 */ 14 #error asm/jump_label.h included on a non-jump-label kernel 15 #endif 16 17 #define JUMP_LABEL_NOP_SIZE 5 18 19 #ifdef CONFIG_X86_64 20 # define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC 21 #else 22 # define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC 23 #endif 24 25 #include <asm/asm.h> 26 #include <asm/nops.h> 27 28 #ifndef __ASSEMBLY__ 29 30 #include <linux/stringify.h> 31 #include <linux/types.h> 32 33 static __always_inline bool arch_static_branch(struct static_key *key, bool branch) 34 { 35 asm_volatile_goto("1:" 36 ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t" 37 ".pushsection __jump_table, \"aw\" \n\t" 38 _ASM_ALIGN "\n\t" 39 _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t" 40 ".popsection \n\t" 41 : : "i" (key), "i" (branch) : : l_yes); 42 43 return false; 44 l_yes: 45 return true; 46 } 47 48 static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) 49 { 50 asm_volatile_goto("1:" 51 ".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t" 52 "2:\n\t" 53 ".pushsection __jump_table, \"aw\" \n\t" 54 _ASM_ALIGN "\n\t" 55 _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t" 56 ".popsection \n\t" 57 : : "i" (key), "i" (branch) : : l_yes); 58 59 return false; 60 l_yes: 61 return true; 62 } 63 64 #ifdef CONFIG_X86_64 65 typedef u64 jump_label_t; 66 #else 67 typedef u32 jump_label_t; 68 #endif 69 70 struct jump_entry { 71 jump_label_t code; 72 jump_label_t target; 73 jump_label_t key; 74 }; 75 76 #else /* __ASSEMBLY__ */ 77 78 .macro STATIC_JUMP_IF_TRUE target, key, def 79 .Lstatic_jump_\@: 80 .if \def 81 /* Equivalent to "jmp.d32 \target" */ 82 .byte 0xe9 83 .long \target - .Lstatic_jump_after_\@ 84 .Lstatic_jump_after_\@: 85 .else 86 .byte STATIC_KEY_INIT_NOP 87 .endif 88 .pushsection __jump_table, "aw" 89 _ASM_ALIGN 90 _ASM_PTR .Lstatic_jump_\@, \target, \key 91 .popsection 92 .endm 93 94 .macro STATIC_JUMP_IF_FALSE target, key, def 95 .Lstatic_jump_\@: 96 .if \def 97 .byte STATIC_KEY_INIT_NOP 98 .else 99 /* Equivalent to "jmp.d32 \target" */ 100 .byte 0xe9 101 .long \target - .Lstatic_jump_after_\@ 102 .Lstatic_jump_after_\@: 103 .endif 104 .pushsection __jump_table, "aw" 105 _ASM_ALIGN 106 _ASM_PTR .Lstatic_jump_\@, \target, \key + 1 107 .popsection 108 .endm 109 110 #endif /* __ASSEMBLY__ */ 111 112 #endif 113