1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/kernel.h> 4 #include <linux/jump_label.h> 5 6 #include "asm/cacheflush.h" 7 8 #define JUMPLABEL_ERR "ARC: jump_label: ERROR: " 9 10 /* Halt system on fatal error to make debug easier */ 11 #define arc_jl_fatal(format...) \ 12 ({ \ 13 pr_err(JUMPLABEL_ERR format); \ 14 BUG(); \ 15 }) 16 17 static inline u32 arc_gen_nop(void) 18 { 19 /* 1x 32bit NOP in middle endian */ 20 return 0x7000264a; 21 } 22 23 /* 24 * Atomic update of patched instruction is only available if this 25 * instruction doesn't cross L1 cache line boundary. You can read about 26 * the way we achieve this in arc/include/asm/jump_label.h 27 */ 28 static inline void instruction_align_assert(void *addr, int len) 29 { 30 unsigned long a = (unsigned long)addr; 31 32 if ((a >> L1_CACHE_SHIFT) != ((a + len - 1) >> L1_CACHE_SHIFT)) 33 arc_jl_fatal("instruction (addr %px) cross L1 cache line border", 34 addr); 35 } 36 37 /* 38 * ARCv2 'Branch unconditionally' instruction: 39 * 00000ssssssssss1SSSSSSSSSSNRtttt 40 * s S[n:0] lower bits signed immediate (number is bitfield size) 41 * S S[m:n+1] upper bits signed immediate (number is bitfield size) 42 * t S[24:21] upper bits signed immediate (branch unconditionally far) 43 * N N <.d> delay slot mode 44 * R R Reserved 45 */ 46 static inline u32 arc_gen_branch(jump_label_t pc, jump_label_t target) 47 { 48 u32 instruction_l, instruction_r; 49 u32 pcl = pc & GENMASK(31, 2); 50 u32 u_offset = target - pcl; 51 u32 s, S, t; 52 53 /* 54 * Offset in 32-bit branch instruction must to fit into s25. 55 * Something is terribly broken if we get such huge offset within one 56 * function. 57 */ 58 if ((s32)u_offset < -16777216 || (s32)u_offset > 16777214) 59 arc_jl_fatal("gen branch with offset (%d) not fit in s25", 60 (s32)u_offset); 61 62 /* 63 * All instructions are aligned by 2 bytes so we should never get offset 64 * here which is not 2 bytes aligned. 65 */ 66 if (u_offset & 0x1) 67 arc_jl_fatal("gen branch with offset (%d) unaligned to 2 bytes", 68 (s32)u_offset); 69 70 s = (u_offset >> 1) & GENMASK(9, 0); 71 S = (u_offset >> 11) & GENMASK(9, 0); 72 t = (u_offset >> 21) & GENMASK(3, 0); 73 74 /* 00000ssssssssss1 */ 75 instruction_l = (s << 1) | 0x1; 76 /* SSSSSSSSSSNRtttt */ 77 instruction_r = (S << 6) | t; 78 79 return (instruction_r << 16) | (instruction_l & GENMASK(15, 0)); 80 } 81 82 void arch_jump_label_transform(struct jump_entry *entry, 83 enum jump_label_type type) 84 { 85 jump_label_t *instr_addr = (jump_label_t *)entry->code; 86 u32 instr; 87 88 instruction_align_assert(instr_addr, JUMP_LABEL_NOP_SIZE); 89 90 if (type == JUMP_LABEL_JMP) 91 instr = arc_gen_branch(entry->code, entry->target); 92 else 93 instr = arc_gen_nop(); 94 95 WRITE_ONCE(*instr_addr, instr); 96 flush_icache_range(entry->code, entry->code + JUMP_LABEL_NOP_SIZE); 97 } 98 99 void arch_jump_label_transform_static(struct jump_entry *entry, 100 enum jump_label_type type) 101 { 102 /* 103 * We use only one NOP type (1x, 4 byte) in arch_static_branch, so 104 * there's no need to patch an identical NOP over the top of it here. 105 * The generic code calls 'arch_jump_label_transform' if the NOP needs 106 * to be replaced by a branch, so 'arch_jump_label_transform_static' is 107 * never called with type other than JUMP_LABEL_NOP. 108 */ 109 BUG_ON(type != JUMP_LABEL_NOP); 110 } 111 112 #ifdef CONFIG_ARC_DBG_JUMP_LABEL 113 #define SELFTEST_MSG "ARC: instruction generation self-test: " 114 115 struct arc_gen_branch_testdata { 116 jump_label_t pc; 117 jump_label_t target_address; 118 u32 expected_instr; 119 }; 120 121 static __init int branch_gen_test(const struct arc_gen_branch_testdata *test) 122 { 123 u32 instr_got; 124 125 instr_got = arc_gen_branch(test->pc, test->target_address); 126 if (instr_got == test->expected_instr) 127 return 0; 128 129 pr_err(SELFTEST_MSG "FAIL:\n arc_gen_branch(0x%08x, 0x%08x) != 0x%08x, got 0x%08x\n", 130 test->pc, test->target_address, 131 test->expected_instr, instr_got); 132 133 return -EFAULT; 134 } 135 136 /* 137 * Offset field in branch instruction is not continuous. Test all 138 * available offset field and sign combinations. Test data is generated 139 * from real working code. 140 */ 141 static const struct arc_gen_branch_testdata arcgenbr_test_data[] __initconst = { 142 {0x90007548, 0x90007514, 0xffcf07cd}, /* tiny (-52) offs */ 143 {0x9000c9c0, 0x9000c782, 0xffcf05c3}, /* tiny (-574) offs */ 144 {0x9000cc1c, 0x9000c782, 0xffcf0367}, /* tiny (-1178) offs */ 145 {0x9009dce0, 0x9009d106, 0xff8f0427}, /* small (-3034) offs */ 146 {0x9000f5de, 0x90007d30, 0xfc0f0755}, /* big (-30892) offs */ 147 {0x900a2444, 0x90035f64, 0xc9cf0321}, /* huge (-443616) offs */ 148 {0x90007514, 0x9000752c, 0x00000019}, /* tiny (+24) offs */ 149 {0x9001a578, 0x9001a77a, 0x00000203}, /* tiny (+514) offs */ 150 {0x90031ed8, 0x90032634, 0x0000075d}, /* tiny (+1884) offs */ 151 {0x9008c7f2, 0x9008d3f0, 0x00400401}, /* small (+3072) offs */ 152 {0x9000bb38, 0x9003b340, 0x17c00009}, /* big (+194568) offs */ 153 {0x90008f44, 0x90578d80, 0xb7c2063d} /* huge (+5701180) offs */ 154 }; 155 156 static __init int instr_gen_test(void) 157 { 158 int i; 159 160 for (i = 0; i < ARRAY_SIZE(arcgenbr_test_data); i++) 161 if (branch_gen_test(&arcgenbr_test_data[i])) 162 return -EFAULT; 163 164 pr_info(SELFTEST_MSG "OK\n"); 165 166 return 0; 167 } 168 early_initcall(instr_gen_test); 169 170 #endif /* CONFIG_ARC_DBG_JUMP_LABEL */ 171