1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (C) 2018 Cadence Design Systems Inc. 3 4 #include <linux/cpu.h> 5 #include <linux/jump_label.h> 6 #include <linux/kernel.h> 7 #include <linux/memory.h> 8 #include <linux/stop_machine.h> 9 #include <linux/types.h> 10 11 #include <asm/cacheflush.h> 12 13 #ifdef HAVE_JUMP_LABEL 14 15 #define J_OFFSET_MASK 0x0003ffff 16 #define J_SIGN_MASK (~(J_OFFSET_MASK >> 1)) 17 18 #if defined(__XTENSA_EL__) 19 #define J_INSN 0x6 20 #define NOP_INSN 0x0020f0 21 #elif defined(__XTENSA_EB__) 22 #define J_INSN 0x60000000 23 #define NOP_INSN 0x0f020000 24 #else 25 #error Unsupported endianness. 26 #endif 27 28 struct patch { 29 atomic_t cpu_count; 30 unsigned long addr; 31 size_t sz; 32 const void *data; 33 }; 34 35 static void local_patch_text(unsigned long addr, const void *data, size_t sz) 36 { 37 memcpy((void *)addr, data, sz); 38 local_flush_icache_range(addr, addr + sz); 39 } 40 41 static int patch_text_stop_machine(void *data) 42 { 43 struct patch *patch = data; 44 45 if (atomic_inc_return(&patch->cpu_count) == 1) { 46 local_patch_text(patch->addr, patch->data, patch->sz); 47 atomic_inc(&patch->cpu_count); 48 } else { 49 while (atomic_read(&patch->cpu_count) <= num_online_cpus()) 50 cpu_relax(); 51 __invalidate_icache_range(patch->addr, patch->sz); 52 } 53 return 0; 54 } 55 56 static void patch_text(unsigned long addr, const void *data, size_t sz) 57 { 58 if (IS_ENABLED(CONFIG_SMP)) { 59 struct patch patch = { 60 .cpu_count = ATOMIC_INIT(0), 61 .addr = addr, 62 .sz = sz, 63 .data = data, 64 }; 65 stop_machine_cpuslocked(patch_text_stop_machine, 66 &patch, NULL); 67 } else { 68 unsigned long flags; 69 70 local_irq_save(flags); 71 local_patch_text(addr, data, sz); 72 local_irq_restore(flags); 73 } 74 } 75 76 void arch_jump_label_transform(struct jump_entry *e, 77 enum jump_label_type type) 78 { 79 u32 d = (jump_entry_target(e) - (jump_entry_code(e) + 4)); 80 u32 insn; 81 82 /* Jump only works within 128K of the J instruction. */ 83 BUG_ON(!((d & J_SIGN_MASK) == 0 || 84 (d & J_SIGN_MASK) == J_SIGN_MASK)); 85 86 if (type == JUMP_LABEL_JMP) { 87 #if defined(__XTENSA_EL__) 88 insn = ((d & J_OFFSET_MASK) << 6) | J_INSN; 89 #elif defined(__XTENSA_EB__) 90 insn = ((d & J_OFFSET_MASK) << 8) | J_INSN; 91 #endif 92 } else { 93 insn = NOP_INSN; 94 } 95 96 patch_text(jump_entry_code(e), &insn, JUMP_LABEL_NOP_SIZE); 97 } 98 99 #endif /* HAVE_JUMP_LABEL */ 100