1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 SiFive 4 */ 5 6 #include <linux/spinlock.h> 7 #include <linux/mm.h> 8 #include <linux/memory.h> 9 #include <linux/uaccess.h> 10 #include <linux/stop_machine.h> 11 #include <asm/kprobes.h> 12 #include <asm/cacheflush.h> 13 #include <asm/fixmap.h> 14 #include <asm/patch.h> 15 16 struct patch_insn { 17 void *addr; 18 u32 insn; 19 atomic_t cpu_count; 20 }; 21 22 #ifdef CONFIG_MMU 23 /* 24 * The fix_to_virt(, idx) needs a const value (not a dynamic variable of 25 * reg-a0) or BUILD_BUG_ON failed with "idx >= __end_of_fixed_addresses". 26 * So use '__always_inline' and 'const unsigned int fixmap' here. 27 */ 28 static __always_inline void *patch_map(void *addr, const unsigned int fixmap) 29 { 30 uintptr_t uintaddr = (uintptr_t) addr; 31 struct page *page; 32 33 if (core_kernel_text(uintaddr)) 34 page = phys_to_page(__pa_symbol(addr)); 35 else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) 36 page = vmalloc_to_page(addr); 37 else 38 return addr; 39 40 BUG_ON(!page); 41 42 return (void *)set_fixmap_offset(fixmap, page_to_phys(page) + 43 (uintaddr & ~PAGE_MASK)); 44 } 45 46 static void patch_unmap(int fixmap) 47 { 48 clear_fixmap(fixmap); 49 } 50 NOKPROBE_SYMBOL(patch_unmap); 51 52 static int patch_insn_write(void *addr, const void *insn, size_t len) 53 { 54 void *waddr = addr; 55 bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE; 56 int ret; 57 58 /* 59 * Before reaching here, it was expected to lock the text_mutex 60 * already, so we don't need to give another lock here and could 61 * ensure that it was safe between each cores. 62 */ 63 lockdep_assert_held(&text_mutex); 64 65 if (across_pages) 66 patch_map(addr + len, FIX_TEXT_POKE1); 67 68 waddr = patch_map(addr, FIX_TEXT_POKE0); 69 70 ret = copy_to_kernel_nofault(waddr, insn, len); 71 72 patch_unmap(FIX_TEXT_POKE0); 73 74 if (across_pages) 75 patch_unmap(FIX_TEXT_POKE1); 76 77 return ret; 78 } 79 NOKPROBE_SYMBOL(patch_insn_write); 80 #else 81 static int patch_insn_write(void *addr, const void *insn, size_t len) 82 { 83 return copy_to_kernel_nofault(addr, insn, len); 84 } 85 NOKPROBE_SYMBOL(patch_insn_write); 86 #endif /* CONFIG_MMU */ 87 88 int patch_text_nosync(void *addr, const void *insns, size_t len) 89 { 90 u32 *tp = addr; 91 int ret; 92 93 ret = patch_insn_write(tp, insns, len); 94 95 if (!ret) 96 flush_icache_range((uintptr_t) tp, (uintptr_t) tp + len); 97 98 return ret; 99 } 100 NOKPROBE_SYMBOL(patch_text_nosync); 101 102 static int patch_text_cb(void *data) 103 { 104 struct patch_insn *patch = data; 105 int ret = 0; 106 107 if (atomic_inc_return(&patch->cpu_count) == num_online_cpus()) { 108 ret = 109 patch_text_nosync(patch->addr, &patch->insn, 110 GET_INSN_LENGTH(patch->insn)); 111 atomic_inc(&patch->cpu_count); 112 } else { 113 while (atomic_read(&patch->cpu_count) <= num_online_cpus()) 114 cpu_relax(); 115 smp_mb(); 116 } 117 118 return ret; 119 } 120 NOKPROBE_SYMBOL(patch_text_cb); 121 122 int patch_text(void *addr, u32 insn) 123 { 124 struct patch_insn patch = { 125 .addr = addr, 126 .insn = insn, 127 .cpu_count = ATOMIC_INIT(0), 128 }; 129 130 return stop_machine_cpuslocked(patch_text_cb, 131 &patch, cpu_online_mask); 132 } 133 NOKPROBE_SYMBOL(patch_text); 134