1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 SiFive 4 */ 5 6 #include <linux/spinlock.h> 7 #include <linux/mm.h> 8 #include <linux/memory.h> 9 #include <linux/string.h> 10 #include <linux/uaccess.h> 11 #include <linux/stop_machine.h> 12 #include <asm/kprobes.h> 13 #include <asm/cacheflush.h> 14 #include <asm/fixmap.h> 15 #include <asm/ftrace.h> 16 #include <asm/patch.h> 17 18 struct patch_insn { 19 void *addr; 20 u32 *insns; 21 int ninsns; 22 atomic_t cpu_count; 23 }; 24 25 int riscv_patch_in_stop_machine = false; 26 27 #ifdef CONFIG_MMU 28 /* 29 * The fix_to_virt(, idx) needs a const value (not a dynamic variable of 30 * reg-a0) or BUILD_BUG_ON failed with "idx >= __end_of_fixed_addresses". 31 * So use '__always_inline' and 'const unsigned int fixmap' here. 32 */ 33 static __always_inline void *patch_map(void *addr, const unsigned int fixmap) 34 { 35 uintptr_t uintaddr = (uintptr_t) addr; 36 struct page *page; 37 38 if (core_kernel_text(uintaddr)) 39 page = phys_to_page(__pa_symbol(addr)); 40 else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) 41 page = vmalloc_to_page(addr); 42 else 43 return addr; 44 45 BUG_ON(!page); 46 47 return (void *)set_fixmap_offset(fixmap, page_to_phys(page) + 48 (uintaddr & ~PAGE_MASK)); 49 } 50 51 static void patch_unmap(int fixmap) 52 { 53 clear_fixmap(fixmap); 54 } 55 NOKPROBE_SYMBOL(patch_unmap); 56 57 static int __patch_insn_set(void *addr, u8 c, size_t len) 58 { 59 void *waddr = addr; 60 bool across_pages = (((uintptr_t)addr & ~PAGE_MASK) + len) > PAGE_SIZE; 61 62 /* 63 * Only two pages can be mapped at a time for writing. 64 */ 65 if (len + offset_in_page(addr) > 2 * PAGE_SIZE) 66 return -EINVAL; 67 /* 68 * Before reaching here, it was expected to lock the text_mutex 69 * already, so we don't need to give another lock here and could 70 * ensure that it was safe between each cores. 71 */ 72 lockdep_assert_held(&text_mutex); 73 74 if (across_pages) 75 patch_map(addr + PAGE_SIZE, FIX_TEXT_POKE1); 76 77 waddr = patch_map(addr, FIX_TEXT_POKE0); 78 79 memset(waddr, c, len); 80 81 patch_unmap(FIX_TEXT_POKE0); 82 83 if (across_pages) 84 patch_unmap(FIX_TEXT_POKE1); 85 86 return 0; 87 } 88 NOKPROBE_SYMBOL(__patch_insn_set); 89 90 static int __patch_insn_write(void *addr, const void *insn, size_t len) 91 { 92 void *waddr = addr; 93 bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE; 94 int ret; 95 96 /* 97 * Only two pages can be mapped at a time for writing. 98 */ 99 if (len + offset_in_page(addr) > 2 * PAGE_SIZE) 100 return -EINVAL; 101 102 /* 103 * Before reaching here, it was expected to lock the text_mutex 104 * already, so we don't need to give another lock here and could 105 * ensure that it was safe between each cores. 106 * 107 * We're currently using stop_machine() for ftrace & kprobes, and while 108 * that ensures text_mutex is held before installing the mappings it 109 * does not ensure text_mutex is held by the calling thread. That's 110 * safe but triggers a lockdep failure, so just elide it for that 111 * specific case. 112 */ 113 if (!riscv_patch_in_stop_machine) 114 lockdep_assert_held(&text_mutex); 115 116 if (across_pages) 117 patch_map(addr + PAGE_SIZE, FIX_TEXT_POKE1); 118 119 waddr = patch_map(addr, FIX_TEXT_POKE0); 120 121 ret = copy_to_kernel_nofault(waddr, insn, len); 122 123 patch_unmap(FIX_TEXT_POKE0); 124 125 if (across_pages) 126 patch_unmap(FIX_TEXT_POKE1); 127 128 return ret; 129 } 130 NOKPROBE_SYMBOL(__patch_insn_write); 131 #else 132 static int __patch_insn_set(void *addr, u8 c, size_t len) 133 { 134 memset(addr, c, len); 135 136 return 0; 137 } 138 NOKPROBE_SYMBOL(__patch_insn_set); 139 140 static int __patch_insn_write(void *addr, const void *insn, size_t len) 141 { 142 return copy_to_kernel_nofault(addr, insn, len); 143 } 144 NOKPROBE_SYMBOL(__patch_insn_write); 145 #endif /* CONFIG_MMU */ 146 147 static int patch_insn_set(void *addr, u8 c, size_t len) 148 { 149 size_t patched = 0; 150 size_t size; 151 int ret = 0; 152 153 /* 154 * __patch_insn_set() can only work on 2 pages at a time so call it in a 155 * loop with len <= 2 * PAGE_SIZE. 156 */ 157 while (patched < len && !ret) { 158 size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched); 159 ret = __patch_insn_set(addr + patched, c, size); 160 161 patched += size; 162 } 163 164 return ret; 165 } 166 NOKPROBE_SYMBOL(patch_insn_set); 167 168 int patch_text_set_nosync(void *addr, u8 c, size_t len) 169 { 170 u32 *tp = addr; 171 int ret; 172 173 ret = patch_insn_set(tp, c, len); 174 175 if (!ret) 176 flush_icache_range((uintptr_t)tp, (uintptr_t)tp + len); 177 178 return ret; 179 } 180 NOKPROBE_SYMBOL(patch_text_set_nosync); 181 182 static int patch_insn_write(void *addr, const void *insn, size_t len) 183 { 184 size_t patched = 0; 185 size_t size; 186 int ret = 0; 187 188 /* 189 * Copy the instructions to the destination address, two pages at a time 190 * because __patch_insn_write() can only handle len <= 2 * PAGE_SIZE. 191 */ 192 while (patched < len && !ret) { 193 size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched); 194 ret = __patch_insn_write(addr + patched, insn + patched, size); 195 196 patched += size; 197 } 198 199 return ret; 200 } 201 NOKPROBE_SYMBOL(patch_insn_write); 202 203 int patch_text_nosync(void *addr, const void *insns, size_t len) 204 { 205 u32 *tp = addr; 206 int ret; 207 208 ret = patch_insn_write(tp, insns, len); 209 210 if (!ret) 211 flush_icache_range((uintptr_t) tp, (uintptr_t) tp + len); 212 213 return ret; 214 } 215 NOKPROBE_SYMBOL(patch_text_nosync); 216 217 static int patch_text_cb(void *data) 218 { 219 struct patch_insn *patch = data; 220 unsigned long len; 221 int i, ret = 0; 222 223 if (atomic_inc_return(&patch->cpu_count) == num_online_cpus()) { 224 for (i = 0; ret == 0 && i < patch->ninsns; i++) { 225 len = GET_INSN_LENGTH(patch->insns[i]); 226 ret = patch_text_nosync(patch->addr + i * len, 227 &patch->insns[i], len); 228 } 229 atomic_inc(&patch->cpu_count); 230 } else { 231 while (atomic_read(&patch->cpu_count) <= num_online_cpus()) 232 cpu_relax(); 233 smp_mb(); 234 } 235 236 return ret; 237 } 238 NOKPROBE_SYMBOL(patch_text_cb); 239 240 int patch_text(void *addr, u32 *insns, int ninsns) 241 { 242 int ret; 243 struct patch_insn patch = { 244 .addr = addr, 245 .insns = insns, 246 .ninsns = ninsns, 247 .cpu_count = ATOMIC_INIT(0), 248 }; 249 250 /* 251 * kprobes takes text_mutex, before calling patch_text(), but as we call 252 * calls stop_machine(), the lockdep assertion in patch_insn_write() 253 * gets confused by the context in which the lock is taken. 254 * Instead, ensure the lock is held before calling stop_machine(), and 255 * set riscv_patch_in_stop_machine to skip the check in 256 * patch_insn_write(). 257 */ 258 lockdep_assert_held(&text_mutex); 259 riscv_patch_in_stop_machine = true; 260 ret = stop_machine_cpuslocked(patch_text_cb, &patch, cpu_online_mask); 261 riscv_patch_in_stop_machine = false; 262 return ret; 263 } 264 NOKPROBE_SYMBOL(patch_text); 265