1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 SiFive 4 */ 5 6 #include <linux/spinlock.h> 7 #include <linux/mm.h> 8 #include <linux/memory.h> 9 #include <linux/uaccess.h> 10 #include <linux/stop_machine.h> 11 #include <asm/kprobes.h> 12 #include <asm/cacheflush.h> 13 #include <asm/fixmap.h> 14 #include <asm/patch.h> 15 16 struct patch_insn { 17 void *addr; 18 u32 *insns; 19 int ninsns; 20 atomic_t cpu_count; 21 }; 22 23 #ifdef CONFIG_MMU 24 /* 25 * The fix_to_virt(, idx) needs a const value (not a dynamic variable of 26 * reg-a0) or BUILD_BUG_ON failed with "idx >= __end_of_fixed_addresses". 27 * So use '__always_inline' and 'const unsigned int fixmap' here. 28 */ 29 static __always_inline void *patch_map(void *addr, const unsigned int fixmap) 30 { 31 uintptr_t uintaddr = (uintptr_t) addr; 32 struct page *page; 33 34 if (core_kernel_text(uintaddr)) 35 page = phys_to_page(__pa_symbol(addr)); 36 else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) 37 page = vmalloc_to_page(addr); 38 else 39 return addr; 40 41 BUG_ON(!page); 42 43 return (void *)set_fixmap_offset(fixmap, page_to_phys(page) + 44 (uintaddr & ~PAGE_MASK)); 45 } 46 47 static void patch_unmap(int fixmap) 48 { 49 clear_fixmap(fixmap); 50 } 51 NOKPROBE_SYMBOL(patch_unmap); 52 53 static int patch_insn_write(void *addr, const void *insn, size_t len) 54 { 55 void *waddr = addr; 56 bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE; 57 int ret; 58 59 /* 60 * Before reaching here, it was expected to lock the text_mutex 61 * already, so we don't need to give another lock here and could 62 * ensure that it was safe between each cores. 63 */ 64 lockdep_assert_held(&text_mutex); 65 66 if (across_pages) 67 patch_map(addr + len, FIX_TEXT_POKE1); 68 69 waddr = patch_map(addr, FIX_TEXT_POKE0); 70 71 ret = copy_to_kernel_nofault(waddr, insn, len); 72 73 patch_unmap(FIX_TEXT_POKE0); 74 75 if (across_pages) 76 patch_unmap(FIX_TEXT_POKE1); 77 78 return ret; 79 } 80 NOKPROBE_SYMBOL(patch_insn_write); 81 #else 82 static int patch_insn_write(void *addr, const void *insn, size_t len) 83 { 84 return copy_to_kernel_nofault(addr, insn, len); 85 } 86 NOKPROBE_SYMBOL(patch_insn_write); 87 #endif /* CONFIG_MMU */ 88 89 int patch_text_nosync(void *addr, const void *insns, size_t len) 90 { 91 u32 *tp = addr; 92 int ret; 93 94 ret = patch_insn_write(tp, insns, len); 95 96 if (!ret) 97 flush_icache_range((uintptr_t) tp, (uintptr_t) tp + len); 98 99 return ret; 100 } 101 NOKPROBE_SYMBOL(patch_text_nosync); 102 103 static int patch_text_cb(void *data) 104 { 105 struct patch_insn *patch = data; 106 unsigned long len; 107 int i, ret = 0; 108 109 if (atomic_inc_return(&patch->cpu_count) == num_online_cpus()) { 110 for (i = 0; ret == 0 && i < patch->ninsns; i++) { 111 len = GET_INSN_LENGTH(patch->insns[i]); 112 ret = patch_text_nosync(patch->addr + i * len, 113 &patch->insns[i], len); 114 } 115 atomic_inc(&patch->cpu_count); 116 } else { 117 while (atomic_read(&patch->cpu_count) <= num_online_cpus()) 118 cpu_relax(); 119 smp_mb(); 120 } 121 122 return ret; 123 } 124 NOKPROBE_SYMBOL(patch_text_cb); 125 126 int patch_text(void *addr, u32 *insns, int ninsns) 127 { 128 struct patch_insn patch = { 129 .addr = addr, 130 .insns = insns, 131 .ninsns = ninsns, 132 .cpu_count = ATOMIC_INIT(0), 133 }; 134 135 return stop_machine_cpuslocked(patch_text_cb, 136 &patch, cpu_online_mask); 137 } 138 NOKPROBE_SYMBOL(patch_text); 139