1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 SiFive 4 */ 5 6 #include <linux/spinlock.h> 7 #include <linux/mm.h> 8 #include <linux/uaccess.h> 9 #include <linux/stop_machine.h> 10 #include <asm/kprobes.h> 11 #include <asm/cacheflush.h> 12 #include <asm/fixmap.h> 13 14 struct riscv_insn_patch { 15 void *addr; 16 u32 insn; 17 atomic_t cpu_count; 18 }; 19 20 #ifdef CONFIG_MMU 21 static DEFINE_RAW_SPINLOCK(patch_lock); 22 23 static void __kprobes *patch_map(void *addr, int fixmap) 24 { 25 uintptr_t uintaddr = (uintptr_t) addr; 26 struct page *page; 27 28 if (core_kernel_text(uintaddr)) 29 page = phys_to_page(__pa_symbol(addr)); 30 else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) 31 page = vmalloc_to_page(addr); 32 else 33 return addr; 34 35 BUG_ON(!page); 36 37 return (void *)set_fixmap_offset(fixmap, page_to_phys(page) + 38 (uintaddr & ~PAGE_MASK)); 39 } 40 41 static void __kprobes patch_unmap(int fixmap) 42 { 43 clear_fixmap(fixmap); 44 } 45 46 static int __kprobes riscv_insn_write(void *addr, const void *insn, size_t len) 47 { 48 void *waddr = addr; 49 bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE; 50 unsigned long flags = 0; 51 int ret; 52 53 raw_spin_lock_irqsave(&patch_lock, flags); 54 55 if (across_pages) 56 patch_map(addr + len, FIX_TEXT_POKE1); 57 58 waddr = patch_map(addr, FIX_TEXT_POKE0); 59 60 ret = probe_kernel_write(waddr, insn, len); 61 62 patch_unmap(FIX_TEXT_POKE0); 63 64 if (across_pages) 65 patch_unmap(FIX_TEXT_POKE1); 66 67 raw_spin_unlock_irqrestore(&patch_lock, flags); 68 69 return ret; 70 } 71 #else 72 static int __kprobes riscv_insn_write(void *addr, const void *insn, size_t len) 73 { 74 return probe_kernel_write(addr, insn, len); 75 } 76 #endif /* CONFIG_MMU */ 77 78 int __kprobes riscv_patch_text_nosync(void *addr, const void *insns, size_t len) 79 { 80 u32 *tp = addr; 81 int ret; 82 83 ret = riscv_insn_write(tp, insns, len); 84 85 if (!ret) 86 flush_icache_range((uintptr_t) tp, (uintptr_t) tp + len); 87 88 return ret; 89 } 90 91 static int __kprobes riscv_patch_text_cb(void *data) 92 { 93 struct riscv_insn_patch *patch = data; 94 int ret = 0; 95 96 if (atomic_inc_return(&patch->cpu_count) == 1) { 97 ret = 98 riscv_patch_text_nosync(patch->addr, &patch->insn, 99 GET_INSN_LENGTH(patch->insn)); 100 atomic_inc(&patch->cpu_count); 101 } else { 102 while (atomic_read(&patch->cpu_count) <= num_online_cpus()) 103 cpu_relax(); 104 smp_mb(); 105 } 106 107 return ret; 108 } 109 110 int __kprobes riscv_patch_text(void *addr, u32 insn) 111 { 112 struct riscv_insn_patch patch = { 113 .addr = addr, 114 .insn = insn, 115 .cpu_count = ATOMIC_INIT(0), 116 }; 117 118 return stop_machine_cpuslocked(riscv_patch_text_cb, 119 &patch, cpu_online_mask); 120 } 121