1620a53d5SSven Schnelle // SPDX-License-Identifier: GPL-2.0 2620a53d5SSven Schnelle /* 3620a53d5SSven Schnelle * functions to patch RO kernel text during runtime 4620a53d5SSven Schnelle * 5620a53d5SSven Schnelle * Copyright (c) 2019 Sven Schnelle <svens@stackframe.org> 6620a53d5SSven Schnelle */ 7620a53d5SSven Schnelle 8620a53d5SSven Schnelle #include <linux/kernel.h> 9620a53d5SSven Schnelle #include <linux/spinlock.h> 10620a53d5SSven Schnelle #include <linux/kprobes.h> 11620a53d5SSven Schnelle #include <linux/mm.h> 12620a53d5SSven Schnelle #include <linux/stop_machine.h> 13620a53d5SSven Schnelle 14620a53d5SSven Schnelle #include <asm/cacheflush.h> 15620a53d5SSven Schnelle #include <asm/fixmap.h> 16620a53d5SSven Schnelle #include <asm/patch.h> 17620a53d5SSven Schnelle 18620a53d5SSven Schnelle struct patch { 19620a53d5SSven Schnelle void *addr; 204e87ace9SSven Schnelle u32 *insn; 214e87ace9SSven Schnelle unsigned int len; 22620a53d5SSven Schnelle }; 23620a53d5SSven Schnelle 244e87ace9SSven Schnelle static DEFINE_RAW_SPINLOCK(patch_lock); 254e87ace9SSven Schnelle 267e923369SSven Schnelle static void __kprobes *patch_map(void *addr, int fixmap, unsigned long *flags, 277e923369SSven Schnelle int *need_unmap) 287e923369SSven Schnelle { 29620a53d5SSven Schnelle unsigned long uintaddr = (uintptr_t) addr; 30620a53d5SSven Schnelle bool module = !core_kernel_text(uintaddr); 31620a53d5SSven Schnelle struct page *page; 32620a53d5SSven Schnelle 334e87ace9SSven Schnelle *need_unmap = 0; 34620a53d5SSven Schnelle if (module && IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) 35620a53d5SSven Schnelle page = vmalloc_to_page(addr); 36620a53d5SSven Schnelle else if (!module && IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) 37620a53d5SSven Schnelle page = virt_to_page(addr); 38620a53d5SSven Schnelle else 39620a53d5SSven Schnelle return addr; 40620a53d5SSven Schnelle 414e87ace9SSven Schnelle *need_unmap = 1; 42620a53d5SSven Schnelle set_fixmap(fixmap, page_to_phys(page)); 437e923369SSven Schnelle if (flags) 447e923369SSven Schnelle raw_spin_lock_irqsave(&patch_lock, *flags); 457e923369SSven Schnelle else 467e923369SSven Schnelle __acquire(&patch_lock); 47620a53d5SSven Schnelle 48620a53d5SSven Schnelle return (void *) (__fix_to_virt(fixmap) + (uintaddr & ~PAGE_MASK)); 49620a53d5SSven Schnelle } 50620a53d5SSven Schnelle 517e923369SSven Schnelle static void __kprobes patch_unmap(int fixmap, unsigned long *flags) 52620a53d5SSven Schnelle { 53620a53d5SSven Schnelle clear_fixmap(fixmap); 547e923369SSven Schnelle 557e923369SSven Schnelle if (flags) 567e923369SSven Schnelle raw_spin_unlock_irqrestore(&patch_lock, *flags); 577e923369SSven Schnelle else 587e923369SSven Schnelle __release(&patch_lock); 59620a53d5SSven Schnelle } 60620a53d5SSven Schnelle 614e87ace9SSven Schnelle void __kprobes __patch_text_multiple(void *addr, u32 *insn, unsigned int len) 62620a53d5SSven Schnelle { 634e87ace9SSven Schnelle unsigned long start = (unsigned long)addr; 644e87ace9SSven Schnelle unsigned long end = (unsigned long)addr + len; 657e923369SSven Schnelle unsigned long flags; 664e87ace9SSven Schnelle u32 *p, *fixmap; 674e87ace9SSven Schnelle int mapped; 68620a53d5SSven Schnelle 694e87ace9SSven Schnelle /* Make sure we don't have any aliases in cache */ 704e87ace9SSven Schnelle flush_kernel_vmap_range(addr, len); 714e87ace9SSven Schnelle flush_icache_range(start, end); 724e87ace9SSven Schnelle 737e923369SSven Schnelle p = fixmap = patch_map(addr, FIX_TEXT_POKE0, &flags, &mapped); 744e87ace9SSven Schnelle 754e87ace9SSven Schnelle while (len >= 4) { 764e87ace9SSven Schnelle *p++ = *insn++; 774e87ace9SSven Schnelle addr += sizeof(u32); 784e87ace9SSven Schnelle len -= sizeof(u32); 794e87ace9SSven Schnelle if (len && offset_in_page(addr) == 0) { 804e87ace9SSven Schnelle /* 814e87ace9SSven Schnelle * We're crossing a page boundary, so 824e87ace9SSven Schnelle * need to remap 834e87ace9SSven Schnelle */ 844e87ace9SSven Schnelle flush_kernel_vmap_range((void *)fixmap, 854e87ace9SSven Schnelle (p-fixmap) * sizeof(*p)); 864e87ace9SSven Schnelle if (mapped) 877e923369SSven Schnelle patch_unmap(FIX_TEXT_POKE0, &flags); 887e923369SSven Schnelle p = fixmap = patch_map(addr, FIX_TEXT_POKE0, &flags, 897e923369SSven Schnelle &mapped); 904e87ace9SSven Schnelle } 914e87ace9SSven Schnelle } 924e87ace9SSven Schnelle 934e87ace9SSven Schnelle flush_kernel_vmap_range((void *)fixmap, (p-fixmap) * sizeof(*p)); 944e87ace9SSven Schnelle if (mapped) 957e923369SSven Schnelle patch_unmap(FIX_TEXT_POKE0, &flags); 964e87ace9SSven Schnelle flush_icache_range(start, end); 974e87ace9SSven Schnelle } 984e87ace9SSven Schnelle 994e87ace9SSven Schnelle void __kprobes __patch_text(void *addr, u32 insn) 1004e87ace9SSven Schnelle { 1014e87ace9SSven Schnelle __patch_text_multiple(addr, &insn, sizeof(insn)); 102620a53d5SSven Schnelle } 103620a53d5SSven Schnelle 104620a53d5SSven Schnelle static int __kprobes patch_text_stop_machine(void *data) 105620a53d5SSven Schnelle { 106620a53d5SSven Schnelle struct patch *patch = data; 107620a53d5SSven Schnelle 1084e87ace9SSven Schnelle __patch_text_multiple(patch->addr, patch->insn, patch->len); 109620a53d5SSven Schnelle return 0; 110620a53d5SSven Schnelle } 111620a53d5SSven Schnelle 112620a53d5SSven Schnelle void __kprobes patch_text(void *addr, unsigned int insn) 113620a53d5SSven Schnelle { 114620a53d5SSven Schnelle struct patch patch = { 115620a53d5SSven Schnelle .addr = addr, 1164e87ace9SSven Schnelle .insn = &insn, 1174e87ace9SSven Schnelle .len = sizeof(insn), 1184e87ace9SSven Schnelle }; 1194e87ace9SSven Schnelle 1204e87ace9SSven Schnelle stop_machine_cpuslocked(patch_text_stop_machine, &patch, NULL); 1214e87ace9SSven Schnelle } 1224e87ace9SSven Schnelle 1234e87ace9SSven Schnelle void __kprobes patch_text_multiple(void *addr, u32 *insn, unsigned int len) 1244e87ace9SSven Schnelle { 1254e87ace9SSven Schnelle 1264e87ace9SSven Schnelle struct patch patch = { 1274e87ace9SSven Schnelle .addr = addr, 128620a53d5SSven Schnelle .insn = insn, 1294e87ace9SSven Schnelle .len = len 130620a53d5SSven Schnelle }; 131620a53d5SSven Schnelle 132620a53d5SSven Schnelle stop_machine_cpuslocked(patch_text_stop_machine, &patch, NULL); 133620a53d5SSven Schnelle } 134