alternative.c (46938cc8ab91354e6d751dc0790ddb4244b6703a) alternative.c (e836673c9b4966bc78e38aeda25f7022c57f0e90)
1#define pr_fmt(fmt) "SMP alternatives: " fmt
2
3#include <linux/module.h>
4#include <linux/sched.h>
5#include <linux/mutex.h>
6#include <linux/list.h>
7#include <linux/stringify.h>
8#include <linux/mm.h>

--- 261 unchanged lines hidden (view full) ---

270 * Are we looking at a near JMP with a 1 or 4-byte displacement.
271 */
272static inline bool is_jmp(const u8 opcode)
273{
274 return opcode == 0xeb || opcode == 0xe9;
275}
276
277static void __init_or_module
1#define pr_fmt(fmt) "SMP alternatives: " fmt
2
3#include <linux/module.h>
4#include <linux/sched.h>
5#include <linux/mutex.h>
6#include <linux/list.h>
7#include <linux/stringify.h>
8#include <linux/mm.h>

--- 261 unchanged lines hidden (view full) ---

270 * Are we looking at a near JMP with a 1 or 4-byte displacement.
271 */
272static inline bool is_jmp(const u8 opcode)
273{
274 return opcode == 0xeb || opcode == 0xe9;
275}
276
277static void __init_or_module
278recompute_jump(struct alt_instr *a, u8 *orig_insn, u8 *repl_insn, u8 *insn_buff)
278recompute_jump(struct alt_instr *a, u8 *orig_insn, u8 *repl_insn, u8 *insnbuf)
279{
280 u8 *next_rip, *tgt_rip;
281 s32 n_dspl, o_dspl;
282 int repl_len;
283
284 if (a->replacementlen != 5)
285 return;
286
279{
280 u8 *next_rip, *tgt_rip;
281 s32 n_dspl, o_dspl;
282 int repl_len;
283
284 if (a->replacementlen != 5)
285 return;
286
287 o_dspl = *(s32 *)(insn_buff + 1);
287 o_dspl = *(s32 *)(insnbuf + 1);
288
289 /* next_rip of the replacement JMP */
290 next_rip = repl_insn + a->replacementlen;
291 /* target rip of the replacement JMP */
292 tgt_rip = next_rip + o_dspl;
293 n_dspl = tgt_rip - orig_insn;
294
295 DPRINTK("target RIP: %px, new_displ: 0x%x", tgt_rip, n_dspl);

--- 9 unchanged lines hidden (view full) ---

305 goto two_byte_jmp;
306 else
307 goto five_byte_jmp;
308 }
309
310two_byte_jmp:
311 n_dspl -= 2;
312
288
289 /* next_rip of the replacement JMP */
290 next_rip = repl_insn + a->replacementlen;
291 /* target rip of the replacement JMP */
292 tgt_rip = next_rip + o_dspl;
293 n_dspl = tgt_rip - orig_insn;
294
295 DPRINTK("target RIP: %px, new_displ: 0x%x", tgt_rip, n_dspl);

--- 9 unchanged lines hidden (view full) ---

305 goto two_byte_jmp;
306 else
307 goto five_byte_jmp;
308 }
309
310two_byte_jmp:
311 n_dspl -= 2;
312
313 insn_buff[0] = 0xeb;
314 insn_buff[1] = (s8)n_dspl;
315 add_nops(insn_buff + 2, 3);
313 insnbuf[0] = 0xeb;
314 insnbuf[1] = (s8)n_dspl;
315 add_nops(insnbuf + 2, 3);
316
317 repl_len = 2;
318 goto done;
319
320five_byte_jmp:
321 n_dspl -= 5;
322
316
317 repl_len = 2;
318 goto done;
319
320five_byte_jmp:
321 n_dspl -= 5;
322
323 insn_buff[0] = 0xe9;
324 *(s32 *)&insn_buff[1] = n_dspl;
323 insnbuf[0] = 0xe9;
324 *(s32 *)&insnbuf[1] = n_dspl;
325
326 repl_len = 5;
327
328done:
329
330 DPRINTK("final displ: 0x%08x, JMP 0x%lx",
331 n_dspl, (unsigned long)orig_insn + n_dspl + repl_len);
332}

--- 30 unchanged lines hidden (view full) ---

363 * Marked "noinline" to cause control flow change and thus insn cache
364 * to refetch changed I$ lines.
365 */
366void __init_or_module noinline apply_alternatives(struct alt_instr *start,
367 struct alt_instr *end)
368{
369 struct alt_instr *a;
370 u8 *instr, *replacement;
325
326 repl_len = 5;
327
328done:
329
330 DPRINTK("final displ: 0x%08x, JMP 0x%lx",
331 n_dspl, (unsigned long)orig_insn + n_dspl + repl_len);
332}

--- 30 unchanged lines hidden (view full) ---

363 * Marked "noinline" to cause control flow change and thus insn cache
364 * to refetch changed I$ lines.
365 */
366void __init_or_module noinline apply_alternatives(struct alt_instr *start,
367 struct alt_instr *end)
368{
369 struct alt_instr *a;
370 u8 *instr, *replacement;
371 u8 insn_buff[MAX_PATCH_LEN];
371 u8 insnbuf[MAX_PATCH_LEN];
372
373 DPRINTK("alt table %px, -> %px", start, end);
374 /*
375 * The scan order should be from start to end. A later scanned
376 * alternative code can overwrite previously scanned alternative code.
377 * Some kernel functions (e.g. memcpy, memset, etc) use this order to
378 * patch code.
379 *
380 * So be careful if you want to change the scan order to any other
381 * order.
382 */
383 for (a = start; a < end; a++) {
372
373 DPRINTK("alt table %px, -> %px", start, end);
374 /*
375 * The scan order should be from start to end. A later scanned
376 * alternative code can overwrite previously scanned alternative code.
377 * Some kernel functions (e.g. memcpy, memset, etc) use this order to
378 * patch code.
379 *
380 * So be careful if you want to change the scan order to any other
381 * order.
382 */
383 for (a = start; a < end; a++) {
384 int insn_buff_sz = 0;
384 int insnbuf_sz = 0;
385
386 instr = (u8 *)&a->instr_offset + a->instr_offset;
387 replacement = (u8 *)&a->repl_offset + a->repl_offset;
385
386 instr = (u8 *)&a->instr_offset + a->instr_offset;
387 replacement = (u8 *)&a->repl_offset + a->repl_offset;
388 BUG_ON(a->instrlen > sizeof(insn_buff));
388 BUG_ON(a->instrlen > sizeof(insnbuf));
389 BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32);
390 if (!boot_cpu_has(a->cpuid)) {
391 if (a->padlen > 1)
392 optimize_nops(a, instr);
393
394 continue;
395 }
396
397 DPRINTK("feat: %d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d), pad: %d",
398 a->cpuid >> 5,
399 a->cpuid & 0x1f,
400 instr, instr, a->instrlen,
401 replacement, a->replacementlen, a->padlen);
402
403 DUMP_BYTES(instr, a->instrlen, "%px: old_insn: ", instr);
404 DUMP_BYTES(replacement, a->replacementlen, "%px: rpl_insn: ", replacement);
405
389 BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32);
390 if (!boot_cpu_has(a->cpuid)) {
391 if (a->padlen > 1)
392 optimize_nops(a, instr);
393
394 continue;
395 }
396
397 DPRINTK("feat: %d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d), pad: %d",
398 a->cpuid >> 5,
399 a->cpuid & 0x1f,
400 instr, instr, a->instrlen,
401 replacement, a->replacementlen, a->padlen);
402
403 DUMP_BYTES(instr, a->instrlen, "%px: old_insn: ", instr);
404 DUMP_BYTES(replacement, a->replacementlen, "%px: rpl_insn: ", replacement);
405
406 memcpy(insn_buff, replacement, a->replacementlen);
407 insn_buff_sz = a->replacementlen;
406 memcpy(insnbuf, replacement, a->replacementlen);
407 insnbuf_sz = a->replacementlen;
408
409 /*
410 * 0xe8 is a relative jump; fix the offset.
411 *
412 * Instruction length is checked before the opcode to avoid
413 * accessing uninitialized bytes for zero-length replacements.
414 */
408
409 /*
410 * 0xe8 is a relative jump; fix the offset.
411 *
412 * Instruction length is checked before the opcode to avoid
413 * accessing uninitialized bytes for zero-length replacements.
414 */
415 if (a->replacementlen == 5 && *insn_buff == 0xe8) {
416 *(s32 *)(insn_buff + 1) += replacement - instr;
415 if (a->replacementlen == 5 && *insnbuf == 0xe8) {
416 *(s32 *)(insnbuf + 1) += replacement - instr;
417 DPRINTK("Fix CALL offset: 0x%x, CALL 0x%lx",
417 DPRINTK("Fix CALL offset: 0x%x, CALL 0x%lx",
418 *(s32 *)(insn_buff + 1),
419 (unsigned long)instr + *(s32 *)(insn_buff + 1) + 5);
418 *(s32 *)(insnbuf + 1),
419 (unsigned long)instr + *(s32 *)(insnbuf + 1) + 5);
420 }
421
422 if (a->replacementlen && is_jmp(replacement[0]))
420 }
421
422 if (a->replacementlen && is_jmp(replacement[0]))
423 recompute_jump(a, instr, replacement, insn_buff);
423 recompute_jump(a, instr, replacement, insnbuf);
424
425 if (a->instrlen > a->replacementlen) {
424
425 if (a->instrlen > a->replacementlen) {
426 add_nops(insn_buff + a->replacementlen,
426 add_nops(insnbuf + a->replacementlen,
427 a->instrlen - a->replacementlen);
427 a->instrlen - a->replacementlen);
428 insn_buff_sz += a->instrlen - a->replacementlen;
428 insnbuf_sz += a->instrlen - a->replacementlen;
429 }
429 }
430 DUMP_BYTES(insn_buff, insn_buff_sz, "%px: final_insn: ", instr);
430 DUMP_BYTES(insnbuf, insnbuf_sz, "%px: final_insn: ", instr);
431
431
432 text_poke_early(instr, insn_buff, insn_buff_sz);
432 text_poke_early(instr, insnbuf, insnbuf_sz);
433 }
434}
435
436#ifdef CONFIG_SMP
437static void alternatives_smp_lock(const s32 *start, const s32 *end,
438 u8 *text, u8 *text_end)
439{
440 const s32 *poff;

--- 145 unchanged lines hidden (view full) ---

586}
587#endif /* CONFIG_SMP */
588
589#ifdef CONFIG_PARAVIRT
590void __init_or_module apply_paravirt(struct paravirt_patch_site *start,
591 struct paravirt_patch_site *end)
592{
593 struct paravirt_patch_site *p;
433 }
434}
435
436#ifdef CONFIG_SMP
437static void alternatives_smp_lock(const s32 *start, const s32 *end,
438 u8 *text, u8 *text_end)
439{
440 const s32 *poff;

--- 145 unchanged lines hidden (view full) ---

586}
587#endif /* CONFIG_SMP */
588
589#ifdef CONFIG_PARAVIRT
590void __init_or_module apply_paravirt(struct paravirt_patch_site *start,
591 struct paravirt_patch_site *end)
592{
593 struct paravirt_patch_site *p;
594 char insn_buff[MAX_PATCH_LEN];
594 char insnbuf[MAX_PATCH_LEN];
595
596 for (p = start; p < end; p++) {
597 unsigned int used;
598
599 BUG_ON(p->len > MAX_PATCH_LEN);
600 /* prep the buffer with the original instructions */
595
596 for (p = start; p < end; p++) {
597 unsigned int used;
598
599 BUG_ON(p->len > MAX_PATCH_LEN);
600 /* prep the buffer with the original instructions */
601 memcpy(insn_buff, p->instr, p->len);
602 used = pv_ops.init.patch(p->type, insn_buff, (unsigned long)p->instr, p->len);
601 memcpy(insnbuf, p->instr, p->len);
602 used = pv_ops.init.patch(p->instrtype, insnbuf,
603 (unsigned long)p->instr, p->len);
603
604 BUG_ON(used > p->len);
605
606 /* Pad the rest with nops */
604
605 BUG_ON(used > p->len);
606
607 /* Pad the rest with nops */
607 add_nops(insn_buff + used, p->len - used);
608 text_poke_early(p->instr, insn_buff, p->len);
608 add_nops(insnbuf + used, p->len - used);
609 text_poke_early(p->instr, insnbuf, p->len);
609 }
610}
611extern struct paravirt_patch_site __start_parainstructions[],
612 __stop_parainstructions[];
613#endif /* CONFIG_PARAVIRT */
614
615void __init alternative_instructions(void)
616{

--- 56 unchanged lines hidden (view full) ---

673 memcpy(addr, opcode, len);
674 local_irq_restore(flags);
675 sync_core();
676 /* Could also do a CLFLUSH here to speed up CPU recovery; but
677 that causes hangs on some VIA CPUs. */
678 return addr;
679}
680
610 }
611}
612extern struct paravirt_patch_site __start_parainstructions[],
613 __stop_parainstructions[];
614#endif /* CONFIG_PARAVIRT */
615
616void __init alternative_instructions(void)
617{

--- 56 unchanged lines hidden (view full) ---

674 memcpy(addr, opcode, len);
675 local_irq_restore(flags);
676 sync_core();
677 /* Could also do a CLFLUSH here to speed up CPU recovery; but
678 that causes hangs on some VIA CPUs. */
679 return addr;
680}
681
681/**
682 * text_poke - Update instructions on a live kernel
683 * @addr: address to modify
684 * @opcode: source of the copy
685 * @len: length to copy
686 *
687 * Only atomic text poke/set should be allowed when not doing early patching.
688 * It means the size must be writable atomically and the address must be aligned
689 * in a way that permits an atomic write. It also makes sure we fit on a single
690 * page.
691 */
692void *text_poke(void *addr, const void *opcode, size_t len)
682static void *__text_poke(void *addr, const void *opcode, size_t len)
693{
694 unsigned long flags;
695 char *vaddr;
696 struct page *pages[2];
697 int i;
698
699 /*
700 * While boot memory allocator is runnig we cannot use struct
701 * pages as they are not yet initialized.
702 */
703 BUG_ON(!after_bootmem);
704
683{
684 unsigned long flags;
685 char *vaddr;
686 struct page *pages[2];
687 int i;
688
689 /*
690 * While boot memory allocator is runnig we cannot use struct
691 * pages as they are not yet initialized.
692 */
693 BUG_ON(!after_bootmem);
694
705 lockdep_assert_held(&text_mutex);
706
707 if (!core_kernel_text((unsigned long)addr)) {
708 pages[0] = vmalloc_to_page(addr);
709 pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
710 } else {
711 pages[0] = virt_to_page(addr);
712 WARN_ON(!PageReserved(pages[0]));
713 pages[1] = virt_to_page(addr + PAGE_SIZE);
714 }

--- 12 unchanged lines hidden (view full) ---

727 /* Could also do a CLFLUSH here to speed up CPU recovery; but
728 that causes hangs on some VIA CPUs. */
729 for (i = 0; i < len; i++)
730 BUG_ON(((char *)addr)[i] != ((char *)opcode)[i]);
731 local_irq_restore(flags);
732 return addr;
733}
734
695 if (!core_kernel_text((unsigned long)addr)) {
696 pages[0] = vmalloc_to_page(addr);
697 pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
698 } else {
699 pages[0] = virt_to_page(addr);
700 WARN_ON(!PageReserved(pages[0]));
701 pages[1] = virt_to_page(addr + PAGE_SIZE);
702 }

--- 12 unchanged lines hidden (view full) ---

715 /* Could also do a CLFLUSH here to speed up CPU recovery; but
716 that causes hangs on some VIA CPUs. */
717 for (i = 0; i < len; i++)
718 BUG_ON(((char *)addr)[i] != ((char *)opcode)[i]);
719 local_irq_restore(flags);
720 return addr;
721}
722
723/**
724 * text_poke - Update instructions on a live kernel
725 * @addr: address to modify
726 * @opcode: source of the copy
727 * @len: length to copy
728 *
729 * Only atomic text poke/set should be allowed when not doing early patching.
730 * It means the size must be writable atomically and the address must be aligned
731 * in a way that permits an atomic write. It also makes sure we fit on a single
732 * page.
733 */
734void *text_poke(void *addr, const void *opcode, size_t len)
735{
736 lockdep_assert_held(&text_mutex);
737
738 return __text_poke(addr, opcode, len);
739}
740
741/**
742 * text_poke_kgdb - Update instructions on a live kernel by kgdb
743 * @addr: address to modify
744 * @opcode: source of the copy
745 * @len: length to copy
746 *
747 * Only atomic text poke/set should be allowed when not doing early patching.
748 * It means the size must be writable atomically and the address must be aligned
749 * in a way that permits an atomic write. It also makes sure we fit on a single
750 * page.
751 *
752 * Context: should only be used by kgdb, which ensures no other core is running,
753 * despite the fact it does not hold the text_mutex.
754 */
755void *text_poke_kgdb(void *addr, const void *opcode, size_t len)
756{
757 return __text_poke(addr, opcode, len);
758}
759
735static void do_sync_core(void *info)
736{
737 sync_core();
738}
739
740static bool bp_patching_in_progress;
741static void *bp_int3_handler, *bp_int3_addr;
742

--- 93 unchanged lines hidden --- 		760static void do_sync_core(void *info)
761{
762 sync_core();
763}
764
765static bool bp_patching_in_progress;
766static void *bp_int3_handler, *bp_int3_addr;
767

--- 93 unchanged lines hidden ---