1 /* 2 * Copyright (C) 2001 Ben. Herrenschmidt (benh@kernel.crashing.org) 3 * 4 * Modifications for ppc64: 5 * Copyright (C) 2003 Dave Engebretsen <engebret@us.ibm.com> 6 * 7 * Copyright 2008 Michael Ellerman, IBM Corporation. 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License 11 * as published by the Free Software Foundation; either version 12 * 2 of the License, or (at your option) any later version. 13 */ 14 15 #include <linux/types.h> 16 #include <linux/jump_label.h> 17 #include <linux/kernel.h> 18 #include <linux/string.h> 19 #include <linux/init.h> 20 #include <linux/sched/mm.h> 21 #include <asm/cputable.h> 22 #include <asm/code-patching.h> 23 #include <asm/page.h> 24 #include <asm/sections.h> 25 #include <asm/setup.h> 26 #include <asm/firmware.h> 27 28 struct fixup_entry { 29 unsigned long mask; 30 unsigned long value; 31 long start_off; 32 long end_off; 33 long alt_start_off; 34 long alt_end_off; 35 }; 36 37 static unsigned int *calc_addr(struct fixup_entry *fcur, long offset) 38 { 39 /* 40 * We store the offset to the code as a negative offset from 41 * the start of the alt_entry, to support the VDSO. This 42 * routine converts that back into an actual address. 43 */ 44 return (unsigned int *)((unsigned long)fcur + offset); 45 } 46 47 static int patch_alt_instruction(unsigned int *src, unsigned int *dest, 48 unsigned int *alt_start, unsigned int *alt_end) 49 { 50 unsigned int instr; 51 52 instr = *src; 53 54 if (instr_is_relative_branch(*src)) { 55 unsigned int *target = (unsigned int *)branch_target(src); 56 57 /* Branch within the section doesn't need translating */ 58 if (target < alt_start || target > alt_end) { 59 instr = translate_branch(dest, src); 60 if (!instr) 61 return 1; 62 } 63 } 64 65 raw_patch_instruction(dest, instr); 66 67 return 0; 68 } 69 70 static int patch_feature_section(unsigned long value, struct fixup_entry *fcur) 71 { 72 unsigned int *start, *end, *alt_start, *alt_end, *src, *dest; 73 74 start = calc_addr(fcur, fcur->start_off); 75 end = calc_addr(fcur, fcur->end_off); 76 alt_start = calc_addr(fcur, fcur->alt_start_off); 77 alt_end = calc_addr(fcur, fcur->alt_end_off); 78 79 if ((alt_end - alt_start) > (end - start)) 80 return 1; 81 82 if ((value & fcur->mask) == fcur->value) 83 return 0; 84 85 src = alt_start; 86 dest = start; 87 88 for (; src < alt_end; src++, dest++) { 89 if (patch_alt_instruction(src, dest, alt_start, alt_end)) 90 return 1; 91 } 92 93 for (; dest < end; dest++) 94 raw_patch_instruction(dest, PPC_INST_NOP); 95 96 return 0; 97 } 98 99 void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end) 100 { 101 struct fixup_entry *fcur, *fend; 102 103 fcur = fixup_start; 104 fend = fixup_end; 105 106 for (; fcur < fend; fcur++) { 107 if (patch_feature_section(value, fcur)) { 108 WARN_ON(1); 109 printk("Unable to patch feature section at %p - %p" \ 110 " with %p - %p\n", 111 calc_addr(fcur, fcur->start_off), 112 calc_addr(fcur, fcur->end_off), 113 calc_addr(fcur, fcur->alt_start_off), 114 calc_addr(fcur, fcur->alt_end_off)); 115 } 116 } 117 } 118 119 #ifdef CONFIG_PPC_BOOK3S_64 120 void do_rfi_flush_fixups(enum l1d_flush_type types) 121 { 122 unsigned int instrs[3], *dest; 123 long *start, *end; 124 int i; 125 126 start = PTRRELOC(&__start___rfi_flush_fixup), 127 end = PTRRELOC(&__stop___rfi_flush_fixup); 128 129 instrs[0] = 0x60000000; /* nop */ 130 instrs[1] = 0x60000000; /* nop */ 131 instrs[2] = 0x60000000; /* nop */ 132 133 if (types & L1D_FLUSH_FALLBACK) 134 /* b .+16 to fallback flush */ 135 instrs[0] = 0x48000010; 136 137 i = 0; 138 if (types & L1D_FLUSH_ORI) { 139 instrs[i++] = 0x63ff0000; /* ori 31,31,0 speculation barrier */ 140 instrs[i++] = 0x63de0000; /* ori 30,30,0 L1d flush*/ 141 } 142 143 if (types & L1D_FLUSH_MTTRIG) 144 instrs[i++] = 0x7c12dba6; /* mtspr TRIG2,r0 (SPR #882) */ 145 146 for (i = 0; start < end; start++, i++) { 147 dest = (void *)start + *start; 148 149 pr_devel("patching dest %lx\n", (unsigned long)dest); 150 151 patch_instruction(dest, instrs[0]); 152 patch_instruction(dest + 1, instrs[1]); 153 patch_instruction(dest + 2, instrs[2]); 154 } 155 156 printk(KERN_DEBUG "rfi-flush: patched %d locations (%s flush)\n", i, 157 (types == L1D_FLUSH_NONE) ? "no" : 158 (types == L1D_FLUSH_FALLBACK) ? "fallback displacement" : 159 (types & L1D_FLUSH_ORI) ? (types & L1D_FLUSH_MTTRIG) 160 ? "ori+mttrig type" 161 : "ori type" : 162 (types & L1D_FLUSH_MTTRIG) ? "mttrig type" 163 : "unknown"); 164 } 165 #endif /* CONFIG_PPC_BOOK3S_64 */ 166 167 void do_lwsync_fixups(unsigned long value, void *fixup_start, void *fixup_end) 168 { 169 long *start, *end; 170 unsigned int *dest; 171 172 if (!(value & CPU_FTR_LWSYNC)) 173 return ; 174 175 start = fixup_start; 176 end = fixup_end; 177 178 for (; start < end; start++) { 179 dest = (void *)start + *start; 180 raw_patch_instruction(dest, PPC_INST_LWSYNC); 181 } 182 } 183 184 static void do_final_fixups(void) 185 { 186 #if defined(CONFIG_PPC64) && defined(CONFIG_RELOCATABLE) 187 int *src, *dest; 188 unsigned long length; 189 190 if (PHYSICAL_START == 0) 191 return; 192 193 src = (int *)(KERNELBASE + PHYSICAL_START); 194 dest = (int *)KERNELBASE; 195 length = (__end_interrupts - _stext) / sizeof(int); 196 197 while (length--) { 198 raw_patch_instruction(dest, *src); 199 src++; 200 dest++; 201 } 202 #endif 203 } 204 205 static unsigned long __initdata saved_cpu_features; 206 static unsigned int __initdata saved_mmu_features; 207 #ifdef CONFIG_PPC64 208 static unsigned long __initdata saved_firmware_features; 209 #endif 210 211 void __init apply_feature_fixups(void) 212 { 213 struct cpu_spec *spec = PTRRELOC(*PTRRELOC(&cur_cpu_spec)); 214 215 *PTRRELOC(&saved_cpu_features) = spec->cpu_features; 216 *PTRRELOC(&saved_mmu_features) = spec->mmu_features; 217 218 /* 219 * Apply the CPU-specific and firmware specific fixups to kernel text 220 * (nop out sections not relevant to this CPU or this firmware). 221 */ 222 do_feature_fixups(spec->cpu_features, 223 PTRRELOC(&__start___ftr_fixup), 224 PTRRELOC(&__stop___ftr_fixup)); 225 226 do_feature_fixups(spec->mmu_features, 227 PTRRELOC(&__start___mmu_ftr_fixup), 228 PTRRELOC(&__stop___mmu_ftr_fixup)); 229 230 do_lwsync_fixups(spec->cpu_features, 231 PTRRELOC(&__start___lwsync_fixup), 232 PTRRELOC(&__stop___lwsync_fixup)); 233 234 #ifdef CONFIG_PPC64 235 saved_firmware_features = powerpc_firmware_features; 236 do_feature_fixups(powerpc_firmware_features, 237 &__start___fw_ftr_fixup, &__stop___fw_ftr_fixup); 238 #endif 239 do_final_fixups(); 240 } 241 242 void __init setup_feature_keys(void) 243 { 244 /* 245 * Initialise jump label. This causes all the cpu/mmu_has_feature() 246 * checks to take on their correct polarity based on the current set of 247 * CPU/MMU features. 248 */ 249 jump_label_init(); 250 cpu_feature_keys_init(); 251 mmu_feature_keys_init(); 252 } 253 254 static int __init check_features(void) 255 { 256 WARN(saved_cpu_features != cur_cpu_spec->cpu_features, 257 "CPU features changed after feature patching!\n"); 258 WARN(saved_mmu_features != cur_cpu_spec->mmu_features, 259 "MMU features changed after feature patching!\n"); 260 #ifdef CONFIG_PPC64 261 WARN(saved_firmware_features != powerpc_firmware_features, 262 "Firmware features changed after feature patching!\n"); 263 #endif 264 265 return 0; 266 } 267 late_initcall(check_features); 268 269 #ifdef CONFIG_FTR_FIXUP_SELFTEST 270 271 #define check(x) \ 272 if (!(x)) printk("feature-fixups: test failed at line %d\n", __LINE__); 273 274 /* This must be after the text it fixes up, vmlinux.lds.S enforces that atm */ 275 static struct fixup_entry fixup; 276 277 static long calc_offset(struct fixup_entry *entry, unsigned int *p) 278 { 279 return (unsigned long)p - (unsigned long)entry; 280 } 281 282 static void test_basic_patching(void) 283 { 284 extern unsigned int ftr_fixup_test1[]; 285 extern unsigned int end_ftr_fixup_test1[]; 286 extern unsigned int ftr_fixup_test1_orig[]; 287 extern unsigned int ftr_fixup_test1_expected[]; 288 int size = end_ftr_fixup_test1 - ftr_fixup_test1; 289 290 fixup.value = fixup.mask = 8; 291 fixup.start_off = calc_offset(&fixup, ftr_fixup_test1 + 1); 292 fixup.end_off = calc_offset(&fixup, ftr_fixup_test1 + 2); 293 fixup.alt_start_off = fixup.alt_end_off = 0; 294 295 /* Sanity check */ 296 check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0); 297 298 /* Check we don't patch if the value matches */ 299 patch_feature_section(8, &fixup); 300 check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0); 301 302 /* Check we do patch if the value doesn't match */ 303 patch_feature_section(0, &fixup); 304 check(memcmp(ftr_fixup_test1, ftr_fixup_test1_expected, size) == 0); 305 306 /* Check we do patch if the mask doesn't match */ 307 memcpy(ftr_fixup_test1, ftr_fixup_test1_orig, size); 308 check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0); 309 patch_feature_section(~8, &fixup); 310 check(memcmp(ftr_fixup_test1, ftr_fixup_test1_expected, size) == 0); 311 } 312 313 static void test_alternative_patching(void) 314 { 315 extern unsigned int ftr_fixup_test2[]; 316 extern unsigned int end_ftr_fixup_test2[]; 317 extern unsigned int ftr_fixup_test2_orig[]; 318 extern unsigned int ftr_fixup_test2_alt[]; 319 extern unsigned int ftr_fixup_test2_expected[]; 320 int size = end_ftr_fixup_test2 - ftr_fixup_test2; 321 322 fixup.value = fixup.mask = 0xF; 323 fixup.start_off = calc_offset(&fixup, ftr_fixup_test2 + 1); 324 fixup.end_off = calc_offset(&fixup, ftr_fixup_test2 + 2); 325 fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_test2_alt); 326 fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_test2_alt + 1); 327 328 /* Sanity check */ 329 check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0); 330 331 /* Check we don't patch if the value matches */ 332 patch_feature_section(0xF, &fixup); 333 check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0); 334 335 /* Check we do patch if the value doesn't match */ 336 patch_feature_section(0, &fixup); 337 check(memcmp(ftr_fixup_test2, ftr_fixup_test2_expected, size) == 0); 338 339 /* Check we do patch if the mask doesn't match */ 340 memcpy(ftr_fixup_test2, ftr_fixup_test2_orig, size); 341 check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0); 342 patch_feature_section(~0xF, &fixup); 343 check(memcmp(ftr_fixup_test2, ftr_fixup_test2_expected, size) == 0); 344 } 345 346 static void test_alternative_case_too_big(void) 347 { 348 extern unsigned int ftr_fixup_test3[]; 349 extern unsigned int end_ftr_fixup_test3[]; 350 extern unsigned int ftr_fixup_test3_orig[]; 351 extern unsigned int ftr_fixup_test3_alt[]; 352 int size = end_ftr_fixup_test3 - ftr_fixup_test3; 353 354 fixup.value = fixup.mask = 0xC; 355 fixup.start_off = calc_offset(&fixup, ftr_fixup_test3 + 1); 356 fixup.end_off = calc_offset(&fixup, ftr_fixup_test3 + 2); 357 fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_test3_alt); 358 fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_test3_alt + 2); 359 360 /* Sanity check */ 361 check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); 362 363 /* Expect nothing to be patched, and the error returned to us */ 364 check(patch_feature_section(0xF, &fixup) == 1); 365 check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); 366 check(patch_feature_section(0, &fixup) == 1); 367 check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); 368 check(patch_feature_section(~0xF, &fixup) == 1); 369 check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); 370 } 371 372 static void test_alternative_case_too_small(void) 373 { 374 extern unsigned int ftr_fixup_test4[]; 375 extern unsigned int end_ftr_fixup_test4[]; 376 extern unsigned int ftr_fixup_test4_orig[]; 377 extern unsigned int ftr_fixup_test4_alt[]; 378 extern unsigned int ftr_fixup_test4_expected[]; 379 int size = end_ftr_fixup_test4 - ftr_fixup_test4; 380 unsigned long flag; 381 382 /* Check a high-bit flag */ 383 flag = 1UL << ((sizeof(unsigned long) - 1) * 8); 384 fixup.value = fixup.mask = flag; 385 fixup.start_off = calc_offset(&fixup, ftr_fixup_test4 + 1); 386 fixup.end_off = calc_offset(&fixup, ftr_fixup_test4 + 5); 387 fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_test4_alt); 388 fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_test4_alt + 2); 389 390 /* Sanity check */ 391 check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0); 392 393 /* Check we don't patch if the value matches */ 394 patch_feature_section(flag, &fixup); 395 check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0); 396 397 /* Check we do patch if the value doesn't match */ 398 patch_feature_section(0, &fixup); 399 check(memcmp(ftr_fixup_test4, ftr_fixup_test4_expected, size) == 0); 400 401 /* Check we do patch if the mask doesn't match */ 402 memcpy(ftr_fixup_test4, ftr_fixup_test4_orig, size); 403 check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0); 404 patch_feature_section(~flag, &fixup); 405 check(memcmp(ftr_fixup_test4, ftr_fixup_test4_expected, size) == 0); 406 } 407 408 static void test_alternative_case_with_branch(void) 409 { 410 extern unsigned int ftr_fixup_test5[]; 411 extern unsigned int end_ftr_fixup_test5[]; 412 extern unsigned int ftr_fixup_test5_expected[]; 413 int size = end_ftr_fixup_test5 - ftr_fixup_test5; 414 415 check(memcmp(ftr_fixup_test5, ftr_fixup_test5_expected, size) == 0); 416 } 417 418 static void test_alternative_case_with_external_branch(void) 419 { 420 extern unsigned int ftr_fixup_test6[]; 421 extern unsigned int end_ftr_fixup_test6[]; 422 extern unsigned int ftr_fixup_test6_expected[]; 423 int size = end_ftr_fixup_test6 - ftr_fixup_test6; 424 425 check(memcmp(ftr_fixup_test6, ftr_fixup_test6_expected, size) == 0); 426 } 427 428 static void test_cpu_macros(void) 429 { 430 extern u8 ftr_fixup_test_FTR_macros[]; 431 extern u8 ftr_fixup_test_FTR_macros_expected[]; 432 unsigned long size = ftr_fixup_test_FTR_macros_expected - 433 ftr_fixup_test_FTR_macros; 434 435 /* The fixups have already been done for us during boot */ 436 check(memcmp(ftr_fixup_test_FTR_macros, 437 ftr_fixup_test_FTR_macros_expected, size) == 0); 438 } 439 440 static void test_fw_macros(void) 441 { 442 #ifdef CONFIG_PPC64 443 extern u8 ftr_fixup_test_FW_FTR_macros[]; 444 extern u8 ftr_fixup_test_FW_FTR_macros_expected[]; 445 unsigned long size = ftr_fixup_test_FW_FTR_macros_expected - 446 ftr_fixup_test_FW_FTR_macros; 447 448 /* The fixups have already been done for us during boot */ 449 check(memcmp(ftr_fixup_test_FW_FTR_macros, 450 ftr_fixup_test_FW_FTR_macros_expected, size) == 0); 451 #endif 452 } 453 454 static void test_lwsync_macros(void) 455 { 456 extern u8 lwsync_fixup_test[]; 457 extern u8 end_lwsync_fixup_test[]; 458 extern u8 lwsync_fixup_test_expected_LWSYNC[]; 459 extern u8 lwsync_fixup_test_expected_SYNC[]; 460 unsigned long size = end_lwsync_fixup_test - 461 lwsync_fixup_test; 462 463 /* The fixups have already been done for us during boot */ 464 if (cur_cpu_spec->cpu_features & CPU_FTR_LWSYNC) { 465 check(memcmp(lwsync_fixup_test, 466 lwsync_fixup_test_expected_LWSYNC, size) == 0); 467 } else { 468 check(memcmp(lwsync_fixup_test, 469 lwsync_fixup_test_expected_SYNC, size) == 0); 470 } 471 } 472 473 static int __init test_feature_fixups(void) 474 { 475 printk(KERN_DEBUG "Running feature fixup self-tests ...\n"); 476 477 test_basic_patching(); 478 test_alternative_patching(); 479 test_alternative_case_too_big(); 480 test_alternative_case_too_small(); 481 test_alternative_case_with_branch(); 482 test_alternative_case_with_external_branch(); 483 test_cpu_macros(); 484 test_fw_macros(); 485 test_lwsync_macros(); 486 487 return 0; 488 } 489 late_initcall(test_feature_fixups); 490 491 #endif /* CONFIG_FTR_FIXUP_SELFTEST */ 492