1 /* 2 * Copyright (C) 2001 Ben. Herrenschmidt (benh@kernel.crashing.org) 3 * 4 * Modifications for ppc64: 5 * Copyright (C) 2003 Dave Engebretsen <engebret@us.ibm.com> 6 * 7 * Copyright 2008 Michael Ellerman, IBM Corporation. 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License 11 * as published by the Free Software Foundation; either version 12 * 2 of the License, or (at your option) any later version. 13 */ 14 15 #include <linux/types.h> 16 #include <linux/jump_label.h> 17 #include <linux/kernel.h> 18 #include <linux/string.h> 19 #include <linux/init.h> 20 #include <linux/sched/mm.h> 21 #include <asm/cputable.h> 22 #include <asm/code-patching.h> 23 #include <asm/page.h> 24 #include <asm/sections.h> 25 #include <asm/setup.h> 26 #include <asm/firmware.h> 27 28 struct fixup_entry { 29 unsigned long mask; 30 unsigned long value; 31 long start_off; 32 long end_off; 33 long alt_start_off; 34 long alt_end_off; 35 }; 36 37 static unsigned int *calc_addr(struct fixup_entry *fcur, long offset) 38 { 39 /* 40 * We store the offset to the code as a negative offset from 41 * the start of the alt_entry, to support the VDSO. This 42 * routine converts that back into an actual address. 43 */ 44 return (unsigned int *)((unsigned long)fcur + offset); 45 } 46 47 static int patch_alt_instruction(unsigned int *src, unsigned int *dest, 48 unsigned int *alt_start, unsigned int *alt_end) 49 { 50 unsigned int instr; 51 52 instr = *src; 53 54 if (instr_is_relative_branch(*src)) { 55 unsigned int *target = (unsigned int *)branch_target(src); 56 57 /* Branch within the section doesn't need translating */ 58 if (target < alt_start || target >= alt_end) { 59 instr = translate_branch(dest, src); 60 if (!instr) 61 return 1; 62 } 63 } 64 65 patch_instruction(dest, instr); 66 67 return 0; 68 } 69 70 static int patch_feature_section(unsigned long value, struct fixup_entry *fcur) 71 { 72 unsigned int *start, *end, *alt_start, *alt_end, *src, *dest; 73 74 start = calc_addr(fcur, fcur->start_off); 75 end = calc_addr(fcur, fcur->end_off); 76 alt_start = calc_addr(fcur, fcur->alt_start_off); 77 alt_end = calc_addr(fcur, fcur->alt_end_off); 78 79 if ((alt_end - alt_start) > (end - start)) 80 return 1; 81 82 if ((value & fcur->mask) == fcur->value) 83 return 0; 84 85 src = alt_start; 86 dest = start; 87 88 for (; src < alt_end; src++, dest++) { 89 if (patch_alt_instruction(src, dest, alt_start, alt_end)) 90 return 1; 91 } 92 93 for (; dest < end; dest++) 94 patch_instruction(dest, PPC_INST_NOP); 95 96 return 0; 97 } 98 99 void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end) 100 { 101 struct fixup_entry *fcur, *fend; 102 103 fcur = fixup_start; 104 fend = fixup_end; 105 106 for (; fcur < fend; fcur++) { 107 if (patch_feature_section(value, fcur)) { 108 WARN_ON(1); 109 printk("Unable to patch feature section at %p - %p" \ 110 " with %p - %p\n", 111 calc_addr(fcur, fcur->start_off), 112 calc_addr(fcur, fcur->end_off), 113 calc_addr(fcur, fcur->alt_start_off), 114 calc_addr(fcur, fcur->alt_end_off)); 115 } 116 } 117 } 118 119 void do_lwsync_fixups(unsigned long value, void *fixup_start, void *fixup_end) 120 { 121 long *start, *end; 122 unsigned int *dest; 123 124 if (!(value & CPU_FTR_LWSYNC)) 125 return ; 126 127 start = fixup_start; 128 end = fixup_end; 129 130 for (; start < end; start++) { 131 dest = (void *)start + *start; 132 patch_instruction(dest, PPC_INST_LWSYNC); 133 } 134 } 135 136 static void do_final_fixups(void) 137 { 138 #if defined(CONFIG_PPC64) && defined(CONFIG_RELOCATABLE) 139 int *src, *dest; 140 unsigned long length; 141 142 if (PHYSICAL_START == 0) 143 return; 144 145 src = (int *)(KERNELBASE + PHYSICAL_START); 146 dest = (int *)KERNELBASE; 147 length = (__end_interrupts - _stext) / sizeof(int); 148 149 while (length--) { 150 patch_instruction(dest, *src); 151 src++; 152 dest++; 153 } 154 #endif 155 } 156 157 static unsigned long __initdata saved_cpu_features; 158 static unsigned int __initdata saved_mmu_features; 159 #ifdef CONFIG_PPC64 160 static unsigned long __initdata saved_firmware_features; 161 #endif 162 163 void __init apply_feature_fixups(void) 164 { 165 struct cpu_spec *spec = PTRRELOC(*PTRRELOC(&cur_cpu_spec)); 166 167 *PTRRELOC(&saved_cpu_features) = spec->cpu_features; 168 *PTRRELOC(&saved_mmu_features) = spec->mmu_features; 169 170 /* 171 * Apply the CPU-specific and firmware specific fixups to kernel text 172 * (nop out sections not relevant to this CPU or this firmware). 173 */ 174 do_feature_fixups(spec->cpu_features, 175 PTRRELOC(&__start___ftr_fixup), 176 PTRRELOC(&__stop___ftr_fixup)); 177 178 do_feature_fixups(spec->mmu_features, 179 PTRRELOC(&__start___mmu_ftr_fixup), 180 PTRRELOC(&__stop___mmu_ftr_fixup)); 181 182 do_lwsync_fixups(spec->cpu_features, 183 PTRRELOC(&__start___lwsync_fixup), 184 PTRRELOC(&__stop___lwsync_fixup)); 185 186 #ifdef CONFIG_PPC64 187 saved_firmware_features = powerpc_firmware_features; 188 do_feature_fixups(powerpc_firmware_features, 189 &__start___fw_ftr_fixup, &__stop___fw_ftr_fixup); 190 #endif 191 do_final_fixups(); 192 } 193 194 void __init setup_feature_keys(void) 195 { 196 /* 197 * Initialise jump label. This causes all the cpu/mmu_has_feature() 198 * checks to take on their correct polarity based on the current set of 199 * CPU/MMU features. 200 */ 201 jump_label_init(); 202 cpu_feature_keys_init(); 203 mmu_feature_keys_init(); 204 } 205 206 static int __init check_features(void) 207 { 208 WARN(saved_cpu_features != cur_cpu_spec->cpu_features, 209 "CPU features changed after feature patching!\n"); 210 WARN(saved_mmu_features != cur_cpu_spec->mmu_features, 211 "MMU features changed after feature patching!\n"); 212 #ifdef CONFIG_PPC64 213 WARN(saved_firmware_features != powerpc_firmware_features, 214 "Firmware features changed after feature patching!\n"); 215 #endif 216 217 return 0; 218 } 219 late_initcall(check_features); 220 221 #ifdef CONFIG_FTR_FIXUP_SELFTEST 222 223 #define check(x) \ 224 if (!(x)) printk("feature-fixups: test failed at line %d\n", __LINE__); 225 226 /* This must be after the text it fixes up, vmlinux.lds.S enforces that atm */ 227 static struct fixup_entry fixup; 228 229 static long calc_offset(struct fixup_entry *entry, unsigned int *p) 230 { 231 return (unsigned long)p - (unsigned long)entry; 232 } 233 234 static void test_basic_patching(void) 235 { 236 extern unsigned int ftr_fixup_test1[]; 237 extern unsigned int end_ftr_fixup_test1[]; 238 extern unsigned int ftr_fixup_test1_orig[]; 239 extern unsigned int ftr_fixup_test1_expected[]; 240 int size = end_ftr_fixup_test1 - ftr_fixup_test1; 241 242 fixup.value = fixup.mask = 8; 243 fixup.start_off = calc_offset(&fixup, ftr_fixup_test1 + 1); 244 fixup.end_off = calc_offset(&fixup, ftr_fixup_test1 + 2); 245 fixup.alt_start_off = fixup.alt_end_off = 0; 246 247 /* Sanity check */ 248 check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0); 249 250 /* Check we don't patch if the value matches */ 251 patch_feature_section(8, &fixup); 252 check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0); 253 254 /* Check we do patch if the value doesn't match */ 255 patch_feature_section(0, &fixup); 256 check(memcmp(ftr_fixup_test1, ftr_fixup_test1_expected, size) == 0); 257 258 /* Check we do patch if the mask doesn't match */ 259 memcpy(ftr_fixup_test1, ftr_fixup_test1_orig, size); 260 check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0); 261 patch_feature_section(~8, &fixup); 262 check(memcmp(ftr_fixup_test1, ftr_fixup_test1_expected, size) == 0); 263 } 264 265 static void test_alternative_patching(void) 266 { 267 extern unsigned int ftr_fixup_test2[]; 268 extern unsigned int end_ftr_fixup_test2[]; 269 extern unsigned int ftr_fixup_test2_orig[]; 270 extern unsigned int ftr_fixup_test2_alt[]; 271 extern unsigned int ftr_fixup_test2_expected[]; 272 int size = end_ftr_fixup_test2 - ftr_fixup_test2; 273 274 fixup.value = fixup.mask = 0xF; 275 fixup.start_off = calc_offset(&fixup, ftr_fixup_test2 + 1); 276 fixup.end_off = calc_offset(&fixup, ftr_fixup_test2 + 2); 277 fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_test2_alt); 278 fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_test2_alt + 1); 279 280 /* Sanity check */ 281 check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0); 282 283 /* Check we don't patch if the value matches */ 284 patch_feature_section(0xF, &fixup); 285 check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0); 286 287 /* Check we do patch if the value doesn't match */ 288 patch_feature_section(0, &fixup); 289 check(memcmp(ftr_fixup_test2, ftr_fixup_test2_expected, size) == 0); 290 291 /* Check we do patch if the mask doesn't match */ 292 memcpy(ftr_fixup_test2, ftr_fixup_test2_orig, size); 293 check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0); 294 patch_feature_section(~0xF, &fixup); 295 check(memcmp(ftr_fixup_test2, ftr_fixup_test2_expected, size) == 0); 296 } 297 298 static void test_alternative_case_too_big(void) 299 { 300 extern unsigned int ftr_fixup_test3[]; 301 extern unsigned int end_ftr_fixup_test3[]; 302 extern unsigned int ftr_fixup_test3_orig[]; 303 extern unsigned int ftr_fixup_test3_alt[]; 304 int size = end_ftr_fixup_test3 - ftr_fixup_test3; 305 306 fixup.value = fixup.mask = 0xC; 307 fixup.start_off = calc_offset(&fixup, ftr_fixup_test3 + 1); 308 fixup.end_off = calc_offset(&fixup, ftr_fixup_test3 + 2); 309 fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_test3_alt); 310 fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_test3_alt + 2); 311 312 /* Sanity check */ 313 check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); 314 315 /* Expect nothing to be patched, and the error returned to us */ 316 check(patch_feature_section(0xF, &fixup) == 1); 317 check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); 318 check(patch_feature_section(0, &fixup) == 1); 319 check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); 320 check(patch_feature_section(~0xF, &fixup) == 1); 321 check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); 322 } 323 324 static void test_alternative_case_too_small(void) 325 { 326 extern unsigned int ftr_fixup_test4[]; 327 extern unsigned int end_ftr_fixup_test4[]; 328 extern unsigned int ftr_fixup_test4_orig[]; 329 extern unsigned int ftr_fixup_test4_alt[]; 330 extern unsigned int ftr_fixup_test4_expected[]; 331 int size = end_ftr_fixup_test4 - ftr_fixup_test4; 332 unsigned long flag; 333 334 /* Check a high-bit flag */ 335 flag = 1UL << ((sizeof(unsigned long) - 1) * 8); 336 fixup.value = fixup.mask = flag; 337 fixup.start_off = calc_offset(&fixup, ftr_fixup_test4 + 1); 338 fixup.end_off = calc_offset(&fixup, ftr_fixup_test4 + 5); 339 fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_test4_alt); 340 fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_test4_alt + 2); 341 342 /* Sanity check */ 343 check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0); 344 345 /* Check we don't patch if the value matches */ 346 patch_feature_section(flag, &fixup); 347 check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0); 348 349 /* Check we do patch if the value doesn't match */ 350 patch_feature_section(0, &fixup); 351 check(memcmp(ftr_fixup_test4, ftr_fixup_test4_expected, size) == 0); 352 353 /* Check we do patch if the mask doesn't match */ 354 memcpy(ftr_fixup_test4, ftr_fixup_test4_orig, size); 355 check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0); 356 patch_feature_section(~flag, &fixup); 357 check(memcmp(ftr_fixup_test4, ftr_fixup_test4_expected, size) == 0); 358 } 359 360 static void test_alternative_case_with_branch(void) 361 { 362 extern unsigned int ftr_fixup_test5[]; 363 extern unsigned int end_ftr_fixup_test5[]; 364 extern unsigned int ftr_fixup_test5_expected[]; 365 int size = end_ftr_fixup_test5 - ftr_fixup_test5; 366 367 check(memcmp(ftr_fixup_test5, ftr_fixup_test5_expected, size) == 0); 368 } 369 370 static void test_alternative_case_with_external_branch(void) 371 { 372 extern unsigned int ftr_fixup_test6[]; 373 extern unsigned int end_ftr_fixup_test6[]; 374 extern unsigned int ftr_fixup_test6_expected[]; 375 int size = end_ftr_fixup_test6 - ftr_fixup_test6; 376 377 check(memcmp(ftr_fixup_test6, ftr_fixup_test6_expected, size) == 0); 378 } 379 380 static void test_cpu_macros(void) 381 { 382 extern u8 ftr_fixup_test_FTR_macros[]; 383 extern u8 ftr_fixup_test_FTR_macros_expected[]; 384 unsigned long size = ftr_fixup_test_FTR_macros_expected - 385 ftr_fixup_test_FTR_macros; 386 387 /* The fixups have already been done for us during boot */ 388 check(memcmp(ftr_fixup_test_FTR_macros, 389 ftr_fixup_test_FTR_macros_expected, size) == 0); 390 } 391 392 static void test_fw_macros(void) 393 { 394 #ifdef CONFIG_PPC64 395 extern u8 ftr_fixup_test_FW_FTR_macros[]; 396 extern u8 ftr_fixup_test_FW_FTR_macros_expected[]; 397 unsigned long size = ftr_fixup_test_FW_FTR_macros_expected - 398 ftr_fixup_test_FW_FTR_macros; 399 400 /* The fixups have already been done for us during boot */ 401 check(memcmp(ftr_fixup_test_FW_FTR_macros, 402 ftr_fixup_test_FW_FTR_macros_expected, size) == 0); 403 #endif 404 } 405 406 static void test_lwsync_macros(void) 407 { 408 extern u8 lwsync_fixup_test[]; 409 extern u8 end_lwsync_fixup_test[]; 410 extern u8 lwsync_fixup_test_expected_LWSYNC[]; 411 extern u8 lwsync_fixup_test_expected_SYNC[]; 412 unsigned long size = end_lwsync_fixup_test - 413 lwsync_fixup_test; 414 415 /* The fixups have already been done for us during boot */ 416 if (cur_cpu_spec->cpu_features & CPU_FTR_LWSYNC) { 417 check(memcmp(lwsync_fixup_test, 418 lwsync_fixup_test_expected_LWSYNC, size) == 0); 419 } else { 420 check(memcmp(lwsync_fixup_test, 421 lwsync_fixup_test_expected_SYNC, size) == 0); 422 } 423 } 424 425 static int __init test_feature_fixups(void) 426 { 427 printk(KERN_DEBUG "Running feature fixup self-tests ...\n"); 428 429 test_basic_patching(); 430 test_alternative_patching(); 431 test_alternative_case_too_big(); 432 test_alternative_case_too_small(); 433 test_alternative_case_with_branch(); 434 test_alternative_case_with_external_branch(); 435 test_cpu_macros(); 436 test_fw_macros(); 437 test_lwsync_macros(); 438 439 return 0; 440 } 441 late_initcall(test_feature_fixups); 442 443 #endif /* CONFIG_FTR_FIXUP_SELFTEST */ 444