1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_POWERPC_HEAD_64_H 3 #define _ASM_POWERPC_HEAD_64_H 4 5 #include <asm/cache.h> 6 7 #ifdef __ASSEMBLY__ 8 /* 9 * We can't do CPP stringification and concatination directly into the section 10 * name for some reason, so these macros can do it for us. 11 */ 12 .macro define_ftsec name 13 .section ".head.text.\name\()","ax",@progbits 14 .endm 15 .macro define_data_ftsec name 16 .section ".head.data.\name\()","a",@progbits 17 .endm 18 .macro use_ftsec name 19 .section ".head.text.\name\()" 20 .endm 21 22 /* 23 * Fixed (location) sections are used by opening fixed sections and emitting 24 * fixed section entries into them before closing them. Multiple fixed sections 25 * can be open at any time. 26 * 27 * Each fixed section created in a .S file must have corresponding linkage 28 * directives including location, added to arch/powerpc/kernel/vmlinux.lds.S 29 * 30 * For each fixed section, code is generated into it in the order which it 31 * appears in the source. Fixed section entries can be placed at a fixed 32 * location within the section using _LOCATION postifx variants. These must 33 * be ordered according to their relative placements within the section. 34 * 35 * OPEN_FIXED_SECTION(section_name, start_address, end_address) 36 * FIXED_SECTION_ENTRY_BEGIN(section_name, label1) 37 * 38 * USE_FIXED_SECTION(section_name) 39 * label3: 40 * li r10,128 41 * mv r11,r10 42 43 * FIXED_SECTION_ENTRY_BEGIN_LOCATION(section_name, label2, start_address, size) 44 * FIXED_SECTION_ENTRY_END_LOCATION(section_name, label2, start_address, size) 45 * CLOSE_FIXED_SECTION(section_name) 46 * 47 * ZERO_FIXED_SECTION can be used to emit zeroed data. 48 * 49 * Troubleshooting: 50 * - If the build dies with "Error: attempt to move .org backwards" at 51 * CLOSE_FIXED_SECTION() or elsewhere, there may be something 52 * unexpected being added there. Remove the '. = x_len' line, rebuild, and 53 * check what is pushing the section down. 54 * - If the build dies in linking, check arch/powerpc/tools/head_check.sh 55 * comments. 56 * - If the kernel crashes or hangs in very early boot, it could be linker 57 * stubs at the start of the main text. 58 */ 59 60 #define OPEN_FIXED_SECTION(sname, start, end) \ 61 sname##_start = (start); \ 62 sname##_end = (end); \ 63 sname##_len = (end) - (start); \ 64 define_ftsec sname; \ 65 . = 0x0; \ 66 start_##sname: 67 68 /* 69 * .linker_stub_catch section is used to catch linker stubs from being 70 * inserted in our .text section, above the start_text label (which breaks 71 * the ABS_ADDR calculation). See kernel/vmlinux.lds.S and tools/head_check.sh 72 * for more details. We would prefer to just keep a cacheline (0x80), but 73 * 0x100 seems to be how the linker aligns branch stub groups. 74 */ 75 #ifdef CONFIG_LD_HEAD_STUB_CATCH 76 #define OPEN_TEXT_SECTION(start) \ 77 .section ".linker_stub_catch","ax",@progbits; \ 78 linker_stub_catch: \ 79 . = 0x4; \ 80 text_start = (start) + 0x100; \ 81 .section ".text","ax",@progbits; \ 82 .balign 0x100; \ 83 start_text: 84 #else 85 #define OPEN_TEXT_SECTION(start) \ 86 text_start = (start); \ 87 .section ".text","ax",@progbits; \ 88 . = 0x0; \ 89 start_text: 90 #endif 91 92 #define ZERO_FIXED_SECTION(sname, start, end) \ 93 sname##_start = (start); \ 94 sname##_end = (end); \ 95 sname##_len = (end) - (start); \ 96 define_data_ftsec sname; \ 97 . = 0x0; \ 98 . = sname##_len; 99 100 #define USE_FIXED_SECTION(sname) \ 101 fs_label = start_##sname; \ 102 fs_start = sname##_start; \ 103 use_ftsec sname; 104 105 #define USE_TEXT_SECTION() \ 106 fs_label = start_text; \ 107 fs_start = text_start; \ 108 .text 109 110 #define CLOSE_FIXED_SECTION(sname) \ 111 USE_FIXED_SECTION(sname); \ 112 . = sname##_len; \ 113 end_##sname: 114 115 116 #define __FIXED_SECTION_ENTRY_BEGIN(sname, name, __align) \ 117 USE_FIXED_SECTION(sname); \ 118 .balign __align; \ 119 .global name; \ 120 name: 121 122 #define FIXED_SECTION_ENTRY_BEGIN(sname, name) \ 123 __FIXED_SECTION_ENTRY_BEGIN(sname, name, IFETCH_ALIGN_BYTES) 124 125 #define FIXED_SECTION_ENTRY_BEGIN_LOCATION(sname, name, start, size) \ 126 USE_FIXED_SECTION(sname); \ 127 name##_start = (start); \ 128 .if ((start) % (size) != 0); \ 129 .error "Fixed section exception vector misalignment"; \ 130 .endif; \ 131 .if ((size) != 0x20) && ((size) != 0x80) && ((size) != 0x100); \ 132 .error "Fixed section exception vector bad size"; \ 133 .endif; \ 134 .if (start) < sname##_start; \ 135 .error "Fixed section underflow"; \ 136 .abort; \ 137 .endif; \ 138 . = (start) - sname##_start; \ 139 .global name; \ 140 name: 141 142 #define FIXED_SECTION_ENTRY_END_LOCATION(sname, name, start, size) \ 143 .if (start) + (size) > sname##_end; \ 144 .error "Fixed section overflow"; \ 145 .abort; \ 146 .endif; \ 147 .if (. - name > (start) + (size) - name##_start); \ 148 .error "Fixed entry overflow"; \ 149 .abort; \ 150 .endif; \ 151 . = ((start) + (size) - sname##_start); \ 152 153 154 /* 155 * These macros are used to change symbols in other fixed sections to be 156 * absolute or related to our current fixed section. 157 * 158 * - DEFINE_FIXED_SYMBOL / FIXED_SYMBOL_ABS_ADDR is used to find the 159 * absolute address of a symbol within a fixed section, from any section. 160 * 161 * - ABS_ADDR is used to find the absolute address of any symbol, from within 162 * a fixed section. 163 */ 164 #define DEFINE_FIXED_SYMBOL(label) \ 165 label##_absolute = (label - fs_label + fs_start) 166 167 #define FIXED_SYMBOL_ABS_ADDR(label) \ 168 (label##_absolute) 169 170 #define ABS_ADDR(label) (label - fs_label + fs_start) 171 172 /* 173 * Following are the BOOK3S exception handler helper macros. 174 * Handlers come in a number of types, and each type has a number of varieties. 175 * 176 * EXC_REAL_* - real, unrelocated exception vectors 177 * EXC_VIRT_* - virt (AIL), unrelocated exception vectors 178 * TRAMP_REAL_* - real, unrelocated helpers (virt can call these) 179 * TRAMP_VIRT_* - virt, unreloc helpers (in practice, real can use) 180 * TRAMP_KVM - KVM handlers that get put into real, unrelocated 181 * EXC_COMMON_* - virt, relocated common handlers 182 * 183 * The EXC handlers are given a name, and branch to name_common, or the 184 * appropriate KVM or masking function. Vector handler verieties are as 185 * follows: 186 * 187 * EXC_{REAL|VIRT}_BEGIN/END - used to open-code the exception 188 * 189 * EXC_{REAL|VIRT} - standard exception 190 * 191 * EXC_{REAL|VIRT}_suffix 192 * where _suffix is: 193 * - _MASKABLE - maskable exception 194 * - _OOL - out of line with trampoline to common handler 195 * - _HV - HV exception 196 * 197 * There can be combinations, e.g., EXC_VIRT_OOL_MASKABLE_HV 198 * 199 * The one unusual case is __EXC_REAL_OOL_HV_DIRECT, which is 200 * an OOL vector that branches to a specified handler rather than the usual 201 * trampoline that goes to common. It, and other underscore macros, should 202 * be used with care. 203 * 204 * KVM handlers come in the following verieties: 205 * TRAMP_KVM 206 * TRAMP_KVM_SKIP 207 * TRAMP_KVM_HV 208 * TRAMP_KVM_HV_SKIP 209 * 210 * COMMON handlers come in the following verieties: 211 * EXC_COMMON_BEGIN/END - used to open-code the handler 212 * EXC_COMMON 213 * EXC_COMMON_ASYNC 214 * EXC_COMMON_HV 215 * 216 * TRAMP_REAL and TRAMP_VIRT can be used with BEGIN/END. KVM 217 * and OOL handlers are implemented as types of TRAMP and TRAMP_VIRT handlers. 218 */ 219 220 #define EXC_REAL_BEGIN(name, start, size) \ 221 FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##name, start, size) 222 223 #define EXC_REAL_END(name, start, size) \ 224 FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##name, start, size) 225 226 #define EXC_VIRT_BEGIN(name, start, size) \ 227 FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) 228 229 #define EXC_VIRT_END(name, start, size) \ 230 FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size) 231 232 #define EXC_COMMON_BEGIN(name) \ 233 USE_TEXT_SECTION(); \ 234 .balign IFETCH_ALIGN_BYTES; \ 235 .global name; \ 236 _ASM_NOKPROBE_SYMBOL(name); \ 237 DEFINE_FIXED_SYMBOL(name); \ 238 name: 239 240 #define TRAMP_REAL_BEGIN(name) \ 241 FIXED_SECTION_ENTRY_BEGIN(real_trampolines, name) 242 243 #define TRAMP_VIRT_BEGIN(name) \ 244 FIXED_SECTION_ENTRY_BEGIN(virt_trampolines, name) 245 246 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER 247 #define TRAMP_KVM_BEGIN(name) \ 248 TRAMP_VIRT_BEGIN(name) 249 #else 250 #define TRAMP_KVM_BEGIN(name) 251 #endif 252 253 #define EXC_REAL_NONE(start, size) \ 254 FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##unused, start, size); \ 255 FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##unused, start, size) 256 257 #define EXC_VIRT_NONE(start, size) \ 258 FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size); \ 259 FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size); 260 261 262 #define EXC_REAL(name, start, size) \ 263 EXC_REAL_BEGIN(name, start, size); \ 264 STD_EXCEPTION_PSERIES(start, name##_common); \ 265 EXC_REAL_END(name, start, size); 266 267 #define EXC_VIRT(name, start, size, realvec) \ 268 EXC_VIRT_BEGIN(name, start, size); \ 269 STD_RELON_EXCEPTION_PSERIES(start, realvec, name##_common); \ 270 EXC_VIRT_END(name, start, size); 271 272 #define EXC_REAL_MASKABLE(name, start, size) \ 273 EXC_REAL_BEGIN(name, start, size); \ 274 MASKABLE_EXCEPTION_PSERIES(start, start, name##_common); \ 275 EXC_REAL_END(name, start, size); 276 277 #define EXC_VIRT_MASKABLE(name, start, size, realvec) \ 278 EXC_VIRT_BEGIN(name, start, size); \ 279 MASKABLE_RELON_EXCEPTION_PSERIES(start, realvec, name##_common); \ 280 EXC_VIRT_END(name, start, size); 281 282 #define EXC_REAL_HV(name, start, size) \ 283 EXC_REAL_BEGIN(name, start, size); \ 284 STD_EXCEPTION_HV(start, start, name##_common); \ 285 EXC_REAL_END(name, start, size); 286 287 #define EXC_VIRT_HV(name, start, size, realvec) \ 288 EXC_VIRT_BEGIN(name, start, size); \ 289 STD_RELON_EXCEPTION_HV(start, realvec, name##_common); \ 290 EXC_VIRT_END(name, start, size); 291 292 #define __EXC_REAL_OOL(name, start, size) \ 293 EXC_REAL_BEGIN(name, start, size); \ 294 __OOL_EXCEPTION(start, label, tramp_real_##name); \ 295 EXC_REAL_END(name, start, size); 296 297 #define __TRAMP_REAL_OOL(name, vec) \ 298 TRAMP_REAL_BEGIN(tramp_real_##name); \ 299 STD_EXCEPTION_PSERIES_OOL(vec, name##_common); \ 300 301 #define EXC_REAL_OOL(name, start, size) \ 302 __EXC_REAL_OOL(name, start, size); \ 303 __TRAMP_REAL_OOL(name, start); 304 305 #define __EXC_REAL_OOL_MASKABLE(name, start, size) \ 306 __EXC_REAL_OOL(name, start, size); 307 308 #define __TRAMP_REAL_OOL_MASKABLE(name, vec) \ 309 TRAMP_REAL_BEGIN(tramp_real_##name); \ 310 MASKABLE_EXCEPTION_PSERIES_OOL(vec, name##_common); \ 311 312 #define EXC_REAL_OOL_MASKABLE(name, start, size) \ 313 __EXC_REAL_OOL_MASKABLE(name, start, size); \ 314 __TRAMP_REAL_OOL_MASKABLE(name, start); 315 316 #define __EXC_REAL_OOL_HV_DIRECT(name, start, size, handler) \ 317 EXC_REAL_BEGIN(name, start, size); \ 318 __OOL_EXCEPTION(start, label, handler); \ 319 EXC_REAL_END(name, start, size); 320 321 #define __EXC_REAL_OOL_HV(name, start, size) \ 322 __EXC_REAL_OOL(name, start, size); 323 324 #define __TRAMP_REAL_OOL_HV(name, vec) \ 325 TRAMP_REAL_BEGIN(tramp_real_##name); \ 326 STD_EXCEPTION_HV_OOL(vec, name##_common); \ 327 328 #define EXC_REAL_OOL_HV(name, start, size) \ 329 __EXC_REAL_OOL_HV(name, start, size); \ 330 __TRAMP_REAL_OOL_HV(name, start); 331 332 #define __EXC_REAL_OOL_MASKABLE_HV(name, start, size) \ 333 __EXC_REAL_OOL(name, start, size); 334 335 #define __TRAMP_REAL_OOL_MASKABLE_HV(name, vec) \ 336 TRAMP_REAL_BEGIN(tramp_real_##name); \ 337 MASKABLE_EXCEPTION_HV_OOL(vec, name##_common); \ 338 339 #define EXC_REAL_OOL_MASKABLE_HV(name, start, size) \ 340 __EXC_REAL_OOL_MASKABLE_HV(name, start, size); \ 341 __TRAMP_REAL_OOL_MASKABLE_HV(name, start); 342 343 #define __EXC_VIRT_OOL(name, start, size) \ 344 EXC_VIRT_BEGIN(name, start, size); \ 345 __OOL_EXCEPTION(start, label, tramp_virt_##name); \ 346 EXC_VIRT_END(name, start, size); 347 348 #define __TRAMP_VIRT_OOL(name, realvec) \ 349 TRAMP_VIRT_BEGIN(tramp_virt_##name); \ 350 STD_RELON_EXCEPTION_PSERIES_OOL(realvec, name##_common); \ 351 352 #define EXC_VIRT_OOL(name, start, size, realvec) \ 353 __EXC_VIRT_OOL(name, start, size); \ 354 __TRAMP_VIRT_OOL(name, realvec); 355 356 #define __EXC_VIRT_OOL_MASKABLE(name, start, size) \ 357 __EXC_VIRT_OOL(name, start, size); 358 359 #define __TRAMP_VIRT_OOL_MASKABLE(name, realvec) \ 360 TRAMP_VIRT_BEGIN(tramp_virt_##name); \ 361 MASKABLE_RELON_EXCEPTION_PSERIES_OOL(realvec, name##_common); \ 362 363 #define EXC_VIRT_OOL_MASKABLE(name, start, size, realvec) \ 364 __EXC_VIRT_OOL_MASKABLE(name, start, size); \ 365 __TRAMP_VIRT_OOL_MASKABLE(name, realvec); 366 367 #define __EXC_VIRT_OOL_HV(name, start, size) \ 368 __EXC_VIRT_OOL(name, start, size); 369 370 #define __TRAMP_VIRT_OOL_HV(name, realvec) \ 371 TRAMP_VIRT_BEGIN(tramp_virt_##name); \ 372 STD_RELON_EXCEPTION_HV_OOL(realvec, name##_common); \ 373 374 #define EXC_VIRT_OOL_HV(name, start, size, realvec) \ 375 __EXC_VIRT_OOL_HV(name, start, size); \ 376 __TRAMP_VIRT_OOL_HV(name, realvec); 377 378 #define __EXC_VIRT_OOL_MASKABLE_HV(name, start, size) \ 379 __EXC_VIRT_OOL(name, start, size); 380 381 #define __TRAMP_VIRT_OOL_MASKABLE_HV(name, realvec) \ 382 TRAMP_VIRT_BEGIN(tramp_virt_##name); \ 383 MASKABLE_RELON_EXCEPTION_HV_OOL(realvec, name##_common); \ 384 385 #define EXC_VIRT_OOL_MASKABLE_HV(name, start, size, realvec) \ 386 __EXC_VIRT_OOL_MASKABLE_HV(name, start, size); \ 387 __TRAMP_VIRT_OOL_MASKABLE_HV(name, realvec); 388 389 #define TRAMP_KVM(area, n) \ 390 TRAMP_KVM_BEGIN(do_kvm_##n); \ 391 KVM_HANDLER(area, EXC_STD, n); \ 392 393 #define TRAMP_KVM_SKIP(area, n) \ 394 TRAMP_KVM_BEGIN(do_kvm_##n); \ 395 KVM_HANDLER_SKIP(area, EXC_STD, n); \ 396 397 /* 398 * HV variant exceptions get the 0x2 bit added to their trap number. 399 */ 400 #define TRAMP_KVM_HV(area, n) \ 401 TRAMP_KVM_BEGIN(do_kvm_H##n); \ 402 KVM_HANDLER(area, EXC_HV, n + 0x2); \ 403 404 #define TRAMP_KVM_HV_SKIP(area, n) \ 405 TRAMP_KVM_BEGIN(do_kvm_H##n); \ 406 KVM_HANDLER_SKIP(area, EXC_HV, n + 0x2); \ 407 408 #define EXC_COMMON(name, realvec, hdlr) \ 409 EXC_COMMON_BEGIN(name); \ 410 STD_EXCEPTION_COMMON(realvec, name, hdlr); \ 411 412 #define EXC_COMMON_ASYNC(name, realvec, hdlr) \ 413 EXC_COMMON_BEGIN(name); \ 414 STD_EXCEPTION_COMMON_ASYNC(realvec, name, hdlr); \ 415 416 #define EXC_COMMON_HV(name, realvec, hdlr) \ 417 EXC_COMMON_BEGIN(name); \ 418 STD_EXCEPTION_COMMON(realvec + 0x2, name, hdlr); \ 419 420 #endif /* __ASSEMBLY__ */ 421 422 #endif /* _ASM_POWERPC_HEAD_64_H */ 423