1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2022 - Google LLC 4 * Author: Ard Biesheuvel <ardb@google.com> 5 */ 6 7 #include <linux/bug.h> 8 #include <linux/errno.h> 9 #include <linux/init.h> 10 #include <linux/linkage.h> 11 #include <linux/printk.h> 12 #include <linux/types.h> 13 14 #include <asm/cacheflush.h> 15 #include <asm/scs.h> 16 17 // 18 // This minimal DWARF CFI parser is partially based on the code in 19 // arch/arc/kernel/unwind.c, and on the document below: 20 // https://refspecs.linuxbase.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html 21 // 22 23 #define DW_CFA_nop 0x00 24 #define DW_CFA_set_loc 0x01 25 #define DW_CFA_advance_loc1 0x02 26 #define DW_CFA_advance_loc2 0x03 27 #define DW_CFA_advance_loc4 0x04 28 #define DW_CFA_offset_extended 0x05 29 #define DW_CFA_restore_extended 0x06 30 #define DW_CFA_undefined 0x07 31 #define DW_CFA_same_value 0x08 32 #define DW_CFA_register 0x09 33 #define DW_CFA_remember_state 0x0a 34 #define DW_CFA_restore_state 0x0b 35 #define DW_CFA_def_cfa 0x0c 36 #define DW_CFA_def_cfa_register 0x0d 37 #define DW_CFA_def_cfa_offset 0x0e 38 #define DW_CFA_def_cfa_expression 0x0f 39 #define DW_CFA_expression 0x10 40 #define DW_CFA_offset_extended_sf 0x11 41 #define DW_CFA_def_cfa_sf 0x12 42 #define DW_CFA_def_cfa_offset_sf 0x13 43 #define DW_CFA_val_offset 0x14 44 #define DW_CFA_val_offset_sf 0x15 45 #define DW_CFA_val_expression 0x16 46 #define DW_CFA_lo_user 0x1c 47 #define DW_CFA_negate_ra_state 0x2d 48 #define DW_CFA_GNU_args_size 0x2e 49 #define DW_CFA_GNU_negative_offset_extended 0x2f 50 #define DW_CFA_hi_user 0x3f 51 52 extern const u8 __eh_frame_start[], __eh_frame_end[]; 53 54 enum { 55 PACIASP = 0xd503233f, 56 AUTIASP = 0xd50323bf, 57 SCS_PUSH = 0xf800865e, 58 SCS_POP = 0xf85f8e5e, 59 }; 60 61 static void __always_inline scs_patch_loc(u64 loc) 62 { 63 u32 insn = le32_to_cpup((void *)loc); 64 65 switch (insn) { 66 case PACIASP: 67 *(u32 *)loc = cpu_to_le32(SCS_PUSH); 68 break; 69 case AUTIASP: 70 *(u32 *)loc = cpu_to_le32(SCS_POP); 71 break; 72 default: 73 /* 74 * While the DW_CFA_negate_ra_state directive is guaranteed to 75 * appear right after a PACIASP/AUTIASP instruction, it may 76 * also appear after a DW_CFA_restore_state directive that 77 * restores a state that is only partially accurate, and is 78 * followed by DW_CFA_negate_ra_state directive to toggle the 79 * PAC bit again. So we permit other instructions here, and ignore 80 * them. 81 */ 82 return; 83 } 84 dcache_clean_pou(loc, loc + sizeof(u32)); 85 } 86 87 /* 88 * Skip one uleb128/sleb128 encoded quantity from the opcode stream. All bytes 89 * except the last one have bit #7 set. 90 */ 91 static int __always_inline skip_xleb128(const u8 **opcode, int size) 92 { 93 u8 c; 94 95 do { 96 c = *(*opcode)++; 97 size--; 98 } while (c & BIT(7)); 99 100 return size; 101 } 102 103 struct eh_frame { 104 /* 105 * The size of this frame if 0 < size < U32_MAX, 0 terminates the list. 106 */ 107 u32 size; 108 109 /* 110 * The first frame is a Common Information Entry (CIE) frame, followed 111 * by one or more Frame Description Entry (FDE) frames. In the former 112 * case, this field is 0, otherwise it is the negated offset relative 113 * to the associated CIE frame. 114 */ 115 u32 cie_id_or_pointer; 116 117 union { 118 struct { // CIE 119 u8 version; 120 u8 augmentation_string[]; 121 }; 122 123 struct { // FDE 124 s32 initial_loc; 125 s32 range; 126 u8 opcodes[]; 127 }; 128 }; 129 }; 130 131 static int noinstr scs_handle_fde_frame(const struct eh_frame *frame, 132 bool fde_has_augmentation_data, 133 int code_alignment_factor) 134 { 135 int size = frame->size - offsetof(struct eh_frame, opcodes) + 4; 136 u64 loc = (u64)offset_to_ptr(&frame->initial_loc); 137 const u8 *opcode = frame->opcodes; 138 139 if (fde_has_augmentation_data) { 140 int l; 141 142 // assume single byte uleb128_t 143 if (WARN_ON(*opcode & BIT(7))) 144 return -ENOEXEC; 145 146 l = *opcode++; 147 opcode += l; 148 size -= l + 1; 149 } 150 151 /* 152 * Starting from 'loc', apply the CFA opcodes that advance the location 153 * pointer, and identify the locations of the PAC instructions. 154 */ 155 while (size-- > 0) { 156 switch (*opcode++) { 157 case DW_CFA_nop: 158 case DW_CFA_remember_state: 159 case DW_CFA_restore_state: 160 break; 161 162 case DW_CFA_advance_loc1: 163 loc += *opcode++ * code_alignment_factor; 164 size--; 165 break; 166 167 case DW_CFA_advance_loc2: 168 loc += *opcode++ * code_alignment_factor; 169 loc += (*opcode++ << 8) * code_alignment_factor; 170 size -= 2; 171 break; 172 173 case DW_CFA_def_cfa: 174 case DW_CFA_offset_extended: 175 size = skip_xleb128(&opcode, size); 176 fallthrough; 177 case DW_CFA_def_cfa_offset: 178 case DW_CFA_def_cfa_offset_sf: 179 case DW_CFA_def_cfa_register: 180 case DW_CFA_same_value: 181 case DW_CFA_restore_extended: 182 case 0x80 ... 0xbf: 183 size = skip_xleb128(&opcode, size); 184 break; 185 186 case DW_CFA_negate_ra_state: 187 scs_patch_loc(loc - 4); 188 break; 189 190 case 0x40 ... 0x7f: 191 // advance loc 192 loc += (opcode[-1] & 0x3f) * code_alignment_factor; 193 break; 194 195 case 0xc0 ... 0xff: 196 break; 197 198 default: 199 pr_err("unhandled opcode: %02x in FDE frame %lx\n", opcode[-1], (uintptr_t)frame); 200 return -ENOEXEC; 201 } 202 } 203 return 0; 204 } 205 206 int noinstr scs_patch(const u8 eh_frame[], int size) 207 { 208 const u8 *p = eh_frame; 209 210 while (size > 4) { 211 const struct eh_frame *frame = (const void *)p; 212 bool fde_has_augmentation_data = true; 213 int code_alignment_factor = 1; 214 int ret; 215 216 if (frame->size == 0 || 217 frame->size == U32_MAX || 218 frame->size > size) 219 break; 220 221 if (frame->cie_id_or_pointer == 0) { 222 const u8 *p = frame->augmentation_string; 223 224 /* a 'z' in the augmentation string must come first */ 225 fde_has_augmentation_data = *p == 'z'; 226 227 /* 228 * The code alignment factor is a uleb128 encoded field 229 * but given that the only sensible values are 1 or 4, 230 * there is no point in decoding the whole thing. 231 */ 232 p += strlen(p) + 1; 233 if (!WARN_ON(*p & BIT(7))) 234 code_alignment_factor = *p; 235 } else { 236 ret = scs_handle_fde_frame(frame, 237 fde_has_augmentation_data, 238 code_alignment_factor); 239 if (ret) 240 return ret; 241 } 242 243 p += sizeof(frame->size) + frame->size; 244 size -= sizeof(frame->size) + frame->size; 245 } 246 return 0; 247 } 248 249 asmlinkage void __init scs_patch_vmlinux(void) 250 { 251 if (!should_patch_pac_into_scs()) 252 return; 253 254 WARN_ON(scs_patch(__eh_frame_start, __eh_frame_end - __eh_frame_start)); 255 icache_inval_all_pou(); 256 isb(); 257 } 258