1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2022 - Google LLC 4 * Author: Ard Biesheuvel <ardb@google.com> 5 */ 6 7 #include <linux/bug.h> 8 #include <linux/errno.h> 9 #include <linux/init.h> 10 #include <linux/linkage.h> 11 #include <linux/printk.h> 12 #include <linux/types.h> 13 14 #include <asm/cacheflush.h> 15 #include <asm/scs.h> 16 17 // 18 // This minimal DWARF CFI parser is partially based on the code in 19 // arch/arc/kernel/unwind.c, and on the document below: 20 // https://refspecs.linuxbase.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html 21 // 22 23 #define DW_CFA_nop 0x00 24 #define DW_CFA_set_loc 0x01 25 #define DW_CFA_advance_loc1 0x02 26 #define DW_CFA_advance_loc2 0x03 27 #define DW_CFA_advance_loc4 0x04 28 #define DW_CFA_offset_extended 0x05 29 #define DW_CFA_restore_extended 0x06 30 #define DW_CFA_undefined 0x07 31 #define DW_CFA_same_value 0x08 32 #define DW_CFA_register 0x09 33 #define DW_CFA_remember_state 0x0a 34 #define DW_CFA_restore_state 0x0b 35 #define DW_CFA_def_cfa 0x0c 36 #define DW_CFA_def_cfa_register 0x0d 37 #define DW_CFA_def_cfa_offset 0x0e 38 #define DW_CFA_def_cfa_expression 0x0f 39 #define DW_CFA_expression 0x10 40 #define DW_CFA_offset_extended_sf 0x11 41 #define DW_CFA_def_cfa_sf 0x12 42 #define DW_CFA_def_cfa_offset_sf 0x13 43 #define DW_CFA_val_offset 0x14 44 #define DW_CFA_val_offset_sf 0x15 45 #define DW_CFA_val_expression 0x16 46 #define DW_CFA_lo_user 0x1c 47 #define DW_CFA_negate_ra_state 0x2d 48 #define DW_CFA_GNU_args_size 0x2e 49 #define DW_CFA_GNU_negative_offset_extended 0x2f 50 #define DW_CFA_hi_user 0x3f 51 52 extern const u8 __eh_frame_start[], __eh_frame_end[]; 53 54 enum { 55 PACIASP = 0xd503233f, 56 AUTIASP = 0xd50323bf, 57 SCS_PUSH = 0xf800865e, 58 SCS_POP = 0xf85f8e5e, 59 }; 60 61 static void __always_inline scs_patch_loc(u64 loc) 62 { 63 u32 insn = le32_to_cpup((void *)loc); 64 65 switch (insn) { 66 case PACIASP: 67 *(u32 *)loc = cpu_to_le32(SCS_PUSH); 68 break; 69 case AUTIASP: 70 *(u32 *)loc = cpu_to_le32(SCS_POP); 71 break; 72 default: 73 /* 74 * While the DW_CFA_negate_ra_state directive is guaranteed to 75 * appear right after a PACIASP/AUTIASP instruction, it may 76 * also appear after a DW_CFA_restore_state directive that 77 * restores a state that is only partially accurate, and is 78 * followed by DW_CFA_negate_ra_state directive to toggle the 79 * PAC bit again. So we permit other instructions here, and ignore 80 * them. 81 */ 82 return; 83 } 84 dcache_clean_pou(loc, loc + sizeof(u32)); 85 } 86 87 /* 88 * Skip one uleb128/sleb128 encoded quantity from the opcode stream. All bytes 89 * except the last one have bit #7 set. 90 */ 91 static int __always_inline skip_xleb128(const u8 **opcode, int size) 92 { 93 u8 c; 94 95 do { 96 c = *(*opcode)++; 97 size--; 98 } while (c & BIT(7)); 99 100 return size; 101 } 102 103 struct eh_frame { 104 /* 105 * The size of this frame if 0 < size < U32_MAX, 0 terminates the list. 106 */ 107 u32 size; 108 109 /* 110 * The first frame is a Common Information Entry (CIE) frame, followed 111 * by one or more Frame Description Entry (FDE) frames. In the former 112 * case, this field is 0, otherwise it is the negated offset relative 113 * to the associated CIE frame. 114 */ 115 u32 cie_id_or_pointer; 116 117 union { 118 struct { // CIE 119 u8 version; 120 u8 augmentation_string[]; 121 }; 122 123 struct { // FDE 124 s32 initial_loc; 125 s32 range; 126 u8 opcodes[]; 127 }; 128 }; 129 }; 130 131 static int noinstr scs_handle_fde_frame(const struct eh_frame *frame, 132 bool fde_has_augmentation_data, 133 int code_alignment_factor, 134 bool dry_run) 135 { 136 int size = frame->size - offsetof(struct eh_frame, opcodes) + 4; 137 u64 loc = (u64)offset_to_ptr(&frame->initial_loc); 138 const u8 *opcode = frame->opcodes; 139 140 if (fde_has_augmentation_data) { 141 int l; 142 143 // assume single byte uleb128_t 144 if (WARN_ON(*opcode & BIT(7))) 145 return -ENOEXEC; 146 147 l = *opcode++; 148 opcode += l; 149 size -= l + 1; 150 } 151 152 /* 153 * Starting from 'loc', apply the CFA opcodes that advance the location 154 * pointer, and identify the locations of the PAC instructions. 155 */ 156 while (size-- > 0) { 157 switch (*opcode++) { 158 case DW_CFA_nop: 159 case DW_CFA_remember_state: 160 case DW_CFA_restore_state: 161 break; 162 163 case DW_CFA_advance_loc1: 164 loc += *opcode++ * code_alignment_factor; 165 size--; 166 break; 167 168 case DW_CFA_advance_loc2: 169 loc += *opcode++ * code_alignment_factor; 170 loc += (*opcode++ << 8) * code_alignment_factor; 171 size -= 2; 172 break; 173 174 case DW_CFA_def_cfa: 175 case DW_CFA_offset_extended: 176 size = skip_xleb128(&opcode, size); 177 fallthrough; 178 case DW_CFA_def_cfa_offset: 179 case DW_CFA_def_cfa_offset_sf: 180 case DW_CFA_def_cfa_register: 181 case DW_CFA_same_value: 182 case DW_CFA_restore_extended: 183 case 0x80 ... 0xbf: 184 size = skip_xleb128(&opcode, size); 185 break; 186 187 case DW_CFA_negate_ra_state: 188 if (!dry_run) 189 scs_patch_loc(loc - 4); 190 break; 191 192 case 0x40 ... 0x7f: 193 // advance loc 194 loc += (opcode[-1] & 0x3f) * code_alignment_factor; 195 break; 196 197 case 0xc0 ... 0xff: 198 break; 199 200 default: 201 pr_err("unhandled opcode: %02x in FDE frame %lx\n", opcode[-1], (uintptr_t)frame); 202 return -ENOEXEC; 203 } 204 } 205 return 0; 206 } 207 208 int noinstr scs_patch(const u8 eh_frame[], int size) 209 { 210 const u8 *p = eh_frame; 211 212 while (size > 4) { 213 const struct eh_frame *frame = (const void *)p; 214 bool fde_has_augmentation_data = true; 215 int code_alignment_factor = 1; 216 int ret; 217 218 if (frame->size == 0 || 219 frame->size == U32_MAX || 220 frame->size > size) 221 break; 222 223 if (frame->cie_id_or_pointer == 0) { 224 const u8 *p = frame->augmentation_string; 225 226 /* a 'z' in the augmentation string must come first */ 227 fde_has_augmentation_data = *p == 'z'; 228 229 /* 230 * The code alignment factor is a uleb128 encoded field 231 * but given that the only sensible values are 1 or 4, 232 * there is no point in decoding the whole thing. 233 */ 234 p += strlen(p) + 1; 235 if (!WARN_ON(*p & BIT(7))) 236 code_alignment_factor = *p; 237 } else { 238 ret = scs_handle_fde_frame(frame, 239 fde_has_augmentation_data, 240 code_alignment_factor, 241 true); 242 if (ret) 243 return ret; 244 scs_handle_fde_frame(frame, fde_has_augmentation_data, 245 code_alignment_factor, false); 246 } 247 248 p += sizeof(frame->size) + frame->size; 249 size -= sizeof(frame->size) + frame->size; 250 } 251 return 0; 252 } 253 254 asmlinkage void __init scs_patch_vmlinux(void) 255 { 256 if (!should_patch_pac_into_scs()) 257 return; 258 259 WARN_ON(scs_patch(__eh_frame_start, __eh_frame_end - __eh_frame_start)); 260 icache_inval_all_pou(); 261 isb(); 262 } 263