1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * eBPF JIT compiler 4 * 5 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com> 6 * IBM Corporation 7 * 8 * Based on the powerpc classic BPF JIT compiler by Matt Evans 9 */ 10 #include <linux/moduleloader.h> 11 #include <asm/cacheflush.h> 12 #include <asm/asm-compat.h> 13 #include <linux/netdevice.h> 14 #include <linux/filter.h> 15 #include <linux/if_vlan.h> 16 #include <asm/kprobes.h> 17 #include <linux/bpf.h> 18 19 #include "bpf_jit.h" 20 21 static void bpf_jit_fill_ill_insns(void *area, unsigned int size) 22 { 23 memset32(area, BREAKPOINT_INSTRUCTION, size / 4); 24 } 25 26 int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr) 27 { 28 if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) { 29 PPC_JMP(exit_addr); 30 } else if (ctx->alt_exit_addr) { 31 if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4)))) 32 return -1; 33 PPC_JMP(ctx->alt_exit_addr); 34 } else { 35 ctx->alt_exit_addr = ctx->idx * 4; 36 bpf_jit_build_epilogue(image, ctx); 37 } 38 39 return 0; 40 } 41 42 struct powerpc_jit_data { 43 /* address of rw header */ 44 struct bpf_binary_header *hdr; 45 /* address of ro final header */ 46 struct bpf_binary_header *fhdr; 47 u32 *addrs; 48 u8 *fimage; 49 u32 proglen; 50 struct codegen_context ctx; 51 }; 52 53 bool bpf_jit_needs_zext(void) 54 { 55 return true; 56 } 57 58 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) 59 { 60 u32 proglen; 61 u32 alloclen; 62 u8 *image = NULL; 63 u32 *code_base; 64 u32 *addrs; 65 struct powerpc_jit_data *jit_data; 66 struct codegen_context cgctx; 67 int pass; 68 int flen; 69 struct bpf_binary_header *fhdr = NULL; 70 struct bpf_binary_header *hdr = NULL; 71 struct bpf_prog *org_fp = fp; 72 struct bpf_prog *tmp_fp; 73 bool bpf_blinded = false; 74 bool extra_pass = false; 75 u8 *fimage = NULL; 76 u32 *fcode_base; 77 u32 extable_len; 78 u32 fixup_len; 79 80 if (!fp->jit_requested) 81 return org_fp; 82 83 tmp_fp = bpf_jit_blind_constants(org_fp); 84 if (IS_ERR(tmp_fp)) 85 return org_fp; 86 87 if (tmp_fp != org_fp) { 88 bpf_blinded = true; 89 fp = tmp_fp; 90 } 91 92 jit_data = fp->aux->jit_data; 93 if (!jit_data) { 94 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); 95 if (!jit_data) { 96 fp = org_fp; 97 goto out; 98 } 99 fp->aux->jit_data = jit_data; 100 } 101 102 flen = fp->len; 103 addrs = jit_data->addrs; 104 if (addrs) { 105 cgctx = jit_data->ctx; 106 /* 107 * JIT compiled to a writable location (image/code_base) first. 108 * It is then moved to the readonly final location (fimage/fcode_base) 109 * using instruction patching. 110 */ 111 fimage = jit_data->fimage; 112 fhdr = jit_data->fhdr; 113 proglen = jit_data->proglen; 114 hdr = jit_data->hdr; 115 image = (void *)hdr + ((void *)fimage - (void *)fhdr); 116 extra_pass = true; 117 /* During extra pass, ensure index is reset before repopulating extable entries */ 118 cgctx.exentry_idx = 0; 119 goto skip_init_ctx; 120 } 121 122 addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL); 123 if (addrs == NULL) { 124 fp = org_fp; 125 goto out_addrs; 126 } 127 128 memset(&cgctx, 0, sizeof(struct codegen_context)); 129 bpf_jit_init_reg_mapping(&cgctx); 130 131 /* Make sure that the stack is quadword aligned. */ 132 cgctx.stack_size = round_up(fp->aux->stack_depth, 16); 133 134 /* Scouting faux-generate pass 0 */ 135 if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) { 136 /* We hit something illegal or unsupported. */ 137 fp = org_fp; 138 goto out_addrs; 139 } 140 141 /* 142 * If we have seen a tail call, we need a second pass. 143 * This is because bpf_jit_emit_common_epilogue() is called 144 * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen. 145 * We also need a second pass if we ended up with too large 146 * a program so as to ensure BPF_EXIT branches are in range. 147 */ 148 if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) { 149 cgctx.idx = 0; 150 if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) { 151 fp = org_fp; 152 goto out_addrs; 153 } 154 } 155 156 bpf_jit_realloc_regs(&cgctx); 157 /* 158 * Pretend to build prologue, given the features we've seen. This will 159 * update ctgtx.idx as it pretends to output instructions, then we can 160 * calculate total size from idx. 161 */ 162 bpf_jit_build_prologue(0, &cgctx); 163 addrs[fp->len] = cgctx.idx * 4; 164 bpf_jit_build_epilogue(0, &cgctx); 165 166 fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4; 167 extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry); 168 169 proglen = cgctx.idx * 4; 170 alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len; 171 172 fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image, 173 bpf_jit_fill_ill_insns); 174 if (!fhdr) { 175 fp = org_fp; 176 goto out_addrs; 177 } 178 179 if (extable_len) 180 fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len; 181 182 skip_init_ctx: 183 code_base = (u32 *)(image + FUNCTION_DESCR_SIZE); 184 fcode_base = (u32 *)(fimage + FUNCTION_DESCR_SIZE); 185 186 /* Code generation passes 1-2 */ 187 for (pass = 1; pass < 3; pass++) { 188 /* Now build the prologue, body code & epilogue for real. */ 189 cgctx.idx = 0; 190 cgctx.alt_exit_addr = 0; 191 bpf_jit_build_prologue(code_base, &cgctx); 192 if (bpf_jit_build_body(fp, code_base, fcode_base, &cgctx, addrs, pass, 193 extra_pass)) { 194 bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size)); 195 bpf_jit_binary_pack_free(fhdr, hdr); 196 fp = org_fp; 197 goto out_addrs; 198 } 199 bpf_jit_build_epilogue(code_base, &cgctx); 200 201 if (bpf_jit_enable > 1) 202 pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass, 203 proglen - (cgctx.idx * 4), cgctx.seen); 204 } 205 206 if (bpf_jit_enable > 1) 207 /* 208 * Note that we output the base address of the code_base 209 * rather than image, since opcodes are in code_base. 210 */ 211 bpf_jit_dump(flen, proglen, pass, code_base); 212 213 #ifdef CONFIG_PPC64_ELF_ABI_V1 214 /* Function descriptor nastiness: Address + TOC */ 215 ((u64 *)image)[0] = (u64)fcode_base; 216 ((u64 *)image)[1] = local_paca->kernel_toc; 217 #endif 218 219 fp->bpf_func = (void *)fimage; 220 fp->jited = 1; 221 fp->jited_len = proglen + FUNCTION_DESCR_SIZE; 222 223 if (!fp->is_func || extra_pass) { 224 if (bpf_jit_binary_pack_finalize(fp, fhdr, hdr)) { 225 fp = org_fp; 226 goto out_addrs; 227 } 228 bpf_prog_fill_jited_linfo(fp, addrs); 229 out_addrs: 230 kfree(addrs); 231 kfree(jit_data); 232 fp->aux->jit_data = NULL; 233 } else { 234 jit_data->addrs = addrs; 235 jit_data->ctx = cgctx; 236 jit_data->proglen = proglen; 237 jit_data->fimage = fimage; 238 jit_data->fhdr = fhdr; 239 jit_data->hdr = hdr; 240 } 241 242 out: 243 if (bpf_blinded) 244 bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp); 245 246 return fp; 247 } 248 249 /* 250 * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling 251 * this function, as this only applies to BPF_PROBE_MEM, for now. 252 */ 253 int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass, 254 struct codegen_context *ctx, int insn_idx, int jmp_off, 255 int dst_reg) 256 { 257 off_t offset; 258 unsigned long pc; 259 struct exception_table_entry *ex, *ex_entry; 260 u32 *fixup; 261 262 /* Populate extable entries only in the last pass */ 263 if (pass != 2) 264 return 0; 265 266 if (!fp->aux->extable || 267 WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries)) 268 return -EINVAL; 269 270 /* 271 * Program is first written to image before copying to the 272 * final location (fimage). Accordingly, update in the image first. 273 * As all offsets used are relative, copying as is to the 274 * final location should be alright. 275 */ 276 pc = (unsigned long)&image[insn_idx]; 277 ex = (void *)fp->aux->extable - (void *)fimage + (void *)image; 278 279 fixup = (void *)ex - 280 (fp->aux->num_exentries * BPF_FIXUP_LEN * 4) + 281 (ctx->exentry_idx * BPF_FIXUP_LEN * 4); 282 283 fixup[0] = PPC_RAW_LI(dst_reg, 0); 284 if (IS_ENABLED(CONFIG_PPC32)) 285 fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */ 286 287 fixup[BPF_FIXUP_LEN - 1] = 288 PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]); 289 290 ex_entry = &ex[ctx->exentry_idx]; 291 292 offset = pc - (long)&ex_entry->insn; 293 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 294 return -ERANGE; 295 ex_entry->insn = offset; 296 297 offset = (long)fixup - (long)&ex_entry->fixup; 298 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 299 return -ERANGE; 300 ex_entry->fixup = offset; 301 302 ctx->exentry_idx++; 303 return 0; 304 } 305 306 void bpf_jit_free(struct bpf_prog *fp) 307 { 308 if (fp->jited) { 309 struct powerpc_jit_data *jit_data = fp->aux->jit_data; 310 struct bpf_binary_header *hdr; 311 312 /* 313 * If we fail the final pass of JIT (from jit_subprogs), 314 * the program may not be finalized yet. Call finalize here 315 * before freeing it. 316 */ 317 if (jit_data) { 318 bpf_jit_binary_pack_finalize(fp, jit_data->fhdr, jit_data->hdr); 319 kvfree(jit_data->addrs); 320 kfree(jit_data); 321 } 322 hdr = bpf_jit_binary_pack_hdr(fp); 323 bpf_jit_binary_pack_free(hdr, NULL); 324 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp)); 325 } 326 327 bpf_prog_unlock_free(fp); 328 } 329