1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Common functionality for RV32 and RV64 BPF JIT compilers 4 * 5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com> 6 * 7 */ 8 9 #include <linux/bpf.h> 10 #include <linux/filter.h> 11 #include "bpf_jit.h" 12 13 /* Number of iterations to try until offsets converge. */ 14 #define NR_JIT_ITERATIONS 32 15 16 static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset) 17 { 18 const struct bpf_prog *prog = ctx->prog; 19 int i; 20 21 for (i = 0; i < prog->len; i++) { 22 const struct bpf_insn *insn = &prog->insnsi[i]; 23 int ret; 24 25 ret = bpf_jit_emit_insn(insn, ctx, extra_pass); 26 /* BPF_LD | BPF_IMM | BPF_DW: skip the next instruction. */ 27 if (ret > 0) 28 i++; 29 if (offset) 30 offset[i] = ctx->ninsns; 31 if (ret < 0) 32 return ret; 33 } 34 return 0; 35 } 36 37 bool bpf_jit_needs_zext(void) 38 { 39 return true; 40 } 41 42 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 43 { 44 unsigned int prog_size = 0, extable_size = 0; 45 bool tmp_blinded = false, extra_pass = false; 46 struct bpf_prog *tmp, *orig_prog = prog; 47 int pass = 0, prev_ninsns = 0, i; 48 struct rv_jit_data *jit_data; 49 struct rv_jit_context *ctx; 50 51 if (!prog->jit_requested) 52 return orig_prog; 53 54 tmp = bpf_jit_blind_constants(prog); 55 if (IS_ERR(tmp)) 56 return orig_prog; 57 if (tmp != prog) { 58 tmp_blinded = true; 59 prog = tmp; 60 } 61 62 jit_data = prog->aux->jit_data; 63 if (!jit_data) { 64 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); 65 if (!jit_data) { 66 prog = orig_prog; 67 goto out; 68 } 69 prog->aux->jit_data = jit_data; 70 } 71 72 ctx = &jit_data->ctx; 73 74 if (ctx->offset) { 75 extra_pass = true; 76 prog_size = sizeof(*ctx->insns) * ctx->ninsns; 77 goto skip_init_ctx; 78 } 79 80 ctx->prog = prog; 81 ctx->offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL); 82 if (!ctx->offset) { 83 prog = orig_prog; 84 goto out_offset; 85 } 86 87 if (build_body(ctx, extra_pass, NULL)) { 88 prog = orig_prog; 89 goto out_offset; 90 } 91 92 for (i = 0; i < prog->len; i++) { 93 prev_ninsns += 32; 94 ctx->offset[i] = prev_ninsns; 95 } 96 97 for (i = 0; i < NR_JIT_ITERATIONS; i++) { 98 pass++; 99 ctx->ninsns = 0; 100 101 bpf_jit_build_prologue(ctx); 102 ctx->prologue_len = ctx->ninsns; 103 104 if (build_body(ctx, extra_pass, ctx->offset)) { 105 prog = orig_prog; 106 goto out_offset; 107 } 108 109 ctx->epilogue_offset = ctx->ninsns; 110 bpf_jit_build_epilogue(ctx); 111 112 if (ctx->ninsns == prev_ninsns) { 113 if (jit_data->header) 114 break; 115 /* obtain the actual image size */ 116 extable_size = prog->aux->num_exentries * 117 sizeof(struct exception_table_entry); 118 prog_size = sizeof(*ctx->insns) * ctx->ninsns; 119 120 jit_data->header = 121 bpf_jit_binary_alloc(prog_size + extable_size, 122 &jit_data->image, 123 sizeof(u32), 124 bpf_fill_ill_insns); 125 if (!jit_data->header) { 126 prog = orig_prog; 127 goto out_offset; 128 } 129 130 ctx->insns = (u16 *)jit_data->image; 131 /* 132 * Now, when the image is allocated, the image can 133 * potentially shrink more (auipc/jalr -> jal). 134 */ 135 } 136 prev_ninsns = ctx->ninsns; 137 } 138 139 if (i == NR_JIT_ITERATIONS) { 140 pr_err("bpf-jit: image did not converge in <%d passes!\n", i); 141 if (jit_data->header) 142 bpf_jit_binary_free(jit_data->header); 143 prog = orig_prog; 144 goto out_offset; 145 } 146 147 if (extable_size) 148 prog->aux->extable = (void *)ctx->insns + prog_size; 149 150 skip_init_ctx: 151 pass++; 152 ctx->ninsns = 0; 153 ctx->nexentries = 0; 154 155 bpf_jit_build_prologue(ctx); 156 if (build_body(ctx, extra_pass, NULL)) { 157 bpf_jit_binary_free(jit_data->header); 158 prog = orig_prog; 159 goto out_offset; 160 } 161 bpf_jit_build_epilogue(ctx); 162 163 if (bpf_jit_enable > 1) 164 bpf_jit_dump(prog->len, prog_size, pass, ctx->insns); 165 166 prog->bpf_func = (void *)ctx->insns; 167 prog->jited = 1; 168 prog->jited_len = prog_size; 169 170 bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns); 171 172 if (!prog->is_func || extra_pass) { 173 bpf_jit_binary_lock_ro(jit_data->header); 174 for (i = 0; i < prog->len; i++) 175 ctx->offset[i] = ninsns_rvoff(ctx->offset[i]); 176 bpf_prog_fill_jited_linfo(prog, ctx->offset); 177 out_offset: 178 kfree(ctx->offset); 179 kfree(jit_data); 180 prog->aux->jit_data = NULL; 181 } 182 out: 183 184 if (tmp_blinded) 185 bpf_jit_prog_release_other(prog, prog == orig_prog ? 186 tmp : orig_prog); 187 return prog; 188 } 189 190 u64 bpf_jit_alloc_exec_limit(void) 191 { 192 return BPF_JIT_REGION_SIZE; 193 } 194 195 void *bpf_jit_alloc_exec(unsigned long size) 196 { 197 return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START, 198 BPF_JIT_REGION_END, GFP_KERNEL, 199 PAGE_KERNEL, 0, NUMA_NO_NODE, 200 __builtin_return_address(0)); 201 } 202 203 void bpf_jit_free_exec(void *addr) 204 { 205 return vfree(addr); 206 } 207