1ca6cb544SLuke Nelson // SPDX-License-Identifier: GPL-2.0 2ca6cb544SLuke Nelson /* BPF JIT compiler for RV64G 3ca6cb544SLuke Nelson * 4ca6cb544SLuke Nelson * Copyright(c) 2019 Björn Töpel <bjorn.topel@gmail.com> 5ca6cb544SLuke Nelson * 6ca6cb544SLuke Nelson */ 7ca6cb544SLuke Nelson 8252c765bSTong Tiangen #include <linux/bitfield.h> 9ca6cb544SLuke Nelson #include <linux/bpf.h> 10ca6cb544SLuke Nelson #include <linux/filter.h> 11*596f2e6fSPu Lehui #include <linux/memory.h> 12*596f2e6fSPu Lehui #include <linux/stop_machine.h> 13ca6cb544SLuke Nelson #include "bpf_jit.h" 14ca6cb544SLuke Nelson 15ca6cb544SLuke Nelson #define RV_REG_TCC RV_REG_A6 16ca6cb544SLuke Nelson #define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */ 17ca6cb544SLuke Nelson 18ca6cb544SLuke Nelson static const int regmap[] = { 19ca6cb544SLuke Nelson [BPF_REG_0] = RV_REG_A5, 20ca6cb544SLuke Nelson [BPF_REG_1] = RV_REG_A0, 21ca6cb544SLuke Nelson [BPF_REG_2] = RV_REG_A1, 22ca6cb544SLuke Nelson [BPF_REG_3] = RV_REG_A2, 23ca6cb544SLuke Nelson [BPF_REG_4] = RV_REG_A3, 24ca6cb544SLuke Nelson [BPF_REG_5] = RV_REG_A4, 25ca6cb544SLuke Nelson [BPF_REG_6] = RV_REG_S1, 26ca6cb544SLuke Nelson [BPF_REG_7] = RV_REG_S2, 27ca6cb544SLuke Nelson [BPF_REG_8] = RV_REG_S3, 28ca6cb544SLuke Nelson [BPF_REG_9] = RV_REG_S4, 29ca6cb544SLuke Nelson [BPF_REG_FP] = RV_REG_S5, 30ca6cb544SLuke Nelson [BPF_REG_AX] = RV_REG_T0, 31ca6cb544SLuke Nelson }; 32ca6cb544SLuke Nelson 33252c765bSTong Tiangen static const int pt_regmap[] = { 34252c765bSTong Tiangen [RV_REG_A0] = offsetof(struct pt_regs, a0), 35252c765bSTong Tiangen [RV_REG_A1] = offsetof(struct pt_regs, a1), 36252c765bSTong Tiangen [RV_REG_A2] = offsetof(struct pt_regs, a2), 37252c765bSTong Tiangen [RV_REG_A3] = offsetof(struct pt_regs, a3), 38252c765bSTong Tiangen [RV_REG_A4] = offsetof(struct pt_regs, a4), 39252c765bSTong Tiangen [RV_REG_A5] = offsetof(struct pt_regs, a5), 40252c765bSTong Tiangen [RV_REG_S1] = offsetof(struct pt_regs, s1), 41252c765bSTong Tiangen [RV_REG_S2] = offsetof(struct pt_regs, s2), 42252c765bSTong Tiangen [RV_REG_S3] = offsetof(struct pt_regs, s3), 43252c765bSTong Tiangen [RV_REG_S4] = offsetof(struct pt_regs, s4), 44252c765bSTong Tiangen [RV_REG_S5] = offsetof(struct pt_regs, s5), 45252c765bSTong Tiangen [RV_REG_T0] = offsetof(struct pt_regs, t0), 46252c765bSTong Tiangen }; 47252c765bSTong Tiangen 48ca6cb544SLuke Nelson enum { 49ca6cb544SLuke Nelson RV_CTX_F_SEEN_TAIL_CALL = 0, 50ca6cb544SLuke Nelson RV_CTX_F_SEEN_CALL = RV_REG_RA, 51ca6cb544SLuke Nelson RV_CTX_F_SEEN_S1 = RV_REG_S1, 52ca6cb544SLuke Nelson RV_CTX_F_SEEN_S2 = RV_REG_S2, 53ca6cb544SLuke Nelson RV_CTX_F_SEEN_S3 = RV_REG_S3, 54ca6cb544SLuke Nelson RV_CTX_F_SEEN_S4 = RV_REG_S4, 55ca6cb544SLuke Nelson RV_CTX_F_SEEN_S5 = RV_REG_S5, 56ca6cb544SLuke Nelson RV_CTX_F_SEEN_S6 = RV_REG_S6, 57ca6cb544SLuke Nelson }; 58ca6cb544SLuke Nelson 59ca6cb544SLuke Nelson static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) 60ca6cb544SLuke Nelson { 61ca6cb544SLuke Nelson u8 reg = regmap[bpf_reg]; 62ca6cb544SLuke Nelson 63ca6cb544SLuke Nelson switch (reg) { 64ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S1: 65ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S2: 66ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S3: 67ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S4: 68ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S5: 69ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S6: 70ca6cb544SLuke Nelson __set_bit(reg, &ctx->flags); 71ca6cb544SLuke Nelson } 72ca6cb544SLuke Nelson return reg; 73ca6cb544SLuke Nelson }; 74ca6cb544SLuke Nelson 75ca6cb544SLuke Nelson static bool seen_reg(int reg, struct rv_jit_context *ctx) 76ca6cb544SLuke Nelson { 77ca6cb544SLuke Nelson switch (reg) { 78ca6cb544SLuke Nelson case RV_CTX_F_SEEN_CALL: 79ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S1: 80ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S2: 81ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S3: 82ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S4: 83ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S5: 84ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S6: 85ca6cb544SLuke Nelson return test_bit(reg, &ctx->flags); 86ca6cb544SLuke Nelson } 87ca6cb544SLuke Nelson return false; 88ca6cb544SLuke Nelson } 89ca6cb544SLuke Nelson 90ca6cb544SLuke Nelson static void mark_fp(struct rv_jit_context *ctx) 91ca6cb544SLuke Nelson { 92ca6cb544SLuke Nelson __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags); 93ca6cb544SLuke Nelson } 94ca6cb544SLuke Nelson 95ca6cb544SLuke Nelson static void mark_call(struct rv_jit_context *ctx) 96ca6cb544SLuke Nelson { 97ca6cb544SLuke Nelson __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 98ca6cb544SLuke Nelson } 99ca6cb544SLuke Nelson 100ca6cb544SLuke Nelson static bool seen_call(struct rv_jit_context *ctx) 101ca6cb544SLuke Nelson { 102ca6cb544SLuke Nelson return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 103ca6cb544SLuke Nelson } 104ca6cb544SLuke Nelson 105ca6cb544SLuke Nelson static void mark_tail_call(struct rv_jit_context *ctx) 106ca6cb544SLuke Nelson { 107ca6cb544SLuke Nelson __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 108ca6cb544SLuke Nelson } 109ca6cb544SLuke Nelson 110ca6cb544SLuke Nelson static bool seen_tail_call(struct rv_jit_context *ctx) 111ca6cb544SLuke Nelson { 112ca6cb544SLuke Nelson return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 113ca6cb544SLuke Nelson } 114ca6cb544SLuke Nelson 115ca6cb544SLuke Nelson static u8 rv_tail_call_reg(struct rv_jit_context *ctx) 116ca6cb544SLuke Nelson { 117ca6cb544SLuke Nelson mark_tail_call(ctx); 118ca6cb544SLuke Nelson 119ca6cb544SLuke Nelson if (seen_call(ctx)) { 120ca6cb544SLuke Nelson __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); 121ca6cb544SLuke Nelson return RV_REG_S6; 122ca6cb544SLuke Nelson } 123ca6cb544SLuke Nelson return RV_REG_A6; 124ca6cb544SLuke Nelson } 125ca6cb544SLuke Nelson 126ca6cb544SLuke Nelson static bool is_32b_int(s64 val) 127ca6cb544SLuke Nelson { 128ca6cb544SLuke Nelson return -(1L << 31) <= val && val < (1L << 31); 129ca6cb544SLuke Nelson } 130ca6cb544SLuke Nelson 131489553ddSLuke Nelson static bool in_auipc_jalr_range(s64 val) 132489553ddSLuke Nelson { 133489553ddSLuke Nelson /* 134489553ddSLuke Nelson * auipc+jalr can reach any signed PC-relative offset in the range 135489553ddSLuke Nelson * [-2^31 - 2^11, 2^31 - 2^11). 136489553ddSLuke Nelson */ 137489553ddSLuke Nelson return (-(1L << 31) - (1L << 11)) <= val && 138489553ddSLuke Nelson val < ((1L << 31) - (1L << 11)); 139489553ddSLuke Nelson } 140489553ddSLuke Nelson 141b54b6003SPu Lehui /* Emit fixed-length instructions for address */ 142b54b6003SPu Lehui static int emit_addr(u8 rd, u64 addr, bool extra_pass, struct rv_jit_context *ctx) 143b54b6003SPu Lehui { 144b54b6003SPu Lehui u64 ip = (u64)(ctx->insns + ctx->ninsns); 145b54b6003SPu Lehui s64 off = addr - ip; 146b54b6003SPu Lehui s64 upper = (off + (1 << 11)) >> 12; 147b54b6003SPu Lehui s64 lower = off & 0xfff; 148b54b6003SPu Lehui 149b54b6003SPu Lehui if (extra_pass && !in_auipc_jalr_range(off)) { 150b54b6003SPu Lehui pr_err("bpf-jit: target offset 0x%llx is out of range\n", off); 151b54b6003SPu Lehui return -ERANGE; 152b54b6003SPu Lehui } 153b54b6003SPu Lehui 154b54b6003SPu Lehui emit(rv_auipc(rd, upper), ctx); 155b54b6003SPu Lehui emit(rv_addi(rd, rd, lower), ctx); 156b54b6003SPu Lehui return 0; 157b54b6003SPu Lehui } 158b54b6003SPu Lehui 159b54b6003SPu Lehui /* Emit variable-length instructions for 32-bit and 64-bit imm */ 160ca6cb544SLuke Nelson static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx) 161ca6cb544SLuke Nelson { 162ca6cb544SLuke Nelson /* Note that the immediate from the add is sign-extended, 163ca6cb544SLuke Nelson * which means that we need to compensate this by adding 2^12, 164ca6cb544SLuke Nelson * when the 12th bit is set. A simpler way of doing this, and 165ca6cb544SLuke Nelson * getting rid of the check, is to just add 2**11 before the 166ca6cb544SLuke Nelson * shift. The "Loading a 32-Bit constant" example from the 167ca6cb544SLuke Nelson * "Computer Organization and Design, RISC-V edition" book by 168ca6cb544SLuke Nelson * Patterson/Hennessy highlights this fact. 169ca6cb544SLuke Nelson * 170ca6cb544SLuke Nelson * This also means that we need to process LSB to MSB. 171ca6cb544SLuke Nelson */ 17218a4d8c9SLuke Nelson s64 upper = (val + (1 << 11)) >> 12; 17318a4d8c9SLuke Nelson /* Sign-extend lower 12 bits to 64 bits since immediates for li, addiw, 17418a4d8c9SLuke Nelson * and addi are signed and RVC checks will perform signed comparisons. 17518a4d8c9SLuke Nelson */ 17618a4d8c9SLuke Nelson s64 lower = ((val & 0xfff) << 52) >> 52; 177ca6cb544SLuke Nelson int shift; 178ca6cb544SLuke Nelson 179ca6cb544SLuke Nelson if (is_32b_int(val)) { 180ca6cb544SLuke Nelson if (upper) 18118a4d8c9SLuke Nelson emit_lui(rd, upper, ctx); 182ca6cb544SLuke Nelson 183ca6cb544SLuke Nelson if (!upper) { 18418a4d8c9SLuke Nelson emit_li(rd, lower, ctx); 185ca6cb544SLuke Nelson return; 186ca6cb544SLuke Nelson } 187ca6cb544SLuke Nelson 18818a4d8c9SLuke Nelson emit_addiw(rd, rd, lower, ctx); 189ca6cb544SLuke Nelson return; 190ca6cb544SLuke Nelson } 191ca6cb544SLuke Nelson 192ca6cb544SLuke Nelson shift = __ffs(upper); 193ca6cb544SLuke Nelson upper >>= shift; 194ca6cb544SLuke Nelson shift += 12; 195ca6cb544SLuke Nelson 196ca6cb544SLuke Nelson emit_imm(rd, upper, ctx); 197ca6cb544SLuke Nelson 19818a4d8c9SLuke Nelson emit_slli(rd, rd, shift, ctx); 199ca6cb544SLuke Nelson if (lower) 20018a4d8c9SLuke Nelson emit_addi(rd, rd, lower, ctx); 201ca6cb544SLuke Nelson } 202ca6cb544SLuke Nelson 203ca6cb544SLuke Nelson static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) 204ca6cb544SLuke Nelson { 205ca6cb544SLuke Nelson int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; 206ca6cb544SLuke Nelson 207ca6cb544SLuke Nelson if (seen_reg(RV_REG_RA, ctx)) { 20818a4d8c9SLuke Nelson emit_ld(RV_REG_RA, store_offset, RV_REG_SP, ctx); 209ca6cb544SLuke Nelson store_offset -= 8; 210ca6cb544SLuke Nelson } 21118a4d8c9SLuke Nelson emit_ld(RV_REG_FP, store_offset, RV_REG_SP, ctx); 212ca6cb544SLuke Nelson store_offset -= 8; 213ca6cb544SLuke Nelson if (seen_reg(RV_REG_S1, ctx)) { 21418a4d8c9SLuke Nelson emit_ld(RV_REG_S1, store_offset, RV_REG_SP, ctx); 215ca6cb544SLuke Nelson store_offset -= 8; 216ca6cb544SLuke Nelson } 217ca6cb544SLuke Nelson if (seen_reg(RV_REG_S2, ctx)) { 21818a4d8c9SLuke Nelson emit_ld(RV_REG_S2, store_offset, RV_REG_SP, ctx); 219ca6cb544SLuke Nelson store_offset -= 8; 220ca6cb544SLuke Nelson } 221ca6cb544SLuke Nelson if (seen_reg(RV_REG_S3, ctx)) { 22218a4d8c9SLuke Nelson emit_ld(RV_REG_S3, store_offset, RV_REG_SP, ctx); 223ca6cb544SLuke Nelson store_offset -= 8; 224ca6cb544SLuke Nelson } 225ca6cb544SLuke Nelson if (seen_reg(RV_REG_S4, ctx)) { 22618a4d8c9SLuke Nelson emit_ld(RV_REG_S4, store_offset, RV_REG_SP, ctx); 227ca6cb544SLuke Nelson store_offset -= 8; 228ca6cb544SLuke Nelson } 229ca6cb544SLuke Nelson if (seen_reg(RV_REG_S5, ctx)) { 23018a4d8c9SLuke Nelson emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx); 231ca6cb544SLuke Nelson store_offset -= 8; 232ca6cb544SLuke Nelson } 233ca6cb544SLuke Nelson if (seen_reg(RV_REG_S6, ctx)) { 23418a4d8c9SLuke Nelson emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx); 235ca6cb544SLuke Nelson store_offset -= 8; 236ca6cb544SLuke Nelson } 237ca6cb544SLuke Nelson 23818a4d8c9SLuke Nelson emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); 239ca6cb544SLuke Nelson /* Set return value. */ 240ca6cb544SLuke Nelson if (!is_tail_call) 24118a4d8c9SLuke Nelson emit_mv(RV_REG_A0, RV_REG_A5, ctx); 24218a4d8c9SLuke Nelson emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, 243*596f2e6fSPu Lehui is_tail_call ? 20 : 0, /* skip reserved nops and TCC init */ 244ca6cb544SLuke Nelson ctx); 245ca6cb544SLuke Nelson } 246ca6cb544SLuke Nelson 247ca6cb544SLuke Nelson static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, 248ca6cb544SLuke Nelson struct rv_jit_context *ctx) 249ca6cb544SLuke Nelson { 250ca6cb544SLuke Nelson switch (cond) { 251ca6cb544SLuke Nelson case BPF_JEQ: 252ca6cb544SLuke Nelson emit(rv_beq(rd, rs, rvoff >> 1), ctx); 253ca6cb544SLuke Nelson return; 254ca6cb544SLuke Nelson case BPF_JGT: 255ca6cb544SLuke Nelson emit(rv_bltu(rs, rd, rvoff >> 1), ctx); 256ca6cb544SLuke Nelson return; 257ca6cb544SLuke Nelson case BPF_JLT: 258ca6cb544SLuke Nelson emit(rv_bltu(rd, rs, rvoff >> 1), ctx); 259ca6cb544SLuke Nelson return; 260ca6cb544SLuke Nelson case BPF_JGE: 261ca6cb544SLuke Nelson emit(rv_bgeu(rd, rs, rvoff >> 1), ctx); 262ca6cb544SLuke Nelson return; 263ca6cb544SLuke Nelson case BPF_JLE: 264ca6cb544SLuke Nelson emit(rv_bgeu(rs, rd, rvoff >> 1), ctx); 265ca6cb544SLuke Nelson return; 266ca6cb544SLuke Nelson case BPF_JNE: 267ca6cb544SLuke Nelson emit(rv_bne(rd, rs, rvoff >> 1), ctx); 268ca6cb544SLuke Nelson return; 269ca6cb544SLuke Nelson case BPF_JSGT: 270ca6cb544SLuke Nelson emit(rv_blt(rs, rd, rvoff >> 1), ctx); 271ca6cb544SLuke Nelson return; 272ca6cb544SLuke Nelson case BPF_JSLT: 273ca6cb544SLuke Nelson emit(rv_blt(rd, rs, rvoff >> 1), ctx); 274ca6cb544SLuke Nelson return; 275ca6cb544SLuke Nelson case BPF_JSGE: 276ca6cb544SLuke Nelson emit(rv_bge(rd, rs, rvoff >> 1), ctx); 277ca6cb544SLuke Nelson return; 278ca6cb544SLuke Nelson case BPF_JSLE: 279ca6cb544SLuke Nelson emit(rv_bge(rs, rd, rvoff >> 1), ctx); 280ca6cb544SLuke Nelson } 281ca6cb544SLuke Nelson } 282ca6cb544SLuke Nelson 283ca6cb544SLuke Nelson static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, 284ca6cb544SLuke Nelson struct rv_jit_context *ctx) 285ca6cb544SLuke Nelson { 286ca6cb544SLuke Nelson s64 upper, lower; 287ca6cb544SLuke Nelson 288ca6cb544SLuke Nelson if (is_13b_int(rvoff)) { 289ca6cb544SLuke Nelson emit_bcc(cond, rd, rs, rvoff, ctx); 290ca6cb544SLuke Nelson return; 291ca6cb544SLuke Nelson } 292ca6cb544SLuke Nelson 293ca6cb544SLuke Nelson /* Adjust for jal */ 294ca6cb544SLuke Nelson rvoff -= 4; 295ca6cb544SLuke Nelson 296ca6cb544SLuke Nelson /* Transform, e.g.: 297ca6cb544SLuke Nelson * bne rd,rs,foo 298ca6cb544SLuke Nelson * to 299ca6cb544SLuke Nelson * beq rd,rs,<.L1> 300ca6cb544SLuke Nelson * (auipc foo) 301ca6cb544SLuke Nelson * jal(r) foo 302ca6cb544SLuke Nelson * .L1 303ca6cb544SLuke Nelson */ 304ca6cb544SLuke Nelson cond = invert_bpf_cond(cond); 305ca6cb544SLuke Nelson if (is_21b_int(rvoff)) { 306ca6cb544SLuke Nelson emit_bcc(cond, rd, rs, 8, ctx); 307ca6cb544SLuke Nelson emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx); 308ca6cb544SLuke Nelson return; 309ca6cb544SLuke Nelson } 310ca6cb544SLuke Nelson 311ca6cb544SLuke Nelson /* 32b No need for an additional rvoff adjustment, since we 312ca6cb544SLuke Nelson * get that from the auipc at PC', where PC = PC' + 4. 313ca6cb544SLuke Nelson */ 314ca6cb544SLuke Nelson upper = (rvoff + (1 << 11)) >> 12; 315ca6cb544SLuke Nelson lower = rvoff & 0xfff; 316ca6cb544SLuke Nelson 317ca6cb544SLuke Nelson emit_bcc(cond, rd, rs, 12, ctx); 318ca6cb544SLuke Nelson emit(rv_auipc(RV_REG_T1, upper), ctx); 319ca6cb544SLuke Nelson emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx); 320ca6cb544SLuke Nelson } 321ca6cb544SLuke Nelson 322ca6cb544SLuke Nelson static void emit_zext_32(u8 reg, struct rv_jit_context *ctx) 323ca6cb544SLuke Nelson { 32418a4d8c9SLuke Nelson emit_slli(reg, reg, 32, ctx); 32518a4d8c9SLuke Nelson emit_srli(reg, reg, 32, ctx); 326ca6cb544SLuke Nelson } 327ca6cb544SLuke Nelson 328ca6cb544SLuke Nelson static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) 329ca6cb544SLuke Nelson { 330ca6cb544SLuke Nelson int tc_ninsn, off, start_insn = ctx->ninsns; 331ca6cb544SLuke Nelson u8 tcc = rv_tail_call_reg(ctx); 332ca6cb544SLuke Nelson 333ca6cb544SLuke Nelson /* a0: &ctx 334ca6cb544SLuke Nelson * a1: &array 335ca6cb544SLuke Nelson * a2: index 336ca6cb544SLuke Nelson * 337ca6cb544SLuke Nelson * if (index >= array->map.max_entries) 338ca6cb544SLuke Nelson * goto out; 339ca6cb544SLuke Nelson */ 340ca6cb544SLuke Nelson tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] : 341ca6cb544SLuke Nelson ctx->offset[0]; 342ca6cb544SLuke Nelson emit_zext_32(RV_REG_A2, ctx); 343ca6cb544SLuke Nelson 344ca6cb544SLuke Nelson off = offsetof(struct bpf_array, map.max_entries); 345ca6cb544SLuke Nelson if (is_12b_check(off, insn)) 346ca6cb544SLuke Nelson return -1; 347ca6cb544SLuke Nelson emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx); 348bfabff3cSLuke Nelson off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 349ca6cb544SLuke Nelson emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx); 350ca6cb544SLuke Nelson 351ebf7f6f0STiezhu Yang /* if (--TCC < 0) 352ca6cb544SLuke Nelson * goto out; 353ca6cb544SLuke Nelson */ 354ebf7f6f0STiezhu Yang emit_addi(RV_REG_TCC, tcc, -1, ctx); 355bfabff3cSLuke Nelson off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 356ebf7f6f0STiezhu Yang emit_branch(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); 357ca6cb544SLuke Nelson 358ca6cb544SLuke Nelson /* prog = array->ptrs[index]; 359ca6cb544SLuke Nelson * if (!prog) 360ca6cb544SLuke Nelson * goto out; 361ca6cb544SLuke Nelson */ 36218a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_A2, 3, ctx); 36318a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_A1, ctx); 364ca6cb544SLuke Nelson off = offsetof(struct bpf_array, ptrs); 365ca6cb544SLuke Nelson if (is_12b_check(off, insn)) 366ca6cb544SLuke Nelson return -1; 36718a4d8c9SLuke Nelson emit_ld(RV_REG_T2, off, RV_REG_T2, ctx); 368bfabff3cSLuke Nelson off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 369ca6cb544SLuke Nelson emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); 370ca6cb544SLuke Nelson 371ca6cb544SLuke Nelson /* goto *(prog->bpf_func + 4); */ 372ca6cb544SLuke Nelson off = offsetof(struct bpf_prog, bpf_func); 373ca6cb544SLuke Nelson if (is_12b_check(off, insn)) 374ca6cb544SLuke Nelson return -1; 37518a4d8c9SLuke Nelson emit_ld(RV_REG_T3, off, RV_REG_T2, ctx); 376ca6cb544SLuke Nelson __build_epilogue(true, ctx); 377ca6cb544SLuke Nelson return 0; 378ca6cb544SLuke Nelson } 379ca6cb544SLuke Nelson 380ca6cb544SLuke Nelson static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn, 381ca6cb544SLuke Nelson struct rv_jit_context *ctx) 382ca6cb544SLuke Nelson { 383ca6cb544SLuke Nelson u8 code = insn->code; 384ca6cb544SLuke Nelson 385ca6cb544SLuke Nelson switch (code) { 386ca6cb544SLuke Nelson case BPF_JMP | BPF_JA: 387ca6cb544SLuke Nelson case BPF_JMP | BPF_CALL: 388ca6cb544SLuke Nelson case BPF_JMP | BPF_EXIT: 389ca6cb544SLuke Nelson case BPF_JMP | BPF_TAIL_CALL: 390ca6cb544SLuke Nelson break; 391ca6cb544SLuke Nelson default: 392ca6cb544SLuke Nelson *rd = bpf_to_rv_reg(insn->dst_reg, ctx); 393ca6cb544SLuke Nelson } 394ca6cb544SLuke Nelson 395ca6cb544SLuke Nelson if (code & (BPF_ALU | BPF_X) || code & (BPF_ALU64 | BPF_X) || 396ca6cb544SLuke Nelson code & (BPF_JMP | BPF_X) || code & (BPF_JMP32 | BPF_X) || 397ca6cb544SLuke Nelson code & BPF_LDX || code & BPF_STX) 398ca6cb544SLuke Nelson *rs = bpf_to_rv_reg(insn->src_reg, ctx); 399ca6cb544SLuke Nelson } 400ca6cb544SLuke Nelson 401ca6cb544SLuke Nelson static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) 402ca6cb544SLuke Nelson { 40318a4d8c9SLuke Nelson emit_mv(RV_REG_T2, *rd, ctx); 404ca6cb544SLuke Nelson emit_zext_32(RV_REG_T2, ctx); 40518a4d8c9SLuke Nelson emit_mv(RV_REG_T1, *rs, ctx); 406ca6cb544SLuke Nelson emit_zext_32(RV_REG_T1, ctx); 407ca6cb544SLuke Nelson *rd = RV_REG_T2; 408ca6cb544SLuke Nelson *rs = RV_REG_T1; 409ca6cb544SLuke Nelson } 410ca6cb544SLuke Nelson 411ca6cb544SLuke Nelson static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) 412ca6cb544SLuke Nelson { 41318a4d8c9SLuke Nelson emit_addiw(RV_REG_T2, *rd, 0, ctx); 41418a4d8c9SLuke Nelson emit_addiw(RV_REG_T1, *rs, 0, ctx); 415ca6cb544SLuke Nelson *rd = RV_REG_T2; 416ca6cb544SLuke Nelson *rs = RV_REG_T1; 417ca6cb544SLuke Nelson } 418ca6cb544SLuke Nelson 419ca6cb544SLuke Nelson static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx) 420ca6cb544SLuke Nelson { 42118a4d8c9SLuke Nelson emit_mv(RV_REG_T2, *rd, ctx); 422ca6cb544SLuke Nelson emit_zext_32(RV_REG_T2, ctx); 423ca6cb544SLuke Nelson emit_zext_32(RV_REG_T1, ctx); 424ca6cb544SLuke Nelson *rd = RV_REG_T2; 425ca6cb544SLuke Nelson } 426ca6cb544SLuke Nelson 427ca6cb544SLuke Nelson static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx) 428ca6cb544SLuke Nelson { 42918a4d8c9SLuke Nelson emit_addiw(RV_REG_T2, *rd, 0, ctx); 430ca6cb544SLuke Nelson *rd = RV_REG_T2; 431ca6cb544SLuke Nelson } 432ca6cb544SLuke Nelson 4330fd1fd01SPu Lehui static int emit_jump_and_link(u8 rd, s64 rvoff, bool fixed_addr, 434ca6cb544SLuke Nelson struct rv_jit_context *ctx) 435ca6cb544SLuke Nelson { 436ca6cb544SLuke Nelson s64 upper, lower; 437ca6cb544SLuke Nelson 4380fd1fd01SPu Lehui if (rvoff && fixed_addr && is_21b_int(rvoff)) { 439ca6cb544SLuke Nelson emit(rv_jal(rd, rvoff >> 1), ctx); 440489553ddSLuke Nelson return 0; 441489553ddSLuke Nelson } else if (in_auipc_jalr_range(rvoff)) { 442ca6cb544SLuke Nelson upper = (rvoff + (1 << 11)) >> 12; 443ca6cb544SLuke Nelson lower = rvoff & 0xfff; 444ca6cb544SLuke Nelson emit(rv_auipc(RV_REG_T1, upper), ctx); 445ca6cb544SLuke Nelson emit(rv_jalr(rd, RV_REG_T1, lower), ctx); 446489553ddSLuke Nelson return 0; 447489553ddSLuke Nelson } 448489553ddSLuke Nelson 449489553ddSLuke Nelson pr_err("bpf-jit: target offset 0x%llx is out of range\n", rvoff); 450489553ddSLuke Nelson return -ERANGE; 451ca6cb544SLuke Nelson } 452ca6cb544SLuke Nelson 453ca6cb544SLuke Nelson static bool is_signed_bpf_cond(u8 cond) 454ca6cb544SLuke Nelson { 455ca6cb544SLuke Nelson return cond == BPF_JSGT || cond == BPF_JSLT || 456ca6cb544SLuke Nelson cond == BPF_JSGE || cond == BPF_JSLE; 457ca6cb544SLuke Nelson } 458ca6cb544SLuke Nelson 4590fd1fd01SPu Lehui static int emit_call(u64 addr, bool fixed_addr, struct rv_jit_context *ctx) 460ca6cb544SLuke Nelson { 461ca6cb544SLuke Nelson s64 off = 0; 462ca6cb544SLuke Nelson u64 ip; 463ca6cb544SLuke Nelson 464ca6cb544SLuke Nelson if (addr && ctx->insns) { 465ca6cb544SLuke Nelson ip = (u64)(long)(ctx->insns + ctx->ninsns); 466ca6cb544SLuke Nelson off = addr - ip; 467ca6cb544SLuke Nelson } 468ca6cb544SLuke Nelson 4690fd1fd01SPu Lehui return emit_jump_and_link(RV_REG_RA, off, fixed_addr, ctx); 470ca6cb544SLuke Nelson } 471ca6cb544SLuke Nelson 472dd642ccbSPu Lehui static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64, 473dd642ccbSPu Lehui struct rv_jit_context *ctx) 474dd642ccbSPu Lehui { 475dd642ccbSPu Lehui u8 r0; 476dd642ccbSPu Lehui int jmp_offset; 477dd642ccbSPu Lehui 478dd642ccbSPu Lehui if (off) { 479dd642ccbSPu Lehui if (is_12b_int(off)) { 480dd642ccbSPu Lehui emit_addi(RV_REG_T1, rd, off, ctx); 481dd642ccbSPu Lehui } else { 482dd642ccbSPu Lehui emit_imm(RV_REG_T1, off, ctx); 483dd642ccbSPu Lehui emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 484dd642ccbSPu Lehui } 485dd642ccbSPu Lehui rd = RV_REG_T1; 486dd642ccbSPu Lehui } 487dd642ccbSPu Lehui 488dd642ccbSPu Lehui switch (imm) { 489dd642ccbSPu Lehui /* lock *(u32/u64 *)(dst_reg + off16) <op>= src_reg */ 490dd642ccbSPu Lehui case BPF_ADD: 491dd642ccbSPu Lehui emit(is64 ? rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0) : 492dd642ccbSPu Lehui rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 493dd642ccbSPu Lehui break; 494dd642ccbSPu Lehui case BPF_AND: 495dd642ccbSPu Lehui emit(is64 ? rv_amoand_d(RV_REG_ZERO, rs, rd, 0, 0) : 496dd642ccbSPu Lehui rv_amoand_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 497dd642ccbSPu Lehui break; 498dd642ccbSPu Lehui case BPF_OR: 499dd642ccbSPu Lehui emit(is64 ? rv_amoor_d(RV_REG_ZERO, rs, rd, 0, 0) : 500dd642ccbSPu Lehui rv_amoor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 501dd642ccbSPu Lehui break; 502dd642ccbSPu Lehui case BPF_XOR: 503dd642ccbSPu Lehui emit(is64 ? rv_amoxor_d(RV_REG_ZERO, rs, rd, 0, 0) : 504dd642ccbSPu Lehui rv_amoxor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 505dd642ccbSPu Lehui break; 506dd642ccbSPu Lehui /* src_reg = atomic_fetch_<op>(dst_reg + off16, src_reg) */ 507dd642ccbSPu Lehui case BPF_ADD | BPF_FETCH: 508dd642ccbSPu Lehui emit(is64 ? rv_amoadd_d(rs, rs, rd, 0, 0) : 509dd642ccbSPu Lehui rv_amoadd_w(rs, rs, rd, 0, 0), ctx); 510dd642ccbSPu Lehui if (!is64) 511dd642ccbSPu Lehui emit_zext_32(rs, ctx); 512dd642ccbSPu Lehui break; 513dd642ccbSPu Lehui case BPF_AND | BPF_FETCH: 514dd642ccbSPu Lehui emit(is64 ? rv_amoand_d(rs, rs, rd, 0, 0) : 515dd642ccbSPu Lehui rv_amoand_w(rs, rs, rd, 0, 0), ctx); 516dd642ccbSPu Lehui if (!is64) 517dd642ccbSPu Lehui emit_zext_32(rs, ctx); 518dd642ccbSPu Lehui break; 519dd642ccbSPu Lehui case BPF_OR | BPF_FETCH: 520dd642ccbSPu Lehui emit(is64 ? rv_amoor_d(rs, rs, rd, 0, 0) : 521dd642ccbSPu Lehui rv_amoor_w(rs, rs, rd, 0, 0), ctx); 522dd642ccbSPu Lehui if (!is64) 523dd642ccbSPu Lehui emit_zext_32(rs, ctx); 524dd642ccbSPu Lehui break; 525dd642ccbSPu Lehui case BPF_XOR | BPF_FETCH: 526dd642ccbSPu Lehui emit(is64 ? rv_amoxor_d(rs, rs, rd, 0, 0) : 527dd642ccbSPu Lehui rv_amoxor_w(rs, rs, rd, 0, 0), ctx); 528dd642ccbSPu Lehui if (!is64) 529dd642ccbSPu Lehui emit_zext_32(rs, ctx); 530dd642ccbSPu Lehui break; 531dd642ccbSPu Lehui /* src_reg = atomic_xchg(dst_reg + off16, src_reg); */ 532dd642ccbSPu Lehui case BPF_XCHG: 533dd642ccbSPu Lehui emit(is64 ? rv_amoswap_d(rs, rs, rd, 0, 0) : 534dd642ccbSPu Lehui rv_amoswap_w(rs, rs, rd, 0, 0), ctx); 535dd642ccbSPu Lehui if (!is64) 536dd642ccbSPu Lehui emit_zext_32(rs, ctx); 537dd642ccbSPu Lehui break; 538dd642ccbSPu Lehui /* r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg); */ 539dd642ccbSPu Lehui case BPF_CMPXCHG: 540dd642ccbSPu Lehui r0 = bpf_to_rv_reg(BPF_REG_0, ctx); 541dd642ccbSPu Lehui emit(is64 ? rv_addi(RV_REG_T2, r0, 0) : 542dd642ccbSPu Lehui rv_addiw(RV_REG_T2, r0, 0), ctx); 543dd642ccbSPu Lehui emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) : 544dd642ccbSPu Lehui rv_lr_w(r0, 0, rd, 0, 0), ctx); 545dd642ccbSPu Lehui jmp_offset = ninsns_rvoff(8); 546dd642ccbSPu Lehui emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx); 547dd642ccbSPu Lehui emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 0) : 548dd642ccbSPu Lehui rv_sc_w(RV_REG_T3, rs, rd, 0, 0), ctx); 549dd642ccbSPu Lehui jmp_offset = ninsns_rvoff(-6); 550dd642ccbSPu Lehui emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx); 551dd642ccbSPu Lehui emit(rv_fence(0x3, 0x3), ctx); 552dd642ccbSPu Lehui break; 553dd642ccbSPu Lehui } 554dd642ccbSPu Lehui } 555dd642ccbSPu Lehui 556252c765bSTong Tiangen #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) 557252c765bSTong Tiangen #define BPF_FIXUP_REG_MASK GENMASK(31, 27) 558252c765bSTong Tiangen 5592bf847dbSJisheng Zhang bool ex_handler_bpf(const struct exception_table_entry *ex, 560252c765bSTong Tiangen struct pt_regs *regs) 561252c765bSTong Tiangen { 562252c765bSTong Tiangen off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); 563252c765bSTong Tiangen int regs_offset = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); 564252c765bSTong Tiangen 565252c765bSTong Tiangen *(unsigned long *)((void *)regs + pt_regmap[regs_offset]) = 0; 566252c765bSTong Tiangen regs->epc = (unsigned long)&ex->fixup - offset; 567252c765bSTong Tiangen 568ef127bcaSJisheng Zhang return true; 569252c765bSTong Tiangen } 570252c765bSTong Tiangen 571252c765bSTong Tiangen /* For accesses to BTF pointers, add an entry to the exception table */ 572252c765bSTong Tiangen static int add_exception_handler(const struct bpf_insn *insn, 573252c765bSTong Tiangen struct rv_jit_context *ctx, 574252c765bSTong Tiangen int dst_reg, int insn_len) 575252c765bSTong Tiangen { 576252c765bSTong Tiangen struct exception_table_entry *ex; 577252c765bSTong Tiangen unsigned long pc; 578252c765bSTong Tiangen off_t offset; 579252c765bSTong Tiangen 580252c765bSTong Tiangen if (!ctx->insns || !ctx->prog->aux->extable || BPF_MODE(insn->code) != BPF_PROBE_MEM) 581252c765bSTong Tiangen return 0; 582252c765bSTong Tiangen 583252c765bSTong Tiangen if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries)) 584252c765bSTong Tiangen return -EINVAL; 585252c765bSTong Tiangen 586252c765bSTong Tiangen if (WARN_ON_ONCE(insn_len > ctx->ninsns)) 587252c765bSTong Tiangen return -EINVAL; 588252c765bSTong Tiangen 589252c765bSTong Tiangen if (WARN_ON_ONCE(!rvc_enabled() && insn_len == 1)) 590252c765bSTong Tiangen return -EINVAL; 591252c765bSTong Tiangen 592252c765bSTong Tiangen ex = &ctx->prog->aux->extable[ctx->nexentries]; 593252c765bSTong Tiangen pc = (unsigned long)&ctx->insns[ctx->ninsns - insn_len]; 594252c765bSTong Tiangen 595252c765bSTong Tiangen offset = pc - (long)&ex->insn; 596252c765bSTong Tiangen if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 597252c765bSTong Tiangen return -ERANGE; 598fc839c6dSJisheng Zhang ex->insn = offset; 599252c765bSTong Tiangen 600252c765bSTong Tiangen /* 601252c765bSTong Tiangen * Since the extable follows the program, the fixup offset is always 602252c765bSTong Tiangen * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value 603252c765bSTong Tiangen * to keep things simple, and put the destination register in the upper 604252c765bSTong Tiangen * bits. We don't need to worry about buildtime or runtime sort 605252c765bSTong Tiangen * modifying the upper bits because the table is already sorted, and 606252c765bSTong Tiangen * isn't part of the main exception table. 607252c765bSTong Tiangen */ 608252c765bSTong Tiangen offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16)); 609252c765bSTong Tiangen if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset)) 610252c765bSTong Tiangen return -ERANGE; 611252c765bSTong Tiangen 612252c765bSTong Tiangen ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | 613252c765bSTong Tiangen FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); 6142bf847dbSJisheng Zhang ex->type = EX_TYPE_BPF; 615252c765bSTong Tiangen 616252c765bSTong Tiangen ctx->nexentries++; 617252c765bSTong Tiangen return 0; 618252c765bSTong Tiangen } 619252c765bSTong Tiangen 620*596f2e6fSPu Lehui static int gen_call_or_nops(void *target, void *ip, u32 *insns) 621*596f2e6fSPu Lehui { 622*596f2e6fSPu Lehui s64 rvoff; 623*596f2e6fSPu Lehui int i, ret; 624*596f2e6fSPu Lehui struct rv_jit_context ctx; 625*596f2e6fSPu Lehui 626*596f2e6fSPu Lehui ctx.ninsns = 0; 627*596f2e6fSPu Lehui ctx.insns = (u16 *)insns; 628*596f2e6fSPu Lehui 629*596f2e6fSPu Lehui if (!target) { 630*596f2e6fSPu Lehui for (i = 0; i < 4; i++) 631*596f2e6fSPu Lehui emit(rv_nop(), &ctx); 632*596f2e6fSPu Lehui return 0; 633*596f2e6fSPu Lehui } 634*596f2e6fSPu Lehui 635*596f2e6fSPu Lehui rvoff = (s64)(target - (ip + 4)); 636*596f2e6fSPu Lehui emit(rv_sd(RV_REG_SP, -8, RV_REG_RA), &ctx); 637*596f2e6fSPu Lehui ret = emit_jump_and_link(RV_REG_RA, rvoff, false, &ctx); 638*596f2e6fSPu Lehui if (ret) 639*596f2e6fSPu Lehui return ret; 640*596f2e6fSPu Lehui emit(rv_ld(RV_REG_RA, -8, RV_REG_SP), &ctx); 641*596f2e6fSPu Lehui 642*596f2e6fSPu Lehui return 0; 643*596f2e6fSPu Lehui } 644*596f2e6fSPu Lehui 645*596f2e6fSPu Lehui static int gen_jump_or_nops(void *target, void *ip, u32 *insns) 646*596f2e6fSPu Lehui { 647*596f2e6fSPu Lehui s64 rvoff; 648*596f2e6fSPu Lehui struct rv_jit_context ctx; 649*596f2e6fSPu Lehui 650*596f2e6fSPu Lehui ctx.ninsns = 0; 651*596f2e6fSPu Lehui ctx.insns = (u16 *)insns; 652*596f2e6fSPu Lehui 653*596f2e6fSPu Lehui if (!target) { 654*596f2e6fSPu Lehui emit(rv_nop(), &ctx); 655*596f2e6fSPu Lehui emit(rv_nop(), &ctx); 656*596f2e6fSPu Lehui return 0; 657*596f2e6fSPu Lehui } 658*596f2e6fSPu Lehui 659*596f2e6fSPu Lehui rvoff = (s64)(target - ip); 660*596f2e6fSPu Lehui return emit_jump_and_link(RV_REG_ZERO, rvoff, false, &ctx); 661*596f2e6fSPu Lehui } 662*596f2e6fSPu Lehui 663*596f2e6fSPu Lehui int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, 664*596f2e6fSPu Lehui void *old_addr, void *new_addr) 665*596f2e6fSPu Lehui { 666*596f2e6fSPu Lehui u32 old_insns[4], new_insns[4]; 667*596f2e6fSPu Lehui bool is_call = poke_type == BPF_MOD_CALL; 668*596f2e6fSPu Lehui int (*gen_insns)(void *target, void *ip, u32 *insns); 669*596f2e6fSPu Lehui int ninsns = is_call ? 4 : 2; 670*596f2e6fSPu Lehui int ret; 671*596f2e6fSPu Lehui 672*596f2e6fSPu Lehui if (!is_bpf_text_address((unsigned long)ip)) 673*596f2e6fSPu Lehui return -ENOTSUPP; 674*596f2e6fSPu Lehui 675*596f2e6fSPu Lehui gen_insns = is_call ? gen_call_or_nops : gen_jump_or_nops; 676*596f2e6fSPu Lehui 677*596f2e6fSPu Lehui ret = gen_insns(old_addr, ip, old_insns); 678*596f2e6fSPu Lehui if (ret) 679*596f2e6fSPu Lehui return ret; 680*596f2e6fSPu Lehui 681*596f2e6fSPu Lehui if (memcmp(ip, old_insns, ninsns * 4)) 682*596f2e6fSPu Lehui return -EFAULT; 683*596f2e6fSPu Lehui 684*596f2e6fSPu Lehui ret = gen_insns(new_addr, ip, new_insns); 685*596f2e6fSPu Lehui if (ret) 686*596f2e6fSPu Lehui return ret; 687*596f2e6fSPu Lehui 688*596f2e6fSPu Lehui cpus_read_lock(); 689*596f2e6fSPu Lehui mutex_lock(&text_mutex); 690*596f2e6fSPu Lehui if (memcmp(ip, new_insns, ninsns * 4)) 691*596f2e6fSPu Lehui ret = patch_text(ip, new_insns, ninsns); 692*596f2e6fSPu Lehui mutex_unlock(&text_mutex); 693*596f2e6fSPu Lehui cpus_read_unlock(); 694*596f2e6fSPu Lehui 695*596f2e6fSPu Lehui return ret; 696*596f2e6fSPu Lehui } 697*596f2e6fSPu Lehui 698ca6cb544SLuke Nelson int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 699ca6cb544SLuke Nelson bool extra_pass) 700ca6cb544SLuke Nelson { 701ca6cb544SLuke Nelson bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || 702ca6cb544SLuke Nelson BPF_CLASS(insn->code) == BPF_JMP; 703489553ddSLuke Nelson int s, e, rvoff, ret, i = insn - ctx->prog->insnsi; 704ca6cb544SLuke Nelson struct bpf_prog_aux *aux = ctx->prog->aux; 705ca6cb544SLuke Nelson u8 rd = -1, rs = -1, code = insn->code; 706ca6cb544SLuke Nelson s16 off = insn->off; 707ca6cb544SLuke Nelson s32 imm = insn->imm; 708ca6cb544SLuke Nelson 709ca6cb544SLuke Nelson init_regs(&rd, &rs, insn, ctx); 710ca6cb544SLuke Nelson 711ca6cb544SLuke Nelson switch (code) { 712ca6cb544SLuke Nelson /* dst = src */ 713ca6cb544SLuke Nelson case BPF_ALU | BPF_MOV | BPF_X: 714ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MOV | BPF_X: 715ca6cb544SLuke Nelson if (imm == 1) { 716ca6cb544SLuke Nelson /* Special mov32 for zext */ 717ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 718ca6cb544SLuke Nelson break; 719ca6cb544SLuke Nelson } 72018a4d8c9SLuke Nelson emit_mv(rd, rs, ctx); 721ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 722ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 723ca6cb544SLuke Nelson break; 724ca6cb544SLuke Nelson 725ca6cb544SLuke Nelson /* dst = dst OP src */ 726ca6cb544SLuke Nelson case BPF_ALU | BPF_ADD | BPF_X: 727ca6cb544SLuke Nelson case BPF_ALU64 | BPF_ADD | BPF_X: 72818a4d8c9SLuke Nelson emit_add(rd, rd, rs, ctx); 729ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 730ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 731ca6cb544SLuke Nelson break; 732ca6cb544SLuke Nelson case BPF_ALU | BPF_SUB | BPF_X: 733ca6cb544SLuke Nelson case BPF_ALU64 | BPF_SUB | BPF_X: 73418a4d8c9SLuke Nelson if (is64) 73518a4d8c9SLuke Nelson emit_sub(rd, rd, rs, ctx); 73618a4d8c9SLuke Nelson else 73718a4d8c9SLuke Nelson emit_subw(rd, rd, rs, ctx); 73818a4d8c9SLuke Nelson 739ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 740ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 741ca6cb544SLuke Nelson break; 742ca6cb544SLuke Nelson case BPF_ALU | BPF_AND | BPF_X: 743ca6cb544SLuke Nelson case BPF_ALU64 | BPF_AND | BPF_X: 74418a4d8c9SLuke Nelson emit_and(rd, rd, rs, ctx); 745ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 746ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 747ca6cb544SLuke Nelson break; 748ca6cb544SLuke Nelson case BPF_ALU | BPF_OR | BPF_X: 749ca6cb544SLuke Nelson case BPF_ALU64 | BPF_OR | BPF_X: 75018a4d8c9SLuke Nelson emit_or(rd, rd, rs, ctx); 751ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 752ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 753ca6cb544SLuke Nelson break; 754ca6cb544SLuke Nelson case BPF_ALU | BPF_XOR | BPF_X: 755ca6cb544SLuke Nelson case BPF_ALU64 | BPF_XOR | BPF_X: 75618a4d8c9SLuke Nelson emit_xor(rd, rd, rs, ctx); 757ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 758ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 759ca6cb544SLuke Nelson break; 760ca6cb544SLuke Nelson case BPF_ALU | BPF_MUL | BPF_X: 761ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MUL | BPF_X: 762ca6cb544SLuke Nelson emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx); 763ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 764ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 765ca6cb544SLuke Nelson break; 766ca6cb544SLuke Nelson case BPF_ALU | BPF_DIV | BPF_X: 767ca6cb544SLuke Nelson case BPF_ALU64 | BPF_DIV | BPF_X: 768ca6cb544SLuke Nelson emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx); 769ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 770ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 771ca6cb544SLuke Nelson break; 772ca6cb544SLuke Nelson case BPF_ALU | BPF_MOD | BPF_X: 773ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MOD | BPF_X: 774ca6cb544SLuke Nelson emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx); 775ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 776ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 777ca6cb544SLuke Nelson break; 778ca6cb544SLuke Nelson case BPF_ALU | BPF_LSH | BPF_X: 779ca6cb544SLuke Nelson case BPF_ALU64 | BPF_LSH | BPF_X: 780ca6cb544SLuke Nelson emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx); 7810224b2acSLuke Nelson if (!is64 && !aux->verifier_zext) 782ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 783ca6cb544SLuke Nelson break; 784ca6cb544SLuke Nelson case BPF_ALU | BPF_RSH | BPF_X: 785ca6cb544SLuke Nelson case BPF_ALU64 | BPF_RSH | BPF_X: 786ca6cb544SLuke Nelson emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx); 787ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 788ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 789ca6cb544SLuke Nelson break; 790ca6cb544SLuke Nelson case BPF_ALU | BPF_ARSH | BPF_X: 791ca6cb544SLuke Nelson case BPF_ALU64 | BPF_ARSH | BPF_X: 792ca6cb544SLuke Nelson emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx); 793ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 794ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 795ca6cb544SLuke Nelson break; 796ca6cb544SLuke Nelson 797ca6cb544SLuke Nelson /* dst = -dst */ 798ca6cb544SLuke Nelson case BPF_ALU | BPF_NEG: 799ca6cb544SLuke Nelson case BPF_ALU64 | BPF_NEG: 80018a4d8c9SLuke Nelson emit_sub(rd, RV_REG_ZERO, rd, ctx); 801ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 802ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 803ca6cb544SLuke Nelson break; 804ca6cb544SLuke Nelson 805ca6cb544SLuke Nelson /* dst = BSWAP##imm(dst) */ 806ca6cb544SLuke Nelson case BPF_ALU | BPF_END | BPF_FROM_LE: 80721a099abSLuke Nelson switch (imm) { 80821a099abSLuke Nelson case 16: 80918a4d8c9SLuke Nelson emit_slli(rd, rd, 48, ctx); 81018a4d8c9SLuke Nelson emit_srli(rd, rd, 48, ctx); 81121a099abSLuke Nelson break; 81221a099abSLuke Nelson case 32: 81321a099abSLuke Nelson if (!aux->verifier_zext) 81421a099abSLuke Nelson emit_zext_32(rd, ctx); 81521a099abSLuke Nelson break; 81621a099abSLuke Nelson case 64: 81721a099abSLuke Nelson /* Do nothing */ 818ca6cb544SLuke Nelson break; 819ca6cb544SLuke Nelson } 82021a099abSLuke Nelson break; 82121a099abSLuke Nelson 822ca6cb544SLuke Nelson case BPF_ALU | BPF_END | BPF_FROM_BE: 82318a4d8c9SLuke Nelson emit_li(RV_REG_T2, 0, ctx); 824ca6cb544SLuke Nelson 82518a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 82618a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 82718a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 82818a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 829ca6cb544SLuke Nelson if (imm == 16) 830ca6cb544SLuke Nelson goto out_be; 831ca6cb544SLuke Nelson 83218a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 83318a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 83418a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 83518a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 836ca6cb544SLuke Nelson 83718a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 83818a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 83918a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 84018a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 841ca6cb544SLuke Nelson if (imm == 32) 842ca6cb544SLuke Nelson goto out_be; 843ca6cb544SLuke Nelson 84418a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 84518a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 84618a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 84718a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 848ca6cb544SLuke Nelson 84918a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 85018a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 85118a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 85218a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 853ca6cb544SLuke Nelson 85418a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 85518a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 85618a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 85718a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 858ca6cb544SLuke Nelson 85918a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 86018a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 86118a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 86218a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 863ca6cb544SLuke Nelson out_be: 86418a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 86518a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 866ca6cb544SLuke Nelson 86718a4d8c9SLuke Nelson emit_mv(rd, RV_REG_T2, ctx); 868ca6cb544SLuke Nelson break; 869ca6cb544SLuke Nelson 870ca6cb544SLuke Nelson /* dst = imm */ 871ca6cb544SLuke Nelson case BPF_ALU | BPF_MOV | BPF_K: 872ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MOV | BPF_K: 873ca6cb544SLuke Nelson emit_imm(rd, imm, ctx); 874ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 875ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 876ca6cb544SLuke Nelson break; 877ca6cb544SLuke Nelson 878ca6cb544SLuke Nelson /* dst = dst OP imm */ 879ca6cb544SLuke Nelson case BPF_ALU | BPF_ADD | BPF_K: 880ca6cb544SLuke Nelson case BPF_ALU64 | BPF_ADD | BPF_K: 881ca6cb544SLuke Nelson if (is_12b_int(imm)) { 88218a4d8c9SLuke Nelson emit_addi(rd, rd, imm, ctx); 883ca6cb544SLuke Nelson } else { 884ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 88518a4d8c9SLuke Nelson emit_add(rd, rd, RV_REG_T1, ctx); 886ca6cb544SLuke Nelson } 887ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 888ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 889ca6cb544SLuke Nelson break; 890ca6cb544SLuke Nelson case BPF_ALU | BPF_SUB | BPF_K: 891ca6cb544SLuke Nelson case BPF_ALU64 | BPF_SUB | BPF_K: 892ca6cb544SLuke Nelson if (is_12b_int(-imm)) { 89318a4d8c9SLuke Nelson emit_addi(rd, rd, -imm, ctx); 894ca6cb544SLuke Nelson } else { 895ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 89618a4d8c9SLuke Nelson emit_sub(rd, rd, RV_REG_T1, ctx); 897ca6cb544SLuke Nelson } 898ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 899ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 900ca6cb544SLuke Nelson break; 901ca6cb544SLuke Nelson case BPF_ALU | BPF_AND | BPF_K: 902ca6cb544SLuke Nelson case BPF_ALU64 | BPF_AND | BPF_K: 903ca6cb544SLuke Nelson if (is_12b_int(imm)) { 90418a4d8c9SLuke Nelson emit_andi(rd, rd, imm, ctx); 905ca6cb544SLuke Nelson } else { 906ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 90718a4d8c9SLuke Nelson emit_and(rd, rd, RV_REG_T1, ctx); 908ca6cb544SLuke Nelson } 909ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 910ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 911ca6cb544SLuke Nelson break; 912ca6cb544SLuke Nelson case BPF_ALU | BPF_OR | BPF_K: 913ca6cb544SLuke Nelson case BPF_ALU64 | BPF_OR | BPF_K: 914ca6cb544SLuke Nelson if (is_12b_int(imm)) { 915ca6cb544SLuke Nelson emit(rv_ori(rd, rd, imm), ctx); 916ca6cb544SLuke Nelson } else { 917ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 91818a4d8c9SLuke Nelson emit_or(rd, rd, RV_REG_T1, ctx); 919ca6cb544SLuke Nelson } 920ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 921ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 922ca6cb544SLuke Nelson break; 923ca6cb544SLuke Nelson case BPF_ALU | BPF_XOR | BPF_K: 924ca6cb544SLuke Nelson case BPF_ALU64 | BPF_XOR | BPF_K: 925ca6cb544SLuke Nelson if (is_12b_int(imm)) { 926ca6cb544SLuke Nelson emit(rv_xori(rd, rd, imm), ctx); 927ca6cb544SLuke Nelson } else { 928ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 92918a4d8c9SLuke Nelson emit_xor(rd, rd, RV_REG_T1, ctx); 930ca6cb544SLuke Nelson } 931ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 932ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 933ca6cb544SLuke Nelson break; 934ca6cb544SLuke Nelson case BPF_ALU | BPF_MUL | BPF_K: 935ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MUL | BPF_K: 936ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 937ca6cb544SLuke Nelson emit(is64 ? rv_mul(rd, rd, RV_REG_T1) : 938ca6cb544SLuke Nelson rv_mulw(rd, rd, RV_REG_T1), ctx); 939ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 940ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 941ca6cb544SLuke Nelson break; 942ca6cb544SLuke Nelson case BPF_ALU | BPF_DIV | BPF_K: 943ca6cb544SLuke Nelson case BPF_ALU64 | BPF_DIV | BPF_K: 944ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 945ca6cb544SLuke Nelson emit(is64 ? rv_divu(rd, rd, RV_REG_T1) : 946ca6cb544SLuke Nelson rv_divuw(rd, rd, RV_REG_T1), ctx); 947ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 948ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 949ca6cb544SLuke Nelson break; 950ca6cb544SLuke Nelson case BPF_ALU | BPF_MOD | BPF_K: 951ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MOD | BPF_K: 952ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 953ca6cb544SLuke Nelson emit(is64 ? rv_remu(rd, rd, RV_REG_T1) : 954ca6cb544SLuke Nelson rv_remuw(rd, rd, RV_REG_T1), ctx); 955ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 956ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 957ca6cb544SLuke Nelson break; 958ca6cb544SLuke Nelson case BPF_ALU | BPF_LSH | BPF_K: 959ca6cb544SLuke Nelson case BPF_ALU64 | BPF_LSH | BPF_K: 96018a4d8c9SLuke Nelson emit_slli(rd, rd, imm, ctx); 96118a4d8c9SLuke Nelson 9620224b2acSLuke Nelson if (!is64 && !aux->verifier_zext) 963ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 964ca6cb544SLuke Nelson break; 965ca6cb544SLuke Nelson case BPF_ALU | BPF_RSH | BPF_K: 966ca6cb544SLuke Nelson case BPF_ALU64 | BPF_RSH | BPF_K: 96718a4d8c9SLuke Nelson if (is64) 96818a4d8c9SLuke Nelson emit_srli(rd, rd, imm, ctx); 96918a4d8c9SLuke Nelson else 97018a4d8c9SLuke Nelson emit(rv_srliw(rd, rd, imm), ctx); 97118a4d8c9SLuke Nelson 9720224b2acSLuke Nelson if (!is64 && !aux->verifier_zext) 973ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 974ca6cb544SLuke Nelson break; 975ca6cb544SLuke Nelson case BPF_ALU | BPF_ARSH | BPF_K: 976ca6cb544SLuke Nelson case BPF_ALU64 | BPF_ARSH | BPF_K: 97718a4d8c9SLuke Nelson if (is64) 97818a4d8c9SLuke Nelson emit_srai(rd, rd, imm, ctx); 97918a4d8c9SLuke Nelson else 98018a4d8c9SLuke Nelson emit(rv_sraiw(rd, rd, imm), ctx); 98118a4d8c9SLuke Nelson 9820224b2acSLuke Nelson if (!is64 && !aux->verifier_zext) 983ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 984ca6cb544SLuke Nelson break; 985ca6cb544SLuke Nelson 986ca6cb544SLuke Nelson /* JUMP off */ 987ca6cb544SLuke Nelson case BPF_JMP | BPF_JA: 988ca6cb544SLuke Nelson rvoff = rv_offset(i, off, ctx); 9890fd1fd01SPu Lehui ret = emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); 990489553ddSLuke Nelson if (ret) 991489553ddSLuke Nelson return ret; 992ca6cb544SLuke Nelson break; 993ca6cb544SLuke Nelson 994ca6cb544SLuke Nelson /* IF (dst COND src) JUMP off */ 995ca6cb544SLuke Nelson case BPF_JMP | BPF_JEQ | BPF_X: 996ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JEQ | BPF_X: 997ca6cb544SLuke Nelson case BPF_JMP | BPF_JGT | BPF_X: 998ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JGT | BPF_X: 999ca6cb544SLuke Nelson case BPF_JMP | BPF_JLT | BPF_X: 1000ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JLT | BPF_X: 1001ca6cb544SLuke Nelson case BPF_JMP | BPF_JGE | BPF_X: 1002ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JGE | BPF_X: 1003ca6cb544SLuke Nelson case BPF_JMP | BPF_JLE | BPF_X: 1004ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JLE | BPF_X: 1005ca6cb544SLuke Nelson case BPF_JMP | BPF_JNE | BPF_X: 1006ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JNE | BPF_X: 1007ca6cb544SLuke Nelson case BPF_JMP | BPF_JSGT | BPF_X: 1008ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSGT | BPF_X: 1009ca6cb544SLuke Nelson case BPF_JMP | BPF_JSLT | BPF_X: 1010ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSLT | BPF_X: 1011ca6cb544SLuke Nelson case BPF_JMP | BPF_JSGE | BPF_X: 1012ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSGE | BPF_X: 1013ca6cb544SLuke Nelson case BPF_JMP | BPF_JSLE | BPF_X: 1014ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSLE | BPF_X: 1015ca6cb544SLuke Nelson case BPF_JMP | BPF_JSET | BPF_X: 1016ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSET | BPF_X: 1017ca6cb544SLuke Nelson rvoff = rv_offset(i, off, ctx); 1018ca6cb544SLuke Nelson if (!is64) { 1019ca6cb544SLuke Nelson s = ctx->ninsns; 1020ca6cb544SLuke Nelson if (is_signed_bpf_cond(BPF_OP(code))) 1021ca6cb544SLuke Nelson emit_sext_32_rd_rs(&rd, &rs, ctx); 1022ca6cb544SLuke Nelson else 1023ca6cb544SLuke Nelson emit_zext_32_rd_rs(&rd, &rs, ctx); 1024ca6cb544SLuke Nelson e = ctx->ninsns; 1025ca6cb544SLuke Nelson 1026ca6cb544SLuke Nelson /* Adjust for extra insns */ 1027bfabff3cSLuke Nelson rvoff -= ninsns_rvoff(e - s); 1028ca6cb544SLuke Nelson } 1029ca6cb544SLuke Nelson 1030ca6cb544SLuke Nelson if (BPF_OP(code) == BPF_JSET) { 1031ca6cb544SLuke Nelson /* Adjust for and */ 1032ca6cb544SLuke Nelson rvoff -= 4; 103318a4d8c9SLuke Nelson emit_and(RV_REG_T1, rd, rs, ctx); 1034ca6cb544SLuke Nelson emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, 1035ca6cb544SLuke Nelson ctx); 1036ca6cb544SLuke Nelson } else { 1037ca6cb544SLuke Nelson emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 1038ca6cb544SLuke Nelson } 1039ca6cb544SLuke Nelson break; 1040ca6cb544SLuke Nelson 1041ca6cb544SLuke Nelson /* IF (dst COND imm) JUMP off */ 1042ca6cb544SLuke Nelson case BPF_JMP | BPF_JEQ | BPF_K: 1043ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JEQ | BPF_K: 1044ca6cb544SLuke Nelson case BPF_JMP | BPF_JGT | BPF_K: 1045ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JGT | BPF_K: 1046ca6cb544SLuke Nelson case BPF_JMP | BPF_JLT | BPF_K: 1047ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JLT | BPF_K: 1048ca6cb544SLuke Nelson case BPF_JMP | BPF_JGE | BPF_K: 1049ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JGE | BPF_K: 1050ca6cb544SLuke Nelson case BPF_JMP | BPF_JLE | BPF_K: 1051ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JLE | BPF_K: 1052ca6cb544SLuke Nelson case BPF_JMP | BPF_JNE | BPF_K: 1053ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JNE | BPF_K: 1054ca6cb544SLuke Nelson case BPF_JMP | BPF_JSGT | BPF_K: 1055ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSGT | BPF_K: 1056ca6cb544SLuke Nelson case BPF_JMP | BPF_JSLT | BPF_K: 1057ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSLT | BPF_K: 1058ca6cb544SLuke Nelson case BPF_JMP | BPF_JSGE | BPF_K: 1059ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSGE | BPF_K: 1060ca6cb544SLuke Nelson case BPF_JMP | BPF_JSLE | BPF_K: 1061ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSLE | BPF_K: 1062ca6cb544SLuke Nelson rvoff = rv_offset(i, off, ctx); 1063ca6cb544SLuke Nelson s = ctx->ninsns; 1064ca349a6aSLuke Nelson if (imm) { 1065ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 1066ca349a6aSLuke Nelson rs = RV_REG_T1; 1067ca349a6aSLuke Nelson } else { 1068ca349a6aSLuke Nelson /* If imm is 0, simply use zero register. */ 1069ca349a6aSLuke Nelson rs = RV_REG_ZERO; 1070ca349a6aSLuke Nelson } 1071ca6cb544SLuke Nelson if (!is64) { 1072ca6cb544SLuke Nelson if (is_signed_bpf_cond(BPF_OP(code))) 1073ca6cb544SLuke Nelson emit_sext_32_rd(&rd, ctx); 1074ca6cb544SLuke Nelson else 1075ca6cb544SLuke Nelson emit_zext_32_rd_t1(&rd, ctx); 1076ca6cb544SLuke Nelson } 1077ca6cb544SLuke Nelson e = ctx->ninsns; 1078ca6cb544SLuke Nelson 1079ca6cb544SLuke Nelson /* Adjust for extra insns */ 1080bfabff3cSLuke Nelson rvoff -= ninsns_rvoff(e - s); 1081ca349a6aSLuke Nelson emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 1082073ca6a0SLuke Nelson break; 1083073ca6a0SLuke Nelson 1084073ca6a0SLuke Nelson case BPF_JMP | BPF_JSET | BPF_K: 1085073ca6a0SLuke Nelson case BPF_JMP32 | BPF_JSET | BPF_K: 1086073ca6a0SLuke Nelson rvoff = rv_offset(i, off, ctx); 1087073ca6a0SLuke Nelson s = ctx->ninsns; 1088073ca6a0SLuke Nelson if (is_12b_int(imm)) { 108918a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, imm, ctx); 1090073ca6a0SLuke Nelson } else { 1091073ca6a0SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 109218a4d8c9SLuke Nelson emit_and(RV_REG_T1, rd, RV_REG_T1, ctx); 1093ca6cb544SLuke Nelson } 1094073ca6a0SLuke Nelson /* For jset32, we should clear the upper 32 bits of t1, but 1095073ca6a0SLuke Nelson * sign-extension is sufficient here and saves one instruction, 1096073ca6a0SLuke Nelson * as t1 is used only in comparison against zero. 1097073ca6a0SLuke Nelson */ 1098073ca6a0SLuke Nelson if (!is64 && imm < 0) 109918a4d8c9SLuke Nelson emit_addiw(RV_REG_T1, RV_REG_T1, 0, ctx); 1100073ca6a0SLuke Nelson e = ctx->ninsns; 1101bfabff3cSLuke Nelson rvoff -= ninsns_rvoff(e - s); 1102073ca6a0SLuke Nelson emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); 1103ca6cb544SLuke Nelson break; 1104ca6cb544SLuke Nelson 1105ca6cb544SLuke Nelson /* function call */ 1106ca6cb544SLuke Nelson case BPF_JMP | BPF_CALL: 1107ca6cb544SLuke Nelson { 11080fd1fd01SPu Lehui bool fixed_addr; 1109ca6cb544SLuke Nelson u64 addr; 1110ca6cb544SLuke Nelson 1111ca6cb544SLuke Nelson mark_call(ctx); 11120fd1fd01SPu Lehui ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, 11130fd1fd01SPu Lehui &addr, &fixed_addr); 1114ca6cb544SLuke Nelson if (ret < 0) 1115ca6cb544SLuke Nelson return ret; 11160fd1fd01SPu Lehui 11170fd1fd01SPu Lehui ret = emit_call(addr, fixed_addr, ctx); 1118ca6cb544SLuke Nelson if (ret) 1119ca6cb544SLuke Nelson return ret; 11200fd1fd01SPu Lehui 11210fd1fd01SPu Lehui emit_mv(bpf_to_rv_reg(BPF_REG_0, ctx), RV_REG_A0, ctx); 1122ca6cb544SLuke Nelson break; 1123ca6cb544SLuke Nelson } 1124ca6cb544SLuke Nelson /* tail call */ 1125ca6cb544SLuke Nelson case BPF_JMP | BPF_TAIL_CALL: 1126ca6cb544SLuke Nelson if (emit_bpf_tail_call(i, ctx)) 1127ca6cb544SLuke Nelson return -1; 1128ca6cb544SLuke Nelson break; 1129ca6cb544SLuke Nelson 1130ca6cb544SLuke Nelson /* function return */ 1131ca6cb544SLuke Nelson case BPF_JMP | BPF_EXIT: 1132ca6cb544SLuke Nelson if (i == ctx->prog->len - 1) 1133ca6cb544SLuke Nelson break; 1134ca6cb544SLuke Nelson 1135ca6cb544SLuke Nelson rvoff = epilogue_offset(ctx); 11360fd1fd01SPu Lehui ret = emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); 1137489553ddSLuke Nelson if (ret) 1138489553ddSLuke Nelson return ret; 1139ca6cb544SLuke Nelson break; 1140ca6cb544SLuke Nelson 1141ca6cb544SLuke Nelson /* dst = imm64 */ 1142ca6cb544SLuke Nelson case BPF_LD | BPF_IMM | BPF_DW: 1143ca6cb544SLuke Nelson { 1144ca6cb544SLuke Nelson struct bpf_insn insn1 = insn[1]; 1145ca6cb544SLuke Nelson u64 imm64; 1146ca6cb544SLuke Nelson 1147ca6cb544SLuke Nelson imm64 = (u64)insn1.imm << 32 | (u32)imm; 1148b54b6003SPu Lehui if (bpf_pseudo_func(insn)) { 1149b54b6003SPu Lehui /* fixed-length insns for extra jit pass */ 1150b54b6003SPu Lehui ret = emit_addr(rd, imm64, extra_pass, ctx); 1151b54b6003SPu Lehui if (ret) 1152b54b6003SPu Lehui return ret; 1153b54b6003SPu Lehui } else { 1154ca6cb544SLuke Nelson emit_imm(rd, imm64, ctx); 1155b54b6003SPu Lehui } 1156b54b6003SPu Lehui 1157ca6cb544SLuke Nelson return 1; 1158ca6cb544SLuke Nelson } 1159ca6cb544SLuke Nelson 1160ca6cb544SLuke Nelson /* LDX: dst = *(size *)(src + off) */ 1161ca6cb544SLuke Nelson case BPF_LDX | BPF_MEM | BPF_B: 1162ca6cb544SLuke Nelson case BPF_LDX | BPF_MEM | BPF_H: 1163ca6cb544SLuke Nelson case BPF_LDX | BPF_MEM | BPF_W: 1164252c765bSTong Tiangen case BPF_LDX | BPF_MEM | BPF_DW: 1165252c765bSTong Tiangen case BPF_LDX | BPF_PROBE_MEM | BPF_B: 1166252c765bSTong Tiangen case BPF_LDX | BPF_PROBE_MEM | BPF_H: 1167252c765bSTong Tiangen case BPF_LDX | BPF_PROBE_MEM | BPF_W: 1168252c765bSTong Tiangen case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 1169252c765bSTong Tiangen { 1170252c765bSTong Tiangen int insn_len, insns_start; 1171252c765bSTong Tiangen 1172252c765bSTong Tiangen switch (BPF_SIZE(code)) { 1173252c765bSTong Tiangen case BPF_B: 1174ca6cb544SLuke Nelson if (is_12b_int(off)) { 1175252c765bSTong Tiangen insns_start = ctx->ninsns; 1176252c765bSTong Tiangen emit(rv_lbu(rd, off, rs), ctx); 1177252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1178ca6cb544SLuke Nelson break; 1179ca6cb544SLuke Nelson } 1180ca6cb544SLuke Nelson 1181ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 118218a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1183252c765bSTong Tiangen insns_start = ctx->ninsns; 1184252c765bSTong Tiangen emit(rv_lbu(rd, 0, RV_REG_T1), ctx); 1185252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1186ca6cb544SLuke Nelson if (insn_is_zext(&insn[1])) 1187ca6cb544SLuke Nelson return 1; 1188ca6cb544SLuke Nelson break; 1189252c765bSTong Tiangen case BPF_H: 1190ca6cb544SLuke Nelson if (is_12b_int(off)) { 1191252c765bSTong Tiangen insns_start = ctx->ninsns; 1192252c765bSTong Tiangen emit(rv_lhu(rd, off, rs), ctx); 1193252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1194ca6cb544SLuke Nelson break; 1195ca6cb544SLuke Nelson } 1196ca6cb544SLuke Nelson 1197ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 119818a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1199252c765bSTong Tiangen insns_start = ctx->ninsns; 1200252c765bSTong Tiangen emit(rv_lhu(rd, 0, RV_REG_T1), ctx); 1201252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1202252c765bSTong Tiangen if (insn_is_zext(&insn[1])) 1203252c765bSTong Tiangen return 1; 1204ca6cb544SLuke Nelson break; 1205252c765bSTong Tiangen case BPF_W: 1206252c765bSTong Tiangen if (is_12b_int(off)) { 1207252c765bSTong Tiangen insns_start = ctx->ninsns; 1208252c765bSTong Tiangen emit(rv_lwu(rd, off, rs), ctx); 1209252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1210252c765bSTong Tiangen break; 1211252c765bSTong Tiangen } 1212ca6cb544SLuke Nelson 1213252c765bSTong Tiangen emit_imm(RV_REG_T1, off, ctx); 1214252c765bSTong Tiangen emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1215252c765bSTong Tiangen insns_start = ctx->ninsns; 1216252c765bSTong Tiangen emit(rv_lwu(rd, 0, RV_REG_T1), ctx); 1217252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1218252c765bSTong Tiangen if (insn_is_zext(&insn[1])) 1219252c765bSTong Tiangen return 1; 1220252c765bSTong Tiangen break; 1221252c765bSTong Tiangen case BPF_DW: 1222252c765bSTong Tiangen if (is_12b_int(off)) { 1223252c765bSTong Tiangen insns_start = ctx->ninsns; 1224252c765bSTong Tiangen emit_ld(rd, off, rs, ctx); 1225252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1226252c765bSTong Tiangen break; 1227252c765bSTong Tiangen } 1228252c765bSTong Tiangen 1229252c765bSTong Tiangen emit_imm(RV_REG_T1, off, ctx); 1230252c765bSTong Tiangen emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1231252c765bSTong Tiangen insns_start = ctx->ninsns; 1232252c765bSTong Tiangen emit_ld(rd, 0, RV_REG_T1, ctx); 1233252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1234252c765bSTong Tiangen break; 1235252c765bSTong Tiangen } 1236252c765bSTong Tiangen 1237252c765bSTong Tiangen ret = add_exception_handler(insn, ctx, rd, insn_len); 1238252c765bSTong Tiangen if (ret) 1239252c765bSTong Tiangen return ret; 1240252c765bSTong Tiangen break; 1241252c765bSTong Tiangen } 1242f5e81d11SDaniel Borkmann /* speculation barrier */ 1243f5e81d11SDaniel Borkmann case BPF_ST | BPF_NOSPEC: 1244f5e81d11SDaniel Borkmann break; 1245f5e81d11SDaniel Borkmann 1246ca6cb544SLuke Nelson /* ST: *(size *)(dst + off) = imm */ 1247ca6cb544SLuke Nelson case BPF_ST | BPF_MEM | BPF_B: 1248ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 1249ca6cb544SLuke Nelson if (is_12b_int(off)) { 1250ca6cb544SLuke Nelson emit(rv_sb(rd, off, RV_REG_T1), ctx); 1251ca6cb544SLuke Nelson break; 1252ca6cb544SLuke Nelson } 1253ca6cb544SLuke Nelson 1254ca6cb544SLuke Nelson emit_imm(RV_REG_T2, off, ctx); 125518a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1256ca6cb544SLuke Nelson emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); 1257ca6cb544SLuke Nelson break; 1258ca6cb544SLuke Nelson 1259ca6cb544SLuke Nelson case BPF_ST | BPF_MEM | BPF_H: 1260ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 1261ca6cb544SLuke Nelson if (is_12b_int(off)) { 1262ca6cb544SLuke Nelson emit(rv_sh(rd, off, RV_REG_T1), ctx); 1263ca6cb544SLuke Nelson break; 1264ca6cb544SLuke Nelson } 1265ca6cb544SLuke Nelson 1266ca6cb544SLuke Nelson emit_imm(RV_REG_T2, off, ctx); 126718a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1268ca6cb544SLuke Nelson emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); 1269ca6cb544SLuke Nelson break; 1270ca6cb544SLuke Nelson case BPF_ST | BPF_MEM | BPF_W: 1271ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 1272ca6cb544SLuke Nelson if (is_12b_int(off)) { 127318a4d8c9SLuke Nelson emit_sw(rd, off, RV_REG_T1, ctx); 1274ca6cb544SLuke Nelson break; 1275ca6cb544SLuke Nelson } 1276ca6cb544SLuke Nelson 1277ca6cb544SLuke Nelson emit_imm(RV_REG_T2, off, ctx); 127818a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 127918a4d8c9SLuke Nelson emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx); 1280ca6cb544SLuke Nelson break; 1281ca6cb544SLuke Nelson case BPF_ST | BPF_MEM | BPF_DW: 1282ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 1283ca6cb544SLuke Nelson if (is_12b_int(off)) { 128418a4d8c9SLuke Nelson emit_sd(rd, off, RV_REG_T1, ctx); 1285ca6cb544SLuke Nelson break; 1286ca6cb544SLuke Nelson } 1287ca6cb544SLuke Nelson 1288ca6cb544SLuke Nelson emit_imm(RV_REG_T2, off, ctx); 128918a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 129018a4d8c9SLuke Nelson emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx); 1291ca6cb544SLuke Nelson break; 1292ca6cb544SLuke Nelson 1293ca6cb544SLuke Nelson /* STX: *(size *)(dst + off) = src */ 1294ca6cb544SLuke Nelson case BPF_STX | BPF_MEM | BPF_B: 1295ca6cb544SLuke Nelson if (is_12b_int(off)) { 1296ca6cb544SLuke Nelson emit(rv_sb(rd, off, rs), ctx); 1297ca6cb544SLuke Nelson break; 1298ca6cb544SLuke Nelson } 1299ca6cb544SLuke Nelson 1300ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 130118a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1302ca6cb544SLuke Nelson emit(rv_sb(RV_REG_T1, 0, rs), ctx); 1303ca6cb544SLuke Nelson break; 1304ca6cb544SLuke Nelson case BPF_STX | BPF_MEM | BPF_H: 1305ca6cb544SLuke Nelson if (is_12b_int(off)) { 1306ca6cb544SLuke Nelson emit(rv_sh(rd, off, rs), ctx); 1307ca6cb544SLuke Nelson break; 1308ca6cb544SLuke Nelson } 1309ca6cb544SLuke Nelson 1310ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 131118a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1312ca6cb544SLuke Nelson emit(rv_sh(RV_REG_T1, 0, rs), ctx); 1313ca6cb544SLuke Nelson break; 1314ca6cb544SLuke Nelson case BPF_STX | BPF_MEM | BPF_W: 1315ca6cb544SLuke Nelson if (is_12b_int(off)) { 131618a4d8c9SLuke Nelson emit_sw(rd, off, rs, ctx); 1317ca6cb544SLuke Nelson break; 1318ca6cb544SLuke Nelson } 1319ca6cb544SLuke Nelson 1320ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 132118a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 132218a4d8c9SLuke Nelson emit_sw(RV_REG_T1, 0, rs, ctx); 1323ca6cb544SLuke Nelson break; 1324ca6cb544SLuke Nelson case BPF_STX | BPF_MEM | BPF_DW: 1325ca6cb544SLuke Nelson if (is_12b_int(off)) { 132618a4d8c9SLuke Nelson emit_sd(rd, off, rs, ctx); 1327ca6cb544SLuke Nelson break; 1328ca6cb544SLuke Nelson } 1329ca6cb544SLuke Nelson 1330ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 133118a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 133218a4d8c9SLuke Nelson emit_sd(RV_REG_T1, 0, rs, ctx); 1333ca6cb544SLuke Nelson break; 133491c960b0SBrendan Jackman case BPF_STX | BPF_ATOMIC | BPF_W: 133591c960b0SBrendan Jackman case BPF_STX | BPF_ATOMIC | BPF_DW: 1336dd642ccbSPu Lehui emit_atomic(rd, rs, off, imm, 1337dd642ccbSPu Lehui BPF_SIZE(code) == BPF_DW, ctx); 1338ca6cb544SLuke Nelson break; 1339ca6cb544SLuke Nelson default: 1340ca6cb544SLuke Nelson pr_err("bpf-jit: unknown opcode %02x\n", code); 1341ca6cb544SLuke Nelson return -EINVAL; 1342ca6cb544SLuke Nelson } 1343ca6cb544SLuke Nelson 1344ca6cb544SLuke Nelson return 0; 1345ca6cb544SLuke Nelson } 1346ca6cb544SLuke Nelson 1347ca6cb544SLuke Nelson void bpf_jit_build_prologue(struct rv_jit_context *ctx) 1348ca6cb544SLuke Nelson { 1349*596f2e6fSPu Lehui int i, stack_adjust = 0, store_offset, bpf_stack_adjust; 1350ca6cb544SLuke Nelson 1351ca6cb544SLuke Nelson bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); 1352ca6cb544SLuke Nelson if (bpf_stack_adjust) 1353ca6cb544SLuke Nelson mark_fp(ctx); 1354ca6cb544SLuke Nelson 1355ca6cb544SLuke Nelson if (seen_reg(RV_REG_RA, ctx)) 1356ca6cb544SLuke Nelson stack_adjust += 8; 1357ca6cb544SLuke Nelson stack_adjust += 8; /* RV_REG_FP */ 1358ca6cb544SLuke Nelson if (seen_reg(RV_REG_S1, ctx)) 1359ca6cb544SLuke Nelson stack_adjust += 8; 1360ca6cb544SLuke Nelson if (seen_reg(RV_REG_S2, ctx)) 1361ca6cb544SLuke Nelson stack_adjust += 8; 1362ca6cb544SLuke Nelson if (seen_reg(RV_REG_S3, ctx)) 1363ca6cb544SLuke Nelson stack_adjust += 8; 1364ca6cb544SLuke Nelson if (seen_reg(RV_REG_S4, ctx)) 1365ca6cb544SLuke Nelson stack_adjust += 8; 1366ca6cb544SLuke Nelson if (seen_reg(RV_REG_S5, ctx)) 1367ca6cb544SLuke Nelson stack_adjust += 8; 1368ca6cb544SLuke Nelson if (seen_reg(RV_REG_S6, ctx)) 1369ca6cb544SLuke Nelson stack_adjust += 8; 1370ca6cb544SLuke Nelson 1371ca6cb544SLuke Nelson stack_adjust = round_up(stack_adjust, 16); 1372ca6cb544SLuke Nelson stack_adjust += bpf_stack_adjust; 1373ca6cb544SLuke Nelson 1374ca6cb544SLuke Nelson store_offset = stack_adjust - 8; 1375ca6cb544SLuke Nelson 1376*596f2e6fSPu Lehui /* reserve 4 nop insns */ 1377*596f2e6fSPu Lehui for (i = 0; i < 4; i++) 1378*596f2e6fSPu Lehui emit(rv_nop(), ctx); 1379*596f2e6fSPu Lehui 1380ca6cb544SLuke Nelson /* First instruction is always setting the tail-call-counter 1381ca6cb544SLuke Nelson * (TCC) register. This instruction is skipped for tail calls. 138218a4d8c9SLuke Nelson * Force using a 4-byte (non-compressed) instruction. 1383ca6cb544SLuke Nelson */ 1384ca6cb544SLuke Nelson emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); 1385ca6cb544SLuke Nelson 138618a4d8c9SLuke Nelson emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); 1387ca6cb544SLuke Nelson 1388ca6cb544SLuke Nelson if (seen_reg(RV_REG_RA, ctx)) { 138918a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_RA, ctx); 1390ca6cb544SLuke Nelson store_offset -= 8; 1391ca6cb544SLuke Nelson } 139218a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_FP, ctx); 1393ca6cb544SLuke Nelson store_offset -= 8; 1394ca6cb544SLuke Nelson if (seen_reg(RV_REG_S1, ctx)) { 139518a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S1, ctx); 1396ca6cb544SLuke Nelson store_offset -= 8; 1397ca6cb544SLuke Nelson } 1398ca6cb544SLuke Nelson if (seen_reg(RV_REG_S2, ctx)) { 139918a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S2, ctx); 1400ca6cb544SLuke Nelson store_offset -= 8; 1401ca6cb544SLuke Nelson } 1402ca6cb544SLuke Nelson if (seen_reg(RV_REG_S3, ctx)) { 140318a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S3, ctx); 1404ca6cb544SLuke Nelson store_offset -= 8; 1405ca6cb544SLuke Nelson } 1406ca6cb544SLuke Nelson if (seen_reg(RV_REG_S4, ctx)) { 140718a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S4, ctx); 1408ca6cb544SLuke Nelson store_offset -= 8; 1409ca6cb544SLuke Nelson } 1410ca6cb544SLuke Nelson if (seen_reg(RV_REG_S5, ctx)) { 141118a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx); 1412ca6cb544SLuke Nelson store_offset -= 8; 1413ca6cb544SLuke Nelson } 1414ca6cb544SLuke Nelson if (seen_reg(RV_REG_S6, ctx)) { 141518a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx); 1416ca6cb544SLuke Nelson store_offset -= 8; 1417ca6cb544SLuke Nelson } 1418ca6cb544SLuke Nelson 141918a4d8c9SLuke Nelson emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx); 1420ca6cb544SLuke Nelson 1421ca6cb544SLuke Nelson if (bpf_stack_adjust) 142218a4d8c9SLuke Nelson emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx); 1423ca6cb544SLuke Nelson 1424ca6cb544SLuke Nelson /* Program contains calls and tail calls, so RV_REG_TCC need 1425ca6cb544SLuke Nelson * to be saved across calls. 1426ca6cb544SLuke Nelson */ 1427ca6cb544SLuke Nelson if (seen_tail_call(ctx) && seen_call(ctx)) 142818a4d8c9SLuke Nelson emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx); 1429ca6cb544SLuke Nelson 1430ca6cb544SLuke Nelson ctx->stack_size = stack_adjust; 1431ca6cb544SLuke Nelson } 1432ca6cb544SLuke Nelson 1433ca6cb544SLuke Nelson void bpf_jit_build_epilogue(struct rv_jit_context *ctx) 1434ca6cb544SLuke Nelson { 1435ca6cb544SLuke Nelson __build_epilogue(false, ctx); 1436ca6cb544SLuke Nelson } 1437