1ca6cb544SLuke Nelson // SPDX-License-Identifier: GPL-2.0 2ca6cb544SLuke Nelson /* BPF JIT compiler for RV64G 3ca6cb544SLuke Nelson * 4ca6cb544SLuke Nelson * Copyright(c) 2019 Björn Töpel <bjorn.topel@gmail.com> 5ca6cb544SLuke Nelson * 6ca6cb544SLuke Nelson */ 7ca6cb544SLuke Nelson 8252c765bSTong Tiangen #include <linux/bitfield.h> 9ca6cb544SLuke Nelson #include <linux/bpf.h> 10ca6cb544SLuke Nelson #include <linux/filter.h> 11ca6cb544SLuke Nelson #include "bpf_jit.h" 12ca6cb544SLuke Nelson 13ca6cb544SLuke Nelson #define RV_REG_TCC RV_REG_A6 14ca6cb544SLuke Nelson #define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */ 15ca6cb544SLuke Nelson 16ca6cb544SLuke Nelson static const int regmap[] = { 17ca6cb544SLuke Nelson [BPF_REG_0] = RV_REG_A5, 18ca6cb544SLuke Nelson [BPF_REG_1] = RV_REG_A0, 19ca6cb544SLuke Nelson [BPF_REG_2] = RV_REG_A1, 20ca6cb544SLuke Nelson [BPF_REG_3] = RV_REG_A2, 21ca6cb544SLuke Nelson [BPF_REG_4] = RV_REG_A3, 22ca6cb544SLuke Nelson [BPF_REG_5] = RV_REG_A4, 23ca6cb544SLuke Nelson [BPF_REG_6] = RV_REG_S1, 24ca6cb544SLuke Nelson [BPF_REG_7] = RV_REG_S2, 25ca6cb544SLuke Nelson [BPF_REG_8] = RV_REG_S3, 26ca6cb544SLuke Nelson [BPF_REG_9] = RV_REG_S4, 27ca6cb544SLuke Nelson [BPF_REG_FP] = RV_REG_S5, 28ca6cb544SLuke Nelson [BPF_REG_AX] = RV_REG_T0, 29ca6cb544SLuke Nelson }; 30ca6cb544SLuke Nelson 31252c765bSTong Tiangen static const int pt_regmap[] = { 32252c765bSTong Tiangen [RV_REG_A0] = offsetof(struct pt_regs, a0), 33252c765bSTong Tiangen [RV_REG_A1] = offsetof(struct pt_regs, a1), 34252c765bSTong Tiangen [RV_REG_A2] = offsetof(struct pt_regs, a2), 35252c765bSTong Tiangen [RV_REG_A3] = offsetof(struct pt_regs, a3), 36252c765bSTong Tiangen [RV_REG_A4] = offsetof(struct pt_regs, a4), 37252c765bSTong Tiangen [RV_REG_A5] = offsetof(struct pt_regs, a5), 38252c765bSTong Tiangen [RV_REG_S1] = offsetof(struct pt_regs, s1), 39252c765bSTong Tiangen [RV_REG_S2] = offsetof(struct pt_regs, s2), 40252c765bSTong Tiangen [RV_REG_S3] = offsetof(struct pt_regs, s3), 41252c765bSTong Tiangen [RV_REG_S4] = offsetof(struct pt_regs, s4), 42252c765bSTong Tiangen [RV_REG_S5] = offsetof(struct pt_regs, s5), 43252c765bSTong Tiangen [RV_REG_T0] = offsetof(struct pt_regs, t0), 44252c765bSTong Tiangen }; 45252c765bSTong Tiangen 46ca6cb544SLuke Nelson enum { 47ca6cb544SLuke Nelson RV_CTX_F_SEEN_TAIL_CALL = 0, 48ca6cb544SLuke Nelson RV_CTX_F_SEEN_CALL = RV_REG_RA, 49ca6cb544SLuke Nelson RV_CTX_F_SEEN_S1 = RV_REG_S1, 50ca6cb544SLuke Nelson RV_CTX_F_SEEN_S2 = RV_REG_S2, 51ca6cb544SLuke Nelson RV_CTX_F_SEEN_S3 = RV_REG_S3, 52ca6cb544SLuke Nelson RV_CTX_F_SEEN_S4 = RV_REG_S4, 53ca6cb544SLuke Nelson RV_CTX_F_SEEN_S5 = RV_REG_S5, 54ca6cb544SLuke Nelson RV_CTX_F_SEEN_S6 = RV_REG_S6, 55ca6cb544SLuke Nelson }; 56ca6cb544SLuke Nelson 57ca6cb544SLuke Nelson static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) 58ca6cb544SLuke Nelson { 59ca6cb544SLuke Nelson u8 reg = regmap[bpf_reg]; 60ca6cb544SLuke Nelson 61ca6cb544SLuke Nelson switch (reg) { 62ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S1: 63ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S2: 64ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S3: 65ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S4: 66ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S5: 67ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S6: 68ca6cb544SLuke Nelson __set_bit(reg, &ctx->flags); 69ca6cb544SLuke Nelson } 70ca6cb544SLuke Nelson return reg; 71ca6cb544SLuke Nelson }; 72ca6cb544SLuke Nelson 73ca6cb544SLuke Nelson static bool seen_reg(int reg, struct rv_jit_context *ctx) 74ca6cb544SLuke Nelson { 75ca6cb544SLuke Nelson switch (reg) { 76ca6cb544SLuke Nelson case RV_CTX_F_SEEN_CALL: 77ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S1: 78ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S2: 79ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S3: 80ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S4: 81ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S5: 82ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S6: 83ca6cb544SLuke Nelson return test_bit(reg, &ctx->flags); 84ca6cb544SLuke Nelson } 85ca6cb544SLuke Nelson return false; 86ca6cb544SLuke Nelson } 87ca6cb544SLuke Nelson 88ca6cb544SLuke Nelson static void mark_fp(struct rv_jit_context *ctx) 89ca6cb544SLuke Nelson { 90ca6cb544SLuke Nelson __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags); 91ca6cb544SLuke Nelson } 92ca6cb544SLuke Nelson 93ca6cb544SLuke Nelson static void mark_call(struct rv_jit_context *ctx) 94ca6cb544SLuke Nelson { 95ca6cb544SLuke Nelson __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 96ca6cb544SLuke Nelson } 97ca6cb544SLuke Nelson 98ca6cb544SLuke Nelson static bool seen_call(struct rv_jit_context *ctx) 99ca6cb544SLuke Nelson { 100ca6cb544SLuke Nelson return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 101ca6cb544SLuke Nelson } 102ca6cb544SLuke Nelson 103ca6cb544SLuke Nelson static void mark_tail_call(struct rv_jit_context *ctx) 104ca6cb544SLuke Nelson { 105ca6cb544SLuke Nelson __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 106ca6cb544SLuke Nelson } 107ca6cb544SLuke Nelson 108ca6cb544SLuke Nelson static bool seen_tail_call(struct rv_jit_context *ctx) 109ca6cb544SLuke Nelson { 110ca6cb544SLuke Nelson return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 111ca6cb544SLuke Nelson } 112ca6cb544SLuke Nelson 113ca6cb544SLuke Nelson static u8 rv_tail_call_reg(struct rv_jit_context *ctx) 114ca6cb544SLuke Nelson { 115ca6cb544SLuke Nelson mark_tail_call(ctx); 116ca6cb544SLuke Nelson 117ca6cb544SLuke Nelson if (seen_call(ctx)) { 118ca6cb544SLuke Nelson __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); 119ca6cb544SLuke Nelson return RV_REG_S6; 120ca6cb544SLuke Nelson } 121ca6cb544SLuke Nelson return RV_REG_A6; 122ca6cb544SLuke Nelson } 123ca6cb544SLuke Nelson 124ca6cb544SLuke Nelson static bool is_32b_int(s64 val) 125ca6cb544SLuke Nelson { 126ca6cb544SLuke Nelson return -(1L << 31) <= val && val < (1L << 31); 127ca6cb544SLuke Nelson } 128ca6cb544SLuke Nelson 129489553ddSLuke Nelson static bool in_auipc_jalr_range(s64 val) 130489553ddSLuke Nelson { 131489553ddSLuke Nelson /* 132489553ddSLuke Nelson * auipc+jalr can reach any signed PC-relative offset in the range 133489553ddSLuke Nelson * [-2^31 - 2^11, 2^31 - 2^11). 134489553ddSLuke Nelson */ 135489553ddSLuke Nelson return (-(1L << 31) - (1L << 11)) <= val && 136489553ddSLuke Nelson val < ((1L << 31) - (1L << 11)); 137489553ddSLuke Nelson } 138489553ddSLuke Nelson 139ca6cb544SLuke Nelson static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx) 140ca6cb544SLuke Nelson { 141ca6cb544SLuke Nelson /* Note that the immediate from the add is sign-extended, 142ca6cb544SLuke Nelson * which means that we need to compensate this by adding 2^12, 143ca6cb544SLuke Nelson * when the 12th bit is set. A simpler way of doing this, and 144ca6cb544SLuke Nelson * getting rid of the check, is to just add 2**11 before the 145ca6cb544SLuke Nelson * shift. The "Loading a 32-Bit constant" example from the 146ca6cb544SLuke Nelson * "Computer Organization and Design, RISC-V edition" book by 147ca6cb544SLuke Nelson * Patterson/Hennessy highlights this fact. 148ca6cb544SLuke Nelson * 149ca6cb544SLuke Nelson * This also means that we need to process LSB to MSB. 150ca6cb544SLuke Nelson */ 15118a4d8c9SLuke Nelson s64 upper = (val + (1 << 11)) >> 12; 15218a4d8c9SLuke Nelson /* Sign-extend lower 12 bits to 64 bits since immediates for li, addiw, 15318a4d8c9SLuke Nelson * and addi are signed and RVC checks will perform signed comparisons. 15418a4d8c9SLuke Nelson */ 15518a4d8c9SLuke Nelson s64 lower = ((val & 0xfff) << 52) >> 52; 156ca6cb544SLuke Nelson int shift; 157ca6cb544SLuke Nelson 158ca6cb544SLuke Nelson if (is_32b_int(val)) { 159ca6cb544SLuke Nelson if (upper) 16018a4d8c9SLuke Nelson emit_lui(rd, upper, ctx); 161ca6cb544SLuke Nelson 162ca6cb544SLuke Nelson if (!upper) { 16318a4d8c9SLuke Nelson emit_li(rd, lower, ctx); 164ca6cb544SLuke Nelson return; 165ca6cb544SLuke Nelson } 166ca6cb544SLuke Nelson 16718a4d8c9SLuke Nelson emit_addiw(rd, rd, lower, ctx); 168ca6cb544SLuke Nelson return; 169ca6cb544SLuke Nelson } 170ca6cb544SLuke Nelson 171ca6cb544SLuke Nelson shift = __ffs(upper); 172ca6cb544SLuke Nelson upper >>= shift; 173ca6cb544SLuke Nelson shift += 12; 174ca6cb544SLuke Nelson 175ca6cb544SLuke Nelson emit_imm(rd, upper, ctx); 176ca6cb544SLuke Nelson 17718a4d8c9SLuke Nelson emit_slli(rd, rd, shift, ctx); 178ca6cb544SLuke Nelson if (lower) 17918a4d8c9SLuke Nelson emit_addi(rd, rd, lower, ctx); 180ca6cb544SLuke Nelson } 181ca6cb544SLuke Nelson 182ca6cb544SLuke Nelson static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) 183ca6cb544SLuke Nelson { 184ca6cb544SLuke Nelson int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; 185ca6cb544SLuke Nelson 186ca6cb544SLuke Nelson if (seen_reg(RV_REG_RA, ctx)) { 18718a4d8c9SLuke Nelson emit_ld(RV_REG_RA, store_offset, RV_REG_SP, ctx); 188ca6cb544SLuke Nelson store_offset -= 8; 189ca6cb544SLuke Nelson } 19018a4d8c9SLuke Nelson emit_ld(RV_REG_FP, store_offset, RV_REG_SP, ctx); 191ca6cb544SLuke Nelson store_offset -= 8; 192ca6cb544SLuke Nelson if (seen_reg(RV_REG_S1, ctx)) { 19318a4d8c9SLuke Nelson emit_ld(RV_REG_S1, store_offset, RV_REG_SP, ctx); 194ca6cb544SLuke Nelson store_offset -= 8; 195ca6cb544SLuke Nelson } 196ca6cb544SLuke Nelson if (seen_reg(RV_REG_S2, ctx)) { 19718a4d8c9SLuke Nelson emit_ld(RV_REG_S2, store_offset, RV_REG_SP, ctx); 198ca6cb544SLuke Nelson store_offset -= 8; 199ca6cb544SLuke Nelson } 200ca6cb544SLuke Nelson if (seen_reg(RV_REG_S3, ctx)) { 20118a4d8c9SLuke Nelson emit_ld(RV_REG_S3, store_offset, RV_REG_SP, ctx); 202ca6cb544SLuke Nelson store_offset -= 8; 203ca6cb544SLuke Nelson } 204ca6cb544SLuke Nelson if (seen_reg(RV_REG_S4, ctx)) { 20518a4d8c9SLuke Nelson emit_ld(RV_REG_S4, store_offset, RV_REG_SP, ctx); 206ca6cb544SLuke Nelson store_offset -= 8; 207ca6cb544SLuke Nelson } 208ca6cb544SLuke Nelson if (seen_reg(RV_REG_S5, ctx)) { 20918a4d8c9SLuke Nelson emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx); 210ca6cb544SLuke Nelson store_offset -= 8; 211ca6cb544SLuke Nelson } 212ca6cb544SLuke Nelson if (seen_reg(RV_REG_S6, ctx)) { 21318a4d8c9SLuke Nelson emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx); 214ca6cb544SLuke Nelson store_offset -= 8; 215ca6cb544SLuke Nelson } 216ca6cb544SLuke Nelson 21718a4d8c9SLuke Nelson emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); 218ca6cb544SLuke Nelson /* Set return value. */ 219ca6cb544SLuke Nelson if (!is_tail_call) 22018a4d8c9SLuke Nelson emit_mv(RV_REG_A0, RV_REG_A5, ctx); 22118a4d8c9SLuke Nelson emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, 22218a4d8c9SLuke Nelson is_tail_call ? 4 : 0, /* skip TCC init */ 223ca6cb544SLuke Nelson ctx); 224ca6cb544SLuke Nelson } 225ca6cb544SLuke Nelson 226ca6cb544SLuke Nelson static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, 227ca6cb544SLuke Nelson struct rv_jit_context *ctx) 228ca6cb544SLuke Nelson { 229ca6cb544SLuke Nelson switch (cond) { 230ca6cb544SLuke Nelson case BPF_JEQ: 231ca6cb544SLuke Nelson emit(rv_beq(rd, rs, rvoff >> 1), ctx); 232ca6cb544SLuke Nelson return; 233ca6cb544SLuke Nelson case BPF_JGT: 234ca6cb544SLuke Nelson emit(rv_bltu(rs, rd, rvoff >> 1), ctx); 235ca6cb544SLuke Nelson return; 236ca6cb544SLuke Nelson case BPF_JLT: 237ca6cb544SLuke Nelson emit(rv_bltu(rd, rs, rvoff >> 1), ctx); 238ca6cb544SLuke Nelson return; 239ca6cb544SLuke Nelson case BPF_JGE: 240ca6cb544SLuke Nelson emit(rv_bgeu(rd, rs, rvoff >> 1), ctx); 241ca6cb544SLuke Nelson return; 242ca6cb544SLuke Nelson case BPF_JLE: 243ca6cb544SLuke Nelson emit(rv_bgeu(rs, rd, rvoff >> 1), ctx); 244ca6cb544SLuke Nelson return; 245ca6cb544SLuke Nelson case BPF_JNE: 246ca6cb544SLuke Nelson emit(rv_bne(rd, rs, rvoff >> 1), ctx); 247ca6cb544SLuke Nelson return; 248ca6cb544SLuke Nelson case BPF_JSGT: 249ca6cb544SLuke Nelson emit(rv_blt(rs, rd, rvoff >> 1), ctx); 250ca6cb544SLuke Nelson return; 251ca6cb544SLuke Nelson case BPF_JSLT: 252ca6cb544SLuke Nelson emit(rv_blt(rd, rs, rvoff >> 1), ctx); 253ca6cb544SLuke Nelson return; 254ca6cb544SLuke Nelson case BPF_JSGE: 255ca6cb544SLuke Nelson emit(rv_bge(rd, rs, rvoff >> 1), ctx); 256ca6cb544SLuke Nelson return; 257ca6cb544SLuke Nelson case BPF_JSLE: 258ca6cb544SLuke Nelson emit(rv_bge(rs, rd, rvoff >> 1), ctx); 259ca6cb544SLuke Nelson } 260ca6cb544SLuke Nelson } 261ca6cb544SLuke Nelson 262ca6cb544SLuke Nelson static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, 263ca6cb544SLuke Nelson struct rv_jit_context *ctx) 264ca6cb544SLuke Nelson { 265ca6cb544SLuke Nelson s64 upper, lower; 266ca6cb544SLuke Nelson 267ca6cb544SLuke Nelson if (is_13b_int(rvoff)) { 268ca6cb544SLuke Nelson emit_bcc(cond, rd, rs, rvoff, ctx); 269ca6cb544SLuke Nelson return; 270ca6cb544SLuke Nelson } 271ca6cb544SLuke Nelson 272ca6cb544SLuke Nelson /* Adjust for jal */ 273ca6cb544SLuke Nelson rvoff -= 4; 274ca6cb544SLuke Nelson 275ca6cb544SLuke Nelson /* Transform, e.g.: 276ca6cb544SLuke Nelson * bne rd,rs,foo 277ca6cb544SLuke Nelson * to 278ca6cb544SLuke Nelson * beq rd,rs,<.L1> 279ca6cb544SLuke Nelson * (auipc foo) 280ca6cb544SLuke Nelson * jal(r) foo 281ca6cb544SLuke Nelson * .L1 282ca6cb544SLuke Nelson */ 283ca6cb544SLuke Nelson cond = invert_bpf_cond(cond); 284ca6cb544SLuke Nelson if (is_21b_int(rvoff)) { 285ca6cb544SLuke Nelson emit_bcc(cond, rd, rs, 8, ctx); 286ca6cb544SLuke Nelson emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx); 287ca6cb544SLuke Nelson return; 288ca6cb544SLuke Nelson } 289ca6cb544SLuke Nelson 290ca6cb544SLuke Nelson /* 32b No need for an additional rvoff adjustment, since we 291ca6cb544SLuke Nelson * get that from the auipc at PC', where PC = PC' + 4. 292ca6cb544SLuke Nelson */ 293ca6cb544SLuke Nelson upper = (rvoff + (1 << 11)) >> 12; 294ca6cb544SLuke Nelson lower = rvoff & 0xfff; 295ca6cb544SLuke Nelson 296ca6cb544SLuke Nelson emit_bcc(cond, rd, rs, 12, ctx); 297ca6cb544SLuke Nelson emit(rv_auipc(RV_REG_T1, upper), ctx); 298ca6cb544SLuke Nelson emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx); 299ca6cb544SLuke Nelson } 300ca6cb544SLuke Nelson 301ca6cb544SLuke Nelson static void emit_zext_32(u8 reg, struct rv_jit_context *ctx) 302ca6cb544SLuke Nelson { 30318a4d8c9SLuke Nelson emit_slli(reg, reg, 32, ctx); 30418a4d8c9SLuke Nelson emit_srli(reg, reg, 32, ctx); 305ca6cb544SLuke Nelson } 306ca6cb544SLuke Nelson 307ca6cb544SLuke Nelson static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) 308ca6cb544SLuke Nelson { 309ca6cb544SLuke Nelson int tc_ninsn, off, start_insn = ctx->ninsns; 310ca6cb544SLuke Nelson u8 tcc = rv_tail_call_reg(ctx); 311ca6cb544SLuke Nelson 312ca6cb544SLuke Nelson /* a0: &ctx 313ca6cb544SLuke Nelson * a1: &array 314ca6cb544SLuke Nelson * a2: index 315ca6cb544SLuke Nelson * 316ca6cb544SLuke Nelson * if (index >= array->map.max_entries) 317ca6cb544SLuke Nelson * goto out; 318ca6cb544SLuke Nelson */ 319ca6cb544SLuke Nelson tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] : 320ca6cb544SLuke Nelson ctx->offset[0]; 321ca6cb544SLuke Nelson emit_zext_32(RV_REG_A2, ctx); 322ca6cb544SLuke Nelson 323ca6cb544SLuke Nelson off = offsetof(struct bpf_array, map.max_entries); 324ca6cb544SLuke Nelson if (is_12b_check(off, insn)) 325ca6cb544SLuke Nelson return -1; 326ca6cb544SLuke Nelson emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx); 327bfabff3cSLuke Nelson off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 328ca6cb544SLuke Nelson emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx); 329ca6cb544SLuke Nelson 330*ebf7f6f0STiezhu Yang /* if (--TCC < 0) 331ca6cb544SLuke Nelson * goto out; 332ca6cb544SLuke Nelson */ 333*ebf7f6f0STiezhu Yang emit_addi(RV_REG_TCC, tcc, -1, ctx); 334bfabff3cSLuke Nelson off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 335*ebf7f6f0STiezhu Yang emit_branch(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); 336ca6cb544SLuke Nelson 337ca6cb544SLuke Nelson /* prog = array->ptrs[index]; 338ca6cb544SLuke Nelson * if (!prog) 339ca6cb544SLuke Nelson * goto out; 340ca6cb544SLuke Nelson */ 34118a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_A2, 3, ctx); 34218a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_A1, ctx); 343ca6cb544SLuke Nelson off = offsetof(struct bpf_array, ptrs); 344ca6cb544SLuke Nelson if (is_12b_check(off, insn)) 345ca6cb544SLuke Nelson return -1; 34618a4d8c9SLuke Nelson emit_ld(RV_REG_T2, off, RV_REG_T2, ctx); 347bfabff3cSLuke Nelson off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 348ca6cb544SLuke Nelson emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); 349ca6cb544SLuke Nelson 350ca6cb544SLuke Nelson /* goto *(prog->bpf_func + 4); */ 351ca6cb544SLuke Nelson off = offsetof(struct bpf_prog, bpf_func); 352ca6cb544SLuke Nelson if (is_12b_check(off, insn)) 353ca6cb544SLuke Nelson return -1; 35418a4d8c9SLuke Nelson emit_ld(RV_REG_T3, off, RV_REG_T2, ctx); 355ca6cb544SLuke Nelson __build_epilogue(true, ctx); 356ca6cb544SLuke Nelson return 0; 357ca6cb544SLuke Nelson } 358ca6cb544SLuke Nelson 359ca6cb544SLuke Nelson static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn, 360ca6cb544SLuke Nelson struct rv_jit_context *ctx) 361ca6cb544SLuke Nelson { 362ca6cb544SLuke Nelson u8 code = insn->code; 363ca6cb544SLuke Nelson 364ca6cb544SLuke Nelson switch (code) { 365ca6cb544SLuke Nelson case BPF_JMP | BPF_JA: 366ca6cb544SLuke Nelson case BPF_JMP | BPF_CALL: 367ca6cb544SLuke Nelson case BPF_JMP | BPF_EXIT: 368ca6cb544SLuke Nelson case BPF_JMP | BPF_TAIL_CALL: 369ca6cb544SLuke Nelson break; 370ca6cb544SLuke Nelson default: 371ca6cb544SLuke Nelson *rd = bpf_to_rv_reg(insn->dst_reg, ctx); 372ca6cb544SLuke Nelson } 373ca6cb544SLuke Nelson 374ca6cb544SLuke Nelson if (code & (BPF_ALU | BPF_X) || code & (BPF_ALU64 | BPF_X) || 375ca6cb544SLuke Nelson code & (BPF_JMP | BPF_X) || code & (BPF_JMP32 | BPF_X) || 376ca6cb544SLuke Nelson code & BPF_LDX || code & BPF_STX) 377ca6cb544SLuke Nelson *rs = bpf_to_rv_reg(insn->src_reg, ctx); 378ca6cb544SLuke Nelson } 379ca6cb544SLuke Nelson 380ca6cb544SLuke Nelson static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) 381ca6cb544SLuke Nelson { 38218a4d8c9SLuke Nelson emit_mv(RV_REG_T2, *rd, ctx); 383ca6cb544SLuke Nelson emit_zext_32(RV_REG_T2, ctx); 38418a4d8c9SLuke Nelson emit_mv(RV_REG_T1, *rs, ctx); 385ca6cb544SLuke Nelson emit_zext_32(RV_REG_T1, ctx); 386ca6cb544SLuke Nelson *rd = RV_REG_T2; 387ca6cb544SLuke Nelson *rs = RV_REG_T1; 388ca6cb544SLuke Nelson } 389ca6cb544SLuke Nelson 390ca6cb544SLuke Nelson static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) 391ca6cb544SLuke Nelson { 39218a4d8c9SLuke Nelson emit_addiw(RV_REG_T2, *rd, 0, ctx); 39318a4d8c9SLuke Nelson emit_addiw(RV_REG_T1, *rs, 0, ctx); 394ca6cb544SLuke Nelson *rd = RV_REG_T2; 395ca6cb544SLuke Nelson *rs = RV_REG_T1; 396ca6cb544SLuke Nelson } 397ca6cb544SLuke Nelson 398ca6cb544SLuke Nelson static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx) 399ca6cb544SLuke Nelson { 40018a4d8c9SLuke Nelson emit_mv(RV_REG_T2, *rd, ctx); 401ca6cb544SLuke Nelson emit_zext_32(RV_REG_T2, ctx); 402ca6cb544SLuke Nelson emit_zext_32(RV_REG_T1, ctx); 403ca6cb544SLuke Nelson *rd = RV_REG_T2; 404ca6cb544SLuke Nelson } 405ca6cb544SLuke Nelson 406ca6cb544SLuke Nelson static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx) 407ca6cb544SLuke Nelson { 40818a4d8c9SLuke Nelson emit_addiw(RV_REG_T2, *rd, 0, ctx); 409ca6cb544SLuke Nelson *rd = RV_REG_T2; 410ca6cb544SLuke Nelson } 411ca6cb544SLuke Nelson 412489553ddSLuke Nelson static int emit_jump_and_link(u8 rd, s64 rvoff, bool force_jalr, 413ca6cb544SLuke Nelson struct rv_jit_context *ctx) 414ca6cb544SLuke Nelson { 415ca6cb544SLuke Nelson s64 upper, lower; 416ca6cb544SLuke Nelson 417ca6cb544SLuke Nelson if (rvoff && is_21b_int(rvoff) && !force_jalr) { 418ca6cb544SLuke Nelson emit(rv_jal(rd, rvoff >> 1), ctx); 419489553ddSLuke Nelson return 0; 420489553ddSLuke Nelson } else if (in_auipc_jalr_range(rvoff)) { 421ca6cb544SLuke Nelson upper = (rvoff + (1 << 11)) >> 12; 422ca6cb544SLuke Nelson lower = rvoff & 0xfff; 423ca6cb544SLuke Nelson emit(rv_auipc(RV_REG_T1, upper), ctx); 424ca6cb544SLuke Nelson emit(rv_jalr(rd, RV_REG_T1, lower), ctx); 425489553ddSLuke Nelson return 0; 426489553ddSLuke Nelson } 427489553ddSLuke Nelson 428489553ddSLuke Nelson pr_err("bpf-jit: target offset 0x%llx is out of range\n", rvoff); 429489553ddSLuke Nelson return -ERANGE; 430ca6cb544SLuke Nelson } 431ca6cb544SLuke Nelson 432ca6cb544SLuke Nelson static bool is_signed_bpf_cond(u8 cond) 433ca6cb544SLuke Nelson { 434ca6cb544SLuke Nelson return cond == BPF_JSGT || cond == BPF_JSLT || 435ca6cb544SLuke Nelson cond == BPF_JSGE || cond == BPF_JSLE; 436ca6cb544SLuke Nelson } 437ca6cb544SLuke Nelson 438ca6cb544SLuke Nelson static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx) 439ca6cb544SLuke Nelson { 440ca6cb544SLuke Nelson s64 off = 0; 441ca6cb544SLuke Nelson u64 ip; 442ca6cb544SLuke Nelson u8 rd; 443489553ddSLuke Nelson int ret; 444ca6cb544SLuke Nelson 445ca6cb544SLuke Nelson if (addr && ctx->insns) { 446ca6cb544SLuke Nelson ip = (u64)(long)(ctx->insns + ctx->ninsns); 447ca6cb544SLuke Nelson off = addr - ip; 448ca6cb544SLuke Nelson } 449ca6cb544SLuke Nelson 450489553ddSLuke Nelson ret = emit_jump_and_link(RV_REG_RA, off, !fixed, ctx); 451489553ddSLuke Nelson if (ret) 452489553ddSLuke Nelson return ret; 453ca6cb544SLuke Nelson rd = bpf_to_rv_reg(BPF_REG_0, ctx); 45418a4d8c9SLuke Nelson emit_mv(rd, RV_REG_A0, ctx); 455ca6cb544SLuke Nelson return 0; 456ca6cb544SLuke Nelson } 457ca6cb544SLuke Nelson 458252c765bSTong Tiangen #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) 459252c765bSTong Tiangen #define BPF_FIXUP_REG_MASK GENMASK(31, 27) 460252c765bSTong Tiangen 461252c765bSTong Tiangen int rv_bpf_fixup_exception(const struct exception_table_entry *ex, 462f47d4ffeSBjörn Töpel struct pt_regs *regs); 463f47d4ffeSBjörn Töpel int rv_bpf_fixup_exception(const struct exception_table_entry *ex, 464252c765bSTong Tiangen struct pt_regs *regs) 465252c765bSTong Tiangen { 466252c765bSTong Tiangen off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); 467252c765bSTong Tiangen int regs_offset = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); 468252c765bSTong Tiangen 469252c765bSTong Tiangen *(unsigned long *)((void *)regs + pt_regmap[regs_offset]) = 0; 470252c765bSTong Tiangen regs->epc = (unsigned long)&ex->fixup - offset; 471252c765bSTong Tiangen 472252c765bSTong Tiangen return 1; 473252c765bSTong Tiangen } 474252c765bSTong Tiangen 475252c765bSTong Tiangen /* For accesses to BTF pointers, add an entry to the exception table */ 476252c765bSTong Tiangen static int add_exception_handler(const struct bpf_insn *insn, 477252c765bSTong Tiangen struct rv_jit_context *ctx, 478252c765bSTong Tiangen int dst_reg, int insn_len) 479252c765bSTong Tiangen { 480252c765bSTong Tiangen struct exception_table_entry *ex; 481252c765bSTong Tiangen unsigned long pc; 482252c765bSTong Tiangen off_t offset; 483252c765bSTong Tiangen 484252c765bSTong Tiangen if (!ctx->insns || !ctx->prog->aux->extable || BPF_MODE(insn->code) != BPF_PROBE_MEM) 485252c765bSTong Tiangen return 0; 486252c765bSTong Tiangen 487252c765bSTong Tiangen if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries)) 488252c765bSTong Tiangen return -EINVAL; 489252c765bSTong Tiangen 490252c765bSTong Tiangen if (WARN_ON_ONCE(insn_len > ctx->ninsns)) 491252c765bSTong Tiangen return -EINVAL; 492252c765bSTong Tiangen 493252c765bSTong Tiangen if (WARN_ON_ONCE(!rvc_enabled() && insn_len == 1)) 494252c765bSTong Tiangen return -EINVAL; 495252c765bSTong Tiangen 496252c765bSTong Tiangen ex = &ctx->prog->aux->extable[ctx->nexentries]; 497252c765bSTong Tiangen pc = (unsigned long)&ctx->insns[ctx->ninsns - insn_len]; 498252c765bSTong Tiangen 499252c765bSTong Tiangen offset = pc - (long)&ex->insn; 500252c765bSTong Tiangen if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 501252c765bSTong Tiangen return -ERANGE; 502252c765bSTong Tiangen ex->insn = pc; 503252c765bSTong Tiangen 504252c765bSTong Tiangen /* 505252c765bSTong Tiangen * Since the extable follows the program, the fixup offset is always 506252c765bSTong Tiangen * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value 507252c765bSTong Tiangen * to keep things simple, and put the destination register in the upper 508252c765bSTong Tiangen * bits. We don't need to worry about buildtime or runtime sort 509252c765bSTong Tiangen * modifying the upper bits because the table is already sorted, and 510252c765bSTong Tiangen * isn't part of the main exception table. 511252c765bSTong Tiangen */ 512252c765bSTong Tiangen offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16)); 513252c765bSTong Tiangen if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset)) 514252c765bSTong Tiangen return -ERANGE; 515252c765bSTong Tiangen 516252c765bSTong Tiangen ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | 517252c765bSTong Tiangen FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); 518252c765bSTong Tiangen 519252c765bSTong Tiangen ctx->nexentries++; 520252c765bSTong Tiangen return 0; 521252c765bSTong Tiangen } 522252c765bSTong Tiangen 523ca6cb544SLuke Nelson int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 524ca6cb544SLuke Nelson bool extra_pass) 525ca6cb544SLuke Nelson { 526ca6cb544SLuke Nelson bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || 527ca6cb544SLuke Nelson BPF_CLASS(insn->code) == BPF_JMP; 528489553ddSLuke Nelson int s, e, rvoff, ret, i = insn - ctx->prog->insnsi; 529ca6cb544SLuke Nelson struct bpf_prog_aux *aux = ctx->prog->aux; 530ca6cb544SLuke Nelson u8 rd = -1, rs = -1, code = insn->code; 531ca6cb544SLuke Nelson s16 off = insn->off; 532ca6cb544SLuke Nelson s32 imm = insn->imm; 533ca6cb544SLuke Nelson 534ca6cb544SLuke Nelson init_regs(&rd, &rs, insn, ctx); 535ca6cb544SLuke Nelson 536ca6cb544SLuke Nelson switch (code) { 537ca6cb544SLuke Nelson /* dst = src */ 538ca6cb544SLuke Nelson case BPF_ALU | BPF_MOV | BPF_X: 539ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MOV | BPF_X: 540ca6cb544SLuke Nelson if (imm == 1) { 541ca6cb544SLuke Nelson /* Special mov32 for zext */ 542ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 543ca6cb544SLuke Nelson break; 544ca6cb544SLuke Nelson } 54518a4d8c9SLuke Nelson emit_mv(rd, rs, ctx); 546ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 547ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 548ca6cb544SLuke Nelson break; 549ca6cb544SLuke Nelson 550ca6cb544SLuke Nelson /* dst = dst OP src */ 551ca6cb544SLuke Nelson case BPF_ALU | BPF_ADD | BPF_X: 552ca6cb544SLuke Nelson case BPF_ALU64 | BPF_ADD | BPF_X: 55318a4d8c9SLuke Nelson emit_add(rd, rd, rs, ctx); 554ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 555ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 556ca6cb544SLuke Nelson break; 557ca6cb544SLuke Nelson case BPF_ALU | BPF_SUB | BPF_X: 558ca6cb544SLuke Nelson case BPF_ALU64 | BPF_SUB | BPF_X: 55918a4d8c9SLuke Nelson if (is64) 56018a4d8c9SLuke Nelson emit_sub(rd, rd, rs, ctx); 56118a4d8c9SLuke Nelson else 56218a4d8c9SLuke Nelson emit_subw(rd, rd, rs, ctx); 56318a4d8c9SLuke Nelson 564ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 565ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 566ca6cb544SLuke Nelson break; 567ca6cb544SLuke Nelson case BPF_ALU | BPF_AND | BPF_X: 568ca6cb544SLuke Nelson case BPF_ALU64 | BPF_AND | BPF_X: 56918a4d8c9SLuke Nelson emit_and(rd, rd, rs, ctx); 570ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 571ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 572ca6cb544SLuke Nelson break; 573ca6cb544SLuke Nelson case BPF_ALU | BPF_OR | BPF_X: 574ca6cb544SLuke Nelson case BPF_ALU64 | BPF_OR | BPF_X: 57518a4d8c9SLuke Nelson emit_or(rd, rd, rs, ctx); 576ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 577ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 578ca6cb544SLuke Nelson break; 579ca6cb544SLuke Nelson case BPF_ALU | BPF_XOR | BPF_X: 580ca6cb544SLuke Nelson case BPF_ALU64 | BPF_XOR | BPF_X: 58118a4d8c9SLuke Nelson emit_xor(rd, rd, rs, ctx); 582ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 583ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 584ca6cb544SLuke Nelson break; 585ca6cb544SLuke Nelson case BPF_ALU | BPF_MUL | BPF_X: 586ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MUL | BPF_X: 587ca6cb544SLuke Nelson emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx); 588ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 589ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 590ca6cb544SLuke Nelson break; 591ca6cb544SLuke Nelson case BPF_ALU | BPF_DIV | BPF_X: 592ca6cb544SLuke Nelson case BPF_ALU64 | BPF_DIV | BPF_X: 593ca6cb544SLuke Nelson emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx); 594ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 595ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 596ca6cb544SLuke Nelson break; 597ca6cb544SLuke Nelson case BPF_ALU | BPF_MOD | BPF_X: 598ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MOD | BPF_X: 599ca6cb544SLuke Nelson emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx); 600ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 601ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 602ca6cb544SLuke Nelson break; 603ca6cb544SLuke Nelson case BPF_ALU | BPF_LSH | BPF_X: 604ca6cb544SLuke Nelson case BPF_ALU64 | BPF_LSH | BPF_X: 605ca6cb544SLuke Nelson emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx); 6060224b2acSLuke Nelson if (!is64 && !aux->verifier_zext) 607ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 608ca6cb544SLuke Nelson break; 609ca6cb544SLuke Nelson case BPF_ALU | BPF_RSH | BPF_X: 610ca6cb544SLuke Nelson case BPF_ALU64 | BPF_RSH | BPF_X: 611ca6cb544SLuke Nelson emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx); 612ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 613ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 614ca6cb544SLuke Nelson break; 615ca6cb544SLuke Nelson case BPF_ALU | BPF_ARSH | BPF_X: 616ca6cb544SLuke Nelson case BPF_ALU64 | BPF_ARSH | BPF_X: 617ca6cb544SLuke Nelson emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx); 618ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 619ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 620ca6cb544SLuke Nelson break; 621ca6cb544SLuke Nelson 622ca6cb544SLuke Nelson /* dst = -dst */ 623ca6cb544SLuke Nelson case BPF_ALU | BPF_NEG: 624ca6cb544SLuke Nelson case BPF_ALU64 | BPF_NEG: 62518a4d8c9SLuke Nelson emit_sub(rd, RV_REG_ZERO, rd, ctx); 626ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 627ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 628ca6cb544SLuke Nelson break; 629ca6cb544SLuke Nelson 630ca6cb544SLuke Nelson /* dst = BSWAP##imm(dst) */ 631ca6cb544SLuke Nelson case BPF_ALU | BPF_END | BPF_FROM_LE: 63221a099abSLuke Nelson switch (imm) { 63321a099abSLuke Nelson case 16: 63418a4d8c9SLuke Nelson emit_slli(rd, rd, 48, ctx); 63518a4d8c9SLuke Nelson emit_srli(rd, rd, 48, ctx); 63621a099abSLuke Nelson break; 63721a099abSLuke Nelson case 32: 63821a099abSLuke Nelson if (!aux->verifier_zext) 63921a099abSLuke Nelson emit_zext_32(rd, ctx); 64021a099abSLuke Nelson break; 64121a099abSLuke Nelson case 64: 64221a099abSLuke Nelson /* Do nothing */ 643ca6cb544SLuke Nelson break; 644ca6cb544SLuke Nelson } 64521a099abSLuke Nelson break; 64621a099abSLuke Nelson 647ca6cb544SLuke Nelson case BPF_ALU | BPF_END | BPF_FROM_BE: 64818a4d8c9SLuke Nelson emit_li(RV_REG_T2, 0, ctx); 649ca6cb544SLuke Nelson 65018a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 65118a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 65218a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 65318a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 654ca6cb544SLuke Nelson if (imm == 16) 655ca6cb544SLuke Nelson goto out_be; 656ca6cb544SLuke Nelson 65718a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 65818a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 65918a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 66018a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 661ca6cb544SLuke Nelson 66218a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 66318a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 66418a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 66518a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 666ca6cb544SLuke Nelson if (imm == 32) 667ca6cb544SLuke Nelson goto out_be; 668ca6cb544SLuke Nelson 66918a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 67018a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 67118a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 67218a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 673ca6cb544SLuke Nelson 67418a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 67518a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 67618a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 67718a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 678ca6cb544SLuke Nelson 67918a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 68018a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 68118a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 68218a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 683ca6cb544SLuke Nelson 68418a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 68518a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 68618a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 68718a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 688ca6cb544SLuke Nelson out_be: 68918a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 69018a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 691ca6cb544SLuke Nelson 69218a4d8c9SLuke Nelson emit_mv(rd, RV_REG_T2, ctx); 693ca6cb544SLuke Nelson break; 694ca6cb544SLuke Nelson 695ca6cb544SLuke Nelson /* dst = imm */ 696ca6cb544SLuke Nelson case BPF_ALU | BPF_MOV | BPF_K: 697ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MOV | BPF_K: 698ca6cb544SLuke Nelson emit_imm(rd, imm, ctx); 699ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 700ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 701ca6cb544SLuke Nelson break; 702ca6cb544SLuke Nelson 703ca6cb544SLuke Nelson /* dst = dst OP imm */ 704ca6cb544SLuke Nelson case BPF_ALU | BPF_ADD | BPF_K: 705ca6cb544SLuke Nelson case BPF_ALU64 | BPF_ADD | BPF_K: 706ca6cb544SLuke Nelson if (is_12b_int(imm)) { 70718a4d8c9SLuke Nelson emit_addi(rd, rd, imm, ctx); 708ca6cb544SLuke Nelson } else { 709ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 71018a4d8c9SLuke Nelson emit_add(rd, rd, RV_REG_T1, ctx); 711ca6cb544SLuke Nelson } 712ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 713ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 714ca6cb544SLuke Nelson break; 715ca6cb544SLuke Nelson case BPF_ALU | BPF_SUB | BPF_K: 716ca6cb544SLuke Nelson case BPF_ALU64 | BPF_SUB | BPF_K: 717ca6cb544SLuke Nelson if (is_12b_int(-imm)) { 71818a4d8c9SLuke Nelson emit_addi(rd, rd, -imm, ctx); 719ca6cb544SLuke Nelson } else { 720ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 72118a4d8c9SLuke Nelson emit_sub(rd, rd, RV_REG_T1, ctx); 722ca6cb544SLuke Nelson } 723ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 724ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 725ca6cb544SLuke Nelson break; 726ca6cb544SLuke Nelson case BPF_ALU | BPF_AND | BPF_K: 727ca6cb544SLuke Nelson case BPF_ALU64 | BPF_AND | BPF_K: 728ca6cb544SLuke Nelson if (is_12b_int(imm)) { 72918a4d8c9SLuke Nelson emit_andi(rd, rd, imm, ctx); 730ca6cb544SLuke Nelson } else { 731ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 73218a4d8c9SLuke Nelson emit_and(rd, rd, RV_REG_T1, ctx); 733ca6cb544SLuke Nelson } 734ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 735ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 736ca6cb544SLuke Nelson break; 737ca6cb544SLuke Nelson case BPF_ALU | BPF_OR | BPF_K: 738ca6cb544SLuke Nelson case BPF_ALU64 | BPF_OR | BPF_K: 739ca6cb544SLuke Nelson if (is_12b_int(imm)) { 740ca6cb544SLuke Nelson emit(rv_ori(rd, rd, imm), ctx); 741ca6cb544SLuke Nelson } else { 742ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 74318a4d8c9SLuke Nelson emit_or(rd, rd, RV_REG_T1, ctx); 744ca6cb544SLuke Nelson } 745ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 746ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 747ca6cb544SLuke Nelson break; 748ca6cb544SLuke Nelson case BPF_ALU | BPF_XOR | BPF_K: 749ca6cb544SLuke Nelson case BPF_ALU64 | BPF_XOR | BPF_K: 750ca6cb544SLuke Nelson if (is_12b_int(imm)) { 751ca6cb544SLuke Nelson emit(rv_xori(rd, rd, imm), ctx); 752ca6cb544SLuke Nelson } else { 753ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 75418a4d8c9SLuke Nelson emit_xor(rd, rd, RV_REG_T1, ctx); 755ca6cb544SLuke Nelson } 756ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 757ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 758ca6cb544SLuke Nelson break; 759ca6cb544SLuke Nelson case BPF_ALU | BPF_MUL | BPF_K: 760ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MUL | BPF_K: 761ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 762ca6cb544SLuke Nelson emit(is64 ? rv_mul(rd, rd, RV_REG_T1) : 763ca6cb544SLuke Nelson rv_mulw(rd, rd, RV_REG_T1), ctx); 764ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 765ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 766ca6cb544SLuke Nelson break; 767ca6cb544SLuke Nelson case BPF_ALU | BPF_DIV | BPF_K: 768ca6cb544SLuke Nelson case BPF_ALU64 | BPF_DIV | BPF_K: 769ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 770ca6cb544SLuke Nelson emit(is64 ? rv_divu(rd, rd, RV_REG_T1) : 771ca6cb544SLuke Nelson rv_divuw(rd, rd, RV_REG_T1), ctx); 772ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 773ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 774ca6cb544SLuke Nelson break; 775ca6cb544SLuke Nelson case BPF_ALU | BPF_MOD | BPF_K: 776ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MOD | BPF_K: 777ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 778ca6cb544SLuke Nelson emit(is64 ? rv_remu(rd, rd, RV_REG_T1) : 779ca6cb544SLuke Nelson rv_remuw(rd, rd, RV_REG_T1), ctx); 780ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 781ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 782ca6cb544SLuke Nelson break; 783ca6cb544SLuke Nelson case BPF_ALU | BPF_LSH | BPF_K: 784ca6cb544SLuke Nelson case BPF_ALU64 | BPF_LSH | BPF_K: 78518a4d8c9SLuke Nelson emit_slli(rd, rd, imm, ctx); 78618a4d8c9SLuke Nelson 7870224b2acSLuke Nelson if (!is64 && !aux->verifier_zext) 788ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 789ca6cb544SLuke Nelson break; 790ca6cb544SLuke Nelson case BPF_ALU | BPF_RSH | BPF_K: 791ca6cb544SLuke Nelson case BPF_ALU64 | BPF_RSH | BPF_K: 79218a4d8c9SLuke Nelson if (is64) 79318a4d8c9SLuke Nelson emit_srli(rd, rd, imm, ctx); 79418a4d8c9SLuke Nelson else 79518a4d8c9SLuke Nelson emit(rv_srliw(rd, rd, imm), ctx); 79618a4d8c9SLuke Nelson 7970224b2acSLuke Nelson if (!is64 && !aux->verifier_zext) 798ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 799ca6cb544SLuke Nelson break; 800ca6cb544SLuke Nelson case BPF_ALU | BPF_ARSH | BPF_K: 801ca6cb544SLuke Nelson case BPF_ALU64 | BPF_ARSH | BPF_K: 80218a4d8c9SLuke Nelson if (is64) 80318a4d8c9SLuke Nelson emit_srai(rd, rd, imm, ctx); 80418a4d8c9SLuke Nelson else 80518a4d8c9SLuke Nelson emit(rv_sraiw(rd, rd, imm), ctx); 80618a4d8c9SLuke Nelson 8070224b2acSLuke Nelson if (!is64 && !aux->verifier_zext) 808ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 809ca6cb544SLuke Nelson break; 810ca6cb544SLuke Nelson 811ca6cb544SLuke Nelson /* JUMP off */ 812ca6cb544SLuke Nelson case BPF_JMP | BPF_JA: 813ca6cb544SLuke Nelson rvoff = rv_offset(i, off, ctx); 814489553ddSLuke Nelson ret = emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx); 815489553ddSLuke Nelson if (ret) 816489553ddSLuke Nelson return ret; 817ca6cb544SLuke Nelson break; 818ca6cb544SLuke Nelson 819ca6cb544SLuke Nelson /* IF (dst COND src) JUMP off */ 820ca6cb544SLuke Nelson case BPF_JMP | BPF_JEQ | BPF_X: 821ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JEQ | BPF_X: 822ca6cb544SLuke Nelson case BPF_JMP | BPF_JGT | BPF_X: 823ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JGT | BPF_X: 824ca6cb544SLuke Nelson case BPF_JMP | BPF_JLT | BPF_X: 825ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JLT | BPF_X: 826ca6cb544SLuke Nelson case BPF_JMP | BPF_JGE | BPF_X: 827ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JGE | BPF_X: 828ca6cb544SLuke Nelson case BPF_JMP | BPF_JLE | BPF_X: 829ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JLE | BPF_X: 830ca6cb544SLuke Nelson case BPF_JMP | BPF_JNE | BPF_X: 831ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JNE | BPF_X: 832ca6cb544SLuke Nelson case BPF_JMP | BPF_JSGT | BPF_X: 833ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSGT | BPF_X: 834ca6cb544SLuke Nelson case BPF_JMP | BPF_JSLT | BPF_X: 835ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSLT | BPF_X: 836ca6cb544SLuke Nelson case BPF_JMP | BPF_JSGE | BPF_X: 837ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSGE | BPF_X: 838ca6cb544SLuke Nelson case BPF_JMP | BPF_JSLE | BPF_X: 839ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSLE | BPF_X: 840ca6cb544SLuke Nelson case BPF_JMP | BPF_JSET | BPF_X: 841ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSET | BPF_X: 842ca6cb544SLuke Nelson rvoff = rv_offset(i, off, ctx); 843ca6cb544SLuke Nelson if (!is64) { 844ca6cb544SLuke Nelson s = ctx->ninsns; 845ca6cb544SLuke Nelson if (is_signed_bpf_cond(BPF_OP(code))) 846ca6cb544SLuke Nelson emit_sext_32_rd_rs(&rd, &rs, ctx); 847ca6cb544SLuke Nelson else 848ca6cb544SLuke Nelson emit_zext_32_rd_rs(&rd, &rs, ctx); 849ca6cb544SLuke Nelson e = ctx->ninsns; 850ca6cb544SLuke Nelson 851ca6cb544SLuke Nelson /* Adjust for extra insns */ 852bfabff3cSLuke Nelson rvoff -= ninsns_rvoff(e - s); 853ca6cb544SLuke Nelson } 854ca6cb544SLuke Nelson 855ca6cb544SLuke Nelson if (BPF_OP(code) == BPF_JSET) { 856ca6cb544SLuke Nelson /* Adjust for and */ 857ca6cb544SLuke Nelson rvoff -= 4; 85818a4d8c9SLuke Nelson emit_and(RV_REG_T1, rd, rs, ctx); 859ca6cb544SLuke Nelson emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, 860ca6cb544SLuke Nelson ctx); 861ca6cb544SLuke Nelson } else { 862ca6cb544SLuke Nelson emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 863ca6cb544SLuke Nelson } 864ca6cb544SLuke Nelson break; 865ca6cb544SLuke Nelson 866ca6cb544SLuke Nelson /* IF (dst COND imm) JUMP off */ 867ca6cb544SLuke Nelson case BPF_JMP | BPF_JEQ | BPF_K: 868ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JEQ | BPF_K: 869ca6cb544SLuke Nelson case BPF_JMP | BPF_JGT | BPF_K: 870ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JGT | BPF_K: 871ca6cb544SLuke Nelson case BPF_JMP | BPF_JLT | BPF_K: 872ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JLT | BPF_K: 873ca6cb544SLuke Nelson case BPF_JMP | BPF_JGE | BPF_K: 874ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JGE | BPF_K: 875ca6cb544SLuke Nelson case BPF_JMP | BPF_JLE | BPF_K: 876ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JLE | BPF_K: 877ca6cb544SLuke Nelson case BPF_JMP | BPF_JNE | BPF_K: 878ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JNE | BPF_K: 879ca6cb544SLuke Nelson case BPF_JMP | BPF_JSGT | BPF_K: 880ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSGT | BPF_K: 881ca6cb544SLuke Nelson case BPF_JMP | BPF_JSLT | BPF_K: 882ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSLT | BPF_K: 883ca6cb544SLuke Nelson case BPF_JMP | BPF_JSGE | BPF_K: 884ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSGE | BPF_K: 885ca6cb544SLuke Nelson case BPF_JMP | BPF_JSLE | BPF_K: 886ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSLE | BPF_K: 887ca6cb544SLuke Nelson rvoff = rv_offset(i, off, ctx); 888ca6cb544SLuke Nelson s = ctx->ninsns; 889ca349a6aSLuke Nelson if (imm) { 890ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 891ca349a6aSLuke Nelson rs = RV_REG_T1; 892ca349a6aSLuke Nelson } else { 893ca349a6aSLuke Nelson /* If imm is 0, simply use zero register. */ 894ca349a6aSLuke Nelson rs = RV_REG_ZERO; 895ca349a6aSLuke Nelson } 896ca6cb544SLuke Nelson if (!is64) { 897ca6cb544SLuke Nelson if (is_signed_bpf_cond(BPF_OP(code))) 898ca6cb544SLuke Nelson emit_sext_32_rd(&rd, ctx); 899ca6cb544SLuke Nelson else 900ca6cb544SLuke Nelson emit_zext_32_rd_t1(&rd, ctx); 901ca6cb544SLuke Nelson } 902ca6cb544SLuke Nelson e = ctx->ninsns; 903ca6cb544SLuke Nelson 904ca6cb544SLuke Nelson /* Adjust for extra insns */ 905bfabff3cSLuke Nelson rvoff -= ninsns_rvoff(e - s); 906ca349a6aSLuke Nelson emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 907073ca6a0SLuke Nelson break; 908073ca6a0SLuke Nelson 909073ca6a0SLuke Nelson case BPF_JMP | BPF_JSET | BPF_K: 910073ca6a0SLuke Nelson case BPF_JMP32 | BPF_JSET | BPF_K: 911073ca6a0SLuke Nelson rvoff = rv_offset(i, off, ctx); 912073ca6a0SLuke Nelson s = ctx->ninsns; 913073ca6a0SLuke Nelson if (is_12b_int(imm)) { 91418a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, imm, ctx); 915073ca6a0SLuke Nelson } else { 916073ca6a0SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 91718a4d8c9SLuke Nelson emit_and(RV_REG_T1, rd, RV_REG_T1, ctx); 918ca6cb544SLuke Nelson } 919073ca6a0SLuke Nelson /* For jset32, we should clear the upper 32 bits of t1, but 920073ca6a0SLuke Nelson * sign-extension is sufficient here and saves one instruction, 921073ca6a0SLuke Nelson * as t1 is used only in comparison against zero. 922073ca6a0SLuke Nelson */ 923073ca6a0SLuke Nelson if (!is64 && imm < 0) 92418a4d8c9SLuke Nelson emit_addiw(RV_REG_T1, RV_REG_T1, 0, ctx); 925073ca6a0SLuke Nelson e = ctx->ninsns; 926bfabff3cSLuke Nelson rvoff -= ninsns_rvoff(e - s); 927073ca6a0SLuke Nelson emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); 928ca6cb544SLuke Nelson break; 929ca6cb544SLuke Nelson 930ca6cb544SLuke Nelson /* function call */ 931ca6cb544SLuke Nelson case BPF_JMP | BPF_CALL: 932ca6cb544SLuke Nelson { 933ca6cb544SLuke Nelson bool fixed; 934ca6cb544SLuke Nelson u64 addr; 935ca6cb544SLuke Nelson 936ca6cb544SLuke Nelson mark_call(ctx); 937ca6cb544SLuke Nelson ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr, 938ca6cb544SLuke Nelson &fixed); 939ca6cb544SLuke Nelson if (ret < 0) 940ca6cb544SLuke Nelson return ret; 941ca6cb544SLuke Nelson ret = emit_call(fixed, addr, ctx); 942ca6cb544SLuke Nelson if (ret) 943ca6cb544SLuke Nelson return ret; 944ca6cb544SLuke Nelson break; 945ca6cb544SLuke Nelson } 946ca6cb544SLuke Nelson /* tail call */ 947ca6cb544SLuke Nelson case BPF_JMP | BPF_TAIL_CALL: 948ca6cb544SLuke Nelson if (emit_bpf_tail_call(i, ctx)) 949ca6cb544SLuke Nelson return -1; 950ca6cb544SLuke Nelson break; 951ca6cb544SLuke Nelson 952ca6cb544SLuke Nelson /* function return */ 953ca6cb544SLuke Nelson case BPF_JMP | BPF_EXIT: 954ca6cb544SLuke Nelson if (i == ctx->prog->len - 1) 955ca6cb544SLuke Nelson break; 956ca6cb544SLuke Nelson 957ca6cb544SLuke Nelson rvoff = epilogue_offset(ctx); 958489553ddSLuke Nelson ret = emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx); 959489553ddSLuke Nelson if (ret) 960489553ddSLuke Nelson return ret; 961ca6cb544SLuke Nelson break; 962ca6cb544SLuke Nelson 963ca6cb544SLuke Nelson /* dst = imm64 */ 964ca6cb544SLuke Nelson case BPF_LD | BPF_IMM | BPF_DW: 965ca6cb544SLuke Nelson { 966ca6cb544SLuke Nelson struct bpf_insn insn1 = insn[1]; 967ca6cb544SLuke Nelson u64 imm64; 968ca6cb544SLuke Nelson 969ca6cb544SLuke Nelson imm64 = (u64)insn1.imm << 32 | (u32)imm; 970ca6cb544SLuke Nelson emit_imm(rd, imm64, ctx); 971ca6cb544SLuke Nelson return 1; 972ca6cb544SLuke Nelson } 973ca6cb544SLuke Nelson 974ca6cb544SLuke Nelson /* LDX: dst = *(size *)(src + off) */ 975ca6cb544SLuke Nelson case BPF_LDX | BPF_MEM | BPF_B: 976ca6cb544SLuke Nelson case BPF_LDX | BPF_MEM | BPF_H: 977ca6cb544SLuke Nelson case BPF_LDX | BPF_MEM | BPF_W: 978252c765bSTong Tiangen case BPF_LDX | BPF_MEM | BPF_DW: 979252c765bSTong Tiangen case BPF_LDX | BPF_PROBE_MEM | BPF_B: 980252c765bSTong Tiangen case BPF_LDX | BPF_PROBE_MEM | BPF_H: 981252c765bSTong Tiangen case BPF_LDX | BPF_PROBE_MEM | BPF_W: 982252c765bSTong Tiangen case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 983252c765bSTong Tiangen { 984252c765bSTong Tiangen int insn_len, insns_start; 985252c765bSTong Tiangen 986252c765bSTong Tiangen switch (BPF_SIZE(code)) { 987252c765bSTong Tiangen case BPF_B: 988ca6cb544SLuke Nelson if (is_12b_int(off)) { 989252c765bSTong Tiangen insns_start = ctx->ninsns; 990252c765bSTong Tiangen emit(rv_lbu(rd, off, rs), ctx); 991252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 992ca6cb544SLuke Nelson break; 993ca6cb544SLuke Nelson } 994ca6cb544SLuke Nelson 995ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 99618a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 997252c765bSTong Tiangen insns_start = ctx->ninsns; 998252c765bSTong Tiangen emit(rv_lbu(rd, 0, RV_REG_T1), ctx); 999252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1000ca6cb544SLuke Nelson if (insn_is_zext(&insn[1])) 1001ca6cb544SLuke Nelson return 1; 1002ca6cb544SLuke Nelson break; 1003252c765bSTong Tiangen case BPF_H: 1004ca6cb544SLuke Nelson if (is_12b_int(off)) { 1005252c765bSTong Tiangen insns_start = ctx->ninsns; 1006252c765bSTong Tiangen emit(rv_lhu(rd, off, rs), ctx); 1007252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1008ca6cb544SLuke Nelson break; 1009ca6cb544SLuke Nelson } 1010ca6cb544SLuke Nelson 1011ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 101218a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1013252c765bSTong Tiangen insns_start = ctx->ninsns; 1014252c765bSTong Tiangen emit(rv_lhu(rd, 0, RV_REG_T1), ctx); 1015252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1016252c765bSTong Tiangen if (insn_is_zext(&insn[1])) 1017252c765bSTong Tiangen return 1; 1018ca6cb544SLuke Nelson break; 1019252c765bSTong Tiangen case BPF_W: 1020252c765bSTong Tiangen if (is_12b_int(off)) { 1021252c765bSTong Tiangen insns_start = ctx->ninsns; 1022252c765bSTong Tiangen emit(rv_lwu(rd, off, rs), ctx); 1023252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1024252c765bSTong Tiangen break; 1025252c765bSTong Tiangen } 1026ca6cb544SLuke Nelson 1027252c765bSTong Tiangen emit_imm(RV_REG_T1, off, ctx); 1028252c765bSTong Tiangen emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1029252c765bSTong Tiangen insns_start = ctx->ninsns; 1030252c765bSTong Tiangen emit(rv_lwu(rd, 0, RV_REG_T1), ctx); 1031252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1032252c765bSTong Tiangen if (insn_is_zext(&insn[1])) 1033252c765bSTong Tiangen return 1; 1034252c765bSTong Tiangen break; 1035252c765bSTong Tiangen case BPF_DW: 1036252c765bSTong Tiangen if (is_12b_int(off)) { 1037252c765bSTong Tiangen insns_start = ctx->ninsns; 1038252c765bSTong Tiangen emit_ld(rd, off, rs, ctx); 1039252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1040252c765bSTong Tiangen break; 1041252c765bSTong Tiangen } 1042252c765bSTong Tiangen 1043252c765bSTong Tiangen emit_imm(RV_REG_T1, off, ctx); 1044252c765bSTong Tiangen emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1045252c765bSTong Tiangen insns_start = ctx->ninsns; 1046252c765bSTong Tiangen emit_ld(rd, 0, RV_REG_T1, ctx); 1047252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1048252c765bSTong Tiangen break; 1049252c765bSTong Tiangen } 1050252c765bSTong Tiangen 1051252c765bSTong Tiangen ret = add_exception_handler(insn, ctx, rd, insn_len); 1052252c765bSTong Tiangen if (ret) 1053252c765bSTong Tiangen return ret; 1054252c765bSTong Tiangen break; 1055252c765bSTong Tiangen } 1056f5e81d11SDaniel Borkmann /* speculation barrier */ 1057f5e81d11SDaniel Borkmann case BPF_ST | BPF_NOSPEC: 1058f5e81d11SDaniel Borkmann break; 1059f5e81d11SDaniel Borkmann 1060ca6cb544SLuke Nelson /* ST: *(size *)(dst + off) = imm */ 1061ca6cb544SLuke Nelson case BPF_ST | BPF_MEM | BPF_B: 1062ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 1063ca6cb544SLuke Nelson if (is_12b_int(off)) { 1064ca6cb544SLuke Nelson emit(rv_sb(rd, off, RV_REG_T1), ctx); 1065ca6cb544SLuke Nelson break; 1066ca6cb544SLuke Nelson } 1067ca6cb544SLuke Nelson 1068ca6cb544SLuke Nelson emit_imm(RV_REG_T2, off, ctx); 106918a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1070ca6cb544SLuke Nelson emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); 1071ca6cb544SLuke Nelson break; 1072ca6cb544SLuke Nelson 1073ca6cb544SLuke Nelson case BPF_ST | BPF_MEM | BPF_H: 1074ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 1075ca6cb544SLuke Nelson if (is_12b_int(off)) { 1076ca6cb544SLuke Nelson emit(rv_sh(rd, off, RV_REG_T1), ctx); 1077ca6cb544SLuke Nelson break; 1078ca6cb544SLuke Nelson } 1079ca6cb544SLuke Nelson 1080ca6cb544SLuke Nelson emit_imm(RV_REG_T2, off, ctx); 108118a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1082ca6cb544SLuke Nelson emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); 1083ca6cb544SLuke Nelson break; 1084ca6cb544SLuke Nelson case BPF_ST | BPF_MEM | BPF_W: 1085ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 1086ca6cb544SLuke Nelson if (is_12b_int(off)) { 108718a4d8c9SLuke Nelson emit_sw(rd, off, RV_REG_T1, ctx); 1088ca6cb544SLuke Nelson break; 1089ca6cb544SLuke Nelson } 1090ca6cb544SLuke Nelson 1091ca6cb544SLuke Nelson emit_imm(RV_REG_T2, off, ctx); 109218a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 109318a4d8c9SLuke Nelson emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx); 1094ca6cb544SLuke Nelson break; 1095ca6cb544SLuke Nelson case BPF_ST | BPF_MEM | BPF_DW: 1096ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 1097ca6cb544SLuke Nelson if (is_12b_int(off)) { 109818a4d8c9SLuke Nelson emit_sd(rd, off, RV_REG_T1, ctx); 1099ca6cb544SLuke Nelson break; 1100ca6cb544SLuke Nelson } 1101ca6cb544SLuke Nelson 1102ca6cb544SLuke Nelson emit_imm(RV_REG_T2, off, ctx); 110318a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 110418a4d8c9SLuke Nelson emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx); 1105ca6cb544SLuke Nelson break; 1106ca6cb544SLuke Nelson 1107ca6cb544SLuke Nelson /* STX: *(size *)(dst + off) = src */ 1108ca6cb544SLuke Nelson case BPF_STX | BPF_MEM | BPF_B: 1109ca6cb544SLuke Nelson if (is_12b_int(off)) { 1110ca6cb544SLuke Nelson emit(rv_sb(rd, off, rs), ctx); 1111ca6cb544SLuke Nelson break; 1112ca6cb544SLuke Nelson } 1113ca6cb544SLuke Nelson 1114ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 111518a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1116ca6cb544SLuke Nelson emit(rv_sb(RV_REG_T1, 0, rs), ctx); 1117ca6cb544SLuke Nelson break; 1118ca6cb544SLuke Nelson case BPF_STX | BPF_MEM | BPF_H: 1119ca6cb544SLuke Nelson if (is_12b_int(off)) { 1120ca6cb544SLuke Nelson emit(rv_sh(rd, off, rs), ctx); 1121ca6cb544SLuke Nelson break; 1122ca6cb544SLuke Nelson } 1123ca6cb544SLuke Nelson 1124ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 112518a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1126ca6cb544SLuke Nelson emit(rv_sh(RV_REG_T1, 0, rs), ctx); 1127ca6cb544SLuke Nelson break; 1128ca6cb544SLuke Nelson case BPF_STX | BPF_MEM | BPF_W: 1129ca6cb544SLuke Nelson if (is_12b_int(off)) { 113018a4d8c9SLuke Nelson emit_sw(rd, off, rs, ctx); 1131ca6cb544SLuke Nelson break; 1132ca6cb544SLuke Nelson } 1133ca6cb544SLuke Nelson 1134ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 113518a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 113618a4d8c9SLuke Nelson emit_sw(RV_REG_T1, 0, rs, ctx); 1137ca6cb544SLuke Nelson break; 1138ca6cb544SLuke Nelson case BPF_STX | BPF_MEM | BPF_DW: 1139ca6cb544SLuke Nelson if (is_12b_int(off)) { 114018a4d8c9SLuke Nelson emit_sd(rd, off, rs, ctx); 1141ca6cb544SLuke Nelson break; 1142ca6cb544SLuke Nelson } 1143ca6cb544SLuke Nelson 1144ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 114518a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 114618a4d8c9SLuke Nelson emit_sd(RV_REG_T1, 0, rs, ctx); 1147ca6cb544SLuke Nelson break; 114891c960b0SBrendan Jackman case BPF_STX | BPF_ATOMIC | BPF_W: 114991c960b0SBrendan Jackman case BPF_STX | BPF_ATOMIC | BPF_DW: 115091c960b0SBrendan Jackman if (insn->imm != BPF_ADD) { 115191c960b0SBrendan Jackman pr_err("bpf-jit: not supported: atomic operation %02x ***\n", 115291c960b0SBrendan Jackman insn->imm); 115391c960b0SBrendan Jackman return -EINVAL; 115491c960b0SBrendan Jackman } 115591c960b0SBrendan Jackman 115691c960b0SBrendan Jackman /* atomic_add: lock *(u32 *)(dst + off) += src 115791c960b0SBrendan Jackman * atomic_add: lock *(u64 *)(dst + off) += src 115891c960b0SBrendan Jackman */ 115991c960b0SBrendan Jackman 1160ca6cb544SLuke Nelson if (off) { 1161ca6cb544SLuke Nelson if (is_12b_int(off)) { 116218a4d8c9SLuke Nelson emit_addi(RV_REG_T1, rd, off, ctx); 1163ca6cb544SLuke Nelson } else { 1164ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 116518a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1166ca6cb544SLuke Nelson } 1167ca6cb544SLuke Nelson 1168ca6cb544SLuke Nelson rd = RV_REG_T1; 1169ca6cb544SLuke Nelson } 1170ca6cb544SLuke Nelson 1171ca6cb544SLuke Nelson emit(BPF_SIZE(code) == BPF_W ? 1172ca6cb544SLuke Nelson rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0) : 1173ca6cb544SLuke Nelson rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0), ctx); 1174ca6cb544SLuke Nelson break; 1175ca6cb544SLuke Nelson default: 1176ca6cb544SLuke Nelson pr_err("bpf-jit: unknown opcode %02x\n", code); 1177ca6cb544SLuke Nelson return -EINVAL; 1178ca6cb544SLuke Nelson } 1179ca6cb544SLuke Nelson 1180ca6cb544SLuke Nelson return 0; 1181ca6cb544SLuke Nelson } 1182ca6cb544SLuke Nelson 1183ca6cb544SLuke Nelson void bpf_jit_build_prologue(struct rv_jit_context *ctx) 1184ca6cb544SLuke Nelson { 1185ca6cb544SLuke Nelson int stack_adjust = 0, store_offset, bpf_stack_adjust; 1186ca6cb544SLuke Nelson 1187ca6cb544SLuke Nelson bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); 1188ca6cb544SLuke Nelson if (bpf_stack_adjust) 1189ca6cb544SLuke Nelson mark_fp(ctx); 1190ca6cb544SLuke Nelson 1191ca6cb544SLuke Nelson if (seen_reg(RV_REG_RA, ctx)) 1192ca6cb544SLuke Nelson stack_adjust += 8; 1193ca6cb544SLuke Nelson stack_adjust += 8; /* RV_REG_FP */ 1194ca6cb544SLuke Nelson if (seen_reg(RV_REG_S1, ctx)) 1195ca6cb544SLuke Nelson stack_adjust += 8; 1196ca6cb544SLuke Nelson if (seen_reg(RV_REG_S2, ctx)) 1197ca6cb544SLuke Nelson stack_adjust += 8; 1198ca6cb544SLuke Nelson if (seen_reg(RV_REG_S3, ctx)) 1199ca6cb544SLuke Nelson stack_adjust += 8; 1200ca6cb544SLuke Nelson if (seen_reg(RV_REG_S4, ctx)) 1201ca6cb544SLuke Nelson stack_adjust += 8; 1202ca6cb544SLuke Nelson if (seen_reg(RV_REG_S5, ctx)) 1203ca6cb544SLuke Nelson stack_adjust += 8; 1204ca6cb544SLuke Nelson if (seen_reg(RV_REG_S6, ctx)) 1205ca6cb544SLuke Nelson stack_adjust += 8; 1206ca6cb544SLuke Nelson 1207ca6cb544SLuke Nelson stack_adjust = round_up(stack_adjust, 16); 1208ca6cb544SLuke Nelson stack_adjust += bpf_stack_adjust; 1209ca6cb544SLuke Nelson 1210ca6cb544SLuke Nelson store_offset = stack_adjust - 8; 1211ca6cb544SLuke Nelson 1212ca6cb544SLuke Nelson /* First instruction is always setting the tail-call-counter 1213ca6cb544SLuke Nelson * (TCC) register. This instruction is skipped for tail calls. 121418a4d8c9SLuke Nelson * Force using a 4-byte (non-compressed) instruction. 1215ca6cb544SLuke Nelson */ 1216ca6cb544SLuke Nelson emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); 1217ca6cb544SLuke Nelson 121818a4d8c9SLuke Nelson emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); 1219ca6cb544SLuke Nelson 1220ca6cb544SLuke Nelson if (seen_reg(RV_REG_RA, ctx)) { 122118a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_RA, ctx); 1222ca6cb544SLuke Nelson store_offset -= 8; 1223ca6cb544SLuke Nelson } 122418a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_FP, ctx); 1225ca6cb544SLuke Nelson store_offset -= 8; 1226ca6cb544SLuke Nelson if (seen_reg(RV_REG_S1, ctx)) { 122718a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S1, ctx); 1228ca6cb544SLuke Nelson store_offset -= 8; 1229ca6cb544SLuke Nelson } 1230ca6cb544SLuke Nelson if (seen_reg(RV_REG_S2, ctx)) { 123118a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S2, ctx); 1232ca6cb544SLuke Nelson store_offset -= 8; 1233ca6cb544SLuke Nelson } 1234ca6cb544SLuke Nelson if (seen_reg(RV_REG_S3, ctx)) { 123518a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S3, ctx); 1236ca6cb544SLuke Nelson store_offset -= 8; 1237ca6cb544SLuke Nelson } 1238ca6cb544SLuke Nelson if (seen_reg(RV_REG_S4, ctx)) { 123918a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S4, ctx); 1240ca6cb544SLuke Nelson store_offset -= 8; 1241ca6cb544SLuke Nelson } 1242ca6cb544SLuke Nelson if (seen_reg(RV_REG_S5, ctx)) { 124318a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx); 1244ca6cb544SLuke Nelson store_offset -= 8; 1245ca6cb544SLuke Nelson } 1246ca6cb544SLuke Nelson if (seen_reg(RV_REG_S6, ctx)) { 124718a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx); 1248ca6cb544SLuke Nelson store_offset -= 8; 1249ca6cb544SLuke Nelson } 1250ca6cb544SLuke Nelson 125118a4d8c9SLuke Nelson emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx); 1252ca6cb544SLuke Nelson 1253ca6cb544SLuke Nelson if (bpf_stack_adjust) 125418a4d8c9SLuke Nelson emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx); 1255ca6cb544SLuke Nelson 1256ca6cb544SLuke Nelson /* Program contains calls and tail calls, so RV_REG_TCC need 1257ca6cb544SLuke Nelson * to be saved across calls. 1258ca6cb544SLuke Nelson */ 1259ca6cb544SLuke Nelson if (seen_tail_call(ctx) && seen_call(ctx)) 126018a4d8c9SLuke Nelson emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx); 1261ca6cb544SLuke Nelson 1262ca6cb544SLuke Nelson ctx->stack_size = stack_adjust; 1263ca6cb544SLuke Nelson } 1264ca6cb544SLuke Nelson 1265ca6cb544SLuke Nelson void bpf_jit_build_epilogue(struct rv_jit_context *ctx) 1266ca6cb544SLuke Nelson { 1267ca6cb544SLuke Nelson __build_epilogue(false, ctx); 1268ca6cb544SLuke Nelson } 1269