1ca6cb544SLuke Nelson // SPDX-License-Identifier: GPL-2.0 2ca6cb544SLuke Nelson /* BPF JIT compiler for RV64G 3ca6cb544SLuke Nelson * 4ca6cb544SLuke Nelson * Copyright(c) 2019 Björn Töpel <bjorn.topel@gmail.com> 5ca6cb544SLuke Nelson * 6ca6cb544SLuke Nelson */ 7ca6cb544SLuke Nelson 8252c765bSTong Tiangen #include <linux/bitfield.h> 9ca6cb544SLuke Nelson #include <linux/bpf.h> 10ca6cb544SLuke Nelson #include <linux/filter.h> 11596f2e6fSPu Lehui #include <linux/memory.h> 12596f2e6fSPu Lehui #include <linux/stop_machine.h> 132d311f48SRandy Dunlap #include <asm/patch.h> 14ca6cb544SLuke Nelson #include "bpf_jit.h" 15ca6cb544SLuke Nelson 1625ad1065SPu Lehui #define RV_FENTRY_NINSNS 2 1725ad1065SPu Lehui 18ca6cb544SLuke Nelson #define RV_REG_TCC RV_REG_A6 19ca6cb544SLuke Nelson #define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */ 20ca6cb544SLuke Nelson 21ca6cb544SLuke Nelson static const int regmap[] = { 22ca6cb544SLuke Nelson [BPF_REG_0] = RV_REG_A5, 23ca6cb544SLuke Nelson [BPF_REG_1] = RV_REG_A0, 24ca6cb544SLuke Nelson [BPF_REG_2] = RV_REG_A1, 25ca6cb544SLuke Nelson [BPF_REG_3] = RV_REG_A2, 26ca6cb544SLuke Nelson [BPF_REG_4] = RV_REG_A3, 27ca6cb544SLuke Nelson [BPF_REG_5] = RV_REG_A4, 28ca6cb544SLuke Nelson [BPF_REG_6] = RV_REG_S1, 29ca6cb544SLuke Nelson [BPF_REG_7] = RV_REG_S2, 30ca6cb544SLuke Nelson [BPF_REG_8] = RV_REG_S3, 31ca6cb544SLuke Nelson [BPF_REG_9] = RV_REG_S4, 32ca6cb544SLuke Nelson [BPF_REG_FP] = RV_REG_S5, 33ca6cb544SLuke Nelson [BPF_REG_AX] = RV_REG_T0, 34ca6cb544SLuke Nelson }; 35ca6cb544SLuke Nelson 36252c765bSTong Tiangen static const int pt_regmap[] = { 37252c765bSTong Tiangen [RV_REG_A0] = offsetof(struct pt_regs, a0), 38252c765bSTong Tiangen [RV_REG_A1] = offsetof(struct pt_regs, a1), 39252c765bSTong Tiangen [RV_REG_A2] = offsetof(struct pt_regs, a2), 40252c765bSTong Tiangen [RV_REG_A3] = offsetof(struct pt_regs, a3), 41252c765bSTong Tiangen [RV_REG_A4] = offsetof(struct pt_regs, a4), 42252c765bSTong Tiangen [RV_REG_A5] = offsetof(struct pt_regs, a5), 43252c765bSTong Tiangen [RV_REG_S1] = offsetof(struct pt_regs, s1), 44252c765bSTong Tiangen [RV_REG_S2] = offsetof(struct pt_regs, s2), 45252c765bSTong Tiangen [RV_REG_S3] = offsetof(struct pt_regs, s3), 46252c765bSTong Tiangen [RV_REG_S4] = offsetof(struct pt_regs, s4), 47252c765bSTong Tiangen [RV_REG_S5] = offsetof(struct pt_regs, s5), 48252c765bSTong Tiangen [RV_REG_T0] = offsetof(struct pt_regs, t0), 49252c765bSTong Tiangen }; 50252c765bSTong Tiangen 51ca6cb544SLuke Nelson enum { 52ca6cb544SLuke Nelson RV_CTX_F_SEEN_TAIL_CALL = 0, 53ca6cb544SLuke Nelson RV_CTX_F_SEEN_CALL = RV_REG_RA, 54ca6cb544SLuke Nelson RV_CTX_F_SEEN_S1 = RV_REG_S1, 55ca6cb544SLuke Nelson RV_CTX_F_SEEN_S2 = RV_REG_S2, 56ca6cb544SLuke Nelson RV_CTX_F_SEEN_S3 = RV_REG_S3, 57ca6cb544SLuke Nelson RV_CTX_F_SEEN_S4 = RV_REG_S4, 58ca6cb544SLuke Nelson RV_CTX_F_SEEN_S5 = RV_REG_S5, 59ca6cb544SLuke Nelson RV_CTX_F_SEEN_S6 = RV_REG_S6, 60ca6cb544SLuke Nelson }; 61ca6cb544SLuke Nelson 62ca6cb544SLuke Nelson static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) 63ca6cb544SLuke Nelson { 64ca6cb544SLuke Nelson u8 reg = regmap[bpf_reg]; 65ca6cb544SLuke Nelson 66ca6cb544SLuke Nelson switch (reg) { 67ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S1: 68ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S2: 69ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S3: 70ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S4: 71ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S5: 72ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S6: 73ca6cb544SLuke Nelson __set_bit(reg, &ctx->flags); 74ca6cb544SLuke Nelson } 75ca6cb544SLuke Nelson return reg; 76ca6cb544SLuke Nelson }; 77ca6cb544SLuke Nelson 78ca6cb544SLuke Nelson static bool seen_reg(int reg, struct rv_jit_context *ctx) 79ca6cb544SLuke Nelson { 80ca6cb544SLuke Nelson switch (reg) { 81ca6cb544SLuke Nelson case RV_CTX_F_SEEN_CALL: 82ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S1: 83ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S2: 84ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S3: 85ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S4: 86ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S5: 87ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S6: 88ca6cb544SLuke Nelson return test_bit(reg, &ctx->flags); 89ca6cb544SLuke Nelson } 90ca6cb544SLuke Nelson return false; 91ca6cb544SLuke Nelson } 92ca6cb544SLuke Nelson 93ca6cb544SLuke Nelson static void mark_fp(struct rv_jit_context *ctx) 94ca6cb544SLuke Nelson { 95ca6cb544SLuke Nelson __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags); 96ca6cb544SLuke Nelson } 97ca6cb544SLuke Nelson 98ca6cb544SLuke Nelson static void mark_call(struct rv_jit_context *ctx) 99ca6cb544SLuke Nelson { 100ca6cb544SLuke Nelson __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 101ca6cb544SLuke Nelson } 102ca6cb544SLuke Nelson 103ca6cb544SLuke Nelson static bool seen_call(struct rv_jit_context *ctx) 104ca6cb544SLuke Nelson { 105ca6cb544SLuke Nelson return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 106ca6cb544SLuke Nelson } 107ca6cb544SLuke Nelson 108ca6cb544SLuke Nelson static void mark_tail_call(struct rv_jit_context *ctx) 109ca6cb544SLuke Nelson { 110ca6cb544SLuke Nelson __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 111ca6cb544SLuke Nelson } 112ca6cb544SLuke Nelson 113ca6cb544SLuke Nelson static bool seen_tail_call(struct rv_jit_context *ctx) 114ca6cb544SLuke Nelson { 115ca6cb544SLuke Nelson return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 116ca6cb544SLuke Nelson } 117ca6cb544SLuke Nelson 118ca6cb544SLuke Nelson static u8 rv_tail_call_reg(struct rv_jit_context *ctx) 119ca6cb544SLuke Nelson { 120ca6cb544SLuke Nelson mark_tail_call(ctx); 121ca6cb544SLuke Nelson 122ca6cb544SLuke Nelson if (seen_call(ctx)) { 123ca6cb544SLuke Nelson __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); 124ca6cb544SLuke Nelson return RV_REG_S6; 125ca6cb544SLuke Nelson } 126ca6cb544SLuke Nelson return RV_REG_A6; 127ca6cb544SLuke Nelson } 128ca6cb544SLuke Nelson 129ca6cb544SLuke Nelson static bool is_32b_int(s64 val) 130ca6cb544SLuke Nelson { 131ca6cb544SLuke Nelson return -(1L << 31) <= val && val < (1L << 31); 132ca6cb544SLuke Nelson } 133ca6cb544SLuke Nelson 134489553ddSLuke Nelson static bool in_auipc_jalr_range(s64 val) 135489553ddSLuke Nelson { 136489553ddSLuke Nelson /* 137489553ddSLuke Nelson * auipc+jalr can reach any signed PC-relative offset in the range 138489553ddSLuke Nelson * [-2^31 - 2^11, 2^31 - 2^11). 139489553ddSLuke Nelson */ 140489553ddSLuke Nelson return (-(1L << 31) - (1L << 11)) <= val && 141489553ddSLuke Nelson val < ((1L << 31) - (1L << 11)); 142489553ddSLuke Nelson } 143489553ddSLuke Nelson 144b54b6003SPu Lehui /* Emit fixed-length instructions for address */ 145b54b6003SPu Lehui static int emit_addr(u8 rd, u64 addr, bool extra_pass, struct rv_jit_context *ctx) 146b54b6003SPu Lehui { 147b54b6003SPu Lehui u64 ip = (u64)(ctx->insns + ctx->ninsns); 148b54b6003SPu Lehui s64 off = addr - ip; 149b54b6003SPu Lehui s64 upper = (off + (1 << 11)) >> 12; 150b54b6003SPu Lehui s64 lower = off & 0xfff; 151b54b6003SPu Lehui 152b54b6003SPu Lehui if (extra_pass && !in_auipc_jalr_range(off)) { 153b54b6003SPu Lehui pr_err("bpf-jit: target offset 0x%llx is out of range\n", off); 154b54b6003SPu Lehui return -ERANGE; 155b54b6003SPu Lehui } 156b54b6003SPu Lehui 157b54b6003SPu Lehui emit(rv_auipc(rd, upper), ctx); 158b54b6003SPu Lehui emit(rv_addi(rd, rd, lower), ctx); 159b54b6003SPu Lehui return 0; 160b54b6003SPu Lehui } 161b54b6003SPu Lehui 162b54b6003SPu Lehui /* Emit variable-length instructions for 32-bit and 64-bit imm */ 163ca6cb544SLuke Nelson static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx) 164ca6cb544SLuke Nelson { 165ca6cb544SLuke Nelson /* Note that the immediate from the add is sign-extended, 166ca6cb544SLuke Nelson * which means that we need to compensate this by adding 2^12, 167ca6cb544SLuke Nelson * when the 12th bit is set. A simpler way of doing this, and 168ca6cb544SLuke Nelson * getting rid of the check, is to just add 2**11 before the 169ca6cb544SLuke Nelson * shift. The "Loading a 32-Bit constant" example from the 170ca6cb544SLuke Nelson * "Computer Organization and Design, RISC-V edition" book by 171ca6cb544SLuke Nelson * Patterson/Hennessy highlights this fact. 172ca6cb544SLuke Nelson * 173ca6cb544SLuke Nelson * This also means that we need to process LSB to MSB. 174ca6cb544SLuke Nelson */ 17518a4d8c9SLuke Nelson s64 upper = (val + (1 << 11)) >> 12; 17618a4d8c9SLuke Nelson /* Sign-extend lower 12 bits to 64 bits since immediates for li, addiw, 17718a4d8c9SLuke Nelson * and addi are signed and RVC checks will perform signed comparisons. 17818a4d8c9SLuke Nelson */ 17918a4d8c9SLuke Nelson s64 lower = ((val & 0xfff) << 52) >> 52; 180ca6cb544SLuke Nelson int shift; 181ca6cb544SLuke Nelson 182ca6cb544SLuke Nelson if (is_32b_int(val)) { 183ca6cb544SLuke Nelson if (upper) 18418a4d8c9SLuke Nelson emit_lui(rd, upper, ctx); 185ca6cb544SLuke Nelson 186ca6cb544SLuke Nelson if (!upper) { 18718a4d8c9SLuke Nelson emit_li(rd, lower, ctx); 188ca6cb544SLuke Nelson return; 189ca6cb544SLuke Nelson } 190ca6cb544SLuke Nelson 19118a4d8c9SLuke Nelson emit_addiw(rd, rd, lower, ctx); 192ca6cb544SLuke Nelson return; 193ca6cb544SLuke Nelson } 194ca6cb544SLuke Nelson 195ca6cb544SLuke Nelson shift = __ffs(upper); 196ca6cb544SLuke Nelson upper >>= shift; 197ca6cb544SLuke Nelson shift += 12; 198ca6cb544SLuke Nelson 199ca6cb544SLuke Nelson emit_imm(rd, upper, ctx); 200ca6cb544SLuke Nelson 20118a4d8c9SLuke Nelson emit_slli(rd, rd, shift, ctx); 202ca6cb544SLuke Nelson if (lower) 20318a4d8c9SLuke Nelson emit_addi(rd, rd, lower, ctx); 204ca6cb544SLuke Nelson } 205ca6cb544SLuke Nelson 206ca6cb544SLuke Nelson static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) 207ca6cb544SLuke Nelson { 208ca6cb544SLuke Nelson int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; 209ca6cb544SLuke Nelson 210ca6cb544SLuke Nelson if (seen_reg(RV_REG_RA, ctx)) { 21118a4d8c9SLuke Nelson emit_ld(RV_REG_RA, store_offset, RV_REG_SP, ctx); 212ca6cb544SLuke Nelson store_offset -= 8; 213ca6cb544SLuke Nelson } 21418a4d8c9SLuke Nelson emit_ld(RV_REG_FP, store_offset, RV_REG_SP, ctx); 215ca6cb544SLuke Nelson store_offset -= 8; 216ca6cb544SLuke Nelson if (seen_reg(RV_REG_S1, ctx)) { 21718a4d8c9SLuke Nelson emit_ld(RV_REG_S1, store_offset, RV_REG_SP, ctx); 218ca6cb544SLuke Nelson store_offset -= 8; 219ca6cb544SLuke Nelson } 220ca6cb544SLuke Nelson if (seen_reg(RV_REG_S2, ctx)) { 22118a4d8c9SLuke Nelson emit_ld(RV_REG_S2, store_offset, RV_REG_SP, ctx); 222ca6cb544SLuke Nelson store_offset -= 8; 223ca6cb544SLuke Nelson } 224ca6cb544SLuke Nelson if (seen_reg(RV_REG_S3, ctx)) { 22518a4d8c9SLuke Nelson emit_ld(RV_REG_S3, store_offset, RV_REG_SP, ctx); 226ca6cb544SLuke Nelson store_offset -= 8; 227ca6cb544SLuke Nelson } 228ca6cb544SLuke Nelson if (seen_reg(RV_REG_S4, ctx)) { 22918a4d8c9SLuke Nelson emit_ld(RV_REG_S4, store_offset, RV_REG_SP, ctx); 230ca6cb544SLuke Nelson store_offset -= 8; 231ca6cb544SLuke Nelson } 232ca6cb544SLuke Nelson if (seen_reg(RV_REG_S5, ctx)) { 23318a4d8c9SLuke Nelson emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx); 234ca6cb544SLuke Nelson store_offset -= 8; 235ca6cb544SLuke Nelson } 236ca6cb544SLuke Nelson if (seen_reg(RV_REG_S6, ctx)) { 23718a4d8c9SLuke Nelson emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx); 238ca6cb544SLuke Nelson store_offset -= 8; 239ca6cb544SLuke Nelson } 240ca6cb544SLuke Nelson 24118a4d8c9SLuke Nelson emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); 242ca6cb544SLuke Nelson /* Set return value. */ 243ca6cb544SLuke Nelson if (!is_tail_call) 24418a4d8c9SLuke Nelson emit_mv(RV_REG_A0, RV_REG_A5, ctx); 24518a4d8c9SLuke Nelson emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, 24625ad1065SPu Lehui is_tail_call ? (RV_FENTRY_NINSNS + 1) * 4 : 0, /* skip reserved nops and TCC init */ 247ca6cb544SLuke Nelson ctx); 248ca6cb544SLuke Nelson } 249ca6cb544SLuke Nelson 250ca6cb544SLuke Nelson static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, 251ca6cb544SLuke Nelson struct rv_jit_context *ctx) 252ca6cb544SLuke Nelson { 253ca6cb544SLuke Nelson switch (cond) { 254ca6cb544SLuke Nelson case BPF_JEQ: 255ca6cb544SLuke Nelson emit(rv_beq(rd, rs, rvoff >> 1), ctx); 256ca6cb544SLuke Nelson return; 257ca6cb544SLuke Nelson case BPF_JGT: 258ca6cb544SLuke Nelson emit(rv_bltu(rs, rd, rvoff >> 1), ctx); 259ca6cb544SLuke Nelson return; 260ca6cb544SLuke Nelson case BPF_JLT: 261ca6cb544SLuke Nelson emit(rv_bltu(rd, rs, rvoff >> 1), ctx); 262ca6cb544SLuke Nelson return; 263ca6cb544SLuke Nelson case BPF_JGE: 264ca6cb544SLuke Nelson emit(rv_bgeu(rd, rs, rvoff >> 1), ctx); 265ca6cb544SLuke Nelson return; 266ca6cb544SLuke Nelson case BPF_JLE: 267ca6cb544SLuke Nelson emit(rv_bgeu(rs, rd, rvoff >> 1), ctx); 268ca6cb544SLuke Nelson return; 269ca6cb544SLuke Nelson case BPF_JNE: 270ca6cb544SLuke Nelson emit(rv_bne(rd, rs, rvoff >> 1), ctx); 271ca6cb544SLuke Nelson return; 272ca6cb544SLuke Nelson case BPF_JSGT: 273ca6cb544SLuke Nelson emit(rv_blt(rs, rd, rvoff >> 1), ctx); 274ca6cb544SLuke Nelson return; 275ca6cb544SLuke Nelson case BPF_JSLT: 276ca6cb544SLuke Nelson emit(rv_blt(rd, rs, rvoff >> 1), ctx); 277ca6cb544SLuke Nelson return; 278ca6cb544SLuke Nelson case BPF_JSGE: 279ca6cb544SLuke Nelson emit(rv_bge(rd, rs, rvoff >> 1), ctx); 280ca6cb544SLuke Nelson return; 281ca6cb544SLuke Nelson case BPF_JSLE: 282ca6cb544SLuke Nelson emit(rv_bge(rs, rd, rvoff >> 1), ctx); 283ca6cb544SLuke Nelson } 284ca6cb544SLuke Nelson } 285ca6cb544SLuke Nelson 286ca6cb544SLuke Nelson static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, 287ca6cb544SLuke Nelson struct rv_jit_context *ctx) 288ca6cb544SLuke Nelson { 289ca6cb544SLuke Nelson s64 upper, lower; 290ca6cb544SLuke Nelson 291ca6cb544SLuke Nelson if (is_13b_int(rvoff)) { 292ca6cb544SLuke Nelson emit_bcc(cond, rd, rs, rvoff, ctx); 293ca6cb544SLuke Nelson return; 294ca6cb544SLuke Nelson } 295ca6cb544SLuke Nelson 296ca6cb544SLuke Nelson /* Adjust for jal */ 297ca6cb544SLuke Nelson rvoff -= 4; 298ca6cb544SLuke Nelson 299ca6cb544SLuke Nelson /* Transform, e.g.: 300ca6cb544SLuke Nelson * bne rd,rs,foo 301ca6cb544SLuke Nelson * to 302ca6cb544SLuke Nelson * beq rd,rs,<.L1> 303ca6cb544SLuke Nelson * (auipc foo) 304ca6cb544SLuke Nelson * jal(r) foo 305ca6cb544SLuke Nelson * .L1 306ca6cb544SLuke Nelson */ 307ca6cb544SLuke Nelson cond = invert_bpf_cond(cond); 308ca6cb544SLuke Nelson if (is_21b_int(rvoff)) { 309ca6cb544SLuke Nelson emit_bcc(cond, rd, rs, 8, ctx); 310ca6cb544SLuke Nelson emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx); 311ca6cb544SLuke Nelson return; 312ca6cb544SLuke Nelson } 313ca6cb544SLuke Nelson 314ca6cb544SLuke Nelson /* 32b No need for an additional rvoff adjustment, since we 315ca6cb544SLuke Nelson * get that from the auipc at PC', where PC = PC' + 4. 316ca6cb544SLuke Nelson */ 317ca6cb544SLuke Nelson upper = (rvoff + (1 << 11)) >> 12; 318ca6cb544SLuke Nelson lower = rvoff & 0xfff; 319ca6cb544SLuke Nelson 320ca6cb544SLuke Nelson emit_bcc(cond, rd, rs, 12, ctx); 321ca6cb544SLuke Nelson emit(rv_auipc(RV_REG_T1, upper), ctx); 322ca6cb544SLuke Nelson emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx); 323ca6cb544SLuke Nelson } 324ca6cb544SLuke Nelson 325ca6cb544SLuke Nelson static void emit_zext_32(u8 reg, struct rv_jit_context *ctx) 326ca6cb544SLuke Nelson { 32718a4d8c9SLuke Nelson emit_slli(reg, reg, 32, ctx); 32818a4d8c9SLuke Nelson emit_srli(reg, reg, 32, ctx); 329ca6cb544SLuke Nelson } 330ca6cb544SLuke Nelson 331ca6cb544SLuke Nelson static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) 332ca6cb544SLuke Nelson { 333ca6cb544SLuke Nelson int tc_ninsn, off, start_insn = ctx->ninsns; 334ca6cb544SLuke Nelson u8 tcc = rv_tail_call_reg(ctx); 335ca6cb544SLuke Nelson 336ca6cb544SLuke Nelson /* a0: &ctx 337ca6cb544SLuke Nelson * a1: &array 338ca6cb544SLuke Nelson * a2: index 339ca6cb544SLuke Nelson * 340ca6cb544SLuke Nelson * if (index >= array->map.max_entries) 341ca6cb544SLuke Nelson * goto out; 342ca6cb544SLuke Nelson */ 343ca6cb544SLuke Nelson tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] : 344ca6cb544SLuke Nelson ctx->offset[0]; 345ca6cb544SLuke Nelson emit_zext_32(RV_REG_A2, ctx); 346ca6cb544SLuke Nelson 347ca6cb544SLuke Nelson off = offsetof(struct bpf_array, map.max_entries); 348ca6cb544SLuke Nelson if (is_12b_check(off, insn)) 349ca6cb544SLuke Nelson return -1; 350ca6cb544SLuke Nelson emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx); 351bfabff3cSLuke Nelson off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 352ca6cb544SLuke Nelson emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx); 353ca6cb544SLuke Nelson 354ebf7f6f0STiezhu Yang /* if (--TCC < 0) 355ca6cb544SLuke Nelson * goto out; 356ca6cb544SLuke Nelson */ 357ebf7f6f0STiezhu Yang emit_addi(RV_REG_TCC, tcc, -1, ctx); 358bfabff3cSLuke Nelson off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 359ebf7f6f0STiezhu Yang emit_branch(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); 360ca6cb544SLuke Nelson 361ca6cb544SLuke Nelson /* prog = array->ptrs[index]; 362ca6cb544SLuke Nelson * if (!prog) 363ca6cb544SLuke Nelson * goto out; 364ca6cb544SLuke Nelson */ 36518a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_A2, 3, ctx); 36618a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_A1, ctx); 367ca6cb544SLuke Nelson off = offsetof(struct bpf_array, ptrs); 368ca6cb544SLuke Nelson if (is_12b_check(off, insn)) 369ca6cb544SLuke Nelson return -1; 37018a4d8c9SLuke Nelson emit_ld(RV_REG_T2, off, RV_REG_T2, ctx); 371bfabff3cSLuke Nelson off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 372ca6cb544SLuke Nelson emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); 373ca6cb544SLuke Nelson 374ca6cb544SLuke Nelson /* goto *(prog->bpf_func + 4); */ 375ca6cb544SLuke Nelson off = offsetof(struct bpf_prog, bpf_func); 376ca6cb544SLuke Nelson if (is_12b_check(off, insn)) 377ca6cb544SLuke Nelson return -1; 37818a4d8c9SLuke Nelson emit_ld(RV_REG_T3, off, RV_REG_T2, ctx); 379ca6cb544SLuke Nelson __build_epilogue(true, ctx); 380ca6cb544SLuke Nelson return 0; 381ca6cb544SLuke Nelson } 382ca6cb544SLuke Nelson 383ca6cb544SLuke Nelson static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn, 384ca6cb544SLuke Nelson struct rv_jit_context *ctx) 385ca6cb544SLuke Nelson { 386ca6cb544SLuke Nelson u8 code = insn->code; 387ca6cb544SLuke Nelson 388ca6cb544SLuke Nelson switch (code) { 389ca6cb544SLuke Nelson case BPF_JMP | BPF_JA: 390ca6cb544SLuke Nelson case BPF_JMP | BPF_CALL: 391ca6cb544SLuke Nelson case BPF_JMP | BPF_EXIT: 392ca6cb544SLuke Nelson case BPF_JMP | BPF_TAIL_CALL: 393ca6cb544SLuke Nelson break; 394ca6cb544SLuke Nelson default: 395ca6cb544SLuke Nelson *rd = bpf_to_rv_reg(insn->dst_reg, ctx); 396ca6cb544SLuke Nelson } 397ca6cb544SLuke Nelson 398ca6cb544SLuke Nelson if (code & (BPF_ALU | BPF_X) || code & (BPF_ALU64 | BPF_X) || 399ca6cb544SLuke Nelson code & (BPF_JMP | BPF_X) || code & (BPF_JMP32 | BPF_X) || 400ca6cb544SLuke Nelson code & BPF_LDX || code & BPF_STX) 401ca6cb544SLuke Nelson *rs = bpf_to_rv_reg(insn->src_reg, ctx); 402ca6cb544SLuke Nelson } 403ca6cb544SLuke Nelson 404ca6cb544SLuke Nelson static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) 405ca6cb544SLuke Nelson { 40618a4d8c9SLuke Nelson emit_mv(RV_REG_T2, *rd, ctx); 407ca6cb544SLuke Nelson emit_zext_32(RV_REG_T2, ctx); 40818a4d8c9SLuke Nelson emit_mv(RV_REG_T1, *rs, ctx); 409ca6cb544SLuke Nelson emit_zext_32(RV_REG_T1, ctx); 410ca6cb544SLuke Nelson *rd = RV_REG_T2; 411ca6cb544SLuke Nelson *rs = RV_REG_T1; 412ca6cb544SLuke Nelson } 413ca6cb544SLuke Nelson 414ca6cb544SLuke Nelson static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) 415ca6cb544SLuke Nelson { 41618a4d8c9SLuke Nelson emit_addiw(RV_REG_T2, *rd, 0, ctx); 41718a4d8c9SLuke Nelson emit_addiw(RV_REG_T1, *rs, 0, ctx); 418ca6cb544SLuke Nelson *rd = RV_REG_T2; 419ca6cb544SLuke Nelson *rs = RV_REG_T1; 420ca6cb544SLuke Nelson } 421ca6cb544SLuke Nelson 422ca6cb544SLuke Nelson static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx) 423ca6cb544SLuke Nelson { 42418a4d8c9SLuke Nelson emit_mv(RV_REG_T2, *rd, ctx); 425ca6cb544SLuke Nelson emit_zext_32(RV_REG_T2, ctx); 426ca6cb544SLuke Nelson emit_zext_32(RV_REG_T1, ctx); 427ca6cb544SLuke Nelson *rd = RV_REG_T2; 428ca6cb544SLuke Nelson } 429ca6cb544SLuke Nelson 430ca6cb544SLuke Nelson static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx) 431ca6cb544SLuke Nelson { 43218a4d8c9SLuke Nelson emit_addiw(RV_REG_T2, *rd, 0, ctx); 433ca6cb544SLuke Nelson *rd = RV_REG_T2; 434ca6cb544SLuke Nelson } 435ca6cb544SLuke Nelson 4360fd1fd01SPu Lehui static int emit_jump_and_link(u8 rd, s64 rvoff, bool fixed_addr, 437ca6cb544SLuke Nelson struct rv_jit_context *ctx) 438ca6cb544SLuke Nelson { 439ca6cb544SLuke Nelson s64 upper, lower; 440ca6cb544SLuke Nelson 4410fd1fd01SPu Lehui if (rvoff && fixed_addr && is_21b_int(rvoff)) { 442ca6cb544SLuke Nelson emit(rv_jal(rd, rvoff >> 1), ctx); 443489553ddSLuke Nelson return 0; 444489553ddSLuke Nelson } else if (in_auipc_jalr_range(rvoff)) { 445ca6cb544SLuke Nelson upper = (rvoff + (1 << 11)) >> 12; 446ca6cb544SLuke Nelson lower = rvoff & 0xfff; 447ca6cb544SLuke Nelson emit(rv_auipc(RV_REG_T1, upper), ctx); 448ca6cb544SLuke Nelson emit(rv_jalr(rd, RV_REG_T1, lower), ctx); 449489553ddSLuke Nelson return 0; 450489553ddSLuke Nelson } 451489553ddSLuke Nelson 452489553ddSLuke Nelson pr_err("bpf-jit: target offset 0x%llx is out of range\n", rvoff); 453489553ddSLuke Nelson return -ERANGE; 454ca6cb544SLuke Nelson } 455ca6cb544SLuke Nelson 456ca6cb544SLuke Nelson static bool is_signed_bpf_cond(u8 cond) 457ca6cb544SLuke Nelson { 458ca6cb544SLuke Nelson return cond == BPF_JSGT || cond == BPF_JSLT || 459ca6cb544SLuke Nelson cond == BPF_JSGE || cond == BPF_JSLE; 460ca6cb544SLuke Nelson } 461ca6cb544SLuke Nelson 4620fd1fd01SPu Lehui static int emit_call(u64 addr, bool fixed_addr, struct rv_jit_context *ctx) 463ca6cb544SLuke Nelson { 464ca6cb544SLuke Nelson s64 off = 0; 465ca6cb544SLuke Nelson u64 ip; 466ca6cb544SLuke Nelson 467ca6cb544SLuke Nelson if (addr && ctx->insns) { 468ca6cb544SLuke Nelson ip = (u64)(long)(ctx->insns + ctx->ninsns); 469ca6cb544SLuke Nelson off = addr - ip; 470ca6cb544SLuke Nelson } 471ca6cb544SLuke Nelson 4720fd1fd01SPu Lehui return emit_jump_and_link(RV_REG_RA, off, fixed_addr, ctx); 473ca6cb544SLuke Nelson } 474ca6cb544SLuke Nelson 475dd642ccbSPu Lehui static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64, 476dd642ccbSPu Lehui struct rv_jit_context *ctx) 477dd642ccbSPu Lehui { 478dd642ccbSPu Lehui u8 r0; 479dd642ccbSPu Lehui int jmp_offset; 480dd642ccbSPu Lehui 481dd642ccbSPu Lehui if (off) { 482dd642ccbSPu Lehui if (is_12b_int(off)) { 483dd642ccbSPu Lehui emit_addi(RV_REG_T1, rd, off, ctx); 484dd642ccbSPu Lehui } else { 485dd642ccbSPu Lehui emit_imm(RV_REG_T1, off, ctx); 486dd642ccbSPu Lehui emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 487dd642ccbSPu Lehui } 488dd642ccbSPu Lehui rd = RV_REG_T1; 489dd642ccbSPu Lehui } 490dd642ccbSPu Lehui 491dd642ccbSPu Lehui switch (imm) { 492dd642ccbSPu Lehui /* lock *(u32/u64 *)(dst_reg + off16) <op>= src_reg */ 493dd642ccbSPu Lehui case BPF_ADD: 494dd642ccbSPu Lehui emit(is64 ? rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0) : 495dd642ccbSPu Lehui rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 496dd642ccbSPu Lehui break; 497dd642ccbSPu Lehui case BPF_AND: 498dd642ccbSPu Lehui emit(is64 ? rv_amoand_d(RV_REG_ZERO, rs, rd, 0, 0) : 499dd642ccbSPu Lehui rv_amoand_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 500dd642ccbSPu Lehui break; 501dd642ccbSPu Lehui case BPF_OR: 502dd642ccbSPu Lehui emit(is64 ? rv_amoor_d(RV_REG_ZERO, rs, rd, 0, 0) : 503dd642ccbSPu Lehui rv_amoor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 504dd642ccbSPu Lehui break; 505dd642ccbSPu Lehui case BPF_XOR: 506dd642ccbSPu Lehui emit(is64 ? rv_amoxor_d(RV_REG_ZERO, rs, rd, 0, 0) : 507dd642ccbSPu Lehui rv_amoxor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 508dd642ccbSPu Lehui break; 509dd642ccbSPu Lehui /* src_reg = atomic_fetch_<op>(dst_reg + off16, src_reg) */ 510dd642ccbSPu Lehui case BPF_ADD | BPF_FETCH: 511dd642ccbSPu Lehui emit(is64 ? rv_amoadd_d(rs, rs, rd, 0, 0) : 512dd642ccbSPu Lehui rv_amoadd_w(rs, rs, rd, 0, 0), ctx); 513dd642ccbSPu Lehui if (!is64) 514dd642ccbSPu Lehui emit_zext_32(rs, ctx); 515dd642ccbSPu Lehui break; 516dd642ccbSPu Lehui case BPF_AND | BPF_FETCH: 517dd642ccbSPu Lehui emit(is64 ? rv_amoand_d(rs, rs, rd, 0, 0) : 518dd642ccbSPu Lehui rv_amoand_w(rs, rs, rd, 0, 0), ctx); 519dd642ccbSPu Lehui if (!is64) 520dd642ccbSPu Lehui emit_zext_32(rs, ctx); 521dd642ccbSPu Lehui break; 522dd642ccbSPu Lehui case BPF_OR | BPF_FETCH: 523dd642ccbSPu Lehui emit(is64 ? rv_amoor_d(rs, rs, rd, 0, 0) : 524dd642ccbSPu Lehui rv_amoor_w(rs, rs, rd, 0, 0), ctx); 525dd642ccbSPu Lehui if (!is64) 526dd642ccbSPu Lehui emit_zext_32(rs, ctx); 527dd642ccbSPu Lehui break; 528dd642ccbSPu Lehui case BPF_XOR | BPF_FETCH: 529dd642ccbSPu Lehui emit(is64 ? rv_amoxor_d(rs, rs, rd, 0, 0) : 530dd642ccbSPu Lehui rv_amoxor_w(rs, rs, rd, 0, 0), ctx); 531dd642ccbSPu Lehui if (!is64) 532dd642ccbSPu Lehui emit_zext_32(rs, ctx); 533dd642ccbSPu Lehui break; 534dd642ccbSPu Lehui /* src_reg = atomic_xchg(dst_reg + off16, src_reg); */ 535dd642ccbSPu Lehui case BPF_XCHG: 536dd642ccbSPu Lehui emit(is64 ? rv_amoswap_d(rs, rs, rd, 0, 0) : 537dd642ccbSPu Lehui rv_amoswap_w(rs, rs, rd, 0, 0), ctx); 538dd642ccbSPu Lehui if (!is64) 539dd642ccbSPu Lehui emit_zext_32(rs, ctx); 540dd642ccbSPu Lehui break; 541dd642ccbSPu Lehui /* r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg); */ 542dd642ccbSPu Lehui case BPF_CMPXCHG: 543dd642ccbSPu Lehui r0 = bpf_to_rv_reg(BPF_REG_0, ctx); 544dd642ccbSPu Lehui emit(is64 ? rv_addi(RV_REG_T2, r0, 0) : 545dd642ccbSPu Lehui rv_addiw(RV_REG_T2, r0, 0), ctx); 546dd642ccbSPu Lehui emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) : 547dd642ccbSPu Lehui rv_lr_w(r0, 0, rd, 0, 0), ctx); 548dd642ccbSPu Lehui jmp_offset = ninsns_rvoff(8); 549dd642ccbSPu Lehui emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx); 550dd642ccbSPu Lehui emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 0) : 551dd642ccbSPu Lehui rv_sc_w(RV_REG_T3, rs, rd, 0, 0), ctx); 552dd642ccbSPu Lehui jmp_offset = ninsns_rvoff(-6); 553dd642ccbSPu Lehui emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx); 554dd642ccbSPu Lehui emit(rv_fence(0x3, 0x3), ctx); 555dd642ccbSPu Lehui break; 556dd642ccbSPu Lehui } 557dd642ccbSPu Lehui } 558dd642ccbSPu Lehui 559252c765bSTong Tiangen #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) 560252c765bSTong Tiangen #define BPF_FIXUP_REG_MASK GENMASK(31, 27) 561252c765bSTong Tiangen 5622bf847dbSJisheng Zhang bool ex_handler_bpf(const struct exception_table_entry *ex, 563252c765bSTong Tiangen struct pt_regs *regs) 564252c765bSTong Tiangen { 565252c765bSTong Tiangen off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); 566252c765bSTong Tiangen int regs_offset = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); 567252c765bSTong Tiangen 568252c765bSTong Tiangen *(unsigned long *)((void *)regs + pt_regmap[regs_offset]) = 0; 569252c765bSTong Tiangen regs->epc = (unsigned long)&ex->fixup - offset; 570252c765bSTong Tiangen 571ef127bcaSJisheng Zhang return true; 572252c765bSTong Tiangen } 573252c765bSTong Tiangen 574252c765bSTong Tiangen /* For accesses to BTF pointers, add an entry to the exception table */ 575252c765bSTong Tiangen static int add_exception_handler(const struct bpf_insn *insn, 576252c765bSTong Tiangen struct rv_jit_context *ctx, 577252c765bSTong Tiangen int dst_reg, int insn_len) 578252c765bSTong Tiangen { 579252c765bSTong Tiangen struct exception_table_entry *ex; 580252c765bSTong Tiangen unsigned long pc; 581252c765bSTong Tiangen off_t offset; 582252c765bSTong Tiangen 5833d06d816SPu Lehui if (!ctx->insns || !ctx->prog->aux->extable || 5843d06d816SPu Lehui (BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX)) 585252c765bSTong Tiangen return 0; 586252c765bSTong Tiangen 587252c765bSTong Tiangen if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries)) 588252c765bSTong Tiangen return -EINVAL; 589252c765bSTong Tiangen 590252c765bSTong Tiangen if (WARN_ON_ONCE(insn_len > ctx->ninsns)) 591252c765bSTong Tiangen return -EINVAL; 592252c765bSTong Tiangen 593252c765bSTong Tiangen if (WARN_ON_ONCE(!rvc_enabled() && insn_len == 1)) 594252c765bSTong Tiangen return -EINVAL; 595252c765bSTong Tiangen 596252c765bSTong Tiangen ex = &ctx->prog->aux->extable[ctx->nexentries]; 597252c765bSTong Tiangen pc = (unsigned long)&ctx->insns[ctx->ninsns - insn_len]; 598252c765bSTong Tiangen 599252c765bSTong Tiangen offset = pc - (long)&ex->insn; 600252c765bSTong Tiangen if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 601252c765bSTong Tiangen return -ERANGE; 602fc839c6dSJisheng Zhang ex->insn = offset; 603252c765bSTong Tiangen 604252c765bSTong Tiangen /* 605252c765bSTong Tiangen * Since the extable follows the program, the fixup offset is always 606252c765bSTong Tiangen * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value 607252c765bSTong Tiangen * to keep things simple, and put the destination register in the upper 608252c765bSTong Tiangen * bits. We don't need to worry about buildtime or runtime sort 609252c765bSTong Tiangen * modifying the upper bits because the table is already sorted, and 610252c765bSTong Tiangen * isn't part of the main exception table. 611252c765bSTong Tiangen */ 612252c765bSTong Tiangen offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16)); 613252c765bSTong Tiangen if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset)) 614252c765bSTong Tiangen return -ERANGE; 615252c765bSTong Tiangen 616252c765bSTong Tiangen ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | 617252c765bSTong Tiangen FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); 6182bf847dbSJisheng Zhang ex->type = EX_TYPE_BPF; 619252c765bSTong Tiangen 620252c765bSTong Tiangen ctx->nexentries++; 621252c765bSTong Tiangen return 0; 622252c765bSTong Tiangen } 623252c765bSTong Tiangen 62425ad1065SPu Lehui static int gen_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call) 625596f2e6fSPu Lehui { 626596f2e6fSPu Lehui s64 rvoff; 627596f2e6fSPu Lehui struct rv_jit_context ctx; 628596f2e6fSPu Lehui 629596f2e6fSPu Lehui ctx.ninsns = 0; 630596f2e6fSPu Lehui ctx.insns = (u16 *)insns; 631596f2e6fSPu Lehui 632596f2e6fSPu Lehui if (!target) { 633596f2e6fSPu Lehui emit(rv_nop(), &ctx); 634596f2e6fSPu Lehui emit(rv_nop(), &ctx); 635596f2e6fSPu Lehui return 0; 636596f2e6fSPu Lehui } 637596f2e6fSPu Lehui 638596f2e6fSPu Lehui rvoff = (s64)(target - ip); 63925ad1065SPu Lehui return emit_jump_and_link(is_call ? RV_REG_T0 : RV_REG_ZERO, rvoff, false, &ctx); 640596f2e6fSPu Lehui } 641596f2e6fSPu Lehui 642596f2e6fSPu Lehui int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, 643596f2e6fSPu Lehui void *old_addr, void *new_addr) 644596f2e6fSPu Lehui { 64525ad1065SPu Lehui u32 old_insns[RV_FENTRY_NINSNS], new_insns[RV_FENTRY_NINSNS]; 646596f2e6fSPu Lehui bool is_call = poke_type == BPF_MOD_CALL; 647596f2e6fSPu Lehui int ret; 648596f2e6fSPu Lehui 64925ad1065SPu Lehui if (!is_kernel_text((unsigned long)ip) && 65025ad1065SPu Lehui !is_bpf_text_address((unsigned long)ip)) 651596f2e6fSPu Lehui return -ENOTSUPP; 652596f2e6fSPu Lehui 65325ad1065SPu Lehui ret = gen_jump_or_nops(old_addr, ip, old_insns, is_call); 654596f2e6fSPu Lehui if (ret) 655596f2e6fSPu Lehui return ret; 656596f2e6fSPu Lehui 65725ad1065SPu Lehui if (memcmp(ip, old_insns, RV_FENTRY_NINSNS * 4)) 658596f2e6fSPu Lehui return -EFAULT; 659596f2e6fSPu Lehui 66025ad1065SPu Lehui ret = gen_jump_or_nops(new_addr, ip, new_insns, is_call); 661596f2e6fSPu Lehui if (ret) 662596f2e6fSPu Lehui return ret; 663596f2e6fSPu Lehui 664596f2e6fSPu Lehui cpus_read_lock(); 665596f2e6fSPu Lehui mutex_lock(&text_mutex); 66625ad1065SPu Lehui if (memcmp(ip, new_insns, RV_FENTRY_NINSNS * 4)) 66725ad1065SPu Lehui ret = patch_text(ip, new_insns, RV_FENTRY_NINSNS); 668596f2e6fSPu Lehui mutex_unlock(&text_mutex); 669596f2e6fSPu Lehui cpus_read_unlock(); 670596f2e6fSPu Lehui 671596f2e6fSPu Lehui return ret; 672596f2e6fSPu Lehui } 673596f2e6fSPu Lehui 67449b5e77aSPu Lehui static void store_args(int nregs, int args_off, struct rv_jit_context *ctx) 67549b5e77aSPu Lehui { 67649b5e77aSPu Lehui int i; 67749b5e77aSPu Lehui 67849b5e77aSPu Lehui for (i = 0; i < nregs; i++) { 67949b5e77aSPu Lehui emit_sd(RV_REG_FP, -args_off, RV_REG_A0 + i, ctx); 68049b5e77aSPu Lehui args_off -= 8; 68149b5e77aSPu Lehui } 68249b5e77aSPu Lehui } 68349b5e77aSPu Lehui 68449b5e77aSPu Lehui static void restore_args(int nregs, int args_off, struct rv_jit_context *ctx) 68549b5e77aSPu Lehui { 68649b5e77aSPu Lehui int i; 68749b5e77aSPu Lehui 68849b5e77aSPu Lehui for (i = 0; i < nregs; i++) { 68949b5e77aSPu Lehui emit_ld(RV_REG_A0 + i, -args_off, RV_REG_FP, ctx); 69049b5e77aSPu Lehui args_off -= 8; 69149b5e77aSPu Lehui } 69249b5e77aSPu Lehui } 69349b5e77aSPu Lehui 69449b5e77aSPu Lehui static int invoke_bpf_prog(struct bpf_tramp_link *l, int args_off, int retval_off, 69549b5e77aSPu Lehui int run_ctx_off, bool save_ret, struct rv_jit_context *ctx) 69649b5e77aSPu Lehui { 69749b5e77aSPu Lehui int ret, branch_off; 69849b5e77aSPu Lehui struct bpf_prog *p = l->link.prog; 69949b5e77aSPu Lehui int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); 70049b5e77aSPu Lehui 70149b5e77aSPu Lehui if (l->cookie) { 70249b5e77aSPu Lehui emit_imm(RV_REG_T1, l->cookie, ctx); 70349b5e77aSPu Lehui emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_T1, ctx); 70449b5e77aSPu Lehui } else { 70549b5e77aSPu Lehui emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_ZERO, ctx); 70649b5e77aSPu Lehui } 70749b5e77aSPu Lehui 70849b5e77aSPu Lehui /* arg1: prog */ 70949b5e77aSPu Lehui emit_imm(RV_REG_A0, (const s64)p, ctx); 71049b5e77aSPu Lehui /* arg2: &run_ctx */ 71149b5e77aSPu Lehui emit_addi(RV_REG_A1, RV_REG_FP, -run_ctx_off, ctx); 71249b5e77aSPu Lehui ret = emit_call((const u64)bpf_trampoline_enter(p), true, ctx); 71349b5e77aSPu Lehui if (ret) 71449b5e77aSPu Lehui return ret; 71549b5e77aSPu Lehui 71649b5e77aSPu Lehui /* if (__bpf_prog_enter(prog) == 0) 71749b5e77aSPu Lehui * goto skip_exec_of_prog; 71849b5e77aSPu Lehui */ 71949b5e77aSPu Lehui branch_off = ctx->ninsns; 72049b5e77aSPu Lehui /* nop reserved for conditional jump */ 72149b5e77aSPu Lehui emit(rv_nop(), ctx); 72249b5e77aSPu Lehui 72349b5e77aSPu Lehui /* store prog start time */ 72449b5e77aSPu Lehui emit_mv(RV_REG_S1, RV_REG_A0, ctx); 72549b5e77aSPu Lehui 72649b5e77aSPu Lehui /* arg1: &args_off */ 72749b5e77aSPu Lehui emit_addi(RV_REG_A0, RV_REG_FP, -args_off, ctx); 72849b5e77aSPu Lehui if (!p->jited) 72949b5e77aSPu Lehui /* arg2: progs[i]->insnsi for interpreter */ 73049b5e77aSPu Lehui emit_imm(RV_REG_A1, (const s64)p->insnsi, ctx); 73149b5e77aSPu Lehui ret = emit_call((const u64)p->bpf_func, true, ctx); 73249b5e77aSPu Lehui if (ret) 73349b5e77aSPu Lehui return ret; 73449b5e77aSPu Lehui 73549b5e77aSPu Lehui if (save_ret) 73649b5e77aSPu Lehui emit_sd(RV_REG_FP, -retval_off, regmap[BPF_REG_0], ctx); 73749b5e77aSPu Lehui 73849b5e77aSPu Lehui /* update branch with beqz */ 73949b5e77aSPu Lehui if (ctx->insns) { 74049b5e77aSPu Lehui int offset = ninsns_rvoff(ctx->ninsns - branch_off); 74149b5e77aSPu Lehui u32 insn = rv_beq(RV_REG_A0, RV_REG_ZERO, offset >> 1); 74249b5e77aSPu Lehui *(u32 *)(ctx->insns + branch_off) = insn; 74349b5e77aSPu Lehui } 74449b5e77aSPu Lehui 74549b5e77aSPu Lehui /* arg1: prog */ 74649b5e77aSPu Lehui emit_imm(RV_REG_A0, (const s64)p, ctx); 74749b5e77aSPu Lehui /* arg2: prog start time */ 74849b5e77aSPu Lehui emit_mv(RV_REG_A1, RV_REG_S1, ctx); 74949b5e77aSPu Lehui /* arg3: &run_ctx */ 75049b5e77aSPu Lehui emit_addi(RV_REG_A2, RV_REG_FP, -run_ctx_off, ctx); 75149b5e77aSPu Lehui ret = emit_call((const u64)bpf_trampoline_exit(p), true, ctx); 75249b5e77aSPu Lehui 75349b5e77aSPu Lehui return ret; 75449b5e77aSPu Lehui } 75549b5e77aSPu Lehui 75649b5e77aSPu Lehui static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, 75749b5e77aSPu Lehui const struct btf_func_model *m, 75849b5e77aSPu Lehui struct bpf_tramp_links *tlinks, 75949b5e77aSPu Lehui void *func_addr, u32 flags, 76049b5e77aSPu Lehui struct rv_jit_context *ctx) 76149b5e77aSPu Lehui { 76249b5e77aSPu Lehui int i, ret, offset; 76349b5e77aSPu Lehui int *branches_off = NULL; 76449b5e77aSPu Lehui int stack_size = 0, nregs = m->nr_args; 76525ad1065SPu Lehui int retval_off, args_off, nregs_off, ip_off, run_ctx_off, sreg_off; 76649b5e77aSPu Lehui struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; 76749b5e77aSPu Lehui struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; 76849b5e77aSPu Lehui struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; 76949b5e77aSPu Lehui void *orig_call = func_addr; 77049b5e77aSPu Lehui bool save_ret; 77149b5e77aSPu Lehui u32 insn; 77249b5e77aSPu Lehui 77325ad1065SPu Lehui /* Two types of generated trampoline stack layout: 77449b5e77aSPu Lehui * 77525ad1065SPu Lehui * 1. trampoline called from function entry 77625ad1065SPu Lehui * -------------------------------------- 77725ad1065SPu Lehui * FP + 8 [ RA to parent func ] return address to parent 77849b5e77aSPu Lehui * function 77925ad1065SPu Lehui * FP + 0 [ FP of parent func ] frame pointer of parent 78049b5e77aSPu Lehui * function 78125ad1065SPu Lehui * FP - 8 [ T0 to traced func ] return address of traced 78225ad1065SPu Lehui * function 78325ad1065SPu Lehui * FP - 16 [ FP of traced func ] frame pointer of traced 78425ad1065SPu Lehui * function 78525ad1065SPu Lehui * -------------------------------------- 78625ad1065SPu Lehui * 78725ad1065SPu Lehui * 2. trampoline called directly 78825ad1065SPu Lehui * -------------------------------------- 78925ad1065SPu Lehui * FP - 8 [ RA to caller func ] return address to caller 79025ad1065SPu Lehui * function 79125ad1065SPu Lehui * FP - 16 [ FP of caller func ] frame pointer of caller 79225ad1065SPu Lehui * function 79325ad1065SPu Lehui * -------------------------------------- 79449b5e77aSPu Lehui * 79549b5e77aSPu Lehui * FP - retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or 79649b5e77aSPu Lehui * BPF_TRAMP_F_RET_FENTRY_RET 79749b5e77aSPu Lehui * [ argN ] 79849b5e77aSPu Lehui * [ ... ] 79949b5e77aSPu Lehui * FP - args_off [ arg1 ] 80049b5e77aSPu Lehui * 80149b5e77aSPu Lehui * FP - nregs_off [ regs count ] 80249b5e77aSPu Lehui * 80349b5e77aSPu Lehui * FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG 80449b5e77aSPu Lehui * 80549b5e77aSPu Lehui * FP - run_ctx_off [ bpf_tramp_run_ctx ] 80649b5e77aSPu Lehui * 80749b5e77aSPu Lehui * FP - sreg_off [ callee saved reg ] 80849b5e77aSPu Lehui * 80949b5e77aSPu Lehui * [ pads ] pads for 16 bytes alignment 81049b5e77aSPu Lehui */ 81149b5e77aSPu Lehui 81249b5e77aSPu Lehui if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY)) 81349b5e77aSPu Lehui return -ENOTSUPP; 81449b5e77aSPu Lehui 81549b5e77aSPu Lehui /* extra regiters for struct arguments */ 81649b5e77aSPu Lehui for (i = 0; i < m->nr_args; i++) 81749b5e77aSPu Lehui if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) 81849b5e77aSPu Lehui nregs += round_up(m->arg_size[i], 8) / 8 - 1; 81949b5e77aSPu Lehui 82049b5e77aSPu Lehui /* 8 arguments passed by registers */ 82149b5e77aSPu Lehui if (nregs > 8) 82249b5e77aSPu Lehui return -ENOTSUPP; 82349b5e77aSPu Lehui 82425ad1065SPu Lehui /* room of trampoline frame to store return address and frame pointer */ 82525ad1065SPu Lehui stack_size += 16; 82649b5e77aSPu Lehui 82749b5e77aSPu Lehui save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); 82849b5e77aSPu Lehui if (save_ret) { 82949b5e77aSPu Lehui stack_size += 8; 83049b5e77aSPu Lehui retval_off = stack_size; 83149b5e77aSPu Lehui } 83249b5e77aSPu Lehui 83349b5e77aSPu Lehui stack_size += nregs * 8; 83449b5e77aSPu Lehui args_off = stack_size; 83549b5e77aSPu Lehui 83649b5e77aSPu Lehui stack_size += 8; 83749b5e77aSPu Lehui nregs_off = stack_size; 83849b5e77aSPu Lehui 83949b5e77aSPu Lehui if (flags & BPF_TRAMP_F_IP_ARG) { 84049b5e77aSPu Lehui stack_size += 8; 84149b5e77aSPu Lehui ip_off = stack_size; 84249b5e77aSPu Lehui } 84349b5e77aSPu Lehui 84449b5e77aSPu Lehui stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8); 84549b5e77aSPu Lehui run_ctx_off = stack_size; 84649b5e77aSPu Lehui 84749b5e77aSPu Lehui stack_size += 8; 84849b5e77aSPu Lehui sreg_off = stack_size; 84949b5e77aSPu Lehui 85049b5e77aSPu Lehui stack_size = round_up(stack_size, 16); 85149b5e77aSPu Lehui 85225ad1065SPu Lehui if (func_addr) { 85325ad1065SPu Lehui /* For the trampoline called from function entry, 85425ad1065SPu Lehui * the frame of traced function and the frame of 85525ad1065SPu Lehui * trampoline need to be considered. 85625ad1065SPu Lehui */ 85725ad1065SPu Lehui emit_addi(RV_REG_SP, RV_REG_SP, -16, ctx); 85825ad1065SPu Lehui emit_sd(RV_REG_SP, 8, RV_REG_RA, ctx); 85925ad1065SPu Lehui emit_sd(RV_REG_SP, 0, RV_REG_FP, ctx); 86025ad1065SPu Lehui emit_addi(RV_REG_FP, RV_REG_SP, 16, ctx); 86125ad1065SPu Lehui 86249b5e77aSPu Lehui emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx); 86325ad1065SPu Lehui emit_sd(RV_REG_SP, stack_size - 8, RV_REG_T0, ctx); 86425ad1065SPu Lehui emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx); 86549b5e77aSPu Lehui emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx); 86625ad1065SPu Lehui } else { 86725ad1065SPu Lehui /* For the trampoline called directly, just handle 86825ad1065SPu Lehui * the frame of trampoline. 86925ad1065SPu Lehui */ 87025ad1065SPu Lehui emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx); 87125ad1065SPu Lehui emit_sd(RV_REG_SP, stack_size - 8, RV_REG_RA, ctx); 87225ad1065SPu Lehui emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx); 87325ad1065SPu Lehui emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx); 87425ad1065SPu Lehui } 87549b5e77aSPu Lehui 87649b5e77aSPu Lehui /* callee saved register S1 to pass start time */ 87749b5e77aSPu Lehui emit_sd(RV_REG_FP, -sreg_off, RV_REG_S1, ctx); 87849b5e77aSPu Lehui 87949b5e77aSPu Lehui /* store ip address of the traced function */ 88049b5e77aSPu Lehui if (flags & BPF_TRAMP_F_IP_ARG) { 88149b5e77aSPu Lehui emit_imm(RV_REG_T1, (const s64)func_addr, ctx); 88249b5e77aSPu Lehui emit_sd(RV_REG_FP, -ip_off, RV_REG_T1, ctx); 88349b5e77aSPu Lehui } 88449b5e77aSPu Lehui 88549b5e77aSPu Lehui emit_li(RV_REG_T1, nregs, ctx); 88649b5e77aSPu Lehui emit_sd(RV_REG_FP, -nregs_off, RV_REG_T1, ctx); 88749b5e77aSPu Lehui 88849b5e77aSPu Lehui store_args(nregs, args_off, ctx); 88949b5e77aSPu Lehui 89049b5e77aSPu Lehui /* skip to actual body of traced function */ 89149b5e77aSPu Lehui if (flags & BPF_TRAMP_F_SKIP_FRAME) 89225ad1065SPu Lehui orig_call += RV_FENTRY_NINSNS * 4; 89349b5e77aSPu Lehui 89449b5e77aSPu Lehui if (flags & BPF_TRAMP_F_CALL_ORIG) { 89549b5e77aSPu Lehui emit_imm(RV_REG_A0, (const s64)im, ctx); 89649b5e77aSPu Lehui ret = emit_call((const u64)__bpf_tramp_enter, true, ctx); 89749b5e77aSPu Lehui if (ret) 89849b5e77aSPu Lehui return ret; 89949b5e77aSPu Lehui } 90049b5e77aSPu Lehui 90149b5e77aSPu Lehui for (i = 0; i < fentry->nr_links; i++) { 90249b5e77aSPu Lehui ret = invoke_bpf_prog(fentry->links[i], args_off, retval_off, run_ctx_off, 90349b5e77aSPu Lehui flags & BPF_TRAMP_F_RET_FENTRY_RET, ctx); 90449b5e77aSPu Lehui if (ret) 90549b5e77aSPu Lehui return ret; 90649b5e77aSPu Lehui } 90749b5e77aSPu Lehui 90849b5e77aSPu Lehui if (fmod_ret->nr_links) { 90949b5e77aSPu Lehui branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL); 91049b5e77aSPu Lehui if (!branches_off) 91149b5e77aSPu Lehui return -ENOMEM; 91249b5e77aSPu Lehui 91349b5e77aSPu Lehui /* cleanup to avoid garbage return value confusion */ 91449b5e77aSPu Lehui emit_sd(RV_REG_FP, -retval_off, RV_REG_ZERO, ctx); 91549b5e77aSPu Lehui for (i = 0; i < fmod_ret->nr_links; i++) { 91649b5e77aSPu Lehui ret = invoke_bpf_prog(fmod_ret->links[i], args_off, retval_off, 91749b5e77aSPu Lehui run_ctx_off, true, ctx); 91849b5e77aSPu Lehui if (ret) 91949b5e77aSPu Lehui goto out; 92049b5e77aSPu Lehui emit_ld(RV_REG_T1, -retval_off, RV_REG_FP, ctx); 92149b5e77aSPu Lehui branches_off[i] = ctx->ninsns; 92249b5e77aSPu Lehui /* nop reserved for conditional jump */ 92349b5e77aSPu Lehui emit(rv_nop(), ctx); 92449b5e77aSPu Lehui } 92549b5e77aSPu Lehui } 92649b5e77aSPu Lehui 92749b5e77aSPu Lehui if (flags & BPF_TRAMP_F_CALL_ORIG) { 92849b5e77aSPu Lehui restore_args(nregs, args_off, ctx); 92949b5e77aSPu Lehui ret = emit_call((const u64)orig_call, true, ctx); 93049b5e77aSPu Lehui if (ret) 93149b5e77aSPu Lehui goto out; 93249b5e77aSPu Lehui emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); 93349b5e77aSPu Lehui im->ip_after_call = ctx->insns + ctx->ninsns; 93449b5e77aSPu Lehui /* 2 nops reserved for auipc+jalr pair */ 93549b5e77aSPu Lehui emit(rv_nop(), ctx); 93649b5e77aSPu Lehui emit(rv_nop(), ctx); 93749b5e77aSPu Lehui } 93849b5e77aSPu Lehui 93949b5e77aSPu Lehui /* update branches saved in invoke_bpf_mod_ret with bnez */ 94049b5e77aSPu Lehui for (i = 0; ctx->insns && i < fmod_ret->nr_links; i++) { 94149b5e77aSPu Lehui offset = ninsns_rvoff(ctx->ninsns - branches_off[i]); 94249b5e77aSPu Lehui insn = rv_bne(RV_REG_T1, RV_REG_ZERO, offset >> 1); 94349b5e77aSPu Lehui *(u32 *)(ctx->insns + branches_off[i]) = insn; 94449b5e77aSPu Lehui } 94549b5e77aSPu Lehui 94649b5e77aSPu Lehui for (i = 0; i < fexit->nr_links; i++) { 94749b5e77aSPu Lehui ret = invoke_bpf_prog(fexit->links[i], args_off, retval_off, 94849b5e77aSPu Lehui run_ctx_off, false, ctx); 94949b5e77aSPu Lehui if (ret) 95049b5e77aSPu Lehui goto out; 95149b5e77aSPu Lehui } 95249b5e77aSPu Lehui 95349b5e77aSPu Lehui if (flags & BPF_TRAMP_F_CALL_ORIG) { 95449b5e77aSPu Lehui im->ip_epilogue = ctx->insns + ctx->ninsns; 95549b5e77aSPu Lehui emit_imm(RV_REG_A0, (const s64)im, ctx); 95649b5e77aSPu Lehui ret = emit_call((const u64)__bpf_tramp_exit, true, ctx); 95749b5e77aSPu Lehui if (ret) 95849b5e77aSPu Lehui goto out; 95949b5e77aSPu Lehui } 96049b5e77aSPu Lehui 96149b5e77aSPu Lehui if (flags & BPF_TRAMP_F_RESTORE_REGS) 96249b5e77aSPu Lehui restore_args(nregs, args_off, ctx); 96349b5e77aSPu Lehui 96449b5e77aSPu Lehui if (save_ret) 96549b5e77aSPu Lehui emit_ld(RV_REG_A0, -retval_off, RV_REG_FP, ctx); 96649b5e77aSPu Lehui 96749b5e77aSPu Lehui emit_ld(RV_REG_S1, -sreg_off, RV_REG_FP, ctx); 96849b5e77aSPu Lehui 96925ad1065SPu Lehui if (func_addr) { 97025ad1065SPu Lehui /* trampoline called from function entry */ 97125ad1065SPu Lehui emit_ld(RV_REG_T0, stack_size - 8, RV_REG_SP, ctx); 97225ad1065SPu Lehui emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx); 97325ad1065SPu Lehui emit_addi(RV_REG_SP, RV_REG_SP, stack_size, ctx); 97449b5e77aSPu Lehui 97525ad1065SPu Lehui emit_ld(RV_REG_RA, 8, RV_REG_SP, ctx); 97625ad1065SPu Lehui emit_ld(RV_REG_FP, 0, RV_REG_SP, ctx); 97725ad1065SPu Lehui emit_addi(RV_REG_SP, RV_REG_SP, 16, ctx); 97825ad1065SPu Lehui 97925ad1065SPu Lehui if (flags & BPF_TRAMP_F_SKIP_FRAME) 98025ad1065SPu Lehui /* return to parent function */ 98125ad1065SPu Lehui emit_jalr(RV_REG_ZERO, RV_REG_RA, 0, ctx); 98225ad1065SPu Lehui else 98325ad1065SPu Lehui /* return to traced function */ 98425ad1065SPu Lehui emit_jalr(RV_REG_ZERO, RV_REG_T0, 0, ctx); 98525ad1065SPu Lehui } else { 98625ad1065SPu Lehui /* trampoline called directly */ 98725ad1065SPu Lehui emit_ld(RV_REG_RA, stack_size - 8, RV_REG_SP, ctx); 98825ad1065SPu Lehui emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx); 98949b5e77aSPu Lehui emit_addi(RV_REG_SP, RV_REG_SP, stack_size, ctx); 99049b5e77aSPu Lehui 99149b5e77aSPu Lehui emit_jalr(RV_REG_ZERO, RV_REG_RA, 0, ctx); 99225ad1065SPu Lehui } 99349b5e77aSPu Lehui 99449b5e77aSPu Lehui ret = ctx->ninsns; 99549b5e77aSPu Lehui out: 99649b5e77aSPu Lehui kfree(branches_off); 99749b5e77aSPu Lehui return ret; 99849b5e77aSPu Lehui } 99949b5e77aSPu Lehui 100049b5e77aSPu Lehui int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, 100149b5e77aSPu Lehui void *image_end, const struct btf_func_model *m, 100249b5e77aSPu Lehui u32 flags, struct bpf_tramp_links *tlinks, 100349b5e77aSPu Lehui void *func_addr) 100449b5e77aSPu Lehui { 100549b5e77aSPu Lehui int ret; 100649b5e77aSPu Lehui struct rv_jit_context ctx; 100749b5e77aSPu Lehui 100849b5e77aSPu Lehui ctx.ninsns = 0; 100949b5e77aSPu Lehui ctx.insns = NULL; 101049b5e77aSPu Lehui ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx); 101149b5e77aSPu Lehui if (ret < 0) 101249b5e77aSPu Lehui return ret; 101349b5e77aSPu Lehui 101449b5e77aSPu Lehui if (ninsns_rvoff(ret) > (long)image_end - (long)image) 101549b5e77aSPu Lehui return -EFBIG; 101649b5e77aSPu Lehui 101749b5e77aSPu Lehui ctx.ninsns = 0; 101849b5e77aSPu Lehui ctx.insns = image; 101949b5e77aSPu Lehui ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx); 102049b5e77aSPu Lehui if (ret < 0) 102149b5e77aSPu Lehui return ret; 102249b5e77aSPu Lehui 102349b5e77aSPu Lehui bpf_flush_icache(ctx.insns, ctx.insns + ctx.ninsns); 102449b5e77aSPu Lehui 102549b5e77aSPu Lehui return ninsns_rvoff(ret); 102649b5e77aSPu Lehui } 102749b5e77aSPu Lehui 1028ca6cb544SLuke Nelson int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 1029ca6cb544SLuke Nelson bool extra_pass) 1030ca6cb544SLuke Nelson { 1031ca6cb544SLuke Nelson bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || 1032ca6cb544SLuke Nelson BPF_CLASS(insn->code) == BPF_JMP; 1033489553ddSLuke Nelson int s, e, rvoff, ret, i = insn - ctx->prog->insnsi; 1034ca6cb544SLuke Nelson struct bpf_prog_aux *aux = ctx->prog->aux; 1035ca6cb544SLuke Nelson u8 rd = -1, rs = -1, code = insn->code; 1036ca6cb544SLuke Nelson s16 off = insn->off; 1037ca6cb544SLuke Nelson s32 imm = insn->imm; 1038ca6cb544SLuke Nelson 1039ca6cb544SLuke Nelson init_regs(&rd, &rs, insn, ctx); 1040ca6cb544SLuke Nelson 1041ca6cb544SLuke Nelson switch (code) { 1042ca6cb544SLuke Nelson /* dst = src */ 1043ca6cb544SLuke Nelson case BPF_ALU | BPF_MOV | BPF_X: 1044ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MOV | BPF_X: 1045ca6cb544SLuke Nelson if (imm == 1) { 1046ca6cb544SLuke Nelson /* Special mov32 for zext */ 1047ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1048ca6cb544SLuke Nelson break; 1049ca6cb544SLuke Nelson } 1050694896adSPu Lehui switch (insn->off) { 1051694896adSPu Lehui case 0: 105218a4d8c9SLuke Nelson emit_mv(rd, rs, ctx); 1053694896adSPu Lehui break; 1054694896adSPu Lehui case 8: 1055694896adSPu Lehui case 16: 1056694896adSPu Lehui emit_slli(RV_REG_T1, rs, 64 - insn->off, ctx); 1057694896adSPu Lehui emit_srai(rd, RV_REG_T1, 64 - insn->off, ctx); 1058694896adSPu Lehui break; 1059694896adSPu Lehui case 32: 1060694896adSPu Lehui emit_addiw(rd, rs, 0, ctx); 1061694896adSPu Lehui break; 1062694896adSPu Lehui } 1063ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 1064ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1065ca6cb544SLuke Nelson break; 1066ca6cb544SLuke Nelson 1067ca6cb544SLuke Nelson /* dst = dst OP src */ 1068ca6cb544SLuke Nelson case BPF_ALU | BPF_ADD | BPF_X: 1069ca6cb544SLuke Nelson case BPF_ALU64 | BPF_ADD | BPF_X: 107018a4d8c9SLuke Nelson emit_add(rd, rd, rs, ctx); 1071ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 1072ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1073ca6cb544SLuke Nelson break; 1074ca6cb544SLuke Nelson case BPF_ALU | BPF_SUB | BPF_X: 1075ca6cb544SLuke Nelson case BPF_ALU64 | BPF_SUB | BPF_X: 107618a4d8c9SLuke Nelson if (is64) 107718a4d8c9SLuke Nelson emit_sub(rd, rd, rs, ctx); 107818a4d8c9SLuke Nelson else 107918a4d8c9SLuke Nelson emit_subw(rd, rd, rs, ctx); 108018a4d8c9SLuke Nelson 1081ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 1082ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1083ca6cb544SLuke Nelson break; 1084ca6cb544SLuke Nelson case BPF_ALU | BPF_AND | BPF_X: 1085ca6cb544SLuke Nelson case BPF_ALU64 | BPF_AND | BPF_X: 108618a4d8c9SLuke Nelson emit_and(rd, rd, rs, ctx); 1087ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 1088ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1089ca6cb544SLuke Nelson break; 1090ca6cb544SLuke Nelson case BPF_ALU | BPF_OR | BPF_X: 1091ca6cb544SLuke Nelson case BPF_ALU64 | BPF_OR | BPF_X: 109218a4d8c9SLuke Nelson emit_or(rd, rd, rs, ctx); 1093ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 1094ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1095ca6cb544SLuke Nelson break; 1096ca6cb544SLuke Nelson case BPF_ALU | BPF_XOR | BPF_X: 1097ca6cb544SLuke Nelson case BPF_ALU64 | BPF_XOR | BPF_X: 109818a4d8c9SLuke Nelson emit_xor(rd, rd, rs, ctx); 1099ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 1100ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1101ca6cb544SLuke Nelson break; 1102ca6cb544SLuke Nelson case BPF_ALU | BPF_MUL | BPF_X: 1103ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MUL | BPF_X: 1104ca6cb544SLuke Nelson emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx); 1105ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 1106ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1107ca6cb544SLuke Nelson break; 1108ca6cb544SLuke Nelson case BPF_ALU | BPF_DIV | BPF_X: 1109ca6cb544SLuke Nelson case BPF_ALU64 | BPF_DIV | BPF_X: 1110*3e18ff4bSPu Lehui if (off) 1111*3e18ff4bSPu Lehui emit(is64 ? rv_div(rd, rd, rs) : rv_divw(rd, rd, rs), ctx); 1112*3e18ff4bSPu Lehui else 1113ca6cb544SLuke Nelson emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx); 1114ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 1115ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1116ca6cb544SLuke Nelson break; 1117ca6cb544SLuke Nelson case BPF_ALU | BPF_MOD | BPF_X: 1118ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MOD | BPF_X: 1119*3e18ff4bSPu Lehui if (off) 1120*3e18ff4bSPu Lehui emit(is64 ? rv_rem(rd, rd, rs) : rv_remw(rd, rd, rs), ctx); 1121*3e18ff4bSPu Lehui else 1122ca6cb544SLuke Nelson emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx); 1123ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 1124ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1125ca6cb544SLuke Nelson break; 1126ca6cb544SLuke Nelson case BPF_ALU | BPF_LSH | BPF_X: 1127ca6cb544SLuke Nelson case BPF_ALU64 | BPF_LSH | BPF_X: 1128ca6cb544SLuke Nelson emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx); 11290224b2acSLuke Nelson if (!is64 && !aux->verifier_zext) 1130ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1131ca6cb544SLuke Nelson break; 1132ca6cb544SLuke Nelson case BPF_ALU | BPF_RSH | BPF_X: 1133ca6cb544SLuke Nelson case BPF_ALU64 | BPF_RSH | BPF_X: 1134ca6cb544SLuke Nelson emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx); 1135ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 1136ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1137ca6cb544SLuke Nelson break; 1138ca6cb544SLuke Nelson case BPF_ALU | BPF_ARSH | BPF_X: 1139ca6cb544SLuke Nelson case BPF_ALU64 | BPF_ARSH | BPF_X: 1140ca6cb544SLuke Nelson emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx); 1141ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 1142ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1143ca6cb544SLuke Nelson break; 1144ca6cb544SLuke Nelson 1145ca6cb544SLuke Nelson /* dst = -dst */ 1146ca6cb544SLuke Nelson case BPF_ALU | BPF_NEG: 1147ca6cb544SLuke Nelson case BPF_ALU64 | BPF_NEG: 114818a4d8c9SLuke Nelson emit_sub(rd, RV_REG_ZERO, rd, ctx); 1149ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 1150ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1151ca6cb544SLuke Nelson break; 1152ca6cb544SLuke Nelson 1153ca6cb544SLuke Nelson /* dst = BSWAP##imm(dst) */ 1154ca6cb544SLuke Nelson case BPF_ALU | BPF_END | BPF_FROM_LE: 115521a099abSLuke Nelson switch (imm) { 115621a099abSLuke Nelson case 16: 115718a4d8c9SLuke Nelson emit_slli(rd, rd, 48, ctx); 115818a4d8c9SLuke Nelson emit_srli(rd, rd, 48, ctx); 115921a099abSLuke Nelson break; 116021a099abSLuke Nelson case 32: 116121a099abSLuke Nelson if (!aux->verifier_zext) 116221a099abSLuke Nelson emit_zext_32(rd, ctx); 116321a099abSLuke Nelson break; 116421a099abSLuke Nelson case 64: 116521a099abSLuke Nelson /* Do nothing */ 1166ca6cb544SLuke Nelson break; 1167ca6cb544SLuke Nelson } 116821a099abSLuke Nelson break; 116921a099abSLuke Nelson 1170ca6cb544SLuke Nelson case BPF_ALU | BPF_END | BPF_FROM_BE: 117118a4d8c9SLuke Nelson emit_li(RV_REG_T2, 0, ctx); 1172ca6cb544SLuke Nelson 117318a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 117418a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 117518a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 117618a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 1177ca6cb544SLuke Nelson if (imm == 16) 1178ca6cb544SLuke Nelson goto out_be; 1179ca6cb544SLuke Nelson 118018a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 118118a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 118218a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 118318a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 1184ca6cb544SLuke Nelson 118518a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 118618a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 118718a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 118818a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 1189ca6cb544SLuke Nelson if (imm == 32) 1190ca6cb544SLuke Nelson goto out_be; 1191ca6cb544SLuke Nelson 119218a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 119318a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 119418a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 119518a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 1196ca6cb544SLuke Nelson 119718a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 119818a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 119918a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 120018a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 1201ca6cb544SLuke Nelson 120218a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 120318a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 120418a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 120518a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 1206ca6cb544SLuke Nelson 120718a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 120818a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 120918a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 121018a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 1211ca6cb544SLuke Nelson out_be: 121218a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 121318a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 1214ca6cb544SLuke Nelson 121518a4d8c9SLuke Nelson emit_mv(rd, RV_REG_T2, ctx); 1216ca6cb544SLuke Nelson break; 1217ca6cb544SLuke Nelson 1218ca6cb544SLuke Nelson /* dst = imm */ 1219ca6cb544SLuke Nelson case BPF_ALU | BPF_MOV | BPF_K: 1220ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MOV | BPF_K: 1221ca6cb544SLuke Nelson emit_imm(rd, imm, ctx); 1222ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 1223ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1224ca6cb544SLuke Nelson break; 1225ca6cb544SLuke Nelson 1226ca6cb544SLuke Nelson /* dst = dst OP imm */ 1227ca6cb544SLuke Nelson case BPF_ALU | BPF_ADD | BPF_K: 1228ca6cb544SLuke Nelson case BPF_ALU64 | BPF_ADD | BPF_K: 1229ca6cb544SLuke Nelson if (is_12b_int(imm)) { 123018a4d8c9SLuke Nelson emit_addi(rd, rd, imm, ctx); 1231ca6cb544SLuke Nelson } else { 1232ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 123318a4d8c9SLuke Nelson emit_add(rd, rd, RV_REG_T1, ctx); 1234ca6cb544SLuke Nelson } 1235ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 1236ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1237ca6cb544SLuke Nelson break; 1238ca6cb544SLuke Nelson case BPF_ALU | BPF_SUB | BPF_K: 1239ca6cb544SLuke Nelson case BPF_ALU64 | BPF_SUB | BPF_K: 1240ca6cb544SLuke Nelson if (is_12b_int(-imm)) { 124118a4d8c9SLuke Nelson emit_addi(rd, rd, -imm, ctx); 1242ca6cb544SLuke Nelson } else { 1243ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 124418a4d8c9SLuke Nelson emit_sub(rd, rd, RV_REG_T1, ctx); 1245ca6cb544SLuke Nelson } 1246ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 1247ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1248ca6cb544SLuke Nelson break; 1249ca6cb544SLuke Nelson case BPF_ALU | BPF_AND | BPF_K: 1250ca6cb544SLuke Nelson case BPF_ALU64 | BPF_AND | BPF_K: 1251ca6cb544SLuke Nelson if (is_12b_int(imm)) { 125218a4d8c9SLuke Nelson emit_andi(rd, rd, imm, ctx); 1253ca6cb544SLuke Nelson } else { 1254ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 125518a4d8c9SLuke Nelson emit_and(rd, rd, RV_REG_T1, ctx); 1256ca6cb544SLuke Nelson } 1257ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 1258ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1259ca6cb544SLuke Nelson break; 1260ca6cb544SLuke Nelson case BPF_ALU | BPF_OR | BPF_K: 1261ca6cb544SLuke Nelson case BPF_ALU64 | BPF_OR | BPF_K: 1262ca6cb544SLuke Nelson if (is_12b_int(imm)) { 1263ca6cb544SLuke Nelson emit(rv_ori(rd, rd, imm), ctx); 1264ca6cb544SLuke Nelson } else { 1265ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 126618a4d8c9SLuke Nelson emit_or(rd, rd, RV_REG_T1, ctx); 1267ca6cb544SLuke Nelson } 1268ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 1269ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1270ca6cb544SLuke Nelson break; 1271ca6cb544SLuke Nelson case BPF_ALU | BPF_XOR | BPF_K: 1272ca6cb544SLuke Nelson case BPF_ALU64 | BPF_XOR | BPF_K: 1273ca6cb544SLuke Nelson if (is_12b_int(imm)) { 1274ca6cb544SLuke Nelson emit(rv_xori(rd, rd, imm), ctx); 1275ca6cb544SLuke Nelson } else { 1276ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 127718a4d8c9SLuke Nelson emit_xor(rd, rd, RV_REG_T1, ctx); 1278ca6cb544SLuke Nelson } 1279ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 1280ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1281ca6cb544SLuke Nelson break; 1282ca6cb544SLuke Nelson case BPF_ALU | BPF_MUL | BPF_K: 1283ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MUL | BPF_K: 1284ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 1285ca6cb544SLuke Nelson emit(is64 ? rv_mul(rd, rd, RV_REG_T1) : 1286ca6cb544SLuke Nelson rv_mulw(rd, rd, RV_REG_T1), ctx); 1287ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 1288ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1289ca6cb544SLuke Nelson break; 1290ca6cb544SLuke Nelson case BPF_ALU | BPF_DIV | BPF_K: 1291ca6cb544SLuke Nelson case BPF_ALU64 | BPF_DIV | BPF_K: 1292ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 1293*3e18ff4bSPu Lehui if (off) 1294*3e18ff4bSPu Lehui emit(is64 ? rv_div(rd, rd, RV_REG_T1) : 1295*3e18ff4bSPu Lehui rv_divw(rd, rd, RV_REG_T1), ctx); 1296*3e18ff4bSPu Lehui else 1297ca6cb544SLuke Nelson emit(is64 ? rv_divu(rd, rd, RV_REG_T1) : 1298ca6cb544SLuke Nelson rv_divuw(rd, rd, RV_REG_T1), ctx); 1299ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 1300ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1301ca6cb544SLuke Nelson break; 1302ca6cb544SLuke Nelson case BPF_ALU | BPF_MOD | BPF_K: 1303ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MOD | BPF_K: 1304ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 1305*3e18ff4bSPu Lehui if (off) 1306*3e18ff4bSPu Lehui emit(is64 ? rv_rem(rd, rd, RV_REG_T1) : 1307*3e18ff4bSPu Lehui rv_remw(rd, rd, RV_REG_T1), ctx); 1308*3e18ff4bSPu Lehui else 1309ca6cb544SLuke Nelson emit(is64 ? rv_remu(rd, rd, RV_REG_T1) : 1310ca6cb544SLuke Nelson rv_remuw(rd, rd, RV_REG_T1), ctx); 1311ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 1312ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1313ca6cb544SLuke Nelson break; 1314ca6cb544SLuke Nelson case BPF_ALU | BPF_LSH | BPF_K: 1315ca6cb544SLuke Nelson case BPF_ALU64 | BPF_LSH | BPF_K: 131618a4d8c9SLuke Nelson emit_slli(rd, rd, imm, ctx); 131718a4d8c9SLuke Nelson 13180224b2acSLuke Nelson if (!is64 && !aux->verifier_zext) 1319ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1320ca6cb544SLuke Nelson break; 1321ca6cb544SLuke Nelson case BPF_ALU | BPF_RSH | BPF_K: 1322ca6cb544SLuke Nelson case BPF_ALU64 | BPF_RSH | BPF_K: 132318a4d8c9SLuke Nelson if (is64) 132418a4d8c9SLuke Nelson emit_srli(rd, rd, imm, ctx); 132518a4d8c9SLuke Nelson else 132618a4d8c9SLuke Nelson emit(rv_srliw(rd, rd, imm), ctx); 132718a4d8c9SLuke Nelson 13280224b2acSLuke Nelson if (!is64 && !aux->verifier_zext) 1329ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1330ca6cb544SLuke Nelson break; 1331ca6cb544SLuke Nelson case BPF_ALU | BPF_ARSH | BPF_K: 1332ca6cb544SLuke Nelson case BPF_ALU64 | BPF_ARSH | BPF_K: 133318a4d8c9SLuke Nelson if (is64) 133418a4d8c9SLuke Nelson emit_srai(rd, rd, imm, ctx); 133518a4d8c9SLuke Nelson else 133618a4d8c9SLuke Nelson emit(rv_sraiw(rd, rd, imm), ctx); 133718a4d8c9SLuke Nelson 13380224b2acSLuke Nelson if (!is64 && !aux->verifier_zext) 1339ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 1340ca6cb544SLuke Nelson break; 1341ca6cb544SLuke Nelson 1342ca6cb544SLuke Nelson /* JUMP off */ 1343ca6cb544SLuke Nelson case BPF_JMP | BPF_JA: 1344d9839f16SPu Lehui case BPF_JMP32 | BPF_JA: 1345d9839f16SPu Lehui if (BPF_CLASS(code) == BPF_JMP) 1346ca6cb544SLuke Nelson rvoff = rv_offset(i, off, ctx); 1347d9839f16SPu Lehui else 1348d9839f16SPu Lehui rvoff = rv_offset(i, imm, ctx); 13490fd1fd01SPu Lehui ret = emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); 1350489553ddSLuke Nelson if (ret) 1351489553ddSLuke Nelson return ret; 1352ca6cb544SLuke Nelson break; 1353ca6cb544SLuke Nelson 1354ca6cb544SLuke Nelson /* IF (dst COND src) JUMP off */ 1355ca6cb544SLuke Nelson case BPF_JMP | BPF_JEQ | BPF_X: 1356ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JEQ | BPF_X: 1357ca6cb544SLuke Nelson case BPF_JMP | BPF_JGT | BPF_X: 1358ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JGT | BPF_X: 1359ca6cb544SLuke Nelson case BPF_JMP | BPF_JLT | BPF_X: 1360ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JLT | BPF_X: 1361ca6cb544SLuke Nelson case BPF_JMP | BPF_JGE | BPF_X: 1362ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JGE | BPF_X: 1363ca6cb544SLuke Nelson case BPF_JMP | BPF_JLE | BPF_X: 1364ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JLE | BPF_X: 1365ca6cb544SLuke Nelson case BPF_JMP | BPF_JNE | BPF_X: 1366ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JNE | BPF_X: 1367ca6cb544SLuke Nelson case BPF_JMP | BPF_JSGT | BPF_X: 1368ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSGT | BPF_X: 1369ca6cb544SLuke Nelson case BPF_JMP | BPF_JSLT | BPF_X: 1370ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSLT | BPF_X: 1371ca6cb544SLuke Nelson case BPF_JMP | BPF_JSGE | BPF_X: 1372ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSGE | BPF_X: 1373ca6cb544SLuke Nelson case BPF_JMP | BPF_JSLE | BPF_X: 1374ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSLE | BPF_X: 1375ca6cb544SLuke Nelson case BPF_JMP | BPF_JSET | BPF_X: 1376ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSET | BPF_X: 1377ca6cb544SLuke Nelson rvoff = rv_offset(i, off, ctx); 1378ca6cb544SLuke Nelson if (!is64) { 1379ca6cb544SLuke Nelson s = ctx->ninsns; 1380ca6cb544SLuke Nelson if (is_signed_bpf_cond(BPF_OP(code))) 1381ca6cb544SLuke Nelson emit_sext_32_rd_rs(&rd, &rs, ctx); 1382ca6cb544SLuke Nelson else 1383ca6cb544SLuke Nelson emit_zext_32_rd_rs(&rd, &rs, ctx); 1384ca6cb544SLuke Nelson e = ctx->ninsns; 1385ca6cb544SLuke Nelson 1386ca6cb544SLuke Nelson /* Adjust for extra insns */ 1387bfabff3cSLuke Nelson rvoff -= ninsns_rvoff(e - s); 1388ca6cb544SLuke Nelson } 1389ca6cb544SLuke Nelson 1390ca6cb544SLuke Nelson if (BPF_OP(code) == BPF_JSET) { 1391ca6cb544SLuke Nelson /* Adjust for and */ 1392ca6cb544SLuke Nelson rvoff -= 4; 139318a4d8c9SLuke Nelson emit_and(RV_REG_T1, rd, rs, ctx); 1394ca6cb544SLuke Nelson emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, 1395ca6cb544SLuke Nelson ctx); 1396ca6cb544SLuke Nelson } else { 1397ca6cb544SLuke Nelson emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 1398ca6cb544SLuke Nelson } 1399ca6cb544SLuke Nelson break; 1400ca6cb544SLuke Nelson 1401ca6cb544SLuke Nelson /* IF (dst COND imm) JUMP off */ 1402ca6cb544SLuke Nelson case BPF_JMP | BPF_JEQ | BPF_K: 1403ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JEQ | BPF_K: 1404ca6cb544SLuke Nelson case BPF_JMP | BPF_JGT | BPF_K: 1405ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JGT | BPF_K: 1406ca6cb544SLuke Nelson case BPF_JMP | BPF_JLT | BPF_K: 1407ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JLT | BPF_K: 1408ca6cb544SLuke Nelson case BPF_JMP | BPF_JGE | BPF_K: 1409ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JGE | BPF_K: 1410ca6cb544SLuke Nelson case BPF_JMP | BPF_JLE | BPF_K: 1411ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JLE | BPF_K: 1412ca6cb544SLuke Nelson case BPF_JMP | BPF_JNE | BPF_K: 1413ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JNE | BPF_K: 1414ca6cb544SLuke Nelson case BPF_JMP | BPF_JSGT | BPF_K: 1415ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSGT | BPF_K: 1416ca6cb544SLuke Nelson case BPF_JMP | BPF_JSLT | BPF_K: 1417ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSLT | BPF_K: 1418ca6cb544SLuke Nelson case BPF_JMP | BPF_JSGE | BPF_K: 1419ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSGE | BPF_K: 1420ca6cb544SLuke Nelson case BPF_JMP | BPF_JSLE | BPF_K: 1421ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSLE | BPF_K: 1422ca6cb544SLuke Nelson rvoff = rv_offset(i, off, ctx); 1423ca6cb544SLuke Nelson s = ctx->ninsns; 1424ca349a6aSLuke Nelson if (imm) { 1425ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 1426ca349a6aSLuke Nelson rs = RV_REG_T1; 1427ca349a6aSLuke Nelson } else { 1428ca349a6aSLuke Nelson /* If imm is 0, simply use zero register. */ 1429ca349a6aSLuke Nelson rs = RV_REG_ZERO; 1430ca349a6aSLuke Nelson } 1431ca6cb544SLuke Nelson if (!is64) { 1432ca6cb544SLuke Nelson if (is_signed_bpf_cond(BPF_OP(code))) 1433ca6cb544SLuke Nelson emit_sext_32_rd(&rd, ctx); 1434ca6cb544SLuke Nelson else 1435ca6cb544SLuke Nelson emit_zext_32_rd_t1(&rd, ctx); 1436ca6cb544SLuke Nelson } 1437ca6cb544SLuke Nelson e = ctx->ninsns; 1438ca6cb544SLuke Nelson 1439ca6cb544SLuke Nelson /* Adjust for extra insns */ 1440bfabff3cSLuke Nelson rvoff -= ninsns_rvoff(e - s); 1441ca349a6aSLuke Nelson emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 1442073ca6a0SLuke Nelson break; 1443073ca6a0SLuke Nelson 1444073ca6a0SLuke Nelson case BPF_JMP | BPF_JSET | BPF_K: 1445073ca6a0SLuke Nelson case BPF_JMP32 | BPF_JSET | BPF_K: 1446073ca6a0SLuke Nelson rvoff = rv_offset(i, off, ctx); 1447073ca6a0SLuke Nelson s = ctx->ninsns; 1448073ca6a0SLuke Nelson if (is_12b_int(imm)) { 144918a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, imm, ctx); 1450073ca6a0SLuke Nelson } else { 1451073ca6a0SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 145218a4d8c9SLuke Nelson emit_and(RV_REG_T1, rd, RV_REG_T1, ctx); 1453ca6cb544SLuke Nelson } 1454073ca6a0SLuke Nelson /* For jset32, we should clear the upper 32 bits of t1, but 1455073ca6a0SLuke Nelson * sign-extension is sufficient here and saves one instruction, 1456073ca6a0SLuke Nelson * as t1 is used only in comparison against zero. 1457073ca6a0SLuke Nelson */ 1458073ca6a0SLuke Nelson if (!is64 && imm < 0) 145918a4d8c9SLuke Nelson emit_addiw(RV_REG_T1, RV_REG_T1, 0, ctx); 1460073ca6a0SLuke Nelson e = ctx->ninsns; 1461bfabff3cSLuke Nelson rvoff -= ninsns_rvoff(e - s); 1462073ca6a0SLuke Nelson emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); 1463ca6cb544SLuke Nelson break; 1464ca6cb544SLuke Nelson 1465ca6cb544SLuke Nelson /* function call */ 1466ca6cb544SLuke Nelson case BPF_JMP | BPF_CALL: 1467ca6cb544SLuke Nelson { 14680fd1fd01SPu Lehui bool fixed_addr; 1469ca6cb544SLuke Nelson u64 addr; 1470ca6cb544SLuke Nelson 1471ca6cb544SLuke Nelson mark_call(ctx); 14720fd1fd01SPu Lehui ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, 14730fd1fd01SPu Lehui &addr, &fixed_addr); 1474ca6cb544SLuke Nelson if (ret < 0) 1475ca6cb544SLuke Nelson return ret; 14760fd1fd01SPu Lehui 14770fd1fd01SPu Lehui ret = emit_call(addr, fixed_addr, ctx); 1478ca6cb544SLuke Nelson if (ret) 1479ca6cb544SLuke Nelson return ret; 14800fd1fd01SPu Lehui 14810fd1fd01SPu Lehui emit_mv(bpf_to_rv_reg(BPF_REG_0, ctx), RV_REG_A0, ctx); 1482ca6cb544SLuke Nelson break; 1483ca6cb544SLuke Nelson } 1484ca6cb544SLuke Nelson /* tail call */ 1485ca6cb544SLuke Nelson case BPF_JMP | BPF_TAIL_CALL: 1486ca6cb544SLuke Nelson if (emit_bpf_tail_call(i, ctx)) 1487ca6cb544SLuke Nelson return -1; 1488ca6cb544SLuke Nelson break; 1489ca6cb544SLuke Nelson 1490ca6cb544SLuke Nelson /* function return */ 1491ca6cb544SLuke Nelson case BPF_JMP | BPF_EXIT: 1492ca6cb544SLuke Nelson if (i == ctx->prog->len - 1) 1493ca6cb544SLuke Nelson break; 1494ca6cb544SLuke Nelson 1495ca6cb544SLuke Nelson rvoff = epilogue_offset(ctx); 14960fd1fd01SPu Lehui ret = emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); 1497489553ddSLuke Nelson if (ret) 1498489553ddSLuke Nelson return ret; 1499ca6cb544SLuke Nelson break; 1500ca6cb544SLuke Nelson 1501ca6cb544SLuke Nelson /* dst = imm64 */ 1502ca6cb544SLuke Nelson case BPF_LD | BPF_IMM | BPF_DW: 1503ca6cb544SLuke Nelson { 1504ca6cb544SLuke Nelson struct bpf_insn insn1 = insn[1]; 1505ca6cb544SLuke Nelson u64 imm64; 1506ca6cb544SLuke Nelson 1507ca6cb544SLuke Nelson imm64 = (u64)insn1.imm << 32 | (u32)imm; 1508b54b6003SPu Lehui if (bpf_pseudo_func(insn)) { 1509b54b6003SPu Lehui /* fixed-length insns for extra jit pass */ 1510b54b6003SPu Lehui ret = emit_addr(rd, imm64, extra_pass, ctx); 1511b54b6003SPu Lehui if (ret) 1512b54b6003SPu Lehui return ret; 1513b54b6003SPu Lehui } else { 1514ca6cb544SLuke Nelson emit_imm(rd, imm64, ctx); 1515b54b6003SPu Lehui } 1516b54b6003SPu Lehui 1517ca6cb544SLuke Nelson return 1; 1518ca6cb544SLuke Nelson } 1519ca6cb544SLuke Nelson 15203d06d816SPu Lehui /* LDX: dst = *(unsigned size *)(src + off) */ 1521ca6cb544SLuke Nelson case BPF_LDX | BPF_MEM | BPF_B: 1522ca6cb544SLuke Nelson case BPF_LDX | BPF_MEM | BPF_H: 1523ca6cb544SLuke Nelson case BPF_LDX | BPF_MEM | BPF_W: 1524252c765bSTong Tiangen case BPF_LDX | BPF_MEM | BPF_DW: 1525252c765bSTong Tiangen case BPF_LDX | BPF_PROBE_MEM | BPF_B: 1526252c765bSTong Tiangen case BPF_LDX | BPF_PROBE_MEM | BPF_H: 1527252c765bSTong Tiangen case BPF_LDX | BPF_PROBE_MEM | BPF_W: 1528252c765bSTong Tiangen case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 15293d06d816SPu Lehui /* LDSX: dst = *(signed size *)(src + off) */ 15303d06d816SPu Lehui case BPF_LDX | BPF_MEMSX | BPF_B: 15313d06d816SPu Lehui case BPF_LDX | BPF_MEMSX | BPF_H: 15323d06d816SPu Lehui case BPF_LDX | BPF_MEMSX | BPF_W: 15333d06d816SPu Lehui case BPF_LDX | BPF_PROBE_MEMSX | BPF_B: 15343d06d816SPu Lehui case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: 15353d06d816SPu Lehui case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: 1536252c765bSTong Tiangen { 1537252c765bSTong Tiangen int insn_len, insns_start; 15383d06d816SPu Lehui bool sign_ext; 15393d06d816SPu Lehui 15403d06d816SPu Lehui sign_ext = BPF_MODE(insn->code) == BPF_MEMSX || 15413d06d816SPu Lehui BPF_MODE(insn->code) == BPF_PROBE_MEMSX; 1542252c765bSTong Tiangen 1543252c765bSTong Tiangen switch (BPF_SIZE(code)) { 1544252c765bSTong Tiangen case BPF_B: 1545ca6cb544SLuke Nelson if (is_12b_int(off)) { 1546252c765bSTong Tiangen insns_start = ctx->ninsns; 15473d06d816SPu Lehui if (sign_ext) 15483d06d816SPu Lehui emit(rv_lb(rd, off, rs), ctx); 15493d06d816SPu Lehui else 1550252c765bSTong Tiangen emit(rv_lbu(rd, off, rs), ctx); 1551252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1552ca6cb544SLuke Nelson break; 1553ca6cb544SLuke Nelson } 1554ca6cb544SLuke Nelson 1555ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 155618a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1557252c765bSTong Tiangen insns_start = ctx->ninsns; 15583d06d816SPu Lehui if (sign_ext) 15593d06d816SPu Lehui emit(rv_lb(rd, 0, RV_REG_T1), ctx); 15603d06d816SPu Lehui else 1561252c765bSTong Tiangen emit(rv_lbu(rd, 0, RV_REG_T1), ctx); 1562252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1563ca6cb544SLuke Nelson break; 1564252c765bSTong Tiangen case BPF_H: 1565ca6cb544SLuke Nelson if (is_12b_int(off)) { 1566252c765bSTong Tiangen insns_start = ctx->ninsns; 15673d06d816SPu Lehui if (sign_ext) 15683d06d816SPu Lehui emit(rv_lh(rd, off, rs), ctx); 15693d06d816SPu Lehui else 1570252c765bSTong Tiangen emit(rv_lhu(rd, off, rs), ctx); 1571252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1572ca6cb544SLuke Nelson break; 1573ca6cb544SLuke Nelson } 1574ca6cb544SLuke Nelson 1575ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 157618a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1577252c765bSTong Tiangen insns_start = ctx->ninsns; 15783d06d816SPu Lehui if (sign_ext) 15793d06d816SPu Lehui emit(rv_lh(rd, 0, RV_REG_T1), ctx); 15803d06d816SPu Lehui else 1581252c765bSTong Tiangen emit(rv_lhu(rd, 0, RV_REG_T1), ctx); 1582252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1583ca6cb544SLuke Nelson break; 1584252c765bSTong Tiangen case BPF_W: 1585252c765bSTong Tiangen if (is_12b_int(off)) { 1586252c765bSTong Tiangen insns_start = ctx->ninsns; 15873d06d816SPu Lehui if (sign_ext) 15883d06d816SPu Lehui emit(rv_lw(rd, off, rs), ctx); 15893d06d816SPu Lehui else 1590252c765bSTong Tiangen emit(rv_lwu(rd, off, rs), ctx); 1591252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1592252c765bSTong Tiangen break; 1593252c765bSTong Tiangen } 1594ca6cb544SLuke Nelson 1595252c765bSTong Tiangen emit_imm(RV_REG_T1, off, ctx); 1596252c765bSTong Tiangen emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1597252c765bSTong Tiangen insns_start = ctx->ninsns; 15983d06d816SPu Lehui if (sign_ext) 15993d06d816SPu Lehui emit(rv_lw(rd, 0, RV_REG_T1), ctx); 16003d06d816SPu Lehui else 1601252c765bSTong Tiangen emit(rv_lwu(rd, 0, RV_REG_T1), ctx); 1602252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1603252c765bSTong Tiangen break; 1604252c765bSTong Tiangen case BPF_DW: 1605252c765bSTong Tiangen if (is_12b_int(off)) { 1606252c765bSTong Tiangen insns_start = ctx->ninsns; 1607252c765bSTong Tiangen emit_ld(rd, off, rs, ctx); 1608252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1609252c765bSTong Tiangen break; 1610252c765bSTong Tiangen } 1611252c765bSTong Tiangen 1612252c765bSTong Tiangen emit_imm(RV_REG_T1, off, ctx); 1613252c765bSTong Tiangen emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1614252c765bSTong Tiangen insns_start = ctx->ninsns; 1615252c765bSTong Tiangen emit_ld(rd, 0, RV_REG_T1, ctx); 1616252c765bSTong Tiangen insn_len = ctx->ninsns - insns_start; 1617252c765bSTong Tiangen break; 1618252c765bSTong Tiangen } 1619252c765bSTong Tiangen 1620252c765bSTong Tiangen ret = add_exception_handler(insn, ctx, rd, insn_len); 1621252c765bSTong Tiangen if (ret) 1622252c765bSTong Tiangen return ret; 1623469fb2c3SPu Lehui 1624469fb2c3SPu Lehui if (BPF_SIZE(code) != BPF_DW && insn_is_zext(&insn[1])) 1625469fb2c3SPu Lehui return 1; 1626252c765bSTong Tiangen break; 1627252c765bSTong Tiangen } 1628f5e81d11SDaniel Borkmann /* speculation barrier */ 1629f5e81d11SDaniel Borkmann case BPF_ST | BPF_NOSPEC: 1630f5e81d11SDaniel Borkmann break; 1631f5e81d11SDaniel Borkmann 1632ca6cb544SLuke Nelson /* ST: *(size *)(dst + off) = imm */ 1633ca6cb544SLuke Nelson case BPF_ST | BPF_MEM | BPF_B: 1634ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 1635ca6cb544SLuke Nelson if (is_12b_int(off)) { 1636ca6cb544SLuke Nelson emit(rv_sb(rd, off, RV_REG_T1), ctx); 1637ca6cb544SLuke Nelson break; 1638ca6cb544SLuke Nelson } 1639ca6cb544SLuke Nelson 1640ca6cb544SLuke Nelson emit_imm(RV_REG_T2, off, ctx); 164118a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1642ca6cb544SLuke Nelson emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); 1643ca6cb544SLuke Nelson break; 1644ca6cb544SLuke Nelson 1645ca6cb544SLuke Nelson case BPF_ST | BPF_MEM | BPF_H: 1646ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 1647ca6cb544SLuke Nelson if (is_12b_int(off)) { 1648ca6cb544SLuke Nelson emit(rv_sh(rd, off, RV_REG_T1), ctx); 1649ca6cb544SLuke Nelson break; 1650ca6cb544SLuke Nelson } 1651ca6cb544SLuke Nelson 1652ca6cb544SLuke Nelson emit_imm(RV_REG_T2, off, ctx); 165318a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1654ca6cb544SLuke Nelson emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); 1655ca6cb544SLuke Nelson break; 1656ca6cb544SLuke Nelson case BPF_ST | BPF_MEM | BPF_W: 1657ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 1658ca6cb544SLuke Nelson if (is_12b_int(off)) { 165918a4d8c9SLuke Nelson emit_sw(rd, off, RV_REG_T1, ctx); 1660ca6cb544SLuke Nelson break; 1661ca6cb544SLuke Nelson } 1662ca6cb544SLuke Nelson 1663ca6cb544SLuke Nelson emit_imm(RV_REG_T2, off, ctx); 166418a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 166518a4d8c9SLuke Nelson emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx); 1666ca6cb544SLuke Nelson break; 1667ca6cb544SLuke Nelson case BPF_ST | BPF_MEM | BPF_DW: 1668ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 1669ca6cb544SLuke Nelson if (is_12b_int(off)) { 167018a4d8c9SLuke Nelson emit_sd(rd, off, RV_REG_T1, ctx); 1671ca6cb544SLuke Nelson break; 1672ca6cb544SLuke Nelson } 1673ca6cb544SLuke Nelson 1674ca6cb544SLuke Nelson emit_imm(RV_REG_T2, off, ctx); 167518a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 167618a4d8c9SLuke Nelson emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx); 1677ca6cb544SLuke Nelson break; 1678ca6cb544SLuke Nelson 1679ca6cb544SLuke Nelson /* STX: *(size *)(dst + off) = src */ 1680ca6cb544SLuke Nelson case BPF_STX | BPF_MEM | BPF_B: 1681ca6cb544SLuke Nelson if (is_12b_int(off)) { 1682ca6cb544SLuke Nelson emit(rv_sb(rd, off, rs), ctx); 1683ca6cb544SLuke Nelson break; 1684ca6cb544SLuke Nelson } 1685ca6cb544SLuke Nelson 1686ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 168718a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1688ca6cb544SLuke Nelson emit(rv_sb(RV_REG_T1, 0, rs), ctx); 1689ca6cb544SLuke Nelson break; 1690ca6cb544SLuke Nelson case BPF_STX | BPF_MEM | BPF_H: 1691ca6cb544SLuke Nelson if (is_12b_int(off)) { 1692ca6cb544SLuke Nelson emit(rv_sh(rd, off, rs), ctx); 1693ca6cb544SLuke Nelson break; 1694ca6cb544SLuke Nelson } 1695ca6cb544SLuke Nelson 1696ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 169718a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1698ca6cb544SLuke Nelson emit(rv_sh(RV_REG_T1, 0, rs), ctx); 1699ca6cb544SLuke Nelson break; 1700ca6cb544SLuke Nelson case BPF_STX | BPF_MEM | BPF_W: 1701ca6cb544SLuke Nelson if (is_12b_int(off)) { 170218a4d8c9SLuke Nelson emit_sw(rd, off, rs, ctx); 1703ca6cb544SLuke Nelson break; 1704ca6cb544SLuke Nelson } 1705ca6cb544SLuke Nelson 1706ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 170718a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 170818a4d8c9SLuke Nelson emit_sw(RV_REG_T1, 0, rs, ctx); 1709ca6cb544SLuke Nelson break; 1710ca6cb544SLuke Nelson case BPF_STX | BPF_MEM | BPF_DW: 1711ca6cb544SLuke Nelson if (is_12b_int(off)) { 171218a4d8c9SLuke Nelson emit_sd(rd, off, rs, ctx); 1713ca6cb544SLuke Nelson break; 1714ca6cb544SLuke Nelson } 1715ca6cb544SLuke Nelson 1716ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 171718a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 171818a4d8c9SLuke Nelson emit_sd(RV_REG_T1, 0, rs, ctx); 1719ca6cb544SLuke Nelson break; 172091c960b0SBrendan Jackman case BPF_STX | BPF_ATOMIC | BPF_W: 172191c960b0SBrendan Jackman case BPF_STX | BPF_ATOMIC | BPF_DW: 1722dd642ccbSPu Lehui emit_atomic(rd, rs, off, imm, 1723dd642ccbSPu Lehui BPF_SIZE(code) == BPF_DW, ctx); 1724ca6cb544SLuke Nelson break; 1725ca6cb544SLuke Nelson default: 1726ca6cb544SLuke Nelson pr_err("bpf-jit: unknown opcode %02x\n", code); 1727ca6cb544SLuke Nelson return -EINVAL; 1728ca6cb544SLuke Nelson } 1729ca6cb544SLuke Nelson 1730ca6cb544SLuke Nelson return 0; 1731ca6cb544SLuke Nelson } 1732ca6cb544SLuke Nelson 1733ca6cb544SLuke Nelson void bpf_jit_build_prologue(struct rv_jit_context *ctx) 1734ca6cb544SLuke Nelson { 1735596f2e6fSPu Lehui int i, stack_adjust = 0, store_offset, bpf_stack_adjust; 1736ca6cb544SLuke Nelson 1737ca6cb544SLuke Nelson bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); 1738ca6cb544SLuke Nelson if (bpf_stack_adjust) 1739ca6cb544SLuke Nelson mark_fp(ctx); 1740ca6cb544SLuke Nelson 1741ca6cb544SLuke Nelson if (seen_reg(RV_REG_RA, ctx)) 1742ca6cb544SLuke Nelson stack_adjust += 8; 1743ca6cb544SLuke Nelson stack_adjust += 8; /* RV_REG_FP */ 1744ca6cb544SLuke Nelson if (seen_reg(RV_REG_S1, ctx)) 1745ca6cb544SLuke Nelson stack_adjust += 8; 1746ca6cb544SLuke Nelson if (seen_reg(RV_REG_S2, ctx)) 1747ca6cb544SLuke Nelson stack_adjust += 8; 1748ca6cb544SLuke Nelson if (seen_reg(RV_REG_S3, ctx)) 1749ca6cb544SLuke Nelson stack_adjust += 8; 1750ca6cb544SLuke Nelson if (seen_reg(RV_REG_S4, ctx)) 1751ca6cb544SLuke Nelson stack_adjust += 8; 1752ca6cb544SLuke Nelson if (seen_reg(RV_REG_S5, ctx)) 1753ca6cb544SLuke Nelson stack_adjust += 8; 1754ca6cb544SLuke Nelson if (seen_reg(RV_REG_S6, ctx)) 1755ca6cb544SLuke Nelson stack_adjust += 8; 1756ca6cb544SLuke Nelson 1757ca6cb544SLuke Nelson stack_adjust = round_up(stack_adjust, 16); 1758ca6cb544SLuke Nelson stack_adjust += bpf_stack_adjust; 1759ca6cb544SLuke Nelson 1760ca6cb544SLuke Nelson store_offset = stack_adjust - 8; 1761ca6cb544SLuke Nelson 176225ad1065SPu Lehui /* nops reserved for auipc+jalr pair */ 176325ad1065SPu Lehui for (i = 0; i < RV_FENTRY_NINSNS; i++) 1764596f2e6fSPu Lehui emit(rv_nop(), ctx); 1765596f2e6fSPu Lehui 1766ca6cb544SLuke Nelson /* First instruction is always setting the tail-call-counter 1767ca6cb544SLuke Nelson * (TCC) register. This instruction is skipped for tail calls. 176818a4d8c9SLuke Nelson * Force using a 4-byte (non-compressed) instruction. 1769ca6cb544SLuke Nelson */ 1770ca6cb544SLuke Nelson emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); 1771ca6cb544SLuke Nelson 177218a4d8c9SLuke Nelson emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); 1773ca6cb544SLuke Nelson 1774ca6cb544SLuke Nelson if (seen_reg(RV_REG_RA, ctx)) { 177518a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_RA, ctx); 1776ca6cb544SLuke Nelson store_offset -= 8; 1777ca6cb544SLuke Nelson } 177818a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_FP, ctx); 1779ca6cb544SLuke Nelson store_offset -= 8; 1780ca6cb544SLuke Nelson if (seen_reg(RV_REG_S1, ctx)) { 178118a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S1, ctx); 1782ca6cb544SLuke Nelson store_offset -= 8; 1783ca6cb544SLuke Nelson } 1784ca6cb544SLuke Nelson if (seen_reg(RV_REG_S2, ctx)) { 178518a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S2, ctx); 1786ca6cb544SLuke Nelson store_offset -= 8; 1787ca6cb544SLuke Nelson } 1788ca6cb544SLuke Nelson if (seen_reg(RV_REG_S3, ctx)) { 178918a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S3, ctx); 1790ca6cb544SLuke Nelson store_offset -= 8; 1791ca6cb544SLuke Nelson } 1792ca6cb544SLuke Nelson if (seen_reg(RV_REG_S4, ctx)) { 179318a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S4, ctx); 1794ca6cb544SLuke Nelson store_offset -= 8; 1795ca6cb544SLuke Nelson } 1796ca6cb544SLuke Nelson if (seen_reg(RV_REG_S5, ctx)) { 179718a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx); 1798ca6cb544SLuke Nelson store_offset -= 8; 1799ca6cb544SLuke Nelson } 1800ca6cb544SLuke Nelson if (seen_reg(RV_REG_S6, ctx)) { 180118a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx); 1802ca6cb544SLuke Nelson store_offset -= 8; 1803ca6cb544SLuke Nelson } 1804ca6cb544SLuke Nelson 180518a4d8c9SLuke Nelson emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx); 1806ca6cb544SLuke Nelson 1807ca6cb544SLuke Nelson if (bpf_stack_adjust) 180818a4d8c9SLuke Nelson emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx); 1809ca6cb544SLuke Nelson 1810ca6cb544SLuke Nelson /* Program contains calls and tail calls, so RV_REG_TCC need 1811ca6cb544SLuke Nelson * to be saved across calls. 1812ca6cb544SLuke Nelson */ 1813ca6cb544SLuke Nelson if (seen_tail_call(ctx) && seen_call(ctx)) 181418a4d8c9SLuke Nelson emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx); 1815ca6cb544SLuke Nelson 1816ca6cb544SLuke Nelson ctx->stack_size = stack_adjust; 1817ca6cb544SLuke Nelson } 1818ca6cb544SLuke Nelson 1819ca6cb544SLuke Nelson void bpf_jit_build_epilogue(struct rv_jit_context *ctx) 1820ca6cb544SLuke Nelson { 1821ca6cb544SLuke Nelson __build_epilogue(false, ctx); 1822ca6cb544SLuke Nelson } 1823d40c3847SPu Lehui 1824d40c3847SPu Lehui bool bpf_jit_supports_kfunc_call(void) 1825d40c3847SPu Lehui { 1826d40c3847SPu Lehui return true; 1827d40c3847SPu Lehui } 1828