1ca6cb544SLuke Nelson // SPDX-License-Identifier: GPL-2.0 2ca6cb544SLuke Nelson /* BPF JIT compiler for RV64G 3ca6cb544SLuke Nelson * 4ca6cb544SLuke Nelson * Copyright(c) 2019 Björn Töpel <bjorn.topel@gmail.com> 5ca6cb544SLuke Nelson * 6ca6cb544SLuke Nelson */ 7ca6cb544SLuke Nelson 8ca6cb544SLuke Nelson #include <linux/bpf.h> 9ca6cb544SLuke Nelson #include <linux/filter.h> 10ca6cb544SLuke Nelson #include "bpf_jit.h" 11ca6cb544SLuke Nelson 12ca6cb544SLuke Nelson #define RV_REG_TCC RV_REG_A6 13ca6cb544SLuke Nelson #define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */ 14ca6cb544SLuke Nelson 15ca6cb544SLuke Nelson static const int regmap[] = { 16ca6cb544SLuke Nelson [BPF_REG_0] = RV_REG_A5, 17ca6cb544SLuke Nelson [BPF_REG_1] = RV_REG_A0, 18ca6cb544SLuke Nelson [BPF_REG_2] = RV_REG_A1, 19ca6cb544SLuke Nelson [BPF_REG_3] = RV_REG_A2, 20ca6cb544SLuke Nelson [BPF_REG_4] = RV_REG_A3, 21ca6cb544SLuke Nelson [BPF_REG_5] = RV_REG_A4, 22ca6cb544SLuke Nelson [BPF_REG_6] = RV_REG_S1, 23ca6cb544SLuke Nelson [BPF_REG_7] = RV_REG_S2, 24ca6cb544SLuke Nelson [BPF_REG_8] = RV_REG_S3, 25ca6cb544SLuke Nelson [BPF_REG_9] = RV_REG_S4, 26ca6cb544SLuke Nelson [BPF_REG_FP] = RV_REG_S5, 27ca6cb544SLuke Nelson [BPF_REG_AX] = RV_REG_T0, 28ca6cb544SLuke Nelson }; 29ca6cb544SLuke Nelson 30ca6cb544SLuke Nelson enum { 31ca6cb544SLuke Nelson RV_CTX_F_SEEN_TAIL_CALL = 0, 32ca6cb544SLuke Nelson RV_CTX_F_SEEN_CALL = RV_REG_RA, 33ca6cb544SLuke Nelson RV_CTX_F_SEEN_S1 = RV_REG_S1, 34ca6cb544SLuke Nelson RV_CTX_F_SEEN_S2 = RV_REG_S2, 35ca6cb544SLuke Nelson RV_CTX_F_SEEN_S3 = RV_REG_S3, 36ca6cb544SLuke Nelson RV_CTX_F_SEEN_S4 = RV_REG_S4, 37ca6cb544SLuke Nelson RV_CTX_F_SEEN_S5 = RV_REG_S5, 38ca6cb544SLuke Nelson RV_CTX_F_SEEN_S6 = RV_REG_S6, 39ca6cb544SLuke Nelson }; 40ca6cb544SLuke Nelson 41ca6cb544SLuke Nelson static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) 42ca6cb544SLuke Nelson { 43ca6cb544SLuke Nelson u8 reg = regmap[bpf_reg]; 44ca6cb544SLuke Nelson 45ca6cb544SLuke Nelson switch (reg) { 46ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S1: 47ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S2: 48ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S3: 49ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S4: 50ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S5: 51ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S6: 52ca6cb544SLuke Nelson __set_bit(reg, &ctx->flags); 53ca6cb544SLuke Nelson } 54ca6cb544SLuke Nelson return reg; 55ca6cb544SLuke Nelson }; 56ca6cb544SLuke Nelson 57ca6cb544SLuke Nelson static bool seen_reg(int reg, struct rv_jit_context *ctx) 58ca6cb544SLuke Nelson { 59ca6cb544SLuke Nelson switch (reg) { 60ca6cb544SLuke Nelson case RV_CTX_F_SEEN_CALL: 61ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S1: 62ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S2: 63ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S3: 64ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S4: 65ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S5: 66ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S6: 67ca6cb544SLuke Nelson return test_bit(reg, &ctx->flags); 68ca6cb544SLuke Nelson } 69ca6cb544SLuke Nelson return false; 70ca6cb544SLuke Nelson } 71ca6cb544SLuke Nelson 72ca6cb544SLuke Nelson static void mark_fp(struct rv_jit_context *ctx) 73ca6cb544SLuke Nelson { 74ca6cb544SLuke Nelson __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags); 75ca6cb544SLuke Nelson } 76ca6cb544SLuke Nelson 77ca6cb544SLuke Nelson static void mark_call(struct rv_jit_context *ctx) 78ca6cb544SLuke Nelson { 79ca6cb544SLuke Nelson __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 80ca6cb544SLuke Nelson } 81ca6cb544SLuke Nelson 82ca6cb544SLuke Nelson static bool seen_call(struct rv_jit_context *ctx) 83ca6cb544SLuke Nelson { 84ca6cb544SLuke Nelson return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 85ca6cb544SLuke Nelson } 86ca6cb544SLuke Nelson 87ca6cb544SLuke Nelson static void mark_tail_call(struct rv_jit_context *ctx) 88ca6cb544SLuke Nelson { 89ca6cb544SLuke Nelson __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 90ca6cb544SLuke Nelson } 91ca6cb544SLuke Nelson 92ca6cb544SLuke Nelson static bool seen_tail_call(struct rv_jit_context *ctx) 93ca6cb544SLuke Nelson { 94ca6cb544SLuke Nelson return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 95ca6cb544SLuke Nelson } 96ca6cb544SLuke Nelson 97ca6cb544SLuke Nelson static u8 rv_tail_call_reg(struct rv_jit_context *ctx) 98ca6cb544SLuke Nelson { 99ca6cb544SLuke Nelson mark_tail_call(ctx); 100ca6cb544SLuke Nelson 101ca6cb544SLuke Nelson if (seen_call(ctx)) { 102ca6cb544SLuke Nelson __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); 103ca6cb544SLuke Nelson return RV_REG_S6; 104ca6cb544SLuke Nelson } 105ca6cb544SLuke Nelson return RV_REG_A6; 106ca6cb544SLuke Nelson } 107ca6cb544SLuke Nelson 108ca6cb544SLuke Nelson static bool is_32b_int(s64 val) 109ca6cb544SLuke Nelson { 110ca6cb544SLuke Nelson return -(1L << 31) <= val && val < (1L << 31); 111ca6cb544SLuke Nelson } 112ca6cb544SLuke Nelson 113489553ddSLuke Nelson static bool in_auipc_jalr_range(s64 val) 114489553ddSLuke Nelson { 115489553ddSLuke Nelson /* 116489553ddSLuke Nelson * auipc+jalr can reach any signed PC-relative offset in the range 117489553ddSLuke Nelson * [-2^31 - 2^11, 2^31 - 2^11). 118489553ddSLuke Nelson */ 119489553ddSLuke Nelson return (-(1L << 31) - (1L << 11)) <= val && 120489553ddSLuke Nelson val < ((1L << 31) - (1L << 11)); 121489553ddSLuke Nelson } 122489553ddSLuke Nelson 123ca6cb544SLuke Nelson static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx) 124ca6cb544SLuke Nelson { 125ca6cb544SLuke Nelson /* Note that the immediate from the add is sign-extended, 126ca6cb544SLuke Nelson * which means that we need to compensate this by adding 2^12, 127ca6cb544SLuke Nelson * when the 12th bit is set. A simpler way of doing this, and 128ca6cb544SLuke Nelson * getting rid of the check, is to just add 2**11 before the 129ca6cb544SLuke Nelson * shift. The "Loading a 32-Bit constant" example from the 130ca6cb544SLuke Nelson * "Computer Organization and Design, RISC-V edition" book by 131ca6cb544SLuke Nelson * Patterson/Hennessy highlights this fact. 132ca6cb544SLuke Nelson * 133ca6cb544SLuke Nelson * This also means that we need to process LSB to MSB. 134ca6cb544SLuke Nelson */ 135ca6cb544SLuke Nelson s64 upper = (val + (1 << 11)) >> 12, lower = val & 0xfff; 136ca6cb544SLuke Nelson int shift; 137ca6cb544SLuke Nelson 138ca6cb544SLuke Nelson if (is_32b_int(val)) { 139ca6cb544SLuke Nelson if (upper) 140ca6cb544SLuke Nelson emit(rv_lui(rd, upper), ctx); 141ca6cb544SLuke Nelson 142ca6cb544SLuke Nelson if (!upper) { 143ca6cb544SLuke Nelson emit(rv_addi(rd, RV_REG_ZERO, lower), ctx); 144ca6cb544SLuke Nelson return; 145ca6cb544SLuke Nelson } 146ca6cb544SLuke Nelson 147ca6cb544SLuke Nelson emit(rv_addiw(rd, rd, lower), ctx); 148ca6cb544SLuke Nelson return; 149ca6cb544SLuke Nelson } 150ca6cb544SLuke Nelson 151ca6cb544SLuke Nelson shift = __ffs(upper); 152ca6cb544SLuke Nelson upper >>= shift; 153ca6cb544SLuke Nelson shift += 12; 154ca6cb544SLuke Nelson 155ca6cb544SLuke Nelson emit_imm(rd, upper, ctx); 156ca6cb544SLuke Nelson 157ca6cb544SLuke Nelson emit(rv_slli(rd, rd, shift), ctx); 158ca6cb544SLuke Nelson if (lower) 159ca6cb544SLuke Nelson emit(rv_addi(rd, rd, lower), ctx); 160ca6cb544SLuke Nelson } 161ca6cb544SLuke Nelson 162ca6cb544SLuke Nelson static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) 163ca6cb544SLuke Nelson { 164ca6cb544SLuke Nelson int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; 165ca6cb544SLuke Nelson 166ca6cb544SLuke Nelson if (seen_reg(RV_REG_RA, ctx)) { 167ca6cb544SLuke Nelson emit(rv_ld(RV_REG_RA, store_offset, RV_REG_SP), ctx); 168ca6cb544SLuke Nelson store_offset -= 8; 169ca6cb544SLuke Nelson } 170ca6cb544SLuke Nelson emit(rv_ld(RV_REG_FP, store_offset, RV_REG_SP), ctx); 171ca6cb544SLuke Nelson store_offset -= 8; 172ca6cb544SLuke Nelson if (seen_reg(RV_REG_S1, ctx)) { 173ca6cb544SLuke Nelson emit(rv_ld(RV_REG_S1, store_offset, RV_REG_SP), ctx); 174ca6cb544SLuke Nelson store_offset -= 8; 175ca6cb544SLuke Nelson } 176ca6cb544SLuke Nelson if (seen_reg(RV_REG_S2, ctx)) { 177ca6cb544SLuke Nelson emit(rv_ld(RV_REG_S2, store_offset, RV_REG_SP), ctx); 178ca6cb544SLuke Nelson store_offset -= 8; 179ca6cb544SLuke Nelson } 180ca6cb544SLuke Nelson if (seen_reg(RV_REG_S3, ctx)) { 181ca6cb544SLuke Nelson emit(rv_ld(RV_REG_S3, store_offset, RV_REG_SP), ctx); 182ca6cb544SLuke Nelson store_offset -= 8; 183ca6cb544SLuke Nelson } 184ca6cb544SLuke Nelson if (seen_reg(RV_REG_S4, ctx)) { 185ca6cb544SLuke Nelson emit(rv_ld(RV_REG_S4, store_offset, RV_REG_SP), ctx); 186ca6cb544SLuke Nelson store_offset -= 8; 187ca6cb544SLuke Nelson } 188ca6cb544SLuke Nelson if (seen_reg(RV_REG_S5, ctx)) { 189ca6cb544SLuke Nelson emit(rv_ld(RV_REG_S5, store_offset, RV_REG_SP), ctx); 190ca6cb544SLuke Nelson store_offset -= 8; 191ca6cb544SLuke Nelson } 192ca6cb544SLuke Nelson if (seen_reg(RV_REG_S6, ctx)) { 193ca6cb544SLuke Nelson emit(rv_ld(RV_REG_S6, store_offset, RV_REG_SP), ctx); 194ca6cb544SLuke Nelson store_offset -= 8; 195ca6cb544SLuke Nelson } 196ca6cb544SLuke Nelson 197ca6cb544SLuke Nelson emit(rv_addi(RV_REG_SP, RV_REG_SP, stack_adjust), ctx); 198ca6cb544SLuke Nelson /* Set return value. */ 199ca6cb544SLuke Nelson if (!is_tail_call) 200ca6cb544SLuke Nelson emit(rv_addi(RV_REG_A0, RV_REG_A5, 0), ctx); 201ca6cb544SLuke Nelson emit(rv_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, 202ca6cb544SLuke Nelson is_tail_call ? 4 : 0), /* skip TCC init */ 203ca6cb544SLuke Nelson ctx); 204ca6cb544SLuke Nelson } 205ca6cb544SLuke Nelson 206ca6cb544SLuke Nelson static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, 207ca6cb544SLuke Nelson struct rv_jit_context *ctx) 208ca6cb544SLuke Nelson { 209ca6cb544SLuke Nelson switch (cond) { 210ca6cb544SLuke Nelson case BPF_JEQ: 211ca6cb544SLuke Nelson emit(rv_beq(rd, rs, rvoff >> 1), ctx); 212ca6cb544SLuke Nelson return; 213ca6cb544SLuke Nelson case BPF_JGT: 214ca6cb544SLuke Nelson emit(rv_bltu(rs, rd, rvoff >> 1), ctx); 215ca6cb544SLuke Nelson return; 216ca6cb544SLuke Nelson case BPF_JLT: 217ca6cb544SLuke Nelson emit(rv_bltu(rd, rs, rvoff >> 1), ctx); 218ca6cb544SLuke Nelson return; 219ca6cb544SLuke Nelson case BPF_JGE: 220ca6cb544SLuke Nelson emit(rv_bgeu(rd, rs, rvoff >> 1), ctx); 221ca6cb544SLuke Nelson return; 222ca6cb544SLuke Nelson case BPF_JLE: 223ca6cb544SLuke Nelson emit(rv_bgeu(rs, rd, rvoff >> 1), ctx); 224ca6cb544SLuke Nelson return; 225ca6cb544SLuke Nelson case BPF_JNE: 226ca6cb544SLuke Nelson emit(rv_bne(rd, rs, rvoff >> 1), ctx); 227ca6cb544SLuke Nelson return; 228ca6cb544SLuke Nelson case BPF_JSGT: 229ca6cb544SLuke Nelson emit(rv_blt(rs, rd, rvoff >> 1), ctx); 230ca6cb544SLuke Nelson return; 231ca6cb544SLuke Nelson case BPF_JSLT: 232ca6cb544SLuke Nelson emit(rv_blt(rd, rs, rvoff >> 1), ctx); 233ca6cb544SLuke Nelson return; 234ca6cb544SLuke Nelson case BPF_JSGE: 235ca6cb544SLuke Nelson emit(rv_bge(rd, rs, rvoff >> 1), ctx); 236ca6cb544SLuke Nelson return; 237ca6cb544SLuke Nelson case BPF_JSLE: 238ca6cb544SLuke Nelson emit(rv_bge(rs, rd, rvoff >> 1), ctx); 239ca6cb544SLuke Nelson } 240ca6cb544SLuke Nelson } 241ca6cb544SLuke Nelson 242ca6cb544SLuke Nelson static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, 243ca6cb544SLuke Nelson struct rv_jit_context *ctx) 244ca6cb544SLuke Nelson { 245ca6cb544SLuke Nelson s64 upper, lower; 246ca6cb544SLuke Nelson 247ca6cb544SLuke Nelson if (is_13b_int(rvoff)) { 248ca6cb544SLuke Nelson emit_bcc(cond, rd, rs, rvoff, ctx); 249ca6cb544SLuke Nelson return; 250ca6cb544SLuke Nelson } 251ca6cb544SLuke Nelson 252ca6cb544SLuke Nelson /* Adjust for jal */ 253ca6cb544SLuke Nelson rvoff -= 4; 254ca6cb544SLuke Nelson 255ca6cb544SLuke Nelson /* Transform, e.g.: 256ca6cb544SLuke Nelson * bne rd,rs,foo 257ca6cb544SLuke Nelson * to 258ca6cb544SLuke Nelson * beq rd,rs,<.L1> 259ca6cb544SLuke Nelson * (auipc foo) 260ca6cb544SLuke Nelson * jal(r) foo 261ca6cb544SLuke Nelson * .L1 262ca6cb544SLuke Nelson */ 263ca6cb544SLuke Nelson cond = invert_bpf_cond(cond); 264ca6cb544SLuke Nelson if (is_21b_int(rvoff)) { 265ca6cb544SLuke Nelson emit_bcc(cond, rd, rs, 8, ctx); 266ca6cb544SLuke Nelson emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx); 267ca6cb544SLuke Nelson return; 268ca6cb544SLuke Nelson } 269ca6cb544SLuke Nelson 270ca6cb544SLuke Nelson /* 32b No need for an additional rvoff adjustment, since we 271ca6cb544SLuke Nelson * get that from the auipc at PC', where PC = PC' + 4. 272ca6cb544SLuke Nelson */ 273ca6cb544SLuke Nelson upper = (rvoff + (1 << 11)) >> 12; 274ca6cb544SLuke Nelson lower = rvoff & 0xfff; 275ca6cb544SLuke Nelson 276ca6cb544SLuke Nelson emit_bcc(cond, rd, rs, 12, ctx); 277ca6cb544SLuke Nelson emit(rv_auipc(RV_REG_T1, upper), ctx); 278ca6cb544SLuke Nelson emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx); 279ca6cb544SLuke Nelson } 280ca6cb544SLuke Nelson 281ca6cb544SLuke Nelson static void emit_zext_32(u8 reg, struct rv_jit_context *ctx) 282ca6cb544SLuke Nelson { 283ca6cb544SLuke Nelson emit(rv_slli(reg, reg, 32), ctx); 284ca6cb544SLuke Nelson emit(rv_srli(reg, reg, 32), ctx); 285ca6cb544SLuke Nelson } 286ca6cb544SLuke Nelson 287ca6cb544SLuke Nelson static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) 288ca6cb544SLuke Nelson { 289ca6cb544SLuke Nelson int tc_ninsn, off, start_insn = ctx->ninsns; 290ca6cb544SLuke Nelson u8 tcc = rv_tail_call_reg(ctx); 291ca6cb544SLuke Nelson 292ca6cb544SLuke Nelson /* a0: &ctx 293ca6cb544SLuke Nelson * a1: &array 294ca6cb544SLuke Nelson * a2: index 295ca6cb544SLuke Nelson * 296ca6cb544SLuke Nelson * if (index >= array->map.max_entries) 297ca6cb544SLuke Nelson * goto out; 298ca6cb544SLuke Nelson */ 299ca6cb544SLuke Nelson tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] : 300ca6cb544SLuke Nelson ctx->offset[0]; 301ca6cb544SLuke Nelson emit_zext_32(RV_REG_A2, ctx); 302ca6cb544SLuke Nelson 303ca6cb544SLuke Nelson off = offsetof(struct bpf_array, map.max_entries); 304ca6cb544SLuke Nelson if (is_12b_check(off, insn)) 305ca6cb544SLuke Nelson return -1; 306ca6cb544SLuke Nelson emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx); 307ca6cb544SLuke Nelson off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; 308ca6cb544SLuke Nelson emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx); 309ca6cb544SLuke Nelson 310ca6cb544SLuke Nelson /* if (TCC-- < 0) 311ca6cb544SLuke Nelson * goto out; 312ca6cb544SLuke Nelson */ 313ca6cb544SLuke Nelson emit(rv_addi(RV_REG_T1, tcc, -1), ctx); 314ca6cb544SLuke Nelson off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; 315ca6cb544SLuke Nelson emit_branch(BPF_JSLT, tcc, RV_REG_ZERO, off, ctx); 316ca6cb544SLuke Nelson 317ca6cb544SLuke Nelson /* prog = array->ptrs[index]; 318ca6cb544SLuke Nelson * if (!prog) 319ca6cb544SLuke Nelson * goto out; 320ca6cb544SLuke Nelson */ 321ca6cb544SLuke Nelson emit(rv_slli(RV_REG_T2, RV_REG_A2, 3), ctx); 322ca6cb544SLuke Nelson emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_A1), ctx); 323ca6cb544SLuke Nelson off = offsetof(struct bpf_array, ptrs); 324ca6cb544SLuke Nelson if (is_12b_check(off, insn)) 325ca6cb544SLuke Nelson return -1; 326ca6cb544SLuke Nelson emit(rv_ld(RV_REG_T2, off, RV_REG_T2), ctx); 327ca6cb544SLuke Nelson off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; 328ca6cb544SLuke Nelson emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); 329ca6cb544SLuke Nelson 330ca6cb544SLuke Nelson /* goto *(prog->bpf_func + 4); */ 331ca6cb544SLuke Nelson off = offsetof(struct bpf_prog, bpf_func); 332ca6cb544SLuke Nelson if (is_12b_check(off, insn)) 333ca6cb544SLuke Nelson return -1; 334ca6cb544SLuke Nelson emit(rv_ld(RV_REG_T3, off, RV_REG_T2), ctx); 335ca6cb544SLuke Nelson emit(rv_addi(RV_REG_TCC, RV_REG_T1, 0), ctx); 336ca6cb544SLuke Nelson __build_epilogue(true, ctx); 337ca6cb544SLuke Nelson return 0; 338ca6cb544SLuke Nelson } 339ca6cb544SLuke Nelson 340ca6cb544SLuke Nelson static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn, 341ca6cb544SLuke Nelson struct rv_jit_context *ctx) 342ca6cb544SLuke Nelson { 343ca6cb544SLuke Nelson u8 code = insn->code; 344ca6cb544SLuke Nelson 345ca6cb544SLuke Nelson switch (code) { 346ca6cb544SLuke Nelson case BPF_JMP | BPF_JA: 347ca6cb544SLuke Nelson case BPF_JMP | BPF_CALL: 348ca6cb544SLuke Nelson case BPF_JMP | BPF_EXIT: 349ca6cb544SLuke Nelson case BPF_JMP | BPF_TAIL_CALL: 350ca6cb544SLuke Nelson break; 351ca6cb544SLuke Nelson default: 352ca6cb544SLuke Nelson *rd = bpf_to_rv_reg(insn->dst_reg, ctx); 353ca6cb544SLuke Nelson } 354ca6cb544SLuke Nelson 355ca6cb544SLuke Nelson if (code & (BPF_ALU | BPF_X) || code & (BPF_ALU64 | BPF_X) || 356ca6cb544SLuke Nelson code & (BPF_JMP | BPF_X) || code & (BPF_JMP32 | BPF_X) || 357ca6cb544SLuke Nelson code & BPF_LDX || code & BPF_STX) 358ca6cb544SLuke Nelson *rs = bpf_to_rv_reg(insn->src_reg, ctx); 359ca6cb544SLuke Nelson } 360ca6cb544SLuke Nelson 361ca6cb544SLuke Nelson static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) 362ca6cb544SLuke Nelson { 363ca6cb544SLuke Nelson emit(rv_addi(RV_REG_T2, *rd, 0), ctx); 364ca6cb544SLuke Nelson emit_zext_32(RV_REG_T2, ctx); 365ca6cb544SLuke Nelson emit(rv_addi(RV_REG_T1, *rs, 0), ctx); 366ca6cb544SLuke Nelson emit_zext_32(RV_REG_T1, ctx); 367ca6cb544SLuke Nelson *rd = RV_REG_T2; 368ca6cb544SLuke Nelson *rs = RV_REG_T1; 369ca6cb544SLuke Nelson } 370ca6cb544SLuke Nelson 371ca6cb544SLuke Nelson static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) 372ca6cb544SLuke Nelson { 373ca6cb544SLuke Nelson emit(rv_addiw(RV_REG_T2, *rd, 0), ctx); 374ca6cb544SLuke Nelson emit(rv_addiw(RV_REG_T1, *rs, 0), ctx); 375ca6cb544SLuke Nelson *rd = RV_REG_T2; 376ca6cb544SLuke Nelson *rs = RV_REG_T1; 377ca6cb544SLuke Nelson } 378ca6cb544SLuke Nelson 379ca6cb544SLuke Nelson static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx) 380ca6cb544SLuke Nelson { 381ca6cb544SLuke Nelson emit(rv_addi(RV_REG_T2, *rd, 0), ctx); 382ca6cb544SLuke Nelson emit_zext_32(RV_REG_T2, ctx); 383ca6cb544SLuke Nelson emit_zext_32(RV_REG_T1, ctx); 384ca6cb544SLuke Nelson *rd = RV_REG_T2; 385ca6cb544SLuke Nelson } 386ca6cb544SLuke Nelson 387ca6cb544SLuke Nelson static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx) 388ca6cb544SLuke Nelson { 389ca6cb544SLuke Nelson emit(rv_addiw(RV_REG_T2, *rd, 0), ctx); 390ca6cb544SLuke Nelson *rd = RV_REG_T2; 391ca6cb544SLuke Nelson } 392ca6cb544SLuke Nelson 393489553ddSLuke Nelson static int emit_jump_and_link(u8 rd, s64 rvoff, bool force_jalr, 394ca6cb544SLuke Nelson struct rv_jit_context *ctx) 395ca6cb544SLuke Nelson { 396ca6cb544SLuke Nelson s64 upper, lower; 397ca6cb544SLuke Nelson 398ca6cb544SLuke Nelson if (rvoff && is_21b_int(rvoff) && !force_jalr) { 399ca6cb544SLuke Nelson emit(rv_jal(rd, rvoff >> 1), ctx); 400489553ddSLuke Nelson return 0; 401489553ddSLuke Nelson } else if (in_auipc_jalr_range(rvoff)) { 402ca6cb544SLuke Nelson upper = (rvoff + (1 << 11)) >> 12; 403ca6cb544SLuke Nelson lower = rvoff & 0xfff; 404ca6cb544SLuke Nelson emit(rv_auipc(RV_REG_T1, upper), ctx); 405ca6cb544SLuke Nelson emit(rv_jalr(rd, RV_REG_T1, lower), ctx); 406489553ddSLuke Nelson return 0; 407489553ddSLuke Nelson } 408489553ddSLuke Nelson 409489553ddSLuke Nelson pr_err("bpf-jit: target offset 0x%llx is out of range\n", rvoff); 410489553ddSLuke Nelson return -ERANGE; 411ca6cb544SLuke Nelson } 412ca6cb544SLuke Nelson 413ca6cb544SLuke Nelson static bool is_signed_bpf_cond(u8 cond) 414ca6cb544SLuke Nelson { 415ca6cb544SLuke Nelson return cond == BPF_JSGT || cond == BPF_JSLT || 416ca6cb544SLuke Nelson cond == BPF_JSGE || cond == BPF_JSLE; 417ca6cb544SLuke Nelson } 418ca6cb544SLuke Nelson 419ca6cb544SLuke Nelson static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx) 420ca6cb544SLuke Nelson { 421ca6cb544SLuke Nelson s64 off = 0; 422ca6cb544SLuke Nelson u64 ip; 423ca6cb544SLuke Nelson u8 rd; 424489553ddSLuke Nelson int ret; 425ca6cb544SLuke Nelson 426ca6cb544SLuke Nelson if (addr && ctx->insns) { 427ca6cb544SLuke Nelson ip = (u64)(long)(ctx->insns + ctx->ninsns); 428ca6cb544SLuke Nelson off = addr - ip; 429ca6cb544SLuke Nelson } 430ca6cb544SLuke Nelson 431489553ddSLuke Nelson ret = emit_jump_and_link(RV_REG_RA, off, !fixed, ctx); 432489553ddSLuke Nelson if (ret) 433489553ddSLuke Nelson return ret; 434ca6cb544SLuke Nelson rd = bpf_to_rv_reg(BPF_REG_0, ctx); 435ca6cb544SLuke Nelson emit(rv_addi(rd, RV_REG_A0, 0), ctx); 436ca6cb544SLuke Nelson return 0; 437ca6cb544SLuke Nelson } 438ca6cb544SLuke Nelson 439ca6cb544SLuke Nelson int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 440ca6cb544SLuke Nelson bool extra_pass) 441ca6cb544SLuke Nelson { 442ca6cb544SLuke Nelson bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || 443ca6cb544SLuke Nelson BPF_CLASS(insn->code) == BPF_JMP; 444489553ddSLuke Nelson int s, e, rvoff, ret, i = insn - ctx->prog->insnsi; 445ca6cb544SLuke Nelson struct bpf_prog_aux *aux = ctx->prog->aux; 446ca6cb544SLuke Nelson u8 rd = -1, rs = -1, code = insn->code; 447ca6cb544SLuke Nelson s16 off = insn->off; 448ca6cb544SLuke Nelson s32 imm = insn->imm; 449ca6cb544SLuke Nelson 450ca6cb544SLuke Nelson init_regs(&rd, &rs, insn, ctx); 451ca6cb544SLuke Nelson 452ca6cb544SLuke Nelson switch (code) { 453ca6cb544SLuke Nelson /* dst = src */ 454ca6cb544SLuke Nelson case BPF_ALU | BPF_MOV | BPF_X: 455ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MOV | BPF_X: 456ca6cb544SLuke Nelson if (imm == 1) { 457ca6cb544SLuke Nelson /* Special mov32 for zext */ 458ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 459ca6cb544SLuke Nelson break; 460ca6cb544SLuke Nelson } 461ca6cb544SLuke Nelson emit(is64 ? rv_addi(rd, rs, 0) : rv_addiw(rd, rs, 0), ctx); 462ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 463ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 464ca6cb544SLuke Nelson break; 465ca6cb544SLuke Nelson 466ca6cb544SLuke Nelson /* dst = dst OP src */ 467ca6cb544SLuke Nelson case BPF_ALU | BPF_ADD | BPF_X: 468ca6cb544SLuke Nelson case BPF_ALU64 | BPF_ADD | BPF_X: 469ca6cb544SLuke Nelson emit(is64 ? rv_add(rd, rd, rs) : rv_addw(rd, rd, rs), ctx); 470ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 471ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 472ca6cb544SLuke Nelson break; 473ca6cb544SLuke Nelson case BPF_ALU | BPF_SUB | BPF_X: 474ca6cb544SLuke Nelson case BPF_ALU64 | BPF_SUB | BPF_X: 475ca6cb544SLuke Nelson emit(is64 ? rv_sub(rd, rd, rs) : rv_subw(rd, rd, rs), ctx); 476ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 477ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 478ca6cb544SLuke Nelson break; 479ca6cb544SLuke Nelson case BPF_ALU | BPF_AND | BPF_X: 480ca6cb544SLuke Nelson case BPF_ALU64 | BPF_AND | BPF_X: 481ca6cb544SLuke Nelson emit(rv_and(rd, rd, rs), ctx); 482ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 483ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 484ca6cb544SLuke Nelson break; 485ca6cb544SLuke Nelson case BPF_ALU | BPF_OR | BPF_X: 486ca6cb544SLuke Nelson case BPF_ALU64 | BPF_OR | BPF_X: 487ca6cb544SLuke Nelson emit(rv_or(rd, rd, rs), ctx); 488ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 489ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 490ca6cb544SLuke Nelson break; 491ca6cb544SLuke Nelson case BPF_ALU | BPF_XOR | BPF_X: 492ca6cb544SLuke Nelson case BPF_ALU64 | BPF_XOR | BPF_X: 493ca6cb544SLuke Nelson emit(rv_xor(rd, rd, rs), ctx); 494ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 495ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 496ca6cb544SLuke Nelson break; 497ca6cb544SLuke Nelson case BPF_ALU | BPF_MUL | BPF_X: 498ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MUL | BPF_X: 499ca6cb544SLuke Nelson emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx); 500ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 501ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 502ca6cb544SLuke Nelson break; 503ca6cb544SLuke Nelson case BPF_ALU | BPF_DIV | BPF_X: 504ca6cb544SLuke Nelson case BPF_ALU64 | BPF_DIV | BPF_X: 505ca6cb544SLuke Nelson emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx); 506ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 507ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 508ca6cb544SLuke Nelson break; 509ca6cb544SLuke Nelson case BPF_ALU | BPF_MOD | BPF_X: 510ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MOD | BPF_X: 511ca6cb544SLuke Nelson emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx); 512ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 513ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 514ca6cb544SLuke Nelson break; 515ca6cb544SLuke Nelson case BPF_ALU | BPF_LSH | BPF_X: 516ca6cb544SLuke Nelson case BPF_ALU64 | BPF_LSH | BPF_X: 517ca6cb544SLuke Nelson emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx); 5180224b2acSLuke Nelson if (!is64 && !aux->verifier_zext) 519ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 520ca6cb544SLuke Nelson break; 521ca6cb544SLuke Nelson case BPF_ALU | BPF_RSH | BPF_X: 522ca6cb544SLuke Nelson case BPF_ALU64 | BPF_RSH | BPF_X: 523ca6cb544SLuke Nelson emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx); 524ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 525ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 526ca6cb544SLuke Nelson break; 527ca6cb544SLuke Nelson case BPF_ALU | BPF_ARSH | BPF_X: 528ca6cb544SLuke Nelson case BPF_ALU64 | BPF_ARSH | BPF_X: 529ca6cb544SLuke Nelson emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx); 530ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 531ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 532ca6cb544SLuke Nelson break; 533ca6cb544SLuke Nelson 534ca6cb544SLuke Nelson /* dst = -dst */ 535ca6cb544SLuke Nelson case BPF_ALU | BPF_NEG: 536ca6cb544SLuke Nelson case BPF_ALU64 | BPF_NEG: 537ca6cb544SLuke Nelson emit(is64 ? rv_sub(rd, RV_REG_ZERO, rd) : 538ca6cb544SLuke Nelson rv_subw(rd, RV_REG_ZERO, rd), ctx); 539ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 540ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 541ca6cb544SLuke Nelson break; 542ca6cb544SLuke Nelson 543ca6cb544SLuke Nelson /* dst = BSWAP##imm(dst) */ 544ca6cb544SLuke Nelson case BPF_ALU | BPF_END | BPF_FROM_LE: 54521a099abSLuke Nelson switch (imm) { 54621a099abSLuke Nelson case 16: 54721a099abSLuke Nelson emit(rv_slli(rd, rd, 48), ctx); 54821a099abSLuke Nelson emit(rv_srli(rd, rd, 48), ctx); 54921a099abSLuke Nelson break; 55021a099abSLuke Nelson case 32: 55121a099abSLuke Nelson if (!aux->verifier_zext) 55221a099abSLuke Nelson emit_zext_32(rd, ctx); 55321a099abSLuke Nelson break; 55421a099abSLuke Nelson case 64: 55521a099abSLuke Nelson /* Do nothing */ 556ca6cb544SLuke Nelson break; 557ca6cb544SLuke Nelson } 55821a099abSLuke Nelson break; 55921a099abSLuke Nelson 560ca6cb544SLuke Nelson case BPF_ALU | BPF_END | BPF_FROM_BE: 561ca6cb544SLuke Nelson emit(rv_addi(RV_REG_T2, RV_REG_ZERO, 0), ctx); 562ca6cb544SLuke Nelson 563ca6cb544SLuke Nelson emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); 564ca6cb544SLuke Nelson emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); 565ca6cb544SLuke Nelson emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); 566ca6cb544SLuke Nelson emit(rv_srli(rd, rd, 8), ctx); 567ca6cb544SLuke Nelson if (imm == 16) 568ca6cb544SLuke Nelson goto out_be; 569ca6cb544SLuke Nelson 570ca6cb544SLuke Nelson emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); 571ca6cb544SLuke Nelson emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); 572ca6cb544SLuke Nelson emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); 573ca6cb544SLuke Nelson emit(rv_srli(rd, rd, 8), ctx); 574ca6cb544SLuke Nelson 575ca6cb544SLuke Nelson emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); 576ca6cb544SLuke Nelson emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); 577ca6cb544SLuke Nelson emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); 578ca6cb544SLuke Nelson emit(rv_srli(rd, rd, 8), ctx); 579ca6cb544SLuke Nelson if (imm == 32) 580ca6cb544SLuke Nelson goto out_be; 581ca6cb544SLuke Nelson 582ca6cb544SLuke Nelson emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); 583ca6cb544SLuke Nelson emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); 584ca6cb544SLuke Nelson emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); 585ca6cb544SLuke Nelson emit(rv_srli(rd, rd, 8), ctx); 586ca6cb544SLuke Nelson 587ca6cb544SLuke Nelson emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); 588ca6cb544SLuke Nelson emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); 589ca6cb544SLuke Nelson emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); 590ca6cb544SLuke Nelson emit(rv_srli(rd, rd, 8), ctx); 591ca6cb544SLuke Nelson 592ca6cb544SLuke Nelson emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); 593ca6cb544SLuke Nelson emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); 594ca6cb544SLuke Nelson emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); 595ca6cb544SLuke Nelson emit(rv_srli(rd, rd, 8), ctx); 596ca6cb544SLuke Nelson 597ca6cb544SLuke Nelson emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); 598ca6cb544SLuke Nelson emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); 599ca6cb544SLuke Nelson emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); 600ca6cb544SLuke Nelson emit(rv_srli(rd, rd, 8), ctx); 601ca6cb544SLuke Nelson out_be: 602ca6cb544SLuke Nelson emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); 603ca6cb544SLuke Nelson emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); 604ca6cb544SLuke Nelson 605ca6cb544SLuke Nelson emit(rv_addi(rd, RV_REG_T2, 0), ctx); 606ca6cb544SLuke Nelson break; 607ca6cb544SLuke Nelson 608ca6cb544SLuke Nelson /* dst = imm */ 609ca6cb544SLuke Nelson case BPF_ALU | BPF_MOV | BPF_K: 610ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MOV | BPF_K: 611ca6cb544SLuke Nelson emit_imm(rd, imm, ctx); 612ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 613ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 614ca6cb544SLuke Nelson break; 615ca6cb544SLuke Nelson 616ca6cb544SLuke Nelson /* dst = dst OP imm */ 617ca6cb544SLuke Nelson case BPF_ALU | BPF_ADD | BPF_K: 618ca6cb544SLuke Nelson case BPF_ALU64 | BPF_ADD | BPF_K: 619ca6cb544SLuke Nelson if (is_12b_int(imm)) { 620ca6cb544SLuke Nelson emit(is64 ? rv_addi(rd, rd, imm) : 621ca6cb544SLuke Nelson rv_addiw(rd, rd, imm), ctx); 622ca6cb544SLuke Nelson } else { 623ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 624ca6cb544SLuke Nelson emit(is64 ? rv_add(rd, rd, RV_REG_T1) : 625ca6cb544SLuke Nelson rv_addw(rd, rd, RV_REG_T1), ctx); 626ca6cb544SLuke Nelson } 627ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 628ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 629ca6cb544SLuke Nelson break; 630ca6cb544SLuke Nelson case BPF_ALU | BPF_SUB | BPF_K: 631ca6cb544SLuke Nelson case BPF_ALU64 | BPF_SUB | BPF_K: 632ca6cb544SLuke Nelson if (is_12b_int(-imm)) { 633ca6cb544SLuke Nelson emit(is64 ? rv_addi(rd, rd, -imm) : 634ca6cb544SLuke Nelson rv_addiw(rd, rd, -imm), ctx); 635ca6cb544SLuke Nelson } else { 636ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 637ca6cb544SLuke Nelson emit(is64 ? rv_sub(rd, rd, RV_REG_T1) : 638ca6cb544SLuke Nelson rv_subw(rd, rd, RV_REG_T1), ctx); 639ca6cb544SLuke Nelson } 640ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 641ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 642ca6cb544SLuke Nelson break; 643ca6cb544SLuke Nelson case BPF_ALU | BPF_AND | BPF_K: 644ca6cb544SLuke Nelson case BPF_ALU64 | BPF_AND | BPF_K: 645ca6cb544SLuke Nelson if (is_12b_int(imm)) { 646ca6cb544SLuke Nelson emit(rv_andi(rd, rd, imm), ctx); 647ca6cb544SLuke Nelson } else { 648ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 649ca6cb544SLuke Nelson emit(rv_and(rd, rd, RV_REG_T1), ctx); 650ca6cb544SLuke Nelson } 651ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 652ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 653ca6cb544SLuke Nelson break; 654ca6cb544SLuke Nelson case BPF_ALU | BPF_OR | BPF_K: 655ca6cb544SLuke Nelson case BPF_ALU64 | BPF_OR | BPF_K: 656ca6cb544SLuke Nelson if (is_12b_int(imm)) { 657ca6cb544SLuke Nelson emit(rv_ori(rd, rd, imm), ctx); 658ca6cb544SLuke Nelson } else { 659ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 660ca6cb544SLuke Nelson emit(rv_or(rd, rd, RV_REG_T1), ctx); 661ca6cb544SLuke Nelson } 662ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 663ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 664ca6cb544SLuke Nelson break; 665ca6cb544SLuke Nelson case BPF_ALU | BPF_XOR | BPF_K: 666ca6cb544SLuke Nelson case BPF_ALU64 | BPF_XOR | BPF_K: 667ca6cb544SLuke Nelson if (is_12b_int(imm)) { 668ca6cb544SLuke Nelson emit(rv_xori(rd, rd, imm), ctx); 669ca6cb544SLuke Nelson } else { 670ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 671ca6cb544SLuke Nelson emit(rv_xor(rd, rd, RV_REG_T1), ctx); 672ca6cb544SLuke Nelson } 673ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 674ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 675ca6cb544SLuke Nelson break; 676ca6cb544SLuke Nelson case BPF_ALU | BPF_MUL | BPF_K: 677ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MUL | BPF_K: 678ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 679ca6cb544SLuke Nelson emit(is64 ? rv_mul(rd, rd, RV_REG_T1) : 680ca6cb544SLuke Nelson rv_mulw(rd, rd, RV_REG_T1), ctx); 681ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 682ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 683ca6cb544SLuke Nelson break; 684ca6cb544SLuke Nelson case BPF_ALU | BPF_DIV | BPF_K: 685ca6cb544SLuke Nelson case BPF_ALU64 | BPF_DIV | BPF_K: 686ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 687ca6cb544SLuke Nelson emit(is64 ? rv_divu(rd, rd, RV_REG_T1) : 688ca6cb544SLuke Nelson rv_divuw(rd, rd, RV_REG_T1), ctx); 689ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 690ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 691ca6cb544SLuke Nelson break; 692ca6cb544SLuke Nelson case BPF_ALU | BPF_MOD | BPF_K: 693ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MOD | BPF_K: 694ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 695ca6cb544SLuke Nelson emit(is64 ? rv_remu(rd, rd, RV_REG_T1) : 696ca6cb544SLuke Nelson rv_remuw(rd, rd, RV_REG_T1), ctx); 697ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 698ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 699ca6cb544SLuke Nelson break; 700ca6cb544SLuke Nelson case BPF_ALU | BPF_LSH | BPF_K: 701ca6cb544SLuke Nelson case BPF_ALU64 | BPF_LSH | BPF_K: 702ca6cb544SLuke Nelson emit(is64 ? rv_slli(rd, rd, imm) : rv_slliw(rd, rd, imm), ctx); 7030224b2acSLuke Nelson if (!is64 && !aux->verifier_zext) 704ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 705ca6cb544SLuke Nelson break; 706ca6cb544SLuke Nelson case BPF_ALU | BPF_RSH | BPF_K: 707ca6cb544SLuke Nelson case BPF_ALU64 | BPF_RSH | BPF_K: 708ca6cb544SLuke Nelson emit(is64 ? rv_srli(rd, rd, imm) : rv_srliw(rd, rd, imm), ctx); 7090224b2acSLuke Nelson if (!is64 && !aux->verifier_zext) 710ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 711ca6cb544SLuke Nelson break; 712ca6cb544SLuke Nelson case BPF_ALU | BPF_ARSH | BPF_K: 713ca6cb544SLuke Nelson case BPF_ALU64 | BPF_ARSH | BPF_K: 714ca6cb544SLuke Nelson emit(is64 ? rv_srai(rd, rd, imm) : rv_sraiw(rd, rd, imm), ctx); 7150224b2acSLuke Nelson if (!is64 && !aux->verifier_zext) 716ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 717ca6cb544SLuke Nelson break; 718ca6cb544SLuke Nelson 719ca6cb544SLuke Nelson /* JUMP off */ 720ca6cb544SLuke Nelson case BPF_JMP | BPF_JA: 721ca6cb544SLuke Nelson rvoff = rv_offset(i, off, ctx); 722489553ddSLuke Nelson ret = emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx); 723489553ddSLuke Nelson if (ret) 724489553ddSLuke Nelson return ret; 725ca6cb544SLuke Nelson break; 726ca6cb544SLuke Nelson 727ca6cb544SLuke Nelson /* IF (dst COND src) JUMP off */ 728ca6cb544SLuke Nelson case BPF_JMP | BPF_JEQ | BPF_X: 729ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JEQ | BPF_X: 730ca6cb544SLuke Nelson case BPF_JMP | BPF_JGT | BPF_X: 731ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JGT | BPF_X: 732ca6cb544SLuke Nelson case BPF_JMP | BPF_JLT | BPF_X: 733ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JLT | BPF_X: 734ca6cb544SLuke Nelson case BPF_JMP | BPF_JGE | BPF_X: 735ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JGE | BPF_X: 736ca6cb544SLuke Nelson case BPF_JMP | BPF_JLE | BPF_X: 737ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JLE | BPF_X: 738ca6cb544SLuke Nelson case BPF_JMP | BPF_JNE | BPF_X: 739ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JNE | BPF_X: 740ca6cb544SLuke Nelson case BPF_JMP | BPF_JSGT | BPF_X: 741ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSGT | BPF_X: 742ca6cb544SLuke Nelson case BPF_JMP | BPF_JSLT | BPF_X: 743ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSLT | BPF_X: 744ca6cb544SLuke Nelson case BPF_JMP | BPF_JSGE | BPF_X: 745ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSGE | BPF_X: 746ca6cb544SLuke Nelson case BPF_JMP | BPF_JSLE | BPF_X: 747ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSLE | BPF_X: 748ca6cb544SLuke Nelson case BPF_JMP | BPF_JSET | BPF_X: 749ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSET | BPF_X: 750ca6cb544SLuke Nelson rvoff = rv_offset(i, off, ctx); 751ca6cb544SLuke Nelson if (!is64) { 752ca6cb544SLuke Nelson s = ctx->ninsns; 753ca6cb544SLuke Nelson if (is_signed_bpf_cond(BPF_OP(code))) 754ca6cb544SLuke Nelson emit_sext_32_rd_rs(&rd, &rs, ctx); 755ca6cb544SLuke Nelson else 756ca6cb544SLuke Nelson emit_zext_32_rd_rs(&rd, &rs, ctx); 757ca6cb544SLuke Nelson e = ctx->ninsns; 758ca6cb544SLuke Nelson 759ca6cb544SLuke Nelson /* Adjust for extra insns */ 760ca6cb544SLuke Nelson rvoff -= (e - s) << 2; 761ca6cb544SLuke Nelson } 762ca6cb544SLuke Nelson 763ca6cb544SLuke Nelson if (BPF_OP(code) == BPF_JSET) { 764ca6cb544SLuke Nelson /* Adjust for and */ 765ca6cb544SLuke Nelson rvoff -= 4; 766ca6cb544SLuke Nelson emit(rv_and(RV_REG_T1, rd, rs), ctx); 767ca6cb544SLuke Nelson emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, 768ca6cb544SLuke Nelson ctx); 769ca6cb544SLuke Nelson } else { 770ca6cb544SLuke Nelson emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 771ca6cb544SLuke Nelson } 772ca6cb544SLuke Nelson break; 773ca6cb544SLuke Nelson 774ca6cb544SLuke Nelson /* IF (dst COND imm) JUMP off */ 775ca6cb544SLuke Nelson case BPF_JMP | BPF_JEQ | BPF_K: 776ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JEQ | BPF_K: 777ca6cb544SLuke Nelson case BPF_JMP | BPF_JGT | BPF_K: 778ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JGT | BPF_K: 779ca6cb544SLuke Nelson case BPF_JMP | BPF_JLT | BPF_K: 780ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JLT | BPF_K: 781ca6cb544SLuke Nelson case BPF_JMP | BPF_JGE | BPF_K: 782ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JGE | BPF_K: 783ca6cb544SLuke Nelson case BPF_JMP | BPF_JLE | BPF_K: 784ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JLE | BPF_K: 785ca6cb544SLuke Nelson case BPF_JMP | BPF_JNE | BPF_K: 786ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JNE | BPF_K: 787ca6cb544SLuke Nelson case BPF_JMP | BPF_JSGT | BPF_K: 788ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSGT | BPF_K: 789ca6cb544SLuke Nelson case BPF_JMP | BPF_JSLT | BPF_K: 790ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSLT | BPF_K: 791ca6cb544SLuke Nelson case BPF_JMP | BPF_JSGE | BPF_K: 792ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSGE | BPF_K: 793ca6cb544SLuke Nelson case BPF_JMP | BPF_JSLE | BPF_K: 794ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSLE | BPF_K: 795ca6cb544SLuke Nelson case BPF_JMP | BPF_JSET | BPF_K: 796ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSET | BPF_K: 797ca6cb544SLuke Nelson rvoff = rv_offset(i, off, ctx); 798ca6cb544SLuke Nelson s = ctx->ninsns; 799ca349a6aSLuke Nelson if (imm) { 800ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 801ca349a6aSLuke Nelson rs = RV_REG_T1; 802ca349a6aSLuke Nelson } else { 803ca349a6aSLuke Nelson /* If imm is 0, simply use zero register. */ 804ca349a6aSLuke Nelson rs = RV_REG_ZERO; 805ca349a6aSLuke Nelson } 806ca6cb544SLuke Nelson if (!is64) { 807ca6cb544SLuke Nelson if (is_signed_bpf_cond(BPF_OP(code))) 808ca6cb544SLuke Nelson emit_sext_32_rd(&rd, ctx); 809ca6cb544SLuke Nelson else 810ca6cb544SLuke Nelson emit_zext_32_rd_t1(&rd, ctx); 811ca6cb544SLuke Nelson } 812ca6cb544SLuke Nelson e = ctx->ninsns; 813ca6cb544SLuke Nelson 814ca6cb544SLuke Nelson /* Adjust for extra insns */ 815ca6cb544SLuke Nelson rvoff -= (e - s) << 2; 816ca6cb544SLuke Nelson 817ca6cb544SLuke Nelson if (BPF_OP(code) == BPF_JSET) { 818ca6cb544SLuke Nelson /* Adjust for and */ 819ca6cb544SLuke Nelson rvoff -= 4; 820ca349a6aSLuke Nelson emit(rv_and(rs, rd, rs), ctx); 821ca349a6aSLuke Nelson emit_branch(BPF_JNE, rs, RV_REG_ZERO, rvoff, ctx); 822ca6cb544SLuke Nelson } else { 823ca349a6aSLuke Nelson emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 824ca6cb544SLuke Nelson } 825ca6cb544SLuke Nelson break; 826ca6cb544SLuke Nelson 827ca6cb544SLuke Nelson /* function call */ 828ca6cb544SLuke Nelson case BPF_JMP | BPF_CALL: 829ca6cb544SLuke Nelson { 830ca6cb544SLuke Nelson bool fixed; 831ca6cb544SLuke Nelson u64 addr; 832ca6cb544SLuke Nelson 833ca6cb544SLuke Nelson mark_call(ctx); 834ca6cb544SLuke Nelson ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr, 835ca6cb544SLuke Nelson &fixed); 836ca6cb544SLuke Nelson if (ret < 0) 837ca6cb544SLuke Nelson return ret; 838ca6cb544SLuke Nelson ret = emit_call(fixed, addr, ctx); 839ca6cb544SLuke Nelson if (ret) 840ca6cb544SLuke Nelson return ret; 841ca6cb544SLuke Nelson break; 842ca6cb544SLuke Nelson } 843ca6cb544SLuke Nelson /* tail call */ 844ca6cb544SLuke Nelson case BPF_JMP | BPF_TAIL_CALL: 845ca6cb544SLuke Nelson if (emit_bpf_tail_call(i, ctx)) 846ca6cb544SLuke Nelson return -1; 847ca6cb544SLuke Nelson break; 848ca6cb544SLuke Nelson 849ca6cb544SLuke Nelson /* function return */ 850ca6cb544SLuke Nelson case BPF_JMP | BPF_EXIT: 851ca6cb544SLuke Nelson if (i == ctx->prog->len - 1) 852ca6cb544SLuke Nelson break; 853ca6cb544SLuke Nelson 854ca6cb544SLuke Nelson rvoff = epilogue_offset(ctx); 855489553ddSLuke Nelson ret = emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx); 856489553ddSLuke Nelson if (ret) 857489553ddSLuke Nelson return ret; 858ca6cb544SLuke Nelson break; 859ca6cb544SLuke Nelson 860ca6cb544SLuke Nelson /* dst = imm64 */ 861ca6cb544SLuke Nelson case BPF_LD | BPF_IMM | BPF_DW: 862ca6cb544SLuke Nelson { 863ca6cb544SLuke Nelson struct bpf_insn insn1 = insn[1]; 864ca6cb544SLuke Nelson u64 imm64; 865ca6cb544SLuke Nelson 866ca6cb544SLuke Nelson imm64 = (u64)insn1.imm << 32 | (u32)imm; 867ca6cb544SLuke Nelson emit_imm(rd, imm64, ctx); 868ca6cb544SLuke Nelson return 1; 869ca6cb544SLuke Nelson } 870ca6cb544SLuke Nelson 871ca6cb544SLuke Nelson /* LDX: dst = *(size *)(src + off) */ 872ca6cb544SLuke Nelson case BPF_LDX | BPF_MEM | BPF_B: 873ca6cb544SLuke Nelson if (is_12b_int(off)) { 874ca6cb544SLuke Nelson emit(rv_lbu(rd, off, rs), ctx); 875ca6cb544SLuke Nelson break; 876ca6cb544SLuke Nelson } 877ca6cb544SLuke Nelson 878ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 879ca6cb544SLuke Nelson emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx); 880ca6cb544SLuke Nelson emit(rv_lbu(rd, 0, RV_REG_T1), ctx); 881ca6cb544SLuke Nelson if (insn_is_zext(&insn[1])) 882ca6cb544SLuke Nelson return 1; 883ca6cb544SLuke Nelson break; 884ca6cb544SLuke Nelson case BPF_LDX | BPF_MEM | BPF_H: 885ca6cb544SLuke Nelson if (is_12b_int(off)) { 886ca6cb544SLuke Nelson emit(rv_lhu(rd, off, rs), ctx); 887ca6cb544SLuke Nelson break; 888ca6cb544SLuke Nelson } 889ca6cb544SLuke Nelson 890ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 891ca6cb544SLuke Nelson emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx); 892ca6cb544SLuke Nelson emit(rv_lhu(rd, 0, RV_REG_T1), ctx); 893ca6cb544SLuke Nelson if (insn_is_zext(&insn[1])) 894ca6cb544SLuke Nelson return 1; 895ca6cb544SLuke Nelson break; 896ca6cb544SLuke Nelson case BPF_LDX | BPF_MEM | BPF_W: 897ca6cb544SLuke Nelson if (is_12b_int(off)) { 898ca6cb544SLuke Nelson emit(rv_lwu(rd, off, rs), ctx); 899ca6cb544SLuke Nelson break; 900ca6cb544SLuke Nelson } 901ca6cb544SLuke Nelson 902ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 903ca6cb544SLuke Nelson emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx); 904ca6cb544SLuke Nelson emit(rv_lwu(rd, 0, RV_REG_T1), ctx); 905ca6cb544SLuke Nelson if (insn_is_zext(&insn[1])) 906ca6cb544SLuke Nelson return 1; 907ca6cb544SLuke Nelson break; 908ca6cb544SLuke Nelson case BPF_LDX | BPF_MEM | BPF_DW: 909ca6cb544SLuke Nelson if (is_12b_int(off)) { 910ca6cb544SLuke Nelson emit(rv_ld(rd, off, rs), ctx); 911ca6cb544SLuke Nelson break; 912ca6cb544SLuke Nelson } 913ca6cb544SLuke Nelson 914ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 915ca6cb544SLuke Nelson emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx); 916ca6cb544SLuke Nelson emit(rv_ld(rd, 0, RV_REG_T1), ctx); 917ca6cb544SLuke Nelson break; 918ca6cb544SLuke Nelson 919ca6cb544SLuke Nelson /* ST: *(size *)(dst + off) = imm */ 920ca6cb544SLuke Nelson case BPF_ST | BPF_MEM | BPF_B: 921ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 922ca6cb544SLuke Nelson if (is_12b_int(off)) { 923ca6cb544SLuke Nelson emit(rv_sb(rd, off, RV_REG_T1), ctx); 924ca6cb544SLuke Nelson break; 925ca6cb544SLuke Nelson } 926ca6cb544SLuke Nelson 927ca6cb544SLuke Nelson emit_imm(RV_REG_T2, off, ctx); 928ca6cb544SLuke Nelson emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx); 929ca6cb544SLuke Nelson emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); 930ca6cb544SLuke Nelson break; 931ca6cb544SLuke Nelson 932ca6cb544SLuke Nelson case BPF_ST | BPF_MEM | BPF_H: 933ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 934ca6cb544SLuke Nelson if (is_12b_int(off)) { 935ca6cb544SLuke Nelson emit(rv_sh(rd, off, RV_REG_T1), ctx); 936ca6cb544SLuke Nelson break; 937ca6cb544SLuke Nelson } 938ca6cb544SLuke Nelson 939ca6cb544SLuke Nelson emit_imm(RV_REG_T2, off, ctx); 940ca6cb544SLuke Nelson emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx); 941ca6cb544SLuke Nelson emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); 942ca6cb544SLuke Nelson break; 943ca6cb544SLuke Nelson case BPF_ST | BPF_MEM | BPF_W: 944ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 945ca6cb544SLuke Nelson if (is_12b_int(off)) { 946ca6cb544SLuke Nelson emit(rv_sw(rd, off, RV_REG_T1), ctx); 947ca6cb544SLuke Nelson break; 948ca6cb544SLuke Nelson } 949ca6cb544SLuke Nelson 950ca6cb544SLuke Nelson emit_imm(RV_REG_T2, off, ctx); 951ca6cb544SLuke Nelson emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx); 952ca6cb544SLuke Nelson emit(rv_sw(RV_REG_T2, 0, RV_REG_T1), ctx); 953ca6cb544SLuke Nelson break; 954ca6cb544SLuke Nelson case BPF_ST | BPF_MEM | BPF_DW: 955ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 956ca6cb544SLuke Nelson if (is_12b_int(off)) { 957ca6cb544SLuke Nelson emit(rv_sd(rd, off, RV_REG_T1), ctx); 958ca6cb544SLuke Nelson break; 959ca6cb544SLuke Nelson } 960ca6cb544SLuke Nelson 961ca6cb544SLuke Nelson emit_imm(RV_REG_T2, off, ctx); 962ca6cb544SLuke Nelson emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx); 963ca6cb544SLuke Nelson emit(rv_sd(RV_REG_T2, 0, RV_REG_T1), ctx); 964ca6cb544SLuke Nelson break; 965ca6cb544SLuke Nelson 966ca6cb544SLuke Nelson /* STX: *(size *)(dst + off) = src */ 967ca6cb544SLuke Nelson case BPF_STX | BPF_MEM | BPF_B: 968ca6cb544SLuke Nelson if (is_12b_int(off)) { 969ca6cb544SLuke Nelson emit(rv_sb(rd, off, rs), ctx); 970ca6cb544SLuke Nelson break; 971ca6cb544SLuke Nelson } 972ca6cb544SLuke Nelson 973ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 974ca6cb544SLuke Nelson emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx); 975ca6cb544SLuke Nelson emit(rv_sb(RV_REG_T1, 0, rs), ctx); 976ca6cb544SLuke Nelson break; 977ca6cb544SLuke Nelson case BPF_STX | BPF_MEM | BPF_H: 978ca6cb544SLuke Nelson if (is_12b_int(off)) { 979ca6cb544SLuke Nelson emit(rv_sh(rd, off, rs), ctx); 980ca6cb544SLuke Nelson break; 981ca6cb544SLuke Nelson } 982ca6cb544SLuke Nelson 983ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 984ca6cb544SLuke Nelson emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx); 985ca6cb544SLuke Nelson emit(rv_sh(RV_REG_T1, 0, rs), ctx); 986ca6cb544SLuke Nelson break; 987ca6cb544SLuke Nelson case BPF_STX | BPF_MEM | BPF_W: 988ca6cb544SLuke Nelson if (is_12b_int(off)) { 989ca6cb544SLuke Nelson emit(rv_sw(rd, off, rs), ctx); 990ca6cb544SLuke Nelson break; 991ca6cb544SLuke Nelson } 992ca6cb544SLuke Nelson 993ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 994ca6cb544SLuke Nelson emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx); 995ca6cb544SLuke Nelson emit(rv_sw(RV_REG_T1, 0, rs), ctx); 996ca6cb544SLuke Nelson break; 997ca6cb544SLuke Nelson case BPF_STX | BPF_MEM | BPF_DW: 998ca6cb544SLuke Nelson if (is_12b_int(off)) { 999ca6cb544SLuke Nelson emit(rv_sd(rd, off, rs), ctx); 1000ca6cb544SLuke Nelson break; 1001ca6cb544SLuke Nelson } 1002ca6cb544SLuke Nelson 1003ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 1004ca6cb544SLuke Nelson emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx); 1005ca6cb544SLuke Nelson emit(rv_sd(RV_REG_T1, 0, rs), ctx); 1006ca6cb544SLuke Nelson break; 1007ca6cb544SLuke Nelson /* STX XADD: lock *(u32 *)(dst + off) += src */ 1008ca6cb544SLuke Nelson case BPF_STX | BPF_XADD | BPF_W: 1009ca6cb544SLuke Nelson /* STX XADD: lock *(u64 *)(dst + off) += src */ 1010ca6cb544SLuke Nelson case BPF_STX | BPF_XADD | BPF_DW: 1011ca6cb544SLuke Nelson if (off) { 1012ca6cb544SLuke Nelson if (is_12b_int(off)) { 1013ca6cb544SLuke Nelson emit(rv_addi(RV_REG_T1, rd, off), ctx); 1014ca6cb544SLuke Nelson } else { 1015ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 1016ca6cb544SLuke Nelson emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx); 1017ca6cb544SLuke Nelson } 1018ca6cb544SLuke Nelson 1019ca6cb544SLuke Nelson rd = RV_REG_T1; 1020ca6cb544SLuke Nelson } 1021ca6cb544SLuke Nelson 1022ca6cb544SLuke Nelson emit(BPF_SIZE(code) == BPF_W ? 1023ca6cb544SLuke Nelson rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0) : 1024ca6cb544SLuke Nelson rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0), ctx); 1025ca6cb544SLuke Nelson break; 1026ca6cb544SLuke Nelson default: 1027ca6cb544SLuke Nelson pr_err("bpf-jit: unknown opcode %02x\n", code); 1028ca6cb544SLuke Nelson return -EINVAL; 1029ca6cb544SLuke Nelson } 1030ca6cb544SLuke Nelson 1031ca6cb544SLuke Nelson return 0; 1032ca6cb544SLuke Nelson } 1033ca6cb544SLuke Nelson 1034ca6cb544SLuke Nelson void bpf_jit_build_prologue(struct rv_jit_context *ctx) 1035ca6cb544SLuke Nelson { 1036ca6cb544SLuke Nelson int stack_adjust = 0, store_offset, bpf_stack_adjust; 1037ca6cb544SLuke Nelson 1038ca6cb544SLuke Nelson bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); 1039ca6cb544SLuke Nelson if (bpf_stack_adjust) 1040ca6cb544SLuke Nelson mark_fp(ctx); 1041ca6cb544SLuke Nelson 1042ca6cb544SLuke Nelson if (seen_reg(RV_REG_RA, ctx)) 1043ca6cb544SLuke Nelson stack_adjust += 8; 1044ca6cb544SLuke Nelson stack_adjust += 8; /* RV_REG_FP */ 1045ca6cb544SLuke Nelson if (seen_reg(RV_REG_S1, ctx)) 1046ca6cb544SLuke Nelson stack_adjust += 8; 1047ca6cb544SLuke Nelson if (seen_reg(RV_REG_S2, ctx)) 1048ca6cb544SLuke Nelson stack_adjust += 8; 1049ca6cb544SLuke Nelson if (seen_reg(RV_REG_S3, ctx)) 1050ca6cb544SLuke Nelson stack_adjust += 8; 1051ca6cb544SLuke Nelson if (seen_reg(RV_REG_S4, ctx)) 1052ca6cb544SLuke Nelson stack_adjust += 8; 1053ca6cb544SLuke Nelson if (seen_reg(RV_REG_S5, ctx)) 1054ca6cb544SLuke Nelson stack_adjust += 8; 1055ca6cb544SLuke Nelson if (seen_reg(RV_REG_S6, ctx)) 1056ca6cb544SLuke Nelson stack_adjust += 8; 1057ca6cb544SLuke Nelson 1058ca6cb544SLuke Nelson stack_adjust = round_up(stack_adjust, 16); 1059ca6cb544SLuke Nelson stack_adjust += bpf_stack_adjust; 1060ca6cb544SLuke Nelson 1061ca6cb544SLuke Nelson store_offset = stack_adjust - 8; 1062ca6cb544SLuke Nelson 1063ca6cb544SLuke Nelson /* First instruction is always setting the tail-call-counter 1064ca6cb544SLuke Nelson * (TCC) register. This instruction is skipped for tail calls. 1065ca6cb544SLuke Nelson */ 1066ca6cb544SLuke Nelson emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); 1067ca6cb544SLuke Nelson 1068ca6cb544SLuke Nelson emit(rv_addi(RV_REG_SP, RV_REG_SP, -stack_adjust), ctx); 1069ca6cb544SLuke Nelson 1070ca6cb544SLuke Nelson if (seen_reg(RV_REG_RA, ctx)) { 1071ca6cb544SLuke Nelson emit(rv_sd(RV_REG_SP, store_offset, RV_REG_RA), ctx); 1072ca6cb544SLuke Nelson store_offset -= 8; 1073ca6cb544SLuke Nelson } 1074ca6cb544SLuke Nelson emit(rv_sd(RV_REG_SP, store_offset, RV_REG_FP), ctx); 1075ca6cb544SLuke Nelson store_offset -= 8; 1076ca6cb544SLuke Nelson if (seen_reg(RV_REG_S1, ctx)) { 1077ca6cb544SLuke Nelson emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S1), ctx); 1078ca6cb544SLuke Nelson store_offset -= 8; 1079ca6cb544SLuke Nelson } 1080ca6cb544SLuke Nelson if (seen_reg(RV_REG_S2, ctx)) { 1081ca6cb544SLuke Nelson emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S2), ctx); 1082ca6cb544SLuke Nelson store_offset -= 8; 1083ca6cb544SLuke Nelson } 1084ca6cb544SLuke Nelson if (seen_reg(RV_REG_S3, ctx)) { 1085ca6cb544SLuke Nelson emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S3), ctx); 1086ca6cb544SLuke Nelson store_offset -= 8; 1087ca6cb544SLuke Nelson } 1088ca6cb544SLuke Nelson if (seen_reg(RV_REG_S4, ctx)) { 1089ca6cb544SLuke Nelson emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S4), ctx); 1090ca6cb544SLuke Nelson store_offset -= 8; 1091ca6cb544SLuke Nelson } 1092ca6cb544SLuke Nelson if (seen_reg(RV_REG_S5, ctx)) { 1093ca6cb544SLuke Nelson emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S5), ctx); 1094ca6cb544SLuke Nelson store_offset -= 8; 1095ca6cb544SLuke Nelson } 1096ca6cb544SLuke Nelson if (seen_reg(RV_REG_S6, ctx)) { 1097ca6cb544SLuke Nelson emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S6), ctx); 1098ca6cb544SLuke Nelson store_offset -= 8; 1099ca6cb544SLuke Nelson } 1100ca6cb544SLuke Nelson 1101ca6cb544SLuke Nelson emit(rv_addi(RV_REG_FP, RV_REG_SP, stack_adjust), ctx); 1102ca6cb544SLuke Nelson 1103ca6cb544SLuke Nelson if (bpf_stack_adjust) 1104ca6cb544SLuke Nelson emit(rv_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust), ctx); 1105ca6cb544SLuke Nelson 1106ca6cb544SLuke Nelson /* Program contains calls and tail calls, so RV_REG_TCC need 1107ca6cb544SLuke Nelson * to be saved across calls. 1108ca6cb544SLuke Nelson */ 1109ca6cb544SLuke Nelson if (seen_tail_call(ctx) && seen_call(ctx)) 1110ca6cb544SLuke Nelson emit(rv_addi(RV_REG_TCC_SAVED, RV_REG_TCC, 0), ctx); 1111ca6cb544SLuke Nelson 1112ca6cb544SLuke Nelson ctx->stack_size = stack_adjust; 1113ca6cb544SLuke Nelson } 1114ca6cb544SLuke Nelson 1115ca6cb544SLuke Nelson void bpf_jit_build_epilogue(struct rv_jit_context *ctx) 1116ca6cb544SLuke Nelson { 1117ca6cb544SLuke Nelson __build_epilogue(false, ctx); 1118ca6cb544SLuke Nelson } 1119ca6cb544SLuke Nelson 1120ca6cb544SLuke Nelson void *bpf_jit_alloc_exec(unsigned long size) 1121ca6cb544SLuke Nelson { 1122ca6cb544SLuke Nelson return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START, 1123ca6cb544SLuke Nelson BPF_JIT_REGION_END, GFP_KERNEL, 1124ca6cb544SLuke Nelson PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, 1125ca6cb544SLuke Nelson __builtin_return_address(0)); 1126ca6cb544SLuke Nelson } 1127ca6cb544SLuke Nelson 1128ca6cb544SLuke Nelson void bpf_jit_free_exec(void *addr) 1129ca6cb544SLuke Nelson { 1130ca6cb544SLuke Nelson return vfree(addr); 1131ca6cb544SLuke Nelson } 1132