1ca6cb544SLuke Nelson // SPDX-License-Identifier: GPL-2.0 2ca6cb544SLuke Nelson /* BPF JIT compiler for RV64G 3ca6cb544SLuke Nelson * 4ca6cb544SLuke Nelson * Copyright(c) 2019 Björn Töpel <bjorn.topel@gmail.com> 5ca6cb544SLuke Nelson * 6ca6cb544SLuke Nelson */ 7ca6cb544SLuke Nelson 8ca6cb544SLuke Nelson #include <linux/bpf.h> 9ca6cb544SLuke Nelson #include <linux/filter.h> 10ca6cb544SLuke Nelson #include "bpf_jit.h" 11ca6cb544SLuke Nelson 12ca6cb544SLuke Nelson #define RV_REG_TCC RV_REG_A6 13ca6cb544SLuke Nelson #define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */ 14ca6cb544SLuke Nelson 15ca6cb544SLuke Nelson static const int regmap[] = { 16ca6cb544SLuke Nelson [BPF_REG_0] = RV_REG_A5, 17ca6cb544SLuke Nelson [BPF_REG_1] = RV_REG_A0, 18ca6cb544SLuke Nelson [BPF_REG_2] = RV_REG_A1, 19ca6cb544SLuke Nelson [BPF_REG_3] = RV_REG_A2, 20ca6cb544SLuke Nelson [BPF_REG_4] = RV_REG_A3, 21ca6cb544SLuke Nelson [BPF_REG_5] = RV_REG_A4, 22ca6cb544SLuke Nelson [BPF_REG_6] = RV_REG_S1, 23ca6cb544SLuke Nelson [BPF_REG_7] = RV_REG_S2, 24ca6cb544SLuke Nelson [BPF_REG_8] = RV_REG_S3, 25ca6cb544SLuke Nelson [BPF_REG_9] = RV_REG_S4, 26ca6cb544SLuke Nelson [BPF_REG_FP] = RV_REG_S5, 27ca6cb544SLuke Nelson [BPF_REG_AX] = RV_REG_T0, 28ca6cb544SLuke Nelson }; 29ca6cb544SLuke Nelson 30ca6cb544SLuke Nelson enum { 31ca6cb544SLuke Nelson RV_CTX_F_SEEN_TAIL_CALL = 0, 32ca6cb544SLuke Nelson RV_CTX_F_SEEN_CALL = RV_REG_RA, 33ca6cb544SLuke Nelson RV_CTX_F_SEEN_S1 = RV_REG_S1, 34ca6cb544SLuke Nelson RV_CTX_F_SEEN_S2 = RV_REG_S2, 35ca6cb544SLuke Nelson RV_CTX_F_SEEN_S3 = RV_REG_S3, 36ca6cb544SLuke Nelson RV_CTX_F_SEEN_S4 = RV_REG_S4, 37ca6cb544SLuke Nelson RV_CTX_F_SEEN_S5 = RV_REG_S5, 38ca6cb544SLuke Nelson RV_CTX_F_SEEN_S6 = RV_REG_S6, 39ca6cb544SLuke Nelson }; 40ca6cb544SLuke Nelson 41ca6cb544SLuke Nelson static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) 42ca6cb544SLuke Nelson { 43ca6cb544SLuke Nelson u8 reg = regmap[bpf_reg]; 44ca6cb544SLuke Nelson 45ca6cb544SLuke Nelson switch (reg) { 46ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S1: 47ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S2: 48ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S3: 49ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S4: 50ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S5: 51ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S6: 52ca6cb544SLuke Nelson __set_bit(reg, &ctx->flags); 53ca6cb544SLuke Nelson } 54ca6cb544SLuke Nelson return reg; 55ca6cb544SLuke Nelson }; 56ca6cb544SLuke Nelson 57ca6cb544SLuke Nelson static bool seen_reg(int reg, struct rv_jit_context *ctx) 58ca6cb544SLuke Nelson { 59ca6cb544SLuke Nelson switch (reg) { 60ca6cb544SLuke Nelson case RV_CTX_F_SEEN_CALL: 61ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S1: 62ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S2: 63ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S3: 64ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S4: 65ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S5: 66ca6cb544SLuke Nelson case RV_CTX_F_SEEN_S6: 67ca6cb544SLuke Nelson return test_bit(reg, &ctx->flags); 68ca6cb544SLuke Nelson } 69ca6cb544SLuke Nelson return false; 70ca6cb544SLuke Nelson } 71ca6cb544SLuke Nelson 72ca6cb544SLuke Nelson static void mark_fp(struct rv_jit_context *ctx) 73ca6cb544SLuke Nelson { 74ca6cb544SLuke Nelson __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags); 75ca6cb544SLuke Nelson } 76ca6cb544SLuke Nelson 77ca6cb544SLuke Nelson static void mark_call(struct rv_jit_context *ctx) 78ca6cb544SLuke Nelson { 79ca6cb544SLuke Nelson __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 80ca6cb544SLuke Nelson } 81ca6cb544SLuke Nelson 82ca6cb544SLuke Nelson static bool seen_call(struct rv_jit_context *ctx) 83ca6cb544SLuke Nelson { 84ca6cb544SLuke Nelson return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 85ca6cb544SLuke Nelson } 86ca6cb544SLuke Nelson 87ca6cb544SLuke Nelson static void mark_tail_call(struct rv_jit_context *ctx) 88ca6cb544SLuke Nelson { 89ca6cb544SLuke Nelson __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 90ca6cb544SLuke Nelson } 91ca6cb544SLuke Nelson 92ca6cb544SLuke Nelson static bool seen_tail_call(struct rv_jit_context *ctx) 93ca6cb544SLuke Nelson { 94ca6cb544SLuke Nelson return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 95ca6cb544SLuke Nelson } 96ca6cb544SLuke Nelson 97ca6cb544SLuke Nelson static u8 rv_tail_call_reg(struct rv_jit_context *ctx) 98ca6cb544SLuke Nelson { 99ca6cb544SLuke Nelson mark_tail_call(ctx); 100ca6cb544SLuke Nelson 101ca6cb544SLuke Nelson if (seen_call(ctx)) { 102ca6cb544SLuke Nelson __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); 103ca6cb544SLuke Nelson return RV_REG_S6; 104ca6cb544SLuke Nelson } 105ca6cb544SLuke Nelson return RV_REG_A6; 106ca6cb544SLuke Nelson } 107ca6cb544SLuke Nelson 108ca6cb544SLuke Nelson static bool is_32b_int(s64 val) 109ca6cb544SLuke Nelson { 110ca6cb544SLuke Nelson return -(1L << 31) <= val && val < (1L << 31); 111ca6cb544SLuke Nelson } 112ca6cb544SLuke Nelson 113489553ddSLuke Nelson static bool in_auipc_jalr_range(s64 val) 114489553ddSLuke Nelson { 115489553ddSLuke Nelson /* 116489553ddSLuke Nelson * auipc+jalr can reach any signed PC-relative offset in the range 117489553ddSLuke Nelson * [-2^31 - 2^11, 2^31 - 2^11). 118489553ddSLuke Nelson */ 119489553ddSLuke Nelson return (-(1L << 31) - (1L << 11)) <= val && 120489553ddSLuke Nelson val < ((1L << 31) - (1L << 11)); 121489553ddSLuke Nelson } 122489553ddSLuke Nelson 123ca6cb544SLuke Nelson static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx) 124ca6cb544SLuke Nelson { 125ca6cb544SLuke Nelson /* Note that the immediate from the add is sign-extended, 126ca6cb544SLuke Nelson * which means that we need to compensate this by adding 2^12, 127ca6cb544SLuke Nelson * when the 12th bit is set. A simpler way of doing this, and 128ca6cb544SLuke Nelson * getting rid of the check, is to just add 2**11 before the 129ca6cb544SLuke Nelson * shift. The "Loading a 32-Bit constant" example from the 130ca6cb544SLuke Nelson * "Computer Organization and Design, RISC-V edition" book by 131ca6cb544SLuke Nelson * Patterson/Hennessy highlights this fact. 132ca6cb544SLuke Nelson * 133ca6cb544SLuke Nelson * This also means that we need to process LSB to MSB. 134ca6cb544SLuke Nelson */ 13518a4d8c9SLuke Nelson s64 upper = (val + (1 << 11)) >> 12; 13618a4d8c9SLuke Nelson /* Sign-extend lower 12 bits to 64 bits since immediates for li, addiw, 13718a4d8c9SLuke Nelson * and addi are signed and RVC checks will perform signed comparisons. 13818a4d8c9SLuke Nelson */ 13918a4d8c9SLuke Nelson s64 lower = ((val & 0xfff) << 52) >> 52; 140ca6cb544SLuke Nelson int shift; 141ca6cb544SLuke Nelson 142ca6cb544SLuke Nelson if (is_32b_int(val)) { 143ca6cb544SLuke Nelson if (upper) 14418a4d8c9SLuke Nelson emit_lui(rd, upper, ctx); 145ca6cb544SLuke Nelson 146ca6cb544SLuke Nelson if (!upper) { 14718a4d8c9SLuke Nelson emit_li(rd, lower, ctx); 148ca6cb544SLuke Nelson return; 149ca6cb544SLuke Nelson } 150ca6cb544SLuke Nelson 15118a4d8c9SLuke Nelson emit_addiw(rd, rd, lower, ctx); 152ca6cb544SLuke Nelson return; 153ca6cb544SLuke Nelson } 154ca6cb544SLuke Nelson 155ca6cb544SLuke Nelson shift = __ffs(upper); 156ca6cb544SLuke Nelson upper >>= shift; 157ca6cb544SLuke Nelson shift += 12; 158ca6cb544SLuke Nelson 159ca6cb544SLuke Nelson emit_imm(rd, upper, ctx); 160ca6cb544SLuke Nelson 16118a4d8c9SLuke Nelson emit_slli(rd, rd, shift, ctx); 162ca6cb544SLuke Nelson if (lower) 16318a4d8c9SLuke Nelson emit_addi(rd, rd, lower, ctx); 164ca6cb544SLuke Nelson } 165ca6cb544SLuke Nelson 166ca6cb544SLuke Nelson static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) 167ca6cb544SLuke Nelson { 168ca6cb544SLuke Nelson int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; 169ca6cb544SLuke Nelson 170ca6cb544SLuke Nelson if (seen_reg(RV_REG_RA, ctx)) { 17118a4d8c9SLuke Nelson emit_ld(RV_REG_RA, store_offset, RV_REG_SP, ctx); 172ca6cb544SLuke Nelson store_offset -= 8; 173ca6cb544SLuke Nelson } 17418a4d8c9SLuke Nelson emit_ld(RV_REG_FP, store_offset, RV_REG_SP, ctx); 175ca6cb544SLuke Nelson store_offset -= 8; 176ca6cb544SLuke Nelson if (seen_reg(RV_REG_S1, ctx)) { 17718a4d8c9SLuke Nelson emit_ld(RV_REG_S1, store_offset, RV_REG_SP, ctx); 178ca6cb544SLuke Nelson store_offset -= 8; 179ca6cb544SLuke Nelson } 180ca6cb544SLuke Nelson if (seen_reg(RV_REG_S2, ctx)) { 18118a4d8c9SLuke Nelson emit_ld(RV_REG_S2, store_offset, RV_REG_SP, ctx); 182ca6cb544SLuke Nelson store_offset -= 8; 183ca6cb544SLuke Nelson } 184ca6cb544SLuke Nelson if (seen_reg(RV_REG_S3, ctx)) { 18518a4d8c9SLuke Nelson emit_ld(RV_REG_S3, store_offset, RV_REG_SP, ctx); 186ca6cb544SLuke Nelson store_offset -= 8; 187ca6cb544SLuke Nelson } 188ca6cb544SLuke Nelson if (seen_reg(RV_REG_S4, ctx)) { 18918a4d8c9SLuke Nelson emit_ld(RV_REG_S4, store_offset, RV_REG_SP, ctx); 190ca6cb544SLuke Nelson store_offset -= 8; 191ca6cb544SLuke Nelson } 192ca6cb544SLuke Nelson if (seen_reg(RV_REG_S5, ctx)) { 19318a4d8c9SLuke Nelson emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx); 194ca6cb544SLuke Nelson store_offset -= 8; 195ca6cb544SLuke Nelson } 196ca6cb544SLuke Nelson if (seen_reg(RV_REG_S6, ctx)) { 19718a4d8c9SLuke Nelson emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx); 198ca6cb544SLuke Nelson store_offset -= 8; 199ca6cb544SLuke Nelson } 200ca6cb544SLuke Nelson 20118a4d8c9SLuke Nelson emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); 202ca6cb544SLuke Nelson /* Set return value. */ 203ca6cb544SLuke Nelson if (!is_tail_call) 20418a4d8c9SLuke Nelson emit_mv(RV_REG_A0, RV_REG_A5, ctx); 20518a4d8c9SLuke Nelson emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, 20618a4d8c9SLuke Nelson is_tail_call ? 4 : 0, /* skip TCC init */ 207ca6cb544SLuke Nelson ctx); 208ca6cb544SLuke Nelson } 209ca6cb544SLuke Nelson 210ca6cb544SLuke Nelson static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, 211ca6cb544SLuke Nelson struct rv_jit_context *ctx) 212ca6cb544SLuke Nelson { 213ca6cb544SLuke Nelson switch (cond) { 214ca6cb544SLuke Nelson case BPF_JEQ: 215ca6cb544SLuke Nelson emit(rv_beq(rd, rs, rvoff >> 1), ctx); 216ca6cb544SLuke Nelson return; 217ca6cb544SLuke Nelson case BPF_JGT: 218ca6cb544SLuke Nelson emit(rv_bltu(rs, rd, rvoff >> 1), ctx); 219ca6cb544SLuke Nelson return; 220ca6cb544SLuke Nelson case BPF_JLT: 221ca6cb544SLuke Nelson emit(rv_bltu(rd, rs, rvoff >> 1), ctx); 222ca6cb544SLuke Nelson return; 223ca6cb544SLuke Nelson case BPF_JGE: 224ca6cb544SLuke Nelson emit(rv_bgeu(rd, rs, rvoff >> 1), ctx); 225ca6cb544SLuke Nelson return; 226ca6cb544SLuke Nelson case BPF_JLE: 227ca6cb544SLuke Nelson emit(rv_bgeu(rs, rd, rvoff >> 1), ctx); 228ca6cb544SLuke Nelson return; 229ca6cb544SLuke Nelson case BPF_JNE: 230ca6cb544SLuke Nelson emit(rv_bne(rd, rs, rvoff >> 1), ctx); 231ca6cb544SLuke Nelson return; 232ca6cb544SLuke Nelson case BPF_JSGT: 233ca6cb544SLuke Nelson emit(rv_blt(rs, rd, rvoff >> 1), ctx); 234ca6cb544SLuke Nelson return; 235ca6cb544SLuke Nelson case BPF_JSLT: 236ca6cb544SLuke Nelson emit(rv_blt(rd, rs, rvoff >> 1), ctx); 237ca6cb544SLuke Nelson return; 238ca6cb544SLuke Nelson case BPF_JSGE: 239ca6cb544SLuke Nelson emit(rv_bge(rd, rs, rvoff >> 1), ctx); 240ca6cb544SLuke Nelson return; 241ca6cb544SLuke Nelson case BPF_JSLE: 242ca6cb544SLuke Nelson emit(rv_bge(rs, rd, rvoff >> 1), ctx); 243ca6cb544SLuke Nelson } 244ca6cb544SLuke Nelson } 245ca6cb544SLuke Nelson 246ca6cb544SLuke Nelson static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, 247ca6cb544SLuke Nelson struct rv_jit_context *ctx) 248ca6cb544SLuke Nelson { 249ca6cb544SLuke Nelson s64 upper, lower; 250ca6cb544SLuke Nelson 251ca6cb544SLuke Nelson if (is_13b_int(rvoff)) { 252ca6cb544SLuke Nelson emit_bcc(cond, rd, rs, rvoff, ctx); 253ca6cb544SLuke Nelson return; 254ca6cb544SLuke Nelson } 255ca6cb544SLuke Nelson 256ca6cb544SLuke Nelson /* Adjust for jal */ 257ca6cb544SLuke Nelson rvoff -= 4; 258ca6cb544SLuke Nelson 259ca6cb544SLuke Nelson /* Transform, e.g.: 260ca6cb544SLuke Nelson * bne rd,rs,foo 261ca6cb544SLuke Nelson * to 262ca6cb544SLuke Nelson * beq rd,rs,<.L1> 263ca6cb544SLuke Nelson * (auipc foo) 264ca6cb544SLuke Nelson * jal(r) foo 265ca6cb544SLuke Nelson * .L1 266ca6cb544SLuke Nelson */ 267ca6cb544SLuke Nelson cond = invert_bpf_cond(cond); 268ca6cb544SLuke Nelson if (is_21b_int(rvoff)) { 269ca6cb544SLuke Nelson emit_bcc(cond, rd, rs, 8, ctx); 270ca6cb544SLuke Nelson emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx); 271ca6cb544SLuke Nelson return; 272ca6cb544SLuke Nelson } 273ca6cb544SLuke Nelson 274ca6cb544SLuke Nelson /* 32b No need for an additional rvoff adjustment, since we 275ca6cb544SLuke Nelson * get that from the auipc at PC', where PC = PC' + 4. 276ca6cb544SLuke Nelson */ 277ca6cb544SLuke Nelson upper = (rvoff + (1 << 11)) >> 12; 278ca6cb544SLuke Nelson lower = rvoff & 0xfff; 279ca6cb544SLuke Nelson 280ca6cb544SLuke Nelson emit_bcc(cond, rd, rs, 12, ctx); 281ca6cb544SLuke Nelson emit(rv_auipc(RV_REG_T1, upper), ctx); 282ca6cb544SLuke Nelson emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx); 283ca6cb544SLuke Nelson } 284ca6cb544SLuke Nelson 285ca6cb544SLuke Nelson static void emit_zext_32(u8 reg, struct rv_jit_context *ctx) 286ca6cb544SLuke Nelson { 28718a4d8c9SLuke Nelson emit_slli(reg, reg, 32, ctx); 28818a4d8c9SLuke Nelson emit_srli(reg, reg, 32, ctx); 289ca6cb544SLuke Nelson } 290ca6cb544SLuke Nelson 291ca6cb544SLuke Nelson static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) 292ca6cb544SLuke Nelson { 293ca6cb544SLuke Nelson int tc_ninsn, off, start_insn = ctx->ninsns; 294ca6cb544SLuke Nelson u8 tcc = rv_tail_call_reg(ctx); 295ca6cb544SLuke Nelson 296ca6cb544SLuke Nelson /* a0: &ctx 297ca6cb544SLuke Nelson * a1: &array 298ca6cb544SLuke Nelson * a2: index 299ca6cb544SLuke Nelson * 300ca6cb544SLuke Nelson * if (index >= array->map.max_entries) 301ca6cb544SLuke Nelson * goto out; 302ca6cb544SLuke Nelson */ 303ca6cb544SLuke Nelson tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] : 304ca6cb544SLuke Nelson ctx->offset[0]; 305ca6cb544SLuke Nelson emit_zext_32(RV_REG_A2, ctx); 306ca6cb544SLuke Nelson 307ca6cb544SLuke Nelson off = offsetof(struct bpf_array, map.max_entries); 308ca6cb544SLuke Nelson if (is_12b_check(off, insn)) 309ca6cb544SLuke Nelson return -1; 310ca6cb544SLuke Nelson emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx); 311bfabff3cSLuke Nelson off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 312ca6cb544SLuke Nelson emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx); 313ca6cb544SLuke Nelson 314ca6cb544SLuke Nelson /* if (TCC-- < 0) 315ca6cb544SLuke Nelson * goto out; 316ca6cb544SLuke Nelson */ 31718a4d8c9SLuke Nelson emit_addi(RV_REG_T1, tcc, -1, ctx); 318bfabff3cSLuke Nelson off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 319ca6cb544SLuke Nelson emit_branch(BPF_JSLT, tcc, RV_REG_ZERO, off, ctx); 320ca6cb544SLuke Nelson 321ca6cb544SLuke Nelson /* prog = array->ptrs[index]; 322ca6cb544SLuke Nelson * if (!prog) 323ca6cb544SLuke Nelson * goto out; 324ca6cb544SLuke Nelson */ 32518a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_A2, 3, ctx); 32618a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_A1, ctx); 327ca6cb544SLuke Nelson off = offsetof(struct bpf_array, ptrs); 328ca6cb544SLuke Nelson if (is_12b_check(off, insn)) 329ca6cb544SLuke Nelson return -1; 33018a4d8c9SLuke Nelson emit_ld(RV_REG_T2, off, RV_REG_T2, ctx); 331bfabff3cSLuke Nelson off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 332ca6cb544SLuke Nelson emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); 333ca6cb544SLuke Nelson 334ca6cb544SLuke Nelson /* goto *(prog->bpf_func + 4); */ 335ca6cb544SLuke Nelson off = offsetof(struct bpf_prog, bpf_func); 336ca6cb544SLuke Nelson if (is_12b_check(off, insn)) 337ca6cb544SLuke Nelson return -1; 33818a4d8c9SLuke Nelson emit_ld(RV_REG_T3, off, RV_REG_T2, ctx); 33918a4d8c9SLuke Nelson emit_mv(RV_REG_TCC, RV_REG_T1, ctx); 340ca6cb544SLuke Nelson __build_epilogue(true, ctx); 341ca6cb544SLuke Nelson return 0; 342ca6cb544SLuke Nelson } 343ca6cb544SLuke Nelson 344ca6cb544SLuke Nelson static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn, 345ca6cb544SLuke Nelson struct rv_jit_context *ctx) 346ca6cb544SLuke Nelson { 347ca6cb544SLuke Nelson u8 code = insn->code; 348ca6cb544SLuke Nelson 349ca6cb544SLuke Nelson switch (code) { 350ca6cb544SLuke Nelson case BPF_JMP | BPF_JA: 351ca6cb544SLuke Nelson case BPF_JMP | BPF_CALL: 352ca6cb544SLuke Nelson case BPF_JMP | BPF_EXIT: 353ca6cb544SLuke Nelson case BPF_JMP | BPF_TAIL_CALL: 354ca6cb544SLuke Nelson break; 355ca6cb544SLuke Nelson default: 356ca6cb544SLuke Nelson *rd = bpf_to_rv_reg(insn->dst_reg, ctx); 357ca6cb544SLuke Nelson } 358ca6cb544SLuke Nelson 359ca6cb544SLuke Nelson if (code & (BPF_ALU | BPF_X) || code & (BPF_ALU64 | BPF_X) || 360ca6cb544SLuke Nelson code & (BPF_JMP | BPF_X) || code & (BPF_JMP32 | BPF_X) || 361ca6cb544SLuke Nelson code & BPF_LDX || code & BPF_STX) 362ca6cb544SLuke Nelson *rs = bpf_to_rv_reg(insn->src_reg, ctx); 363ca6cb544SLuke Nelson } 364ca6cb544SLuke Nelson 365ca6cb544SLuke Nelson static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) 366ca6cb544SLuke Nelson { 36718a4d8c9SLuke Nelson emit_mv(RV_REG_T2, *rd, ctx); 368ca6cb544SLuke Nelson emit_zext_32(RV_REG_T2, ctx); 36918a4d8c9SLuke Nelson emit_mv(RV_REG_T1, *rs, ctx); 370ca6cb544SLuke Nelson emit_zext_32(RV_REG_T1, ctx); 371ca6cb544SLuke Nelson *rd = RV_REG_T2; 372ca6cb544SLuke Nelson *rs = RV_REG_T1; 373ca6cb544SLuke Nelson } 374ca6cb544SLuke Nelson 375ca6cb544SLuke Nelson static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) 376ca6cb544SLuke Nelson { 37718a4d8c9SLuke Nelson emit_addiw(RV_REG_T2, *rd, 0, ctx); 37818a4d8c9SLuke Nelson emit_addiw(RV_REG_T1, *rs, 0, ctx); 379ca6cb544SLuke Nelson *rd = RV_REG_T2; 380ca6cb544SLuke Nelson *rs = RV_REG_T1; 381ca6cb544SLuke Nelson } 382ca6cb544SLuke Nelson 383ca6cb544SLuke Nelson static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx) 384ca6cb544SLuke Nelson { 38518a4d8c9SLuke Nelson emit_mv(RV_REG_T2, *rd, ctx); 386ca6cb544SLuke Nelson emit_zext_32(RV_REG_T2, ctx); 387ca6cb544SLuke Nelson emit_zext_32(RV_REG_T1, ctx); 388ca6cb544SLuke Nelson *rd = RV_REG_T2; 389ca6cb544SLuke Nelson } 390ca6cb544SLuke Nelson 391ca6cb544SLuke Nelson static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx) 392ca6cb544SLuke Nelson { 39318a4d8c9SLuke Nelson emit_addiw(RV_REG_T2, *rd, 0, ctx); 394ca6cb544SLuke Nelson *rd = RV_REG_T2; 395ca6cb544SLuke Nelson } 396ca6cb544SLuke Nelson 397489553ddSLuke Nelson static int emit_jump_and_link(u8 rd, s64 rvoff, bool force_jalr, 398ca6cb544SLuke Nelson struct rv_jit_context *ctx) 399ca6cb544SLuke Nelson { 400ca6cb544SLuke Nelson s64 upper, lower; 401ca6cb544SLuke Nelson 402ca6cb544SLuke Nelson if (rvoff && is_21b_int(rvoff) && !force_jalr) { 403ca6cb544SLuke Nelson emit(rv_jal(rd, rvoff >> 1), ctx); 404489553ddSLuke Nelson return 0; 405489553ddSLuke Nelson } else if (in_auipc_jalr_range(rvoff)) { 406ca6cb544SLuke Nelson upper = (rvoff + (1 << 11)) >> 12; 407ca6cb544SLuke Nelson lower = rvoff & 0xfff; 408ca6cb544SLuke Nelson emit(rv_auipc(RV_REG_T1, upper), ctx); 409ca6cb544SLuke Nelson emit(rv_jalr(rd, RV_REG_T1, lower), ctx); 410489553ddSLuke Nelson return 0; 411489553ddSLuke Nelson } 412489553ddSLuke Nelson 413489553ddSLuke Nelson pr_err("bpf-jit: target offset 0x%llx is out of range\n", rvoff); 414489553ddSLuke Nelson return -ERANGE; 415ca6cb544SLuke Nelson } 416ca6cb544SLuke Nelson 417ca6cb544SLuke Nelson static bool is_signed_bpf_cond(u8 cond) 418ca6cb544SLuke Nelson { 419ca6cb544SLuke Nelson return cond == BPF_JSGT || cond == BPF_JSLT || 420ca6cb544SLuke Nelson cond == BPF_JSGE || cond == BPF_JSLE; 421ca6cb544SLuke Nelson } 422ca6cb544SLuke Nelson 423ca6cb544SLuke Nelson static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx) 424ca6cb544SLuke Nelson { 425ca6cb544SLuke Nelson s64 off = 0; 426ca6cb544SLuke Nelson u64 ip; 427ca6cb544SLuke Nelson u8 rd; 428489553ddSLuke Nelson int ret; 429ca6cb544SLuke Nelson 430ca6cb544SLuke Nelson if (addr && ctx->insns) { 431ca6cb544SLuke Nelson ip = (u64)(long)(ctx->insns + ctx->ninsns); 432ca6cb544SLuke Nelson off = addr - ip; 433ca6cb544SLuke Nelson } 434ca6cb544SLuke Nelson 435489553ddSLuke Nelson ret = emit_jump_and_link(RV_REG_RA, off, !fixed, ctx); 436489553ddSLuke Nelson if (ret) 437489553ddSLuke Nelson return ret; 438ca6cb544SLuke Nelson rd = bpf_to_rv_reg(BPF_REG_0, ctx); 43918a4d8c9SLuke Nelson emit_mv(rd, RV_REG_A0, ctx); 440ca6cb544SLuke Nelson return 0; 441ca6cb544SLuke Nelson } 442ca6cb544SLuke Nelson 443ca6cb544SLuke Nelson int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 444ca6cb544SLuke Nelson bool extra_pass) 445ca6cb544SLuke Nelson { 446ca6cb544SLuke Nelson bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || 447ca6cb544SLuke Nelson BPF_CLASS(insn->code) == BPF_JMP; 448489553ddSLuke Nelson int s, e, rvoff, ret, i = insn - ctx->prog->insnsi; 449ca6cb544SLuke Nelson struct bpf_prog_aux *aux = ctx->prog->aux; 450ca6cb544SLuke Nelson u8 rd = -1, rs = -1, code = insn->code; 451ca6cb544SLuke Nelson s16 off = insn->off; 452ca6cb544SLuke Nelson s32 imm = insn->imm; 453ca6cb544SLuke Nelson 454ca6cb544SLuke Nelson init_regs(&rd, &rs, insn, ctx); 455ca6cb544SLuke Nelson 456ca6cb544SLuke Nelson switch (code) { 457ca6cb544SLuke Nelson /* dst = src */ 458ca6cb544SLuke Nelson case BPF_ALU | BPF_MOV | BPF_X: 459ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MOV | BPF_X: 460ca6cb544SLuke Nelson if (imm == 1) { 461ca6cb544SLuke Nelson /* Special mov32 for zext */ 462ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 463ca6cb544SLuke Nelson break; 464ca6cb544SLuke Nelson } 46518a4d8c9SLuke Nelson emit_mv(rd, rs, ctx); 466ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 467ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 468ca6cb544SLuke Nelson break; 469ca6cb544SLuke Nelson 470ca6cb544SLuke Nelson /* dst = dst OP src */ 471ca6cb544SLuke Nelson case BPF_ALU | BPF_ADD | BPF_X: 472ca6cb544SLuke Nelson case BPF_ALU64 | BPF_ADD | BPF_X: 47318a4d8c9SLuke Nelson emit_add(rd, rd, rs, ctx); 474ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 475ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 476ca6cb544SLuke Nelson break; 477ca6cb544SLuke Nelson case BPF_ALU | BPF_SUB | BPF_X: 478ca6cb544SLuke Nelson case BPF_ALU64 | BPF_SUB | BPF_X: 47918a4d8c9SLuke Nelson if (is64) 48018a4d8c9SLuke Nelson emit_sub(rd, rd, rs, ctx); 48118a4d8c9SLuke Nelson else 48218a4d8c9SLuke Nelson emit_subw(rd, rd, rs, ctx); 48318a4d8c9SLuke Nelson 484ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 485ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 486ca6cb544SLuke Nelson break; 487ca6cb544SLuke Nelson case BPF_ALU | BPF_AND | BPF_X: 488ca6cb544SLuke Nelson case BPF_ALU64 | BPF_AND | BPF_X: 48918a4d8c9SLuke Nelson emit_and(rd, rd, rs, ctx); 490ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 491ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 492ca6cb544SLuke Nelson break; 493ca6cb544SLuke Nelson case BPF_ALU | BPF_OR | BPF_X: 494ca6cb544SLuke Nelson case BPF_ALU64 | BPF_OR | BPF_X: 49518a4d8c9SLuke Nelson emit_or(rd, rd, rs, ctx); 496ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 497ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 498ca6cb544SLuke Nelson break; 499ca6cb544SLuke Nelson case BPF_ALU | BPF_XOR | BPF_X: 500ca6cb544SLuke Nelson case BPF_ALU64 | BPF_XOR | BPF_X: 50118a4d8c9SLuke Nelson emit_xor(rd, rd, rs, ctx); 502ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 503ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 504ca6cb544SLuke Nelson break; 505ca6cb544SLuke Nelson case BPF_ALU | BPF_MUL | BPF_X: 506ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MUL | BPF_X: 507ca6cb544SLuke Nelson emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx); 508ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 509ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 510ca6cb544SLuke Nelson break; 511ca6cb544SLuke Nelson case BPF_ALU | BPF_DIV | BPF_X: 512ca6cb544SLuke Nelson case BPF_ALU64 | BPF_DIV | BPF_X: 513ca6cb544SLuke Nelson emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx); 514ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 515ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 516ca6cb544SLuke Nelson break; 517ca6cb544SLuke Nelson case BPF_ALU | BPF_MOD | BPF_X: 518ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MOD | BPF_X: 519ca6cb544SLuke Nelson emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx); 520ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 521ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 522ca6cb544SLuke Nelson break; 523ca6cb544SLuke Nelson case BPF_ALU | BPF_LSH | BPF_X: 524ca6cb544SLuke Nelson case BPF_ALU64 | BPF_LSH | BPF_X: 525ca6cb544SLuke Nelson emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx); 5260224b2acSLuke Nelson if (!is64 && !aux->verifier_zext) 527ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 528ca6cb544SLuke Nelson break; 529ca6cb544SLuke Nelson case BPF_ALU | BPF_RSH | BPF_X: 530ca6cb544SLuke Nelson case BPF_ALU64 | BPF_RSH | BPF_X: 531ca6cb544SLuke Nelson emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx); 532ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 533ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 534ca6cb544SLuke Nelson break; 535ca6cb544SLuke Nelson case BPF_ALU | BPF_ARSH | BPF_X: 536ca6cb544SLuke Nelson case BPF_ALU64 | BPF_ARSH | BPF_X: 537ca6cb544SLuke Nelson emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx); 538ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 539ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 540ca6cb544SLuke Nelson break; 541ca6cb544SLuke Nelson 542ca6cb544SLuke Nelson /* dst = -dst */ 543ca6cb544SLuke Nelson case BPF_ALU | BPF_NEG: 544ca6cb544SLuke Nelson case BPF_ALU64 | BPF_NEG: 54518a4d8c9SLuke Nelson emit_sub(rd, RV_REG_ZERO, rd, ctx); 546ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 547ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 548ca6cb544SLuke Nelson break; 549ca6cb544SLuke Nelson 550ca6cb544SLuke Nelson /* dst = BSWAP##imm(dst) */ 551ca6cb544SLuke Nelson case BPF_ALU | BPF_END | BPF_FROM_LE: 55221a099abSLuke Nelson switch (imm) { 55321a099abSLuke Nelson case 16: 55418a4d8c9SLuke Nelson emit_slli(rd, rd, 48, ctx); 55518a4d8c9SLuke Nelson emit_srli(rd, rd, 48, ctx); 55621a099abSLuke Nelson break; 55721a099abSLuke Nelson case 32: 55821a099abSLuke Nelson if (!aux->verifier_zext) 55921a099abSLuke Nelson emit_zext_32(rd, ctx); 56021a099abSLuke Nelson break; 56121a099abSLuke Nelson case 64: 56221a099abSLuke Nelson /* Do nothing */ 563ca6cb544SLuke Nelson break; 564ca6cb544SLuke Nelson } 56521a099abSLuke Nelson break; 56621a099abSLuke Nelson 567ca6cb544SLuke Nelson case BPF_ALU | BPF_END | BPF_FROM_BE: 56818a4d8c9SLuke Nelson emit_li(RV_REG_T2, 0, ctx); 569ca6cb544SLuke Nelson 57018a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 57118a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 57218a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 57318a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 574ca6cb544SLuke Nelson if (imm == 16) 575ca6cb544SLuke Nelson goto out_be; 576ca6cb544SLuke Nelson 57718a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 57818a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 57918a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 58018a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 581ca6cb544SLuke Nelson 58218a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 58318a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 58418a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 58518a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 586ca6cb544SLuke Nelson if (imm == 32) 587ca6cb544SLuke Nelson goto out_be; 588ca6cb544SLuke Nelson 58918a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 59018a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 59118a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 59218a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 593ca6cb544SLuke Nelson 59418a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 59518a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 59618a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 59718a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 598ca6cb544SLuke Nelson 59918a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 60018a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 60118a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 60218a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 603ca6cb544SLuke Nelson 60418a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 60518a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 60618a4d8c9SLuke Nelson emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 60718a4d8c9SLuke Nelson emit_srli(rd, rd, 8, ctx); 608ca6cb544SLuke Nelson out_be: 60918a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, 0xff, ctx); 61018a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 611ca6cb544SLuke Nelson 61218a4d8c9SLuke Nelson emit_mv(rd, RV_REG_T2, ctx); 613ca6cb544SLuke Nelson break; 614ca6cb544SLuke Nelson 615ca6cb544SLuke Nelson /* dst = imm */ 616ca6cb544SLuke Nelson case BPF_ALU | BPF_MOV | BPF_K: 617ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MOV | BPF_K: 618ca6cb544SLuke Nelson emit_imm(rd, imm, ctx); 619ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 620ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 621ca6cb544SLuke Nelson break; 622ca6cb544SLuke Nelson 623ca6cb544SLuke Nelson /* dst = dst OP imm */ 624ca6cb544SLuke Nelson case BPF_ALU | BPF_ADD | BPF_K: 625ca6cb544SLuke Nelson case BPF_ALU64 | BPF_ADD | BPF_K: 626ca6cb544SLuke Nelson if (is_12b_int(imm)) { 62718a4d8c9SLuke Nelson emit_addi(rd, rd, imm, ctx); 628ca6cb544SLuke Nelson } else { 629ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 63018a4d8c9SLuke Nelson emit_add(rd, rd, RV_REG_T1, ctx); 631ca6cb544SLuke Nelson } 632ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 633ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 634ca6cb544SLuke Nelson break; 635ca6cb544SLuke Nelson case BPF_ALU | BPF_SUB | BPF_K: 636ca6cb544SLuke Nelson case BPF_ALU64 | BPF_SUB | BPF_K: 637ca6cb544SLuke Nelson if (is_12b_int(-imm)) { 63818a4d8c9SLuke Nelson emit_addi(rd, rd, -imm, ctx); 639ca6cb544SLuke Nelson } else { 640ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 64118a4d8c9SLuke Nelson emit_sub(rd, rd, RV_REG_T1, ctx); 642ca6cb544SLuke Nelson } 643ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 644ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 645ca6cb544SLuke Nelson break; 646ca6cb544SLuke Nelson case BPF_ALU | BPF_AND | BPF_K: 647ca6cb544SLuke Nelson case BPF_ALU64 | BPF_AND | BPF_K: 648ca6cb544SLuke Nelson if (is_12b_int(imm)) { 64918a4d8c9SLuke Nelson emit_andi(rd, rd, imm, ctx); 650ca6cb544SLuke Nelson } else { 651ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 65218a4d8c9SLuke Nelson emit_and(rd, rd, RV_REG_T1, ctx); 653ca6cb544SLuke Nelson } 654ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 655ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 656ca6cb544SLuke Nelson break; 657ca6cb544SLuke Nelson case BPF_ALU | BPF_OR | BPF_K: 658ca6cb544SLuke Nelson case BPF_ALU64 | BPF_OR | BPF_K: 659ca6cb544SLuke Nelson if (is_12b_int(imm)) { 660ca6cb544SLuke Nelson emit(rv_ori(rd, rd, imm), ctx); 661ca6cb544SLuke Nelson } else { 662ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 66318a4d8c9SLuke Nelson emit_or(rd, rd, RV_REG_T1, ctx); 664ca6cb544SLuke Nelson } 665ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 666ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 667ca6cb544SLuke Nelson break; 668ca6cb544SLuke Nelson case BPF_ALU | BPF_XOR | BPF_K: 669ca6cb544SLuke Nelson case BPF_ALU64 | BPF_XOR | BPF_K: 670ca6cb544SLuke Nelson if (is_12b_int(imm)) { 671ca6cb544SLuke Nelson emit(rv_xori(rd, rd, imm), ctx); 672ca6cb544SLuke Nelson } else { 673ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 67418a4d8c9SLuke Nelson emit_xor(rd, rd, RV_REG_T1, ctx); 675ca6cb544SLuke Nelson } 676ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 677ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 678ca6cb544SLuke Nelson break; 679ca6cb544SLuke Nelson case BPF_ALU | BPF_MUL | BPF_K: 680ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MUL | BPF_K: 681ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 682ca6cb544SLuke Nelson emit(is64 ? rv_mul(rd, rd, RV_REG_T1) : 683ca6cb544SLuke Nelson rv_mulw(rd, rd, RV_REG_T1), ctx); 684ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 685ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 686ca6cb544SLuke Nelson break; 687ca6cb544SLuke Nelson case BPF_ALU | BPF_DIV | BPF_K: 688ca6cb544SLuke Nelson case BPF_ALU64 | BPF_DIV | BPF_K: 689ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 690ca6cb544SLuke Nelson emit(is64 ? rv_divu(rd, rd, RV_REG_T1) : 691ca6cb544SLuke Nelson rv_divuw(rd, rd, RV_REG_T1), ctx); 692ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 693ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 694ca6cb544SLuke Nelson break; 695ca6cb544SLuke Nelson case BPF_ALU | BPF_MOD | BPF_K: 696ca6cb544SLuke Nelson case BPF_ALU64 | BPF_MOD | BPF_K: 697ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 698ca6cb544SLuke Nelson emit(is64 ? rv_remu(rd, rd, RV_REG_T1) : 699ca6cb544SLuke Nelson rv_remuw(rd, rd, RV_REG_T1), ctx); 700ca6cb544SLuke Nelson if (!is64 && !aux->verifier_zext) 701ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 702ca6cb544SLuke Nelson break; 703ca6cb544SLuke Nelson case BPF_ALU | BPF_LSH | BPF_K: 704ca6cb544SLuke Nelson case BPF_ALU64 | BPF_LSH | BPF_K: 70518a4d8c9SLuke Nelson emit_slli(rd, rd, imm, ctx); 70618a4d8c9SLuke Nelson 7070224b2acSLuke Nelson if (!is64 && !aux->verifier_zext) 708ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 709ca6cb544SLuke Nelson break; 710ca6cb544SLuke Nelson case BPF_ALU | BPF_RSH | BPF_K: 711ca6cb544SLuke Nelson case BPF_ALU64 | BPF_RSH | BPF_K: 71218a4d8c9SLuke Nelson if (is64) 71318a4d8c9SLuke Nelson emit_srli(rd, rd, imm, ctx); 71418a4d8c9SLuke Nelson else 71518a4d8c9SLuke Nelson emit(rv_srliw(rd, rd, imm), ctx); 71618a4d8c9SLuke Nelson 7170224b2acSLuke Nelson if (!is64 && !aux->verifier_zext) 718ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 719ca6cb544SLuke Nelson break; 720ca6cb544SLuke Nelson case BPF_ALU | BPF_ARSH | BPF_K: 721ca6cb544SLuke Nelson case BPF_ALU64 | BPF_ARSH | BPF_K: 72218a4d8c9SLuke Nelson if (is64) 72318a4d8c9SLuke Nelson emit_srai(rd, rd, imm, ctx); 72418a4d8c9SLuke Nelson else 72518a4d8c9SLuke Nelson emit(rv_sraiw(rd, rd, imm), ctx); 72618a4d8c9SLuke Nelson 7270224b2acSLuke Nelson if (!is64 && !aux->verifier_zext) 728ca6cb544SLuke Nelson emit_zext_32(rd, ctx); 729ca6cb544SLuke Nelson break; 730ca6cb544SLuke Nelson 731ca6cb544SLuke Nelson /* JUMP off */ 732ca6cb544SLuke Nelson case BPF_JMP | BPF_JA: 733ca6cb544SLuke Nelson rvoff = rv_offset(i, off, ctx); 734489553ddSLuke Nelson ret = emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx); 735489553ddSLuke Nelson if (ret) 736489553ddSLuke Nelson return ret; 737ca6cb544SLuke Nelson break; 738ca6cb544SLuke Nelson 739ca6cb544SLuke Nelson /* IF (dst COND src) JUMP off */ 740ca6cb544SLuke Nelson case BPF_JMP | BPF_JEQ | BPF_X: 741ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JEQ | BPF_X: 742ca6cb544SLuke Nelson case BPF_JMP | BPF_JGT | BPF_X: 743ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JGT | BPF_X: 744ca6cb544SLuke Nelson case BPF_JMP | BPF_JLT | BPF_X: 745ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JLT | BPF_X: 746ca6cb544SLuke Nelson case BPF_JMP | BPF_JGE | BPF_X: 747ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JGE | BPF_X: 748ca6cb544SLuke Nelson case BPF_JMP | BPF_JLE | BPF_X: 749ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JLE | BPF_X: 750ca6cb544SLuke Nelson case BPF_JMP | BPF_JNE | BPF_X: 751ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JNE | BPF_X: 752ca6cb544SLuke Nelson case BPF_JMP | BPF_JSGT | BPF_X: 753ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSGT | BPF_X: 754ca6cb544SLuke Nelson case BPF_JMP | BPF_JSLT | BPF_X: 755ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSLT | BPF_X: 756ca6cb544SLuke Nelson case BPF_JMP | BPF_JSGE | BPF_X: 757ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSGE | BPF_X: 758ca6cb544SLuke Nelson case BPF_JMP | BPF_JSLE | BPF_X: 759ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSLE | BPF_X: 760ca6cb544SLuke Nelson case BPF_JMP | BPF_JSET | BPF_X: 761ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSET | BPF_X: 762ca6cb544SLuke Nelson rvoff = rv_offset(i, off, ctx); 763ca6cb544SLuke Nelson if (!is64) { 764ca6cb544SLuke Nelson s = ctx->ninsns; 765ca6cb544SLuke Nelson if (is_signed_bpf_cond(BPF_OP(code))) 766ca6cb544SLuke Nelson emit_sext_32_rd_rs(&rd, &rs, ctx); 767ca6cb544SLuke Nelson else 768ca6cb544SLuke Nelson emit_zext_32_rd_rs(&rd, &rs, ctx); 769ca6cb544SLuke Nelson e = ctx->ninsns; 770ca6cb544SLuke Nelson 771ca6cb544SLuke Nelson /* Adjust for extra insns */ 772bfabff3cSLuke Nelson rvoff -= ninsns_rvoff(e - s); 773ca6cb544SLuke Nelson } 774ca6cb544SLuke Nelson 775ca6cb544SLuke Nelson if (BPF_OP(code) == BPF_JSET) { 776ca6cb544SLuke Nelson /* Adjust for and */ 777ca6cb544SLuke Nelson rvoff -= 4; 77818a4d8c9SLuke Nelson emit_and(RV_REG_T1, rd, rs, ctx); 779ca6cb544SLuke Nelson emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, 780ca6cb544SLuke Nelson ctx); 781ca6cb544SLuke Nelson } else { 782ca6cb544SLuke Nelson emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 783ca6cb544SLuke Nelson } 784ca6cb544SLuke Nelson break; 785ca6cb544SLuke Nelson 786ca6cb544SLuke Nelson /* IF (dst COND imm) JUMP off */ 787ca6cb544SLuke Nelson case BPF_JMP | BPF_JEQ | BPF_K: 788ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JEQ | BPF_K: 789ca6cb544SLuke Nelson case BPF_JMP | BPF_JGT | BPF_K: 790ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JGT | BPF_K: 791ca6cb544SLuke Nelson case BPF_JMP | BPF_JLT | BPF_K: 792ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JLT | BPF_K: 793ca6cb544SLuke Nelson case BPF_JMP | BPF_JGE | BPF_K: 794ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JGE | BPF_K: 795ca6cb544SLuke Nelson case BPF_JMP | BPF_JLE | BPF_K: 796ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JLE | BPF_K: 797ca6cb544SLuke Nelson case BPF_JMP | BPF_JNE | BPF_K: 798ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JNE | BPF_K: 799ca6cb544SLuke Nelson case BPF_JMP | BPF_JSGT | BPF_K: 800ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSGT | BPF_K: 801ca6cb544SLuke Nelson case BPF_JMP | BPF_JSLT | BPF_K: 802ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSLT | BPF_K: 803ca6cb544SLuke Nelson case BPF_JMP | BPF_JSGE | BPF_K: 804ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSGE | BPF_K: 805ca6cb544SLuke Nelson case BPF_JMP | BPF_JSLE | BPF_K: 806ca6cb544SLuke Nelson case BPF_JMP32 | BPF_JSLE | BPF_K: 807ca6cb544SLuke Nelson rvoff = rv_offset(i, off, ctx); 808ca6cb544SLuke Nelson s = ctx->ninsns; 809ca349a6aSLuke Nelson if (imm) { 810ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 811ca349a6aSLuke Nelson rs = RV_REG_T1; 812ca349a6aSLuke Nelson } else { 813ca349a6aSLuke Nelson /* If imm is 0, simply use zero register. */ 814ca349a6aSLuke Nelson rs = RV_REG_ZERO; 815ca349a6aSLuke Nelson } 816ca6cb544SLuke Nelson if (!is64) { 817ca6cb544SLuke Nelson if (is_signed_bpf_cond(BPF_OP(code))) 818ca6cb544SLuke Nelson emit_sext_32_rd(&rd, ctx); 819ca6cb544SLuke Nelson else 820ca6cb544SLuke Nelson emit_zext_32_rd_t1(&rd, ctx); 821ca6cb544SLuke Nelson } 822ca6cb544SLuke Nelson e = ctx->ninsns; 823ca6cb544SLuke Nelson 824ca6cb544SLuke Nelson /* Adjust for extra insns */ 825bfabff3cSLuke Nelson rvoff -= ninsns_rvoff(e - s); 826ca349a6aSLuke Nelson emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 827073ca6a0SLuke Nelson break; 828073ca6a0SLuke Nelson 829073ca6a0SLuke Nelson case BPF_JMP | BPF_JSET | BPF_K: 830073ca6a0SLuke Nelson case BPF_JMP32 | BPF_JSET | BPF_K: 831073ca6a0SLuke Nelson rvoff = rv_offset(i, off, ctx); 832073ca6a0SLuke Nelson s = ctx->ninsns; 833073ca6a0SLuke Nelson if (is_12b_int(imm)) { 83418a4d8c9SLuke Nelson emit_andi(RV_REG_T1, rd, imm, ctx); 835073ca6a0SLuke Nelson } else { 836073ca6a0SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 83718a4d8c9SLuke Nelson emit_and(RV_REG_T1, rd, RV_REG_T1, ctx); 838ca6cb544SLuke Nelson } 839073ca6a0SLuke Nelson /* For jset32, we should clear the upper 32 bits of t1, but 840073ca6a0SLuke Nelson * sign-extension is sufficient here and saves one instruction, 841073ca6a0SLuke Nelson * as t1 is used only in comparison against zero. 842073ca6a0SLuke Nelson */ 843073ca6a0SLuke Nelson if (!is64 && imm < 0) 84418a4d8c9SLuke Nelson emit_addiw(RV_REG_T1, RV_REG_T1, 0, ctx); 845073ca6a0SLuke Nelson e = ctx->ninsns; 846bfabff3cSLuke Nelson rvoff -= ninsns_rvoff(e - s); 847073ca6a0SLuke Nelson emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); 848ca6cb544SLuke Nelson break; 849ca6cb544SLuke Nelson 850ca6cb544SLuke Nelson /* function call */ 851ca6cb544SLuke Nelson case BPF_JMP | BPF_CALL: 852ca6cb544SLuke Nelson { 853ca6cb544SLuke Nelson bool fixed; 854ca6cb544SLuke Nelson u64 addr; 855ca6cb544SLuke Nelson 856ca6cb544SLuke Nelson mark_call(ctx); 857ca6cb544SLuke Nelson ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr, 858ca6cb544SLuke Nelson &fixed); 859ca6cb544SLuke Nelson if (ret < 0) 860ca6cb544SLuke Nelson return ret; 861ca6cb544SLuke Nelson ret = emit_call(fixed, addr, ctx); 862ca6cb544SLuke Nelson if (ret) 863ca6cb544SLuke Nelson return ret; 864ca6cb544SLuke Nelson break; 865ca6cb544SLuke Nelson } 866ca6cb544SLuke Nelson /* tail call */ 867ca6cb544SLuke Nelson case BPF_JMP | BPF_TAIL_CALL: 868ca6cb544SLuke Nelson if (emit_bpf_tail_call(i, ctx)) 869ca6cb544SLuke Nelson return -1; 870ca6cb544SLuke Nelson break; 871ca6cb544SLuke Nelson 872ca6cb544SLuke Nelson /* function return */ 873ca6cb544SLuke Nelson case BPF_JMP | BPF_EXIT: 874ca6cb544SLuke Nelson if (i == ctx->prog->len - 1) 875ca6cb544SLuke Nelson break; 876ca6cb544SLuke Nelson 877ca6cb544SLuke Nelson rvoff = epilogue_offset(ctx); 878489553ddSLuke Nelson ret = emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx); 879489553ddSLuke Nelson if (ret) 880489553ddSLuke Nelson return ret; 881ca6cb544SLuke Nelson break; 882ca6cb544SLuke Nelson 883ca6cb544SLuke Nelson /* dst = imm64 */ 884ca6cb544SLuke Nelson case BPF_LD | BPF_IMM | BPF_DW: 885ca6cb544SLuke Nelson { 886ca6cb544SLuke Nelson struct bpf_insn insn1 = insn[1]; 887ca6cb544SLuke Nelson u64 imm64; 888ca6cb544SLuke Nelson 889ca6cb544SLuke Nelson imm64 = (u64)insn1.imm << 32 | (u32)imm; 890ca6cb544SLuke Nelson emit_imm(rd, imm64, ctx); 891ca6cb544SLuke Nelson return 1; 892ca6cb544SLuke Nelson } 893ca6cb544SLuke Nelson 894ca6cb544SLuke Nelson /* LDX: dst = *(size *)(src + off) */ 895ca6cb544SLuke Nelson case BPF_LDX | BPF_MEM | BPF_B: 896ca6cb544SLuke Nelson if (is_12b_int(off)) { 897ca6cb544SLuke Nelson emit(rv_lbu(rd, off, rs), ctx); 898ca6cb544SLuke Nelson break; 899ca6cb544SLuke Nelson } 900ca6cb544SLuke Nelson 901ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 90218a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 903ca6cb544SLuke Nelson emit(rv_lbu(rd, 0, RV_REG_T1), ctx); 904ca6cb544SLuke Nelson if (insn_is_zext(&insn[1])) 905ca6cb544SLuke Nelson return 1; 906ca6cb544SLuke Nelson break; 907ca6cb544SLuke Nelson case BPF_LDX | BPF_MEM | BPF_H: 908ca6cb544SLuke Nelson if (is_12b_int(off)) { 909ca6cb544SLuke Nelson emit(rv_lhu(rd, off, rs), ctx); 910ca6cb544SLuke Nelson break; 911ca6cb544SLuke Nelson } 912ca6cb544SLuke Nelson 913ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 91418a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 915ca6cb544SLuke Nelson emit(rv_lhu(rd, 0, RV_REG_T1), ctx); 916ca6cb544SLuke Nelson if (insn_is_zext(&insn[1])) 917ca6cb544SLuke Nelson return 1; 918ca6cb544SLuke Nelson break; 919ca6cb544SLuke Nelson case BPF_LDX | BPF_MEM | BPF_W: 920ca6cb544SLuke Nelson if (is_12b_int(off)) { 921ca6cb544SLuke Nelson emit(rv_lwu(rd, off, rs), ctx); 922ca6cb544SLuke Nelson break; 923ca6cb544SLuke Nelson } 924ca6cb544SLuke Nelson 925ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 92618a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 927ca6cb544SLuke Nelson emit(rv_lwu(rd, 0, RV_REG_T1), ctx); 928ca6cb544SLuke Nelson if (insn_is_zext(&insn[1])) 929ca6cb544SLuke Nelson return 1; 930ca6cb544SLuke Nelson break; 931ca6cb544SLuke Nelson case BPF_LDX | BPF_MEM | BPF_DW: 932ca6cb544SLuke Nelson if (is_12b_int(off)) { 93318a4d8c9SLuke Nelson emit_ld(rd, off, rs, ctx); 934ca6cb544SLuke Nelson break; 935ca6cb544SLuke Nelson } 936ca6cb544SLuke Nelson 937ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 93818a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 93918a4d8c9SLuke Nelson emit_ld(rd, 0, RV_REG_T1, ctx); 940ca6cb544SLuke Nelson break; 941ca6cb544SLuke Nelson 942ca6cb544SLuke Nelson /* ST: *(size *)(dst + off) = imm */ 943ca6cb544SLuke Nelson case BPF_ST | BPF_MEM | BPF_B: 944ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 945ca6cb544SLuke Nelson if (is_12b_int(off)) { 946ca6cb544SLuke Nelson emit(rv_sb(rd, off, RV_REG_T1), ctx); 947ca6cb544SLuke Nelson break; 948ca6cb544SLuke Nelson } 949ca6cb544SLuke Nelson 950ca6cb544SLuke Nelson emit_imm(RV_REG_T2, off, ctx); 95118a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 952ca6cb544SLuke Nelson emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); 953ca6cb544SLuke Nelson break; 954ca6cb544SLuke Nelson 955ca6cb544SLuke Nelson case BPF_ST | BPF_MEM | BPF_H: 956ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 957ca6cb544SLuke Nelson if (is_12b_int(off)) { 958ca6cb544SLuke Nelson emit(rv_sh(rd, off, RV_REG_T1), ctx); 959ca6cb544SLuke Nelson break; 960ca6cb544SLuke Nelson } 961ca6cb544SLuke Nelson 962ca6cb544SLuke Nelson emit_imm(RV_REG_T2, off, ctx); 96318a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 964ca6cb544SLuke Nelson emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); 965ca6cb544SLuke Nelson break; 966ca6cb544SLuke Nelson case BPF_ST | BPF_MEM | BPF_W: 967ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 968ca6cb544SLuke Nelson if (is_12b_int(off)) { 96918a4d8c9SLuke Nelson emit_sw(rd, off, RV_REG_T1, ctx); 970ca6cb544SLuke Nelson break; 971ca6cb544SLuke Nelson } 972ca6cb544SLuke Nelson 973ca6cb544SLuke Nelson emit_imm(RV_REG_T2, off, ctx); 97418a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 97518a4d8c9SLuke Nelson emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx); 976ca6cb544SLuke Nelson break; 977ca6cb544SLuke Nelson case BPF_ST | BPF_MEM | BPF_DW: 978ca6cb544SLuke Nelson emit_imm(RV_REG_T1, imm, ctx); 979ca6cb544SLuke Nelson if (is_12b_int(off)) { 98018a4d8c9SLuke Nelson emit_sd(rd, off, RV_REG_T1, ctx); 981ca6cb544SLuke Nelson break; 982ca6cb544SLuke Nelson } 983ca6cb544SLuke Nelson 984ca6cb544SLuke Nelson emit_imm(RV_REG_T2, off, ctx); 98518a4d8c9SLuke Nelson emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 98618a4d8c9SLuke Nelson emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx); 987ca6cb544SLuke Nelson break; 988ca6cb544SLuke Nelson 989ca6cb544SLuke Nelson /* STX: *(size *)(dst + off) = src */ 990ca6cb544SLuke Nelson case BPF_STX | BPF_MEM | BPF_B: 991ca6cb544SLuke Nelson if (is_12b_int(off)) { 992ca6cb544SLuke Nelson emit(rv_sb(rd, off, rs), ctx); 993ca6cb544SLuke Nelson break; 994ca6cb544SLuke Nelson } 995ca6cb544SLuke Nelson 996ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 99718a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 998ca6cb544SLuke Nelson emit(rv_sb(RV_REG_T1, 0, rs), ctx); 999ca6cb544SLuke Nelson break; 1000ca6cb544SLuke Nelson case BPF_STX | BPF_MEM | BPF_H: 1001ca6cb544SLuke Nelson if (is_12b_int(off)) { 1002ca6cb544SLuke Nelson emit(rv_sh(rd, off, rs), ctx); 1003ca6cb544SLuke Nelson break; 1004ca6cb544SLuke Nelson } 1005ca6cb544SLuke Nelson 1006ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 100718a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1008ca6cb544SLuke Nelson emit(rv_sh(RV_REG_T1, 0, rs), ctx); 1009ca6cb544SLuke Nelson break; 1010ca6cb544SLuke Nelson case BPF_STX | BPF_MEM | BPF_W: 1011ca6cb544SLuke Nelson if (is_12b_int(off)) { 101218a4d8c9SLuke Nelson emit_sw(rd, off, rs, ctx); 1013ca6cb544SLuke Nelson break; 1014ca6cb544SLuke Nelson } 1015ca6cb544SLuke Nelson 1016ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 101718a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 101818a4d8c9SLuke Nelson emit_sw(RV_REG_T1, 0, rs, ctx); 1019ca6cb544SLuke Nelson break; 1020ca6cb544SLuke Nelson case BPF_STX | BPF_MEM | BPF_DW: 1021ca6cb544SLuke Nelson if (is_12b_int(off)) { 102218a4d8c9SLuke Nelson emit_sd(rd, off, rs, ctx); 1023ca6cb544SLuke Nelson break; 1024ca6cb544SLuke Nelson } 1025ca6cb544SLuke Nelson 1026ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 102718a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 102818a4d8c9SLuke Nelson emit_sd(RV_REG_T1, 0, rs, ctx); 1029ca6cb544SLuke Nelson break; 1030*91c960b0SBrendan Jackman case BPF_STX | BPF_ATOMIC | BPF_W: 1031*91c960b0SBrendan Jackman case BPF_STX | BPF_ATOMIC | BPF_DW: 1032*91c960b0SBrendan Jackman if (insn->imm != BPF_ADD) { 1033*91c960b0SBrendan Jackman pr_err("bpf-jit: not supported: atomic operation %02x ***\n", 1034*91c960b0SBrendan Jackman insn->imm); 1035*91c960b0SBrendan Jackman return -EINVAL; 1036*91c960b0SBrendan Jackman } 1037*91c960b0SBrendan Jackman 1038*91c960b0SBrendan Jackman /* atomic_add: lock *(u32 *)(dst + off) += src 1039*91c960b0SBrendan Jackman * atomic_add: lock *(u64 *)(dst + off) += src 1040*91c960b0SBrendan Jackman */ 1041*91c960b0SBrendan Jackman 1042ca6cb544SLuke Nelson if (off) { 1043ca6cb544SLuke Nelson if (is_12b_int(off)) { 104418a4d8c9SLuke Nelson emit_addi(RV_REG_T1, rd, off, ctx); 1045ca6cb544SLuke Nelson } else { 1046ca6cb544SLuke Nelson emit_imm(RV_REG_T1, off, ctx); 104718a4d8c9SLuke Nelson emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1048ca6cb544SLuke Nelson } 1049ca6cb544SLuke Nelson 1050ca6cb544SLuke Nelson rd = RV_REG_T1; 1051ca6cb544SLuke Nelson } 1052ca6cb544SLuke Nelson 1053ca6cb544SLuke Nelson emit(BPF_SIZE(code) == BPF_W ? 1054ca6cb544SLuke Nelson rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0) : 1055ca6cb544SLuke Nelson rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0), ctx); 1056ca6cb544SLuke Nelson break; 1057ca6cb544SLuke Nelson default: 1058ca6cb544SLuke Nelson pr_err("bpf-jit: unknown opcode %02x\n", code); 1059ca6cb544SLuke Nelson return -EINVAL; 1060ca6cb544SLuke Nelson } 1061ca6cb544SLuke Nelson 1062ca6cb544SLuke Nelson return 0; 1063ca6cb544SLuke Nelson } 1064ca6cb544SLuke Nelson 1065ca6cb544SLuke Nelson void bpf_jit_build_prologue(struct rv_jit_context *ctx) 1066ca6cb544SLuke Nelson { 1067ca6cb544SLuke Nelson int stack_adjust = 0, store_offset, bpf_stack_adjust; 1068ca6cb544SLuke Nelson 1069ca6cb544SLuke Nelson bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); 1070ca6cb544SLuke Nelson if (bpf_stack_adjust) 1071ca6cb544SLuke Nelson mark_fp(ctx); 1072ca6cb544SLuke Nelson 1073ca6cb544SLuke Nelson if (seen_reg(RV_REG_RA, ctx)) 1074ca6cb544SLuke Nelson stack_adjust += 8; 1075ca6cb544SLuke Nelson stack_adjust += 8; /* RV_REG_FP */ 1076ca6cb544SLuke Nelson if (seen_reg(RV_REG_S1, ctx)) 1077ca6cb544SLuke Nelson stack_adjust += 8; 1078ca6cb544SLuke Nelson if (seen_reg(RV_REG_S2, ctx)) 1079ca6cb544SLuke Nelson stack_adjust += 8; 1080ca6cb544SLuke Nelson if (seen_reg(RV_REG_S3, ctx)) 1081ca6cb544SLuke Nelson stack_adjust += 8; 1082ca6cb544SLuke Nelson if (seen_reg(RV_REG_S4, ctx)) 1083ca6cb544SLuke Nelson stack_adjust += 8; 1084ca6cb544SLuke Nelson if (seen_reg(RV_REG_S5, ctx)) 1085ca6cb544SLuke Nelson stack_adjust += 8; 1086ca6cb544SLuke Nelson if (seen_reg(RV_REG_S6, ctx)) 1087ca6cb544SLuke Nelson stack_adjust += 8; 1088ca6cb544SLuke Nelson 1089ca6cb544SLuke Nelson stack_adjust = round_up(stack_adjust, 16); 1090ca6cb544SLuke Nelson stack_adjust += bpf_stack_adjust; 1091ca6cb544SLuke Nelson 1092ca6cb544SLuke Nelson store_offset = stack_adjust - 8; 1093ca6cb544SLuke Nelson 1094ca6cb544SLuke Nelson /* First instruction is always setting the tail-call-counter 1095ca6cb544SLuke Nelson * (TCC) register. This instruction is skipped for tail calls. 109618a4d8c9SLuke Nelson * Force using a 4-byte (non-compressed) instruction. 1097ca6cb544SLuke Nelson */ 1098ca6cb544SLuke Nelson emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); 1099ca6cb544SLuke Nelson 110018a4d8c9SLuke Nelson emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); 1101ca6cb544SLuke Nelson 1102ca6cb544SLuke Nelson if (seen_reg(RV_REG_RA, ctx)) { 110318a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_RA, ctx); 1104ca6cb544SLuke Nelson store_offset -= 8; 1105ca6cb544SLuke Nelson } 110618a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_FP, ctx); 1107ca6cb544SLuke Nelson store_offset -= 8; 1108ca6cb544SLuke Nelson if (seen_reg(RV_REG_S1, ctx)) { 110918a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S1, ctx); 1110ca6cb544SLuke Nelson store_offset -= 8; 1111ca6cb544SLuke Nelson } 1112ca6cb544SLuke Nelson if (seen_reg(RV_REG_S2, ctx)) { 111318a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S2, ctx); 1114ca6cb544SLuke Nelson store_offset -= 8; 1115ca6cb544SLuke Nelson } 1116ca6cb544SLuke Nelson if (seen_reg(RV_REG_S3, ctx)) { 111718a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S3, ctx); 1118ca6cb544SLuke Nelson store_offset -= 8; 1119ca6cb544SLuke Nelson } 1120ca6cb544SLuke Nelson if (seen_reg(RV_REG_S4, ctx)) { 112118a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S4, ctx); 1122ca6cb544SLuke Nelson store_offset -= 8; 1123ca6cb544SLuke Nelson } 1124ca6cb544SLuke Nelson if (seen_reg(RV_REG_S5, ctx)) { 112518a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx); 1126ca6cb544SLuke Nelson store_offset -= 8; 1127ca6cb544SLuke Nelson } 1128ca6cb544SLuke Nelson if (seen_reg(RV_REG_S6, ctx)) { 112918a4d8c9SLuke Nelson emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx); 1130ca6cb544SLuke Nelson store_offset -= 8; 1131ca6cb544SLuke Nelson } 1132ca6cb544SLuke Nelson 113318a4d8c9SLuke Nelson emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx); 1134ca6cb544SLuke Nelson 1135ca6cb544SLuke Nelson if (bpf_stack_adjust) 113618a4d8c9SLuke Nelson emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx); 1137ca6cb544SLuke Nelson 1138ca6cb544SLuke Nelson /* Program contains calls and tail calls, so RV_REG_TCC need 1139ca6cb544SLuke Nelson * to be saved across calls. 1140ca6cb544SLuke Nelson */ 1141ca6cb544SLuke Nelson if (seen_tail_call(ctx) && seen_call(ctx)) 114218a4d8c9SLuke Nelson emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx); 1143ca6cb544SLuke Nelson 1144ca6cb544SLuke Nelson ctx->stack_size = stack_adjust; 1145ca6cb544SLuke Nelson } 1146ca6cb544SLuke Nelson 1147ca6cb544SLuke Nelson void bpf_jit_build_epilogue(struct rv_jit_context *ctx) 1148ca6cb544SLuke Nelson { 1149ca6cb544SLuke Nelson __build_epilogue(false, ctx); 1150ca6cb544SLuke Nelson } 1151ca6cb544SLuke Nelson 1152ca6cb544SLuke Nelson void *bpf_jit_alloc_exec(unsigned long size) 1153ca6cb544SLuke Nelson { 1154ca6cb544SLuke Nelson return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START, 1155ca6cb544SLuke Nelson BPF_JIT_REGION_END, GFP_KERNEL, 1156ca6cb544SLuke Nelson PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, 1157ca6cb544SLuke Nelson __builtin_return_address(0)); 1158ca6cb544SLuke Nelson } 1159ca6cb544SLuke Nelson 1160ca6cb544SLuke Nelson void bpf_jit_free_exec(void *addr) 1161ca6cb544SLuke Nelson { 1162ca6cb544SLuke Nelson return vfree(addr); 1163ca6cb544SLuke Nelson } 1164