1 // SPDX-License-Identifier: GPL-2.0 2 /* BPF JIT compiler for RV64G 3 * 4 * Copyright(c) 2019 Björn Töpel <bjorn.topel@gmail.com> 5 * 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/bpf.h> 10 #include <linux/filter.h> 11 #include "bpf_jit.h" 12 13 #define RV_REG_TCC RV_REG_A6 14 #define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */ 15 16 static const int regmap[] = { 17 [BPF_REG_0] = RV_REG_A5, 18 [BPF_REG_1] = RV_REG_A0, 19 [BPF_REG_2] = RV_REG_A1, 20 [BPF_REG_3] = RV_REG_A2, 21 [BPF_REG_4] = RV_REG_A3, 22 [BPF_REG_5] = RV_REG_A4, 23 [BPF_REG_6] = RV_REG_S1, 24 [BPF_REG_7] = RV_REG_S2, 25 [BPF_REG_8] = RV_REG_S3, 26 [BPF_REG_9] = RV_REG_S4, 27 [BPF_REG_FP] = RV_REG_S5, 28 [BPF_REG_AX] = RV_REG_T0, 29 }; 30 31 static const int pt_regmap[] = { 32 [RV_REG_A0] = offsetof(struct pt_regs, a0), 33 [RV_REG_A1] = offsetof(struct pt_regs, a1), 34 [RV_REG_A2] = offsetof(struct pt_regs, a2), 35 [RV_REG_A3] = offsetof(struct pt_regs, a3), 36 [RV_REG_A4] = offsetof(struct pt_regs, a4), 37 [RV_REG_A5] = offsetof(struct pt_regs, a5), 38 [RV_REG_S1] = offsetof(struct pt_regs, s1), 39 [RV_REG_S2] = offsetof(struct pt_regs, s2), 40 [RV_REG_S3] = offsetof(struct pt_regs, s3), 41 [RV_REG_S4] = offsetof(struct pt_regs, s4), 42 [RV_REG_S5] = offsetof(struct pt_regs, s5), 43 [RV_REG_T0] = offsetof(struct pt_regs, t0), 44 }; 45 46 enum { 47 RV_CTX_F_SEEN_TAIL_CALL = 0, 48 RV_CTX_F_SEEN_CALL = RV_REG_RA, 49 RV_CTX_F_SEEN_S1 = RV_REG_S1, 50 RV_CTX_F_SEEN_S2 = RV_REG_S2, 51 RV_CTX_F_SEEN_S3 = RV_REG_S3, 52 RV_CTX_F_SEEN_S4 = RV_REG_S4, 53 RV_CTX_F_SEEN_S5 = RV_REG_S5, 54 RV_CTX_F_SEEN_S6 = RV_REG_S6, 55 }; 56 57 static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) 58 { 59 u8 reg = regmap[bpf_reg]; 60 61 switch (reg) { 62 case RV_CTX_F_SEEN_S1: 63 case RV_CTX_F_SEEN_S2: 64 case RV_CTX_F_SEEN_S3: 65 case RV_CTX_F_SEEN_S4: 66 case RV_CTX_F_SEEN_S5: 67 case RV_CTX_F_SEEN_S6: 68 __set_bit(reg, &ctx->flags); 69 } 70 return reg; 71 }; 72 73 static bool seen_reg(int reg, struct rv_jit_context *ctx) 74 { 75 switch (reg) { 76 case RV_CTX_F_SEEN_CALL: 77 case RV_CTX_F_SEEN_S1: 78 case RV_CTX_F_SEEN_S2: 79 case RV_CTX_F_SEEN_S3: 80 case RV_CTX_F_SEEN_S4: 81 case RV_CTX_F_SEEN_S5: 82 case RV_CTX_F_SEEN_S6: 83 return test_bit(reg, &ctx->flags); 84 } 85 return false; 86 } 87 88 static void mark_fp(struct rv_jit_context *ctx) 89 { 90 __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags); 91 } 92 93 static void mark_call(struct rv_jit_context *ctx) 94 { 95 __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 96 } 97 98 static bool seen_call(struct rv_jit_context *ctx) 99 { 100 return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 101 } 102 103 static void mark_tail_call(struct rv_jit_context *ctx) 104 { 105 __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 106 } 107 108 static bool seen_tail_call(struct rv_jit_context *ctx) 109 { 110 return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 111 } 112 113 static u8 rv_tail_call_reg(struct rv_jit_context *ctx) 114 { 115 mark_tail_call(ctx); 116 117 if (seen_call(ctx)) { 118 __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); 119 return RV_REG_S6; 120 } 121 return RV_REG_A6; 122 } 123 124 static bool is_32b_int(s64 val) 125 { 126 return -(1L << 31) <= val && val < (1L << 31); 127 } 128 129 static bool in_auipc_jalr_range(s64 val) 130 { 131 /* 132 * auipc+jalr can reach any signed PC-relative offset in the range 133 * [-2^31 - 2^11, 2^31 - 2^11). 134 */ 135 return (-(1L << 31) - (1L << 11)) <= val && 136 val < ((1L << 31) - (1L << 11)); 137 } 138 139 /* Emit fixed-length instructions for address */ 140 static int emit_addr(u8 rd, u64 addr, bool extra_pass, struct rv_jit_context *ctx) 141 { 142 u64 ip = (u64)(ctx->insns + ctx->ninsns); 143 s64 off = addr - ip; 144 s64 upper = (off + (1 << 11)) >> 12; 145 s64 lower = off & 0xfff; 146 147 if (extra_pass && !in_auipc_jalr_range(off)) { 148 pr_err("bpf-jit: target offset 0x%llx is out of range\n", off); 149 return -ERANGE; 150 } 151 152 emit(rv_auipc(rd, upper), ctx); 153 emit(rv_addi(rd, rd, lower), ctx); 154 return 0; 155 } 156 157 /* Emit variable-length instructions for 32-bit and 64-bit imm */ 158 static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx) 159 { 160 /* Note that the immediate from the add is sign-extended, 161 * which means that we need to compensate this by adding 2^12, 162 * when the 12th bit is set. A simpler way of doing this, and 163 * getting rid of the check, is to just add 2**11 before the 164 * shift. The "Loading a 32-Bit constant" example from the 165 * "Computer Organization and Design, RISC-V edition" book by 166 * Patterson/Hennessy highlights this fact. 167 * 168 * This also means that we need to process LSB to MSB. 169 */ 170 s64 upper = (val + (1 << 11)) >> 12; 171 /* Sign-extend lower 12 bits to 64 bits since immediates for li, addiw, 172 * and addi are signed and RVC checks will perform signed comparisons. 173 */ 174 s64 lower = ((val & 0xfff) << 52) >> 52; 175 int shift; 176 177 if (is_32b_int(val)) { 178 if (upper) 179 emit_lui(rd, upper, ctx); 180 181 if (!upper) { 182 emit_li(rd, lower, ctx); 183 return; 184 } 185 186 emit_addiw(rd, rd, lower, ctx); 187 return; 188 } 189 190 shift = __ffs(upper); 191 upper >>= shift; 192 shift += 12; 193 194 emit_imm(rd, upper, ctx); 195 196 emit_slli(rd, rd, shift, ctx); 197 if (lower) 198 emit_addi(rd, rd, lower, ctx); 199 } 200 201 static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) 202 { 203 int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; 204 205 if (seen_reg(RV_REG_RA, ctx)) { 206 emit_ld(RV_REG_RA, store_offset, RV_REG_SP, ctx); 207 store_offset -= 8; 208 } 209 emit_ld(RV_REG_FP, store_offset, RV_REG_SP, ctx); 210 store_offset -= 8; 211 if (seen_reg(RV_REG_S1, ctx)) { 212 emit_ld(RV_REG_S1, store_offset, RV_REG_SP, ctx); 213 store_offset -= 8; 214 } 215 if (seen_reg(RV_REG_S2, ctx)) { 216 emit_ld(RV_REG_S2, store_offset, RV_REG_SP, ctx); 217 store_offset -= 8; 218 } 219 if (seen_reg(RV_REG_S3, ctx)) { 220 emit_ld(RV_REG_S3, store_offset, RV_REG_SP, ctx); 221 store_offset -= 8; 222 } 223 if (seen_reg(RV_REG_S4, ctx)) { 224 emit_ld(RV_REG_S4, store_offset, RV_REG_SP, ctx); 225 store_offset -= 8; 226 } 227 if (seen_reg(RV_REG_S5, ctx)) { 228 emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx); 229 store_offset -= 8; 230 } 231 if (seen_reg(RV_REG_S6, ctx)) { 232 emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx); 233 store_offset -= 8; 234 } 235 236 emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); 237 /* Set return value. */ 238 if (!is_tail_call) 239 emit_mv(RV_REG_A0, RV_REG_A5, ctx); 240 emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, 241 is_tail_call ? 4 : 0, /* skip TCC init */ 242 ctx); 243 } 244 245 static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, 246 struct rv_jit_context *ctx) 247 { 248 switch (cond) { 249 case BPF_JEQ: 250 emit(rv_beq(rd, rs, rvoff >> 1), ctx); 251 return; 252 case BPF_JGT: 253 emit(rv_bltu(rs, rd, rvoff >> 1), ctx); 254 return; 255 case BPF_JLT: 256 emit(rv_bltu(rd, rs, rvoff >> 1), ctx); 257 return; 258 case BPF_JGE: 259 emit(rv_bgeu(rd, rs, rvoff >> 1), ctx); 260 return; 261 case BPF_JLE: 262 emit(rv_bgeu(rs, rd, rvoff >> 1), ctx); 263 return; 264 case BPF_JNE: 265 emit(rv_bne(rd, rs, rvoff >> 1), ctx); 266 return; 267 case BPF_JSGT: 268 emit(rv_blt(rs, rd, rvoff >> 1), ctx); 269 return; 270 case BPF_JSLT: 271 emit(rv_blt(rd, rs, rvoff >> 1), ctx); 272 return; 273 case BPF_JSGE: 274 emit(rv_bge(rd, rs, rvoff >> 1), ctx); 275 return; 276 case BPF_JSLE: 277 emit(rv_bge(rs, rd, rvoff >> 1), ctx); 278 } 279 } 280 281 static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, 282 struct rv_jit_context *ctx) 283 { 284 s64 upper, lower; 285 286 if (is_13b_int(rvoff)) { 287 emit_bcc(cond, rd, rs, rvoff, ctx); 288 return; 289 } 290 291 /* Adjust for jal */ 292 rvoff -= 4; 293 294 /* Transform, e.g.: 295 * bne rd,rs,foo 296 * to 297 * beq rd,rs,<.L1> 298 * (auipc foo) 299 * jal(r) foo 300 * .L1 301 */ 302 cond = invert_bpf_cond(cond); 303 if (is_21b_int(rvoff)) { 304 emit_bcc(cond, rd, rs, 8, ctx); 305 emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx); 306 return; 307 } 308 309 /* 32b No need for an additional rvoff adjustment, since we 310 * get that from the auipc at PC', where PC = PC' + 4. 311 */ 312 upper = (rvoff + (1 << 11)) >> 12; 313 lower = rvoff & 0xfff; 314 315 emit_bcc(cond, rd, rs, 12, ctx); 316 emit(rv_auipc(RV_REG_T1, upper), ctx); 317 emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx); 318 } 319 320 static void emit_zext_32(u8 reg, struct rv_jit_context *ctx) 321 { 322 emit_slli(reg, reg, 32, ctx); 323 emit_srli(reg, reg, 32, ctx); 324 } 325 326 static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) 327 { 328 int tc_ninsn, off, start_insn = ctx->ninsns; 329 u8 tcc = rv_tail_call_reg(ctx); 330 331 /* a0: &ctx 332 * a1: &array 333 * a2: index 334 * 335 * if (index >= array->map.max_entries) 336 * goto out; 337 */ 338 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] : 339 ctx->offset[0]; 340 emit_zext_32(RV_REG_A2, ctx); 341 342 off = offsetof(struct bpf_array, map.max_entries); 343 if (is_12b_check(off, insn)) 344 return -1; 345 emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx); 346 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 347 emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx); 348 349 /* if (--TCC < 0) 350 * goto out; 351 */ 352 emit_addi(RV_REG_TCC, tcc, -1, ctx); 353 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 354 emit_branch(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); 355 356 /* prog = array->ptrs[index]; 357 * if (!prog) 358 * goto out; 359 */ 360 emit_slli(RV_REG_T2, RV_REG_A2, 3, ctx); 361 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_A1, ctx); 362 off = offsetof(struct bpf_array, ptrs); 363 if (is_12b_check(off, insn)) 364 return -1; 365 emit_ld(RV_REG_T2, off, RV_REG_T2, ctx); 366 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); 367 emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); 368 369 /* goto *(prog->bpf_func + 4); */ 370 off = offsetof(struct bpf_prog, bpf_func); 371 if (is_12b_check(off, insn)) 372 return -1; 373 emit_ld(RV_REG_T3, off, RV_REG_T2, ctx); 374 __build_epilogue(true, ctx); 375 return 0; 376 } 377 378 static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn, 379 struct rv_jit_context *ctx) 380 { 381 u8 code = insn->code; 382 383 switch (code) { 384 case BPF_JMP | BPF_JA: 385 case BPF_JMP | BPF_CALL: 386 case BPF_JMP | BPF_EXIT: 387 case BPF_JMP | BPF_TAIL_CALL: 388 break; 389 default: 390 *rd = bpf_to_rv_reg(insn->dst_reg, ctx); 391 } 392 393 if (code & (BPF_ALU | BPF_X) || code & (BPF_ALU64 | BPF_X) || 394 code & (BPF_JMP | BPF_X) || code & (BPF_JMP32 | BPF_X) || 395 code & BPF_LDX || code & BPF_STX) 396 *rs = bpf_to_rv_reg(insn->src_reg, ctx); 397 } 398 399 static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) 400 { 401 emit_mv(RV_REG_T2, *rd, ctx); 402 emit_zext_32(RV_REG_T2, ctx); 403 emit_mv(RV_REG_T1, *rs, ctx); 404 emit_zext_32(RV_REG_T1, ctx); 405 *rd = RV_REG_T2; 406 *rs = RV_REG_T1; 407 } 408 409 static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) 410 { 411 emit_addiw(RV_REG_T2, *rd, 0, ctx); 412 emit_addiw(RV_REG_T1, *rs, 0, ctx); 413 *rd = RV_REG_T2; 414 *rs = RV_REG_T1; 415 } 416 417 static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx) 418 { 419 emit_mv(RV_REG_T2, *rd, ctx); 420 emit_zext_32(RV_REG_T2, ctx); 421 emit_zext_32(RV_REG_T1, ctx); 422 *rd = RV_REG_T2; 423 } 424 425 static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx) 426 { 427 emit_addiw(RV_REG_T2, *rd, 0, ctx); 428 *rd = RV_REG_T2; 429 } 430 431 static int emit_jump_and_link(u8 rd, s64 rvoff, bool fixed_addr, 432 struct rv_jit_context *ctx) 433 { 434 s64 upper, lower; 435 436 if (rvoff && fixed_addr && is_21b_int(rvoff)) { 437 emit(rv_jal(rd, rvoff >> 1), ctx); 438 return 0; 439 } else if (in_auipc_jalr_range(rvoff)) { 440 upper = (rvoff + (1 << 11)) >> 12; 441 lower = rvoff & 0xfff; 442 emit(rv_auipc(RV_REG_T1, upper), ctx); 443 emit(rv_jalr(rd, RV_REG_T1, lower), ctx); 444 return 0; 445 } 446 447 pr_err("bpf-jit: target offset 0x%llx is out of range\n", rvoff); 448 return -ERANGE; 449 } 450 451 static bool is_signed_bpf_cond(u8 cond) 452 { 453 return cond == BPF_JSGT || cond == BPF_JSLT || 454 cond == BPF_JSGE || cond == BPF_JSLE; 455 } 456 457 static int emit_call(u64 addr, bool fixed_addr, struct rv_jit_context *ctx) 458 { 459 s64 off = 0; 460 u64 ip; 461 462 if (addr && ctx->insns) { 463 ip = (u64)(long)(ctx->insns + ctx->ninsns); 464 off = addr - ip; 465 } 466 467 return emit_jump_and_link(RV_REG_RA, off, fixed_addr, ctx); 468 } 469 470 static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64, 471 struct rv_jit_context *ctx) 472 { 473 u8 r0; 474 int jmp_offset; 475 476 if (off) { 477 if (is_12b_int(off)) { 478 emit_addi(RV_REG_T1, rd, off, ctx); 479 } else { 480 emit_imm(RV_REG_T1, off, ctx); 481 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 482 } 483 rd = RV_REG_T1; 484 } 485 486 switch (imm) { 487 /* lock *(u32/u64 *)(dst_reg + off16) <op>= src_reg */ 488 case BPF_ADD: 489 emit(is64 ? rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0) : 490 rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 491 break; 492 case BPF_AND: 493 emit(is64 ? rv_amoand_d(RV_REG_ZERO, rs, rd, 0, 0) : 494 rv_amoand_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 495 break; 496 case BPF_OR: 497 emit(is64 ? rv_amoor_d(RV_REG_ZERO, rs, rd, 0, 0) : 498 rv_amoor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 499 break; 500 case BPF_XOR: 501 emit(is64 ? rv_amoxor_d(RV_REG_ZERO, rs, rd, 0, 0) : 502 rv_amoxor_w(RV_REG_ZERO, rs, rd, 0, 0), ctx); 503 break; 504 /* src_reg = atomic_fetch_<op>(dst_reg + off16, src_reg) */ 505 case BPF_ADD | BPF_FETCH: 506 emit(is64 ? rv_amoadd_d(rs, rs, rd, 0, 0) : 507 rv_amoadd_w(rs, rs, rd, 0, 0), ctx); 508 if (!is64) 509 emit_zext_32(rs, ctx); 510 break; 511 case BPF_AND | BPF_FETCH: 512 emit(is64 ? rv_amoand_d(rs, rs, rd, 0, 0) : 513 rv_amoand_w(rs, rs, rd, 0, 0), ctx); 514 if (!is64) 515 emit_zext_32(rs, ctx); 516 break; 517 case BPF_OR | BPF_FETCH: 518 emit(is64 ? rv_amoor_d(rs, rs, rd, 0, 0) : 519 rv_amoor_w(rs, rs, rd, 0, 0), ctx); 520 if (!is64) 521 emit_zext_32(rs, ctx); 522 break; 523 case BPF_XOR | BPF_FETCH: 524 emit(is64 ? rv_amoxor_d(rs, rs, rd, 0, 0) : 525 rv_amoxor_w(rs, rs, rd, 0, 0), ctx); 526 if (!is64) 527 emit_zext_32(rs, ctx); 528 break; 529 /* src_reg = atomic_xchg(dst_reg + off16, src_reg); */ 530 case BPF_XCHG: 531 emit(is64 ? rv_amoswap_d(rs, rs, rd, 0, 0) : 532 rv_amoswap_w(rs, rs, rd, 0, 0), ctx); 533 if (!is64) 534 emit_zext_32(rs, ctx); 535 break; 536 /* r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg); */ 537 case BPF_CMPXCHG: 538 r0 = bpf_to_rv_reg(BPF_REG_0, ctx); 539 emit(is64 ? rv_addi(RV_REG_T2, r0, 0) : 540 rv_addiw(RV_REG_T2, r0, 0), ctx); 541 emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) : 542 rv_lr_w(r0, 0, rd, 0, 0), ctx); 543 jmp_offset = ninsns_rvoff(8); 544 emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx); 545 emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 0) : 546 rv_sc_w(RV_REG_T3, rs, rd, 0, 0), ctx); 547 jmp_offset = ninsns_rvoff(-6); 548 emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx); 549 emit(rv_fence(0x3, 0x3), ctx); 550 break; 551 } 552 } 553 554 #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) 555 #define BPF_FIXUP_REG_MASK GENMASK(31, 27) 556 557 bool ex_handler_bpf(const struct exception_table_entry *ex, 558 struct pt_regs *regs) 559 { 560 off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); 561 int regs_offset = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); 562 563 *(unsigned long *)((void *)regs + pt_regmap[regs_offset]) = 0; 564 regs->epc = (unsigned long)&ex->fixup - offset; 565 566 return true; 567 } 568 569 /* For accesses to BTF pointers, add an entry to the exception table */ 570 static int add_exception_handler(const struct bpf_insn *insn, 571 struct rv_jit_context *ctx, 572 int dst_reg, int insn_len) 573 { 574 struct exception_table_entry *ex; 575 unsigned long pc; 576 off_t offset; 577 578 if (!ctx->insns || !ctx->prog->aux->extable || BPF_MODE(insn->code) != BPF_PROBE_MEM) 579 return 0; 580 581 if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries)) 582 return -EINVAL; 583 584 if (WARN_ON_ONCE(insn_len > ctx->ninsns)) 585 return -EINVAL; 586 587 if (WARN_ON_ONCE(!rvc_enabled() && insn_len == 1)) 588 return -EINVAL; 589 590 ex = &ctx->prog->aux->extable[ctx->nexentries]; 591 pc = (unsigned long)&ctx->insns[ctx->ninsns - insn_len]; 592 593 offset = pc - (long)&ex->insn; 594 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 595 return -ERANGE; 596 ex->insn = offset; 597 598 /* 599 * Since the extable follows the program, the fixup offset is always 600 * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value 601 * to keep things simple, and put the destination register in the upper 602 * bits. We don't need to worry about buildtime or runtime sort 603 * modifying the upper bits because the table is already sorted, and 604 * isn't part of the main exception table. 605 */ 606 offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16)); 607 if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset)) 608 return -ERANGE; 609 610 ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | 611 FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); 612 ex->type = EX_TYPE_BPF; 613 614 ctx->nexentries++; 615 return 0; 616 } 617 618 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 619 bool extra_pass) 620 { 621 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || 622 BPF_CLASS(insn->code) == BPF_JMP; 623 int s, e, rvoff, ret, i = insn - ctx->prog->insnsi; 624 struct bpf_prog_aux *aux = ctx->prog->aux; 625 u8 rd = -1, rs = -1, code = insn->code; 626 s16 off = insn->off; 627 s32 imm = insn->imm; 628 629 init_regs(&rd, &rs, insn, ctx); 630 631 switch (code) { 632 /* dst = src */ 633 case BPF_ALU | BPF_MOV | BPF_X: 634 case BPF_ALU64 | BPF_MOV | BPF_X: 635 if (imm == 1) { 636 /* Special mov32 for zext */ 637 emit_zext_32(rd, ctx); 638 break; 639 } 640 emit_mv(rd, rs, ctx); 641 if (!is64 && !aux->verifier_zext) 642 emit_zext_32(rd, ctx); 643 break; 644 645 /* dst = dst OP src */ 646 case BPF_ALU | BPF_ADD | BPF_X: 647 case BPF_ALU64 | BPF_ADD | BPF_X: 648 emit_add(rd, rd, rs, ctx); 649 if (!is64 && !aux->verifier_zext) 650 emit_zext_32(rd, ctx); 651 break; 652 case BPF_ALU | BPF_SUB | BPF_X: 653 case BPF_ALU64 | BPF_SUB | BPF_X: 654 if (is64) 655 emit_sub(rd, rd, rs, ctx); 656 else 657 emit_subw(rd, rd, rs, ctx); 658 659 if (!is64 && !aux->verifier_zext) 660 emit_zext_32(rd, ctx); 661 break; 662 case BPF_ALU | BPF_AND | BPF_X: 663 case BPF_ALU64 | BPF_AND | BPF_X: 664 emit_and(rd, rd, rs, ctx); 665 if (!is64 && !aux->verifier_zext) 666 emit_zext_32(rd, ctx); 667 break; 668 case BPF_ALU | BPF_OR | BPF_X: 669 case BPF_ALU64 | BPF_OR | BPF_X: 670 emit_or(rd, rd, rs, ctx); 671 if (!is64 && !aux->verifier_zext) 672 emit_zext_32(rd, ctx); 673 break; 674 case BPF_ALU | BPF_XOR | BPF_X: 675 case BPF_ALU64 | BPF_XOR | BPF_X: 676 emit_xor(rd, rd, rs, ctx); 677 if (!is64 && !aux->verifier_zext) 678 emit_zext_32(rd, ctx); 679 break; 680 case BPF_ALU | BPF_MUL | BPF_X: 681 case BPF_ALU64 | BPF_MUL | BPF_X: 682 emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx); 683 if (!is64 && !aux->verifier_zext) 684 emit_zext_32(rd, ctx); 685 break; 686 case BPF_ALU | BPF_DIV | BPF_X: 687 case BPF_ALU64 | BPF_DIV | BPF_X: 688 emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx); 689 if (!is64 && !aux->verifier_zext) 690 emit_zext_32(rd, ctx); 691 break; 692 case BPF_ALU | BPF_MOD | BPF_X: 693 case BPF_ALU64 | BPF_MOD | BPF_X: 694 emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx); 695 if (!is64 && !aux->verifier_zext) 696 emit_zext_32(rd, ctx); 697 break; 698 case BPF_ALU | BPF_LSH | BPF_X: 699 case BPF_ALU64 | BPF_LSH | BPF_X: 700 emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx); 701 if (!is64 && !aux->verifier_zext) 702 emit_zext_32(rd, ctx); 703 break; 704 case BPF_ALU | BPF_RSH | BPF_X: 705 case BPF_ALU64 | BPF_RSH | BPF_X: 706 emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx); 707 if (!is64 && !aux->verifier_zext) 708 emit_zext_32(rd, ctx); 709 break; 710 case BPF_ALU | BPF_ARSH | BPF_X: 711 case BPF_ALU64 | BPF_ARSH | BPF_X: 712 emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx); 713 if (!is64 && !aux->verifier_zext) 714 emit_zext_32(rd, ctx); 715 break; 716 717 /* dst = -dst */ 718 case BPF_ALU | BPF_NEG: 719 case BPF_ALU64 | BPF_NEG: 720 emit_sub(rd, RV_REG_ZERO, rd, ctx); 721 if (!is64 && !aux->verifier_zext) 722 emit_zext_32(rd, ctx); 723 break; 724 725 /* dst = BSWAP##imm(dst) */ 726 case BPF_ALU | BPF_END | BPF_FROM_LE: 727 switch (imm) { 728 case 16: 729 emit_slli(rd, rd, 48, ctx); 730 emit_srli(rd, rd, 48, ctx); 731 break; 732 case 32: 733 if (!aux->verifier_zext) 734 emit_zext_32(rd, ctx); 735 break; 736 case 64: 737 /* Do nothing */ 738 break; 739 } 740 break; 741 742 case BPF_ALU | BPF_END | BPF_FROM_BE: 743 emit_li(RV_REG_T2, 0, ctx); 744 745 emit_andi(RV_REG_T1, rd, 0xff, ctx); 746 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 747 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 748 emit_srli(rd, rd, 8, ctx); 749 if (imm == 16) 750 goto out_be; 751 752 emit_andi(RV_REG_T1, rd, 0xff, ctx); 753 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 754 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 755 emit_srli(rd, rd, 8, ctx); 756 757 emit_andi(RV_REG_T1, rd, 0xff, ctx); 758 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 759 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 760 emit_srli(rd, rd, 8, ctx); 761 if (imm == 32) 762 goto out_be; 763 764 emit_andi(RV_REG_T1, rd, 0xff, ctx); 765 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 766 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 767 emit_srli(rd, rd, 8, ctx); 768 769 emit_andi(RV_REG_T1, rd, 0xff, ctx); 770 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 771 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 772 emit_srli(rd, rd, 8, ctx); 773 774 emit_andi(RV_REG_T1, rd, 0xff, ctx); 775 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 776 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 777 emit_srli(rd, rd, 8, ctx); 778 779 emit_andi(RV_REG_T1, rd, 0xff, ctx); 780 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 781 emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); 782 emit_srli(rd, rd, 8, ctx); 783 out_be: 784 emit_andi(RV_REG_T1, rd, 0xff, ctx); 785 emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); 786 787 emit_mv(rd, RV_REG_T2, ctx); 788 break; 789 790 /* dst = imm */ 791 case BPF_ALU | BPF_MOV | BPF_K: 792 case BPF_ALU64 | BPF_MOV | BPF_K: 793 emit_imm(rd, imm, ctx); 794 if (!is64 && !aux->verifier_zext) 795 emit_zext_32(rd, ctx); 796 break; 797 798 /* dst = dst OP imm */ 799 case BPF_ALU | BPF_ADD | BPF_K: 800 case BPF_ALU64 | BPF_ADD | BPF_K: 801 if (is_12b_int(imm)) { 802 emit_addi(rd, rd, imm, ctx); 803 } else { 804 emit_imm(RV_REG_T1, imm, ctx); 805 emit_add(rd, rd, RV_REG_T1, ctx); 806 } 807 if (!is64 && !aux->verifier_zext) 808 emit_zext_32(rd, ctx); 809 break; 810 case BPF_ALU | BPF_SUB | BPF_K: 811 case BPF_ALU64 | BPF_SUB | BPF_K: 812 if (is_12b_int(-imm)) { 813 emit_addi(rd, rd, -imm, ctx); 814 } else { 815 emit_imm(RV_REG_T1, imm, ctx); 816 emit_sub(rd, rd, RV_REG_T1, ctx); 817 } 818 if (!is64 && !aux->verifier_zext) 819 emit_zext_32(rd, ctx); 820 break; 821 case BPF_ALU | BPF_AND | BPF_K: 822 case BPF_ALU64 | BPF_AND | BPF_K: 823 if (is_12b_int(imm)) { 824 emit_andi(rd, rd, imm, ctx); 825 } else { 826 emit_imm(RV_REG_T1, imm, ctx); 827 emit_and(rd, rd, RV_REG_T1, ctx); 828 } 829 if (!is64 && !aux->verifier_zext) 830 emit_zext_32(rd, ctx); 831 break; 832 case BPF_ALU | BPF_OR | BPF_K: 833 case BPF_ALU64 | BPF_OR | BPF_K: 834 if (is_12b_int(imm)) { 835 emit(rv_ori(rd, rd, imm), ctx); 836 } else { 837 emit_imm(RV_REG_T1, imm, ctx); 838 emit_or(rd, rd, RV_REG_T1, ctx); 839 } 840 if (!is64 && !aux->verifier_zext) 841 emit_zext_32(rd, ctx); 842 break; 843 case BPF_ALU | BPF_XOR | BPF_K: 844 case BPF_ALU64 | BPF_XOR | BPF_K: 845 if (is_12b_int(imm)) { 846 emit(rv_xori(rd, rd, imm), ctx); 847 } else { 848 emit_imm(RV_REG_T1, imm, ctx); 849 emit_xor(rd, rd, RV_REG_T1, ctx); 850 } 851 if (!is64 && !aux->verifier_zext) 852 emit_zext_32(rd, ctx); 853 break; 854 case BPF_ALU | BPF_MUL | BPF_K: 855 case BPF_ALU64 | BPF_MUL | BPF_K: 856 emit_imm(RV_REG_T1, imm, ctx); 857 emit(is64 ? rv_mul(rd, rd, RV_REG_T1) : 858 rv_mulw(rd, rd, RV_REG_T1), ctx); 859 if (!is64 && !aux->verifier_zext) 860 emit_zext_32(rd, ctx); 861 break; 862 case BPF_ALU | BPF_DIV | BPF_K: 863 case BPF_ALU64 | BPF_DIV | BPF_K: 864 emit_imm(RV_REG_T1, imm, ctx); 865 emit(is64 ? rv_divu(rd, rd, RV_REG_T1) : 866 rv_divuw(rd, rd, RV_REG_T1), ctx); 867 if (!is64 && !aux->verifier_zext) 868 emit_zext_32(rd, ctx); 869 break; 870 case BPF_ALU | BPF_MOD | BPF_K: 871 case BPF_ALU64 | BPF_MOD | BPF_K: 872 emit_imm(RV_REG_T1, imm, ctx); 873 emit(is64 ? rv_remu(rd, rd, RV_REG_T1) : 874 rv_remuw(rd, rd, RV_REG_T1), ctx); 875 if (!is64 && !aux->verifier_zext) 876 emit_zext_32(rd, ctx); 877 break; 878 case BPF_ALU | BPF_LSH | BPF_K: 879 case BPF_ALU64 | BPF_LSH | BPF_K: 880 emit_slli(rd, rd, imm, ctx); 881 882 if (!is64 && !aux->verifier_zext) 883 emit_zext_32(rd, ctx); 884 break; 885 case BPF_ALU | BPF_RSH | BPF_K: 886 case BPF_ALU64 | BPF_RSH | BPF_K: 887 if (is64) 888 emit_srli(rd, rd, imm, ctx); 889 else 890 emit(rv_srliw(rd, rd, imm), ctx); 891 892 if (!is64 && !aux->verifier_zext) 893 emit_zext_32(rd, ctx); 894 break; 895 case BPF_ALU | BPF_ARSH | BPF_K: 896 case BPF_ALU64 | BPF_ARSH | BPF_K: 897 if (is64) 898 emit_srai(rd, rd, imm, ctx); 899 else 900 emit(rv_sraiw(rd, rd, imm), ctx); 901 902 if (!is64 && !aux->verifier_zext) 903 emit_zext_32(rd, ctx); 904 break; 905 906 /* JUMP off */ 907 case BPF_JMP | BPF_JA: 908 rvoff = rv_offset(i, off, ctx); 909 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); 910 if (ret) 911 return ret; 912 break; 913 914 /* IF (dst COND src) JUMP off */ 915 case BPF_JMP | BPF_JEQ | BPF_X: 916 case BPF_JMP32 | BPF_JEQ | BPF_X: 917 case BPF_JMP | BPF_JGT | BPF_X: 918 case BPF_JMP32 | BPF_JGT | BPF_X: 919 case BPF_JMP | BPF_JLT | BPF_X: 920 case BPF_JMP32 | BPF_JLT | BPF_X: 921 case BPF_JMP | BPF_JGE | BPF_X: 922 case BPF_JMP32 | BPF_JGE | BPF_X: 923 case BPF_JMP | BPF_JLE | BPF_X: 924 case BPF_JMP32 | BPF_JLE | BPF_X: 925 case BPF_JMP | BPF_JNE | BPF_X: 926 case BPF_JMP32 | BPF_JNE | BPF_X: 927 case BPF_JMP | BPF_JSGT | BPF_X: 928 case BPF_JMP32 | BPF_JSGT | BPF_X: 929 case BPF_JMP | BPF_JSLT | BPF_X: 930 case BPF_JMP32 | BPF_JSLT | BPF_X: 931 case BPF_JMP | BPF_JSGE | BPF_X: 932 case BPF_JMP32 | BPF_JSGE | BPF_X: 933 case BPF_JMP | BPF_JSLE | BPF_X: 934 case BPF_JMP32 | BPF_JSLE | BPF_X: 935 case BPF_JMP | BPF_JSET | BPF_X: 936 case BPF_JMP32 | BPF_JSET | BPF_X: 937 rvoff = rv_offset(i, off, ctx); 938 if (!is64) { 939 s = ctx->ninsns; 940 if (is_signed_bpf_cond(BPF_OP(code))) 941 emit_sext_32_rd_rs(&rd, &rs, ctx); 942 else 943 emit_zext_32_rd_rs(&rd, &rs, ctx); 944 e = ctx->ninsns; 945 946 /* Adjust for extra insns */ 947 rvoff -= ninsns_rvoff(e - s); 948 } 949 950 if (BPF_OP(code) == BPF_JSET) { 951 /* Adjust for and */ 952 rvoff -= 4; 953 emit_and(RV_REG_T1, rd, rs, ctx); 954 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, 955 ctx); 956 } else { 957 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 958 } 959 break; 960 961 /* IF (dst COND imm) JUMP off */ 962 case BPF_JMP | BPF_JEQ | BPF_K: 963 case BPF_JMP32 | BPF_JEQ | BPF_K: 964 case BPF_JMP | BPF_JGT | BPF_K: 965 case BPF_JMP32 | BPF_JGT | BPF_K: 966 case BPF_JMP | BPF_JLT | BPF_K: 967 case BPF_JMP32 | BPF_JLT | BPF_K: 968 case BPF_JMP | BPF_JGE | BPF_K: 969 case BPF_JMP32 | BPF_JGE | BPF_K: 970 case BPF_JMP | BPF_JLE | BPF_K: 971 case BPF_JMP32 | BPF_JLE | BPF_K: 972 case BPF_JMP | BPF_JNE | BPF_K: 973 case BPF_JMP32 | BPF_JNE | BPF_K: 974 case BPF_JMP | BPF_JSGT | BPF_K: 975 case BPF_JMP32 | BPF_JSGT | BPF_K: 976 case BPF_JMP | BPF_JSLT | BPF_K: 977 case BPF_JMP32 | BPF_JSLT | BPF_K: 978 case BPF_JMP | BPF_JSGE | BPF_K: 979 case BPF_JMP32 | BPF_JSGE | BPF_K: 980 case BPF_JMP | BPF_JSLE | BPF_K: 981 case BPF_JMP32 | BPF_JSLE | BPF_K: 982 rvoff = rv_offset(i, off, ctx); 983 s = ctx->ninsns; 984 if (imm) { 985 emit_imm(RV_REG_T1, imm, ctx); 986 rs = RV_REG_T1; 987 } else { 988 /* If imm is 0, simply use zero register. */ 989 rs = RV_REG_ZERO; 990 } 991 if (!is64) { 992 if (is_signed_bpf_cond(BPF_OP(code))) 993 emit_sext_32_rd(&rd, ctx); 994 else 995 emit_zext_32_rd_t1(&rd, ctx); 996 } 997 e = ctx->ninsns; 998 999 /* Adjust for extra insns */ 1000 rvoff -= ninsns_rvoff(e - s); 1001 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 1002 break; 1003 1004 case BPF_JMP | BPF_JSET | BPF_K: 1005 case BPF_JMP32 | BPF_JSET | BPF_K: 1006 rvoff = rv_offset(i, off, ctx); 1007 s = ctx->ninsns; 1008 if (is_12b_int(imm)) { 1009 emit_andi(RV_REG_T1, rd, imm, ctx); 1010 } else { 1011 emit_imm(RV_REG_T1, imm, ctx); 1012 emit_and(RV_REG_T1, rd, RV_REG_T1, ctx); 1013 } 1014 /* For jset32, we should clear the upper 32 bits of t1, but 1015 * sign-extension is sufficient here and saves one instruction, 1016 * as t1 is used only in comparison against zero. 1017 */ 1018 if (!is64 && imm < 0) 1019 emit_addiw(RV_REG_T1, RV_REG_T1, 0, ctx); 1020 e = ctx->ninsns; 1021 rvoff -= ninsns_rvoff(e - s); 1022 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); 1023 break; 1024 1025 /* function call */ 1026 case BPF_JMP | BPF_CALL: 1027 { 1028 bool fixed_addr; 1029 u64 addr; 1030 1031 mark_call(ctx); 1032 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, 1033 &addr, &fixed_addr); 1034 if (ret < 0) 1035 return ret; 1036 1037 ret = emit_call(addr, fixed_addr, ctx); 1038 if (ret) 1039 return ret; 1040 1041 emit_mv(bpf_to_rv_reg(BPF_REG_0, ctx), RV_REG_A0, ctx); 1042 break; 1043 } 1044 /* tail call */ 1045 case BPF_JMP | BPF_TAIL_CALL: 1046 if (emit_bpf_tail_call(i, ctx)) 1047 return -1; 1048 break; 1049 1050 /* function return */ 1051 case BPF_JMP | BPF_EXIT: 1052 if (i == ctx->prog->len - 1) 1053 break; 1054 1055 rvoff = epilogue_offset(ctx); 1056 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx); 1057 if (ret) 1058 return ret; 1059 break; 1060 1061 /* dst = imm64 */ 1062 case BPF_LD | BPF_IMM | BPF_DW: 1063 { 1064 struct bpf_insn insn1 = insn[1]; 1065 u64 imm64; 1066 1067 imm64 = (u64)insn1.imm << 32 | (u32)imm; 1068 if (bpf_pseudo_func(insn)) { 1069 /* fixed-length insns for extra jit pass */ 1070 ret = emit_addr(rd, imm64, extra_pass, ctx); 1071 if (ret) 1072 return ret; 1073 } else { 1074 emit_imm(rd, imm64, ctx); 1075 } 1076 1077 return 1; 1078 } 1079 1080 /* LDX: dst = *(size *)(src + off) */ 1081 case BPF_LDX | BPF_MEM | BPF_B: 1082 case BPF_LDX | BPF_MEM | BPF_H: 1083 case BPF_LDX | BPF_MEM | BPF_W: 1084 case BPF_LDX | BPF_MEM | BPF_DW: 1085 case BPF_LDX | BPF_PROBE_MEM | BPF_B: 1086 case BPF_LDX | BPF_PROBE_MEM | BPF_H: 1087 case BPF_LDX | BPF_PROBE_MEM | BPF_W: 1088 case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 1089 { 1090 int insn_len, insns_start; 1091 1092 switch (BPF_SIZE(code)) { 1093 case BPF_B: 1094 if (is_12b_int(off)) { 1095 insns_start = ctx->ninsns; 1096 emit(rv_lbu(rd, off, rs), ctx); 1097 insn_len = ctx->ninsns - insns_start; 1098 break; 1099 } 1100 1101 emit_imm(RV_REG_T1, off, ctx); 1102 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1103 insns_start = ctx->ninsns; 1104 emit(rv_lbu(rd, 0, RV_REG_T1), ctx); 1105 insn_len = ctx->ninsns - insns_start; 1106 if (insn_is_zext(&insn[1])) 1107 return 1; 1108 break; 1109 case BPF_H: 1110 if (is_12b_int(off)) { 1111 insns_start = ctx->ninsns; 1112 emit(rv_lhu(rd, off, rs), ctx); 1113 insn_len = ctx->ninsns - insns_start; 1114 break; 1115 } 1116 1117 emit_imm(RV_REG_T1, off, ctx); 1118 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1119 insns_start = ctx->ninsns; 1120 emit(rv_lhu(rd, 0, RV_REG_T1), ctx); 1121 insn_len = ctx->ninsns - insns_start; 1122 if (insn_is_zext(&insn[1])) 1123 return 1; 1124 break; 1125 case BPF_W: 1126 if (is_12b_int(off)) { 1127 insns_start = ctx->ninsns; 1128 emit(rv_lwu(rd, off, rs), ctx); 1129 insn_len = ctx->ninsns - insns_start; 1130 break; 1131 } 1132 1133 emit_imm(RV_REG_T1, off, ctx); 1134 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1135 insns_start = ctx->ninsns; 1136 emit(rv_lwu(rd, 0, RV_REG_T1), ctx); 1137 insn_len = ctx->ninsns - insns_start; 1138 if (insn_is_zext(&insn[1])) 1139 return 1; 1140 break; 1141 case BPF_DW: 1142 if (is_12b_int(off)) { 1143 insns_start = ctx->ninsns; 1144 emit_ld(rd, off, rs, ctx); 1145 insn_len = ctx->ninsns - insns_start; 1146 break; 1147 } 1148 1149 emit_imm(RV_REG_T1, off, ctx); 1150 emit_add(RV_REG_T1, RV_REG_T1, rs, ctx); 1151 insns_start = ctx->ninsns; 1152 emit_ld(rd, 0, RV_REG_T1, ctx); 1153 insn_len = ctx->ninsns - insns_start; 1154 break; 1155 } 1156 1157 ret = add_exception_handler(insn, ctx, rd, insn_len); 1158 if (ret) 1159 return ret; 1160 break; 1161 } 1162 /* speculation barrier */ 1163 case BPF_ST | BPF_NOSPEC: 1164 break; 1165 1166 /* ST: *(size *)(dst + off) = imm */ 1167 case BPF_ST | BPF_MEM | BPF_B: 1168 emit_imm(RV_REG_T1, imm, ctx); 1169 if (is_12b_int(off)) { 1170 emit(rv_sb(rd, off, RV_REG_T1), ctx); 1171 break; 1172 } 1173 1174 emit_imm(RV_REG_T2, off, ctx); 1175 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1176 emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); 1177 break; 1178 1179 case BPF_ST | BPF_MEM | BPF_H: 1180 emit_imm(RV_REG_T1, imm, ctx); 1181 if (is_12b_int(off)) { 1182 emit(rv_sh(rd, off, RV_REG_T1), ctx); 1183 break; 1184 } 1185 1186 emit_imm(RV_REG_T2, off, ctx); 1187 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1188 emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); 1189 break; 1190 case BPF_ST | BPF_MEM | BPF_W: 1191 emit_imm(RV_REG_T1, imm, ctx); 1192 if (is_12b_int(off)) { 1193 emit_sw(rd, off, RV_REG_T1, ctx); 1194 break; 1195 } 1196 1197 emit_imm(RV_REG_T2, off, ctx); 1198 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1199 emit_sw(RV_REG_T2, 0, RV_REG_T1, ctx); 1200 break; 1201 case BPF_ST | BPF_MEM | BPF_DW: 1202 emit_imm(RV_REG_T1, imm, ctx); 1203 if (is_12b_int(off)) { 1204 emit_sd(rd, off, RV_REG_T1, ctx); 1205 break; 1206 } 1207 1208 emit_imm(RV_REG_T2, off, ctx); 1209 emit_add(RV_REG_T2, RV_REG_T2, rd, ctx); 1210 emit_sd(RV_REG_T2, 0, RV_REG_T1, ctx); 1211 break; 1212 1213 /* STX: *(size *)(dst + off) = src */ 1214 case BPF_STX | BPF_MEM | BPF_B: 1215 if (is_12b_int(off)) { 1216 emit(rv_sb(rd, off, rs), ctx); 1217 break; 1218 } 1219 1220 emit_imm(RV_REG_T1, off, ctx); 1221 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1222 emit(rv_sb(RV_REG_T1, 0, rs), ctx); 1223 break; 1224 case BPF_STX | BPF_MEM | BPF_H: 1225 if (is_12b_int(off)) { 1226 emit(rv_sh(rd, off, rs), ctx); 1227 break; 1228 } 1229 1230 emit_imm(RV_REG_T1, off, ctx); 1231 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1232 emit(rv_sh(RV_REG_T1, 0, rs), ctx); 1233 break; 1234 case BPF_STX | BPF_MEM | BPF_W: 1235 if (is_12b_int(off)) { 1236 emit_sw(rd, off, rs, ctx); 1237 break; 1238 } 1239 1240 emit_imm(RV_REG_T1, off, ctx); 1241 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1242 emit_sw(RV_REG_T1, 0, rs, ctx); 1243 break; 1244 case BPF_STX | BPF_MEM | BPF_DW: 1245 if (is_12b_int(off)) { 1246 emit_sd(rd, off, rs, ctx); 1247 break; 1248 } 1249 1250 emit_imm(RV_REG_T1, off, ctx); 1251 emit_add(RV_REG_T1, RV_REG_T1, rd, ctx); 1252 emit_sd(RV_REG_T1, 0, rs, ctx); 1253 break; 1254 case BPF_STX | BPF_ATOMIC | BPF_W: 1255 case BPF_STX | BPF_ATOMIC | BPF_DW: 1256 emit_atomic(rd, rs, off, imm, 1257 BPF_SIZE(code) == BPF_DW, ctx); 1258 break; 1259 default: 1260 pr_err("bpf-jit: unknown opcode %02x\n", code); 1261 return -EINVAL; 1262 } 1263 1264 return 0; 1265 } 1266 1267 void bpf_jit_build_prologue(struct rv_jit_context *ctx) 1268 { 1269 int stack_adjust = 0, store_offset, bpf_stack_adjust; 1270 1271 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); 1272 if (bpf_stack_adjust) 1273 mark_fp(ctx); 1274 1275 if (seen_reg(RV_REG_RA, ctx)) 1276 stack_adjust += 8; 1277 stack_adjust += 8; /* RV_REG_FP */ 1278 if (seen_reg(RV_REG_S1, ctx)) 1279 stack_adjust += 8; 1280 if (seen_reg(RV_REG_S2, ctx)) 1281 stack_adjust += 8; 1282 if (seen_reg(RV_REG_S3, ctx)) 1283 stack_adjust += 8; 1284 if (seen_reg(RV_REG_S4, ctx)) 1285 stack_adjust += 8; 1286 if (seen_reg(RV_REG_S5, ctx)) 1287 stack_adjust += 8; 1288 if (seen_reg(RV_REG_S6, ctx)) 1289 stack_adjust += 8; 1290 1291 stack_adjust = round_up(stack_adjust, 16); 1292 stack_adjust += bpf_stack_adjust; 1293 1294 store_offset = stack_adjust - 8; 1295 1296 /* First instruction is always setting the tail-call-counter 1297 * (TCC) register. This instruction is skipped for tail calls. 1298 * Force using a 4-byte (non-compressed) instruction. 1299 */ 1300 emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); 1301 1302 emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); 1303 1304 if (seen_reg(RV_REG_RA, ctx)) { 1305 emit_sd(RV_REG_SP, store_offset, RV_REG_RA, ctx); 1306 store_offset -= 8; 1307 } 1308 emit_sd(RV_REG_SP, store_offset, RV_REG_FP, ctx); 1309 store_offset -= 8; 1310 if (seen_reg(RV_REG_S1, ctx)) { 1311 emit_sd(RV_REG_SP, store_offset, RV_REG_S1, ctx); 1312 store_offset -= 8; 1313 } 1314 if (seen_reg(RV_REG_S2, ctx)) { 1315 emit_sd(RV_REG_SP, store_offset, RV_REG_S2, ctx); 1316 store_offset -= 8; 1317 } 1318 if (seen_reg(RV_REG_S3, ctx)) { 1319 emit_sd(RV_REG_SP, store_offset, RV_REG_S3, ctx); 1320 store_offset -= 8; 1321 } 1322 if (seen_reg(RV_REG_S4, ctx)) { 1323 emit_sd(RV_REG_SP, store_offset, RV_REG_S4, ctx); 1324 store_offset -= 8; 1325 } 1326 if (seen_reg(RV_REG_S5, ctx)) { 1327 emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx); 1328 store_offset -= 8; 1329 } 1330 if (seen_reg(RV_REG_S6, ctx)) { 1331 emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx); 1332 store_offset -= 8; 1333 } 1334 1335 emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx); 1336 1337 if (bpf_stack_adjust) 1338 emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx); 1339 1340 /* Program contains calls and tail calls, so RV_REG_TCC need 1341 * to be saved across calls. 1342 */ 1343 if (seen_tail_call(ctx) && seen_call(ctx)) 1344 emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx); 1345 1346 ctx->stack_size = stack_adjust; 1347 } 1348 1349 void bpf_jit_build_epilogue(struct rv_jit_context *ctx) 1350 { 1351 __build_epilogue(false, ctx); 1352 } 1353