1 // SPDX-License-Identifier: GPL-2.0 2 /* BPF JIT compiler for RV64G 3 * 4 * Copyright(c) 2019 Björn Töpel <bjorn.topel@gmail.com> 5 * 6 */ 7 8 #include <linux/bpf.h> 9 #include <linux/filter.h> 10 #include "bpf_jit.h" 11 12 #define RV_REG_TCC RV_REG_A6 13 #define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */ 14 15 static const int regmap[] = { 16 [BPF_REG_0] = RV_REG_A5, 17 [BPF_REG_1] = RV_REG_A0, 18 [BPF_REG_2] = RV_REG_A1, 19 [BPF_REG_3] = RV_REG_A2, 20 [BPF_REG_4] = RV_REG_A3, 21 [BPF_REG_5] = RV_REG_A4, 22 [BPF_REG_6] = RV_REG_S1, 23 [BPF_REG_7] = RV_REG_S2, 24 [BPF_REG_8] = RV_REG_S3, 25 [BPF_REG_9] = RV_REG_S4, 26 [BPF_REG_FP] = RV_REG_S5, 27 [BPF_REG_AX] = RV_REG_T0, 28 }; 29 30 enum { 31 RV_CTX_F_SEEN_TAIL_CALL = 0, 32 RV_CTX_F_SEEN_CALL = RV_REG_RA, 33 RV_CTX_F_SEEN_S1 = RV_REG_S1, 34 RV_CTX_F_SEEN_S2 = RV_REG_S2, 35 RV_CTX_F_SEEN_S3 = RV_REG_S3, 36 RV_CTX_F_SEEN_S4 = RV_REG_S4, 37 RV_CTX_F_SEEN_S5 = RV_REG_S5, 38 RV_CTX_F_SEEN_S6 = RV_REG_S6, 39 }; 40 41 static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) 42 { 43 u8 reg = regmap[bpf_reg]; 44 45 switch (reg) { 46 case RV_CTX_F_SEEN_S1: 47 case RV_CTX_F_SEEN_S2: 48 case RV_CTX_F_SEEN_S3: 49 case RV_CTX_F_SEEN_S4: 50 case RV_CTX_F_SEEN_S5: 51 case RV_CTX_F_SEEN_S6: 52 __set_bit(reg, &ctx->flags); 53 } 54 return reg; 55 }; 56 57 static bool seen_reg(int reg, struct rv_jit_context *ctx) 58 { 59 switch (reg) { 60 case RV_CTX_F_SEEN_CALL: 61 case RV_CTX_F_SEEN_S1: 62 case RV_CTX_F_SEEN_S2: 63 case RV_CTX_F_SEEN_S3: 64 case RV_CTX_F_SEEN_S4: 65 case RV_CTX_F_SEEN_S5: 66 case RV_CTX_F_SEEN_S6: 67 return test_bit(reg, &ctx->flags); 68 } 69 return false; 70 } 71 72 static void mark_fp(struct rv_jit_context *ctx) 73 { 74 __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags); 75 } 76 77 static void mark_call(struct rv_jit_context *ctx) 78 { 79 __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 80 } 81 82 static bool seen_call(struct rv_jit_context *ctx) 83 { 84 return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); 85 } 86 87 static void mark_tail_call(struct rv_jit_context *ctx) 88 { 89 __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 90 } 91 92 static bool seen_tail_call(struct rv_jit_context *ctx) 93 { 94 return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); 95 } 96 97 static u8 rv_tail_call_reg(struct rv_jit_context *ctx) 98 { 99 mark_tail_call(ctx); 100 101 if (seen_call(ctx)) { 102 __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); 103 return RV_REG_S6; 104 } 105 return RV_REG_A6; 106 } 107 108 static bool is_32b_int(s64 val) 109 { 110 return -(1L << 31) <= val && val < (1L << 31); 111 } 112 113 static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx) 114 { 115 /* Note that the immediate from the add is sign-extended, 116 * which means that we need to compensate this by adding 2^12, 117 * when the 12th bit is set. A simpler way of doing this, and 118 * getting rid of the check, is to just add 2**11 before the 119 * shift. The "Loading a 32-Bit constant" example from the 120 * "Computer Organization and Design, RISC-V edition" book by 121 * Patterson/Hennessy highlights this fact. 122 * 123 * This also means that we need to process LSB to MSB. 124 */ 125 s64 upper = (val + (1 << 11)) >> 12, lower = val & 0xfff; 126 int shift; 127 128 if (is_32b_int(val)) { 129 if (upper) 130 emit(rv_lui(rd, upper), ctx); 131 132 if (!upper) { 133 emit(rv_addi(rd, RV_REG_ZERO, lower), ctx); 134 return; 135 } 136 137 emit(rv_addiw(rd, rd, lower), ctx); 138 return; 139 } 140 141 shift = __ffs(upper); 142 upper >>= shift; 143 shift += 12; 144 145 emit_imm(rd, upper, ctx); 146 147 emit(rv_slli(rd, rd, shift), ctx); 148 if (lower) 149 emit(rv_addi(rd, rd, lower), ctx); 150 } 151 152 static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) 153 { 154 int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; 155 156 if (seen_reg(RV_REG_RA, ctx)) { 157 emit(rv_ld(RV_REG_RA, store_offset, RV_REG_SP), ctx); 158 store_offset -= 8; 159 } 160 emit(rv_ld(RV_REG_FP, store_offset, RV_REG_SP), ctx); 161 store_offset -= 8; 162 if (seen_reg(RV_REG_S1, ctx)) { 163 emit(rv_ld(RV_REG_S1, store_offset, RV_REG_SP), ctx); 164 store_offset -= 8; 165 } 166 if (seen_reg(RV_REG_S2, ctx)) { 167 emit(rv_ld(RV_REG_S2, store_offset, RV_REG_SP), ctx); 168 store_offset -= 8; 169 } 170 if (seen_reg(RV_REG_S3, ctx)) { 171 emit(rv_ld(RV_REG_S3, store_offset, RV_REG_SP), ctx); 172 store_offset -= 8; 173 } 174 if (seen_reg(RV_REG_S4, ctx)) { 175 emit(rv_ld(RV_REG_S4, store_offset, RV_REG_SP), ctx); 176 store_offset -= 8; 177 } 178 if (seen_reg(RV_REG_S5, ctx)) { 179 emit(rv_ld(RV_REG_S5, store_offset, RV_REG_SP), ctx); 180 store_offset -= 8; 181 } 182 if (seen_reg(RV_REG_S6, ctx)) { 183 emit(rv_ld(RV_REG_S6, store_offset, RV_REG_SP), ctx); 184 store_offset -= 8; 185 } 186 187 emit(rv_addi(RV_REG_SP, RV_REG_SP, stack_adjust), ctx); 188 /* Set return value. */ 189 if (!is_tail_call) 190 emit(rv_addi(RV_REG_A0, RV_REG_A5, 0), ctx); 191 emit(rv_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, 192 is_tail_call ? 4 : 0), /* skip TCC init */ 193 ctx); 194 } 195 196 static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, 197 struct rv_jit_context *ctx) 198 { 199 switch (cond) { 200 case BPF_JEQ: 201 emit(rv_beq(rd, rs, rvoff >> 1), ctx); 202 return; 203 case BPF_JGT: 204 emit(rv_bltu(rs, rd, rvoff >> 1), ctx); 205 return; 206 case BPF_JLT: 207 emit(rv_bltu(rd, rs, rvoff >> 1), ctx); 208 return; 209 case BPF_JGE: 210 emit(rv_bgeu(rd, rs, rvoff >> 1), ctx); 211 return; 212 case BPF_JLE: 213 emit(rv_bgeu(rs, rd, rvoff >> 1), ctx); 214 return; 215 case BPF_JNE: 216 emit(rv_bne(rd, rs, rvoff >> 1), ctx); 217 return; 218 case BPF_JSGT: 219 emit(rv_blt(rs, rd, rvoff >> 1), ctx); 220 return; 221 case BPF_JSLT: 222 emit(rv_blt(rd, rs, rvoff >> 1), ctx); 223 return; 224 case BPF_JSGE: 225 emit(rv_bge(rd, rs, rvoff >> 1), ctx); 226 return; 227 case BPF_JSLE: 228 emit(rv_bge(rs, rd, rvoff >> 1), ctx); 229 } 230 } 231 232 static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, 233 struct rv_jit_context *ctx) 234 { 235 s64 upper, lower; 236 237 if (is_13b_int(rvoff)) { 238 emit_bcc(cond, rd, rs, rvoff, ctx); 239 return; 240 } 241 242 /* Adjust for jal */ 243 rvoff -= 4; 244 245 /* Transform, e.g.: 246 * bne rd,rs,foo 247 * to 248 * beq rd,rs,<.L1> 249 * (auipc foo) 250 * jal(r) foo 251 * .L1 252 */ 253 cond = invert_bpf_cond(cond); 254 if (is_21b_int(rvoff)) { 255 emit_bcc(cond, rd, rs, 8, ctx); 256 emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx); 257 return; 258 } 259 260 /* 32b No need for an additional rvoff adjustment, since we 261 * get that from the auipc at PC', where PC = PC' + 4. 262 */ 263 upper = (rvoff + (1 << 11)) >> 12; 264 lower = rvoff & 0xfff; 265 266 emit_bcc(cond, rd, rs, 12, ctx); 267 emit(rv_auipc(RV_REG_T1, upper), ctx); 268 emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx); 269 } 270 271 static void emit_zext_32(u8 reg, struct rv_jit_context *ctx) 272 { 273 emit(rv_slli(reg, reg, 32), ctx); 274 emit(rv_srli(reg, reg, 32), ctx); 275 } 276 277 static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) 278 { 279 int tc_ninsn, off, start_insn = ctx->ninsns; 280 u8 tcc = rv_tail_call_reg(ctx); 281 282 /* a0: &ctx 283 * a1: &array 284 * a2: index 285 * 286 * if (index >= array->map.max_entries) 287 * goto out; 288 */ 289 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] : 290 ctx->offset[0]; 291 emit_zext_32(RV_REG_A2, ctx); 292 293 off = offsetof(struct bpf_array, map.max_entries); 294 if (is_12b_check(off, insn)) 295 return -1; 296 emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx); 297 off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; 298 emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx); 299 300 /* if (TCC-- < 0) 301 * goto out; 302 */ 303 emit(rv_addi(RV_REG_T1, tcc, -1), ctx); 304 off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; 305 emit_branch(BPF_JSLT, tcc, RV_REG_ZERO, off, ctx); 306 307 /* prog = array->ptrs[index]; 308 * if (!prog) 309 * goto out; 310 */ 311 emit(rv_slli(RV_REG_T2, RV_REG_A2, 3), ctx); 312 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_A1), ctx); 313 off = offsetof(struct bpf_array, ptrs); 314 if (is_12b_check(off, insn)) 315 return -1; 316 emit(rv_ld(RV_REG_T2, off, RV_REG_T2), ctx); 317 off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; 318 emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); 319 320 /* goto *(prog->bpf_func + 4); */ 321 off = offsetof(struct bpf_prog, bpf_func); 322 if (is_12b_check(off, insn)) 323 return -1; 324 emit(rv_ld(RV_REG_T3, off, RV_REG_T2), ctx); 325 emit(rv_addi(RV_REG_TCC, RV_REG_T1, 0), ctx); 326 __build_epilogue(true, ctx); 327 return 0; 328 } 329 330 static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn, 331 struct rv_jit_context *ctx) 332 { 333 u8 code = insn->code; 334 335 switch (code) { 336 case BPF_JMP | BPF_JA: 337 case BPF_JMP | BPF_CALL: 338 case BPF_JMP | BPF_EXIT: 339 case BPF_JMP | BPF_TAIL_CALL: 340 break; 341 default: 342 *rd = bpf_to_rv_reg(insn->dst_reg, ctx); 343 } 344 345 if (code & (BPF_ALU | BPF_X) || code & (BPF_ALU64 | BPF_X) || 346 code & (BPF_JMP | BPF_X) || code & (BPF_JMP32 | BPF_X) || 347 code & BPF_LDX || code & BPF_STX) 348 *rs = bpf_to_rv_reg(insn->src_reg, ctx); 349 } 350 351 static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) 352 { 353 emit(rv_addi(RV_REG_T2, *rd, 0), ctx); 354 emit_zext_32(RV_REG_T2, ctx); 355 emit(rv_addi(RV_REG_T1, *rs, 0), ctx); 356 emit_zext_32(RV_REG_T1, ctx); 357 *rd = RV_REG_T2; 358 *rs = RV_REG_T1; 359 } 360 361 static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) 362 { 363 emit(rv_addiw(RV_REG_T2, *rd, 0), ctx); 364 emit(rv_addiw(RV_REG_T1, *rs, 0), ctx); 365 *rd = RV_REG_T2; 366 *rs = RV_REG_T1; 367 } 368 369 static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx) 370 { 371 emit(rv_addi(RV_REG_T2, *rd, 0), ctx); 372 emit_zext_32(RV_REG_T2, ctx); 373 emit_zext_32(RV_REG_T1, ctx); 374 *rd = RV_REG_T2; 375 } 376 377 static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx) 378 { 379 emit(rv_addiw(RV_REG_T2, *rd, 0), ctx); 380 *rd = RV_REG_T2; 381 } 382 383 static void emit_jump_and_link(u8 rd, s64 rvoff, bool force_jalr, 384 struct rv_jit_context *ctx) 385 { 386 s64 upper, lower; 387 388 if (rvoff && is_21b_int(rvoff) && !force_jalr) { 389 emit(rv_jal(rd, rvoff >> 1), ctx); 390 return; 391 } 392 393 upper = (rvoff + (1 << 11)) >> 12; 394 lower = rvoff & 0xfff; 395 emit(rv_auipc(RV_REG_T1, upper), ctx); 396 emit(rv_jalr(rd, RV_REG_T1, lower), ctx); 397 } 398 399 static bool is_signed_bpf_cond(u8 cond) 400 { 401 return cond == BPF_JSGT || cond == BPF_JSLT || 402 cond == BPF_JSGE || cond == BPF_JSLE; 403 } 404 405 static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx) 406 { 407 s64 off = 0; 408 u64 ip; 409 u8 rd; 410 411 if (addr && ctx->insns) { 412 ip = (u64)(long)(ctx->insns + ctx->ninsns); 413 off = addr - ip; 414 if (!is_32b_int(off)) { 415 pr_err("bpf-jit: target call addr %pK is out of range\n", 416 (void *)addr); 417 return -ERANGE; 418 } 419 } 420 421 emit_jump_and_link(RV_REG_RA, off, !fixed, ctx); 422 rd = bpf_to_rv_reg(BPF_REG_0, ctx); 423 emit(rv_addi(rd, RV_REG_A0, 0), ctx); 424 return 0; 425 } 426 427 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 428 bool extra_pass) 429 { 430 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || 431 BPF_CLASS(insn->code) == BPF_JMP; 432 int s, e, rvoff, i = insn - ctx->prog->insnsi; 433 struct bpf_prog_aux *aux = ctx->prog->aux; 434 u8 rd = -1, rs = -1, code = insn->code; 435 s16 off = insn->off; 436 s32 imm = insn->imm; 437 438 init_regs(&rd, &rs, insn, ctx); 439 440 switch (code) { 441 /* dst = src */ 442 case BPF_ALU | BPF_MOV | BPF_X: 443 case BPF_ALU64 | BPF_MOV | BPF_X: 444 if (imm == 1) { 445 /* Special mov32 for zext */ 446 emit_zext_32(rd, ctx); 447 break; 448 } 449 emit(is64 ? rv_addi(rd, rs, 0) : rv_addiw(rd, rs, 0), ctx); 450 if (!is64 && !aux->verifier_zext) 451 emit_zext_32(rd, ctx); 452 break; 453 454 /* dst = dst OP src */ 455 case BPF_ALU | BPF_ADD | BPF_X: 456 case BPF_ALU64 | BPF_ADD | BPF_X: 457 emit(is64 ? rv_add(rd, rd, rs) : rv_addw(rd, rd, rs), ctx); 458 if (!is64 && !aux->verifier_zext) 459 emit_zext_32(rd, ctx); 460 break; 461 case BPF_ALU | BPF_SUB | BPF_X: 462 case BPF_ALU64 | BPF_SUB | BPF_X: 463 emit(is64 ? rv_sub(rd, rd, rs) : rv_subw(rd, rd, rs), ctx); 464 if (!is64 && !aux->verifier_zext) 465 emit_zext_32(rd, ctx); 466 break; 467 case BPF_ALU | BPF_AND | BPF_X: 468 case BPF_ALU64 | BPF_AND | BPF_X: 469 emit(rv_and(rd, rd, rs), ctx); 470 if (!is64 && !aux->verifier_zext) 471 emit_zext_32(rd, ctx); 472 break; 473 case BPF_ALU | BPF_OR | BPF_X: 474 case BPF_ALU64 | BPF_OR | BPF_X: 475 emit(rv_or(rd, rd, rs), ctx); 476 if (!is64 && !aux->verifier_zext) 477 emit_zext_32(rd, ctx); 478 break; 479 case BPF_ALU | BPF_XOR | BPF_X: 480 case BPF_ALU64 | BPF_XOR | BPF_X: 481 emit(rv_xor(rd, rd, rs), ctx); 482 if (!is64 && !aux->verifier_zext) 483 emit_zext_32(rd, ctx); 484 break; 485 case BPF_ALU | BPF_MUL | BPF_X: 486 case BPF_ALU64 | BPF_MUL | BPF_X: 487 emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx); 488 if (!is64 && !aux->verifier_zext) 489 emit_zext_32(rd, ctx); 490 break; 491 case BPF_ALU | BPF_DIV | BPF_X: 492 case BPF_ALU64 | BPF_DIV | BPF_X: 493 emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx); 494 if (!is64 && !aux->verifier_zext) 495 emit_zext_32(rd, ctx); 496 break; 497 case BPF_ALU | BPF_MOD | BPF_X: 498 case BPF_ALU64 | BPF_MOD | BPF_X: 499 emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx); 500 if (!is64 && !aux->verifier_zext) 501 emit_zext_32(rd, ctx); 502 break; 503 case BPF_ALU | BPF_LSH | BPF_X: 504 case BPF_ALU64 | BPF_LSH | BPF_X: 505 emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx); 506 if (!is64) 507 emit_zext_32(rd, ctx); 508 break; 509 case BPF_ALU | BPF_RSH | BPF_X: 510 case BPF_ALU64 | BPF_RSH | BPF_X: 511 emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx); 512 if (!is64 && !aux->verifier_zext) 513 emit_zext_32(rd, ctx); 514 break; 515 case BPF_ALU | BPF_ARSH | BPF_X: 516 case BPF_ALU64 | BPF_ARSH | BPF_X: 517 emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx); 518 if (!is64 && !aux->verifier_zext) 519 emit_zext_32(rd, ctx); 520 break; 521 522 /* dst = -dst */ 523 case BPF_ALU | BPF_NEG: 524 case BPF_ALU64 | BPF_NEG: 525 emit(is64 ? rv_sub(rd, RV_REG_ZERO, rd) : 526 rv_subw(rd, RV_REG_ZERO, rd), ctx); 527 if (!is64 && !aux->verifier_zext) 528 emit_zext_32(rd, ctx); 529 break; 530 531 /* dst = BSWAP##imm(dst) */ 532 case BPF_ALU | BPF_END | BPF_FROM_LE: 533 { 534 int shift = 64 - imm; 535 536 emit(rv_slli(rd, rd, shift), ctx); 537 emit(rv_srli(rd, rd, shift), ctx); 538 break; 539 } 540 case BPF_ALU | BPF_END | BPF_FROM_BE: 541 emit(rv_addi(RV_REG_T2, RV_REG_ZERO, 0), ctx); 542 543 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); 544 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); 545 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); 546 emit(rv_srli(rd, rd, 8), ctx); 547 if (imm == 16) 548 goto out_be; 549 550 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); 551 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); 552 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); 553 emit(rv_srli(rd, rd, 8), ctx); 554 555 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); 556 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); 557 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); 558 emit(rv_srli(rd, rd, 8), ctx); 559 if (imm == 32) 560 goto out_be; 561 562 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); 563 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); 564 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); 565 emit(rv_srli(rd, rd, 8), ctx); 566 567 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); 568 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); 569 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); 570 emit(rv_srli(rd, rd, 8), ctx); 571 572 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); 573 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); 574 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); 575 emit(rv_srli(rd, rd, 8), ctx); 576 577 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); 578 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); 579 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx); 580 emit(rv_srli(rd, rd, 8), ctx); 581 out_be: 582 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx); 583 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx); 584 585 emit(rv_addi(rd, RV_REG_T2, 0), ctx); 586 break; 587 588 /* dst = imm */ 589 case BPF_ALU | BPF_MOV | BPF_K: 590 case BPF_ALU64 | BPF_MOV | BPF_K: 591 emit_imm(rd, imm, ctx); 592 if (!is64 && !aux->verifier_zext) 593 emit_zext_32(rd, ctx); 594 break; 595 596 /* dst = dst OP imm */ 597 case BPF_ALU | BPF_ADD | BPF_K: 598 case BPF_ALU64 | BPF_ADD | BPF_K: 599 if (is_12b_int(imm)) { 600 emit(is64 ? rv_addi(rd, rd, imm) : 601 rv_addiw(rd, rd, imm), ctx); 602 } else { 603 emit_imm(RV_REG_T1, imm, ctx); 604 emit(is64 ? rv_add(rd, rd, RV_REG_T1) : 605 rv_addw(rd, rd, RV_REG_T1), ctx); 606 } 607 if (!is64 && !aux->verifier_zext) 608 emit_zext_32(rd, ctx); 609 break; 610 case BPF_ALU | BPF_SUB | BPF_K: 611 case BPF_ALU64 | BPF_SUB | BPF_K: 612 if (is_12b_int(-imm)) { 613 emit(is64 ? rv_addi(rd, rd, -imm) : 614 rv_addiw(rd, rd, -imm), ctx); 615 } else { 616 emit_imm(RV_REG_T1, imm, ctx); 617 emit(is64 ? rv_sub(rd, rd, RV_REG_T1) : 618 rv_subw(rd, rd, RV_REG_T1), ctx); 619 } 620 if (!is64 && !aux->verifier_zext) 621 emit_zext_32(rd, ctx); 622 break; 623 case BPF_ALU | BPF_AND | BPF_K: 624 case BPF_ALU64 | BPF_AND | BPF_K: 625 if (is_12b_int(imm)) { 626 emit(rv_andi(rd, rd, imm), ctx); 627 } else { 628 emit_imm(RV_REG_T1, imm, ctx); 629 emit(rv_and(rd, rd, RV_REG_T1), ctx); 630 } 631 if (!is64 && !aux->verifier_zext) 632 emit_zext_32(rd, ctx); 633 break; 634 case BPF_ALU | BPF_OR | BPF_K: 635 case BPF_ALU64 | BPF_OR | BPF_K: 636 if (is_12b_int(imm)) { 637 emit(rv_ori(rd, rd, imm), ctx); 638 } else { 639 emit_imm(RV_REG_T1, imm, ctx); 640 emit(rv_or(rd, rd, RV_REG_T1), ctx); 641 } 642 if (!is64 && !aux->verifier_zext) 643 emit_zext_32(rd, ctx); 644 break; 645 case BPF_ALU | BPF_XOR | BPF_K: 646 case BPF_ALU64 | BPF_XOR | BPF_K: 647 if (is_12b_int(imm)) { 648 emit(rv_xori(rd, rd, imm), ctx); 649 } else { 650 emit_imm(RV_REG_T1, imm, ctx); 651 emit(rv_xor(rd, rd, RV_REG_T1), ctx); 652 } 653 if (!is64 && !aux->verifier_zext) 654 emit_zext_32(rd, ctx); 655 break; 656 case BPF_ALU | BPF_MUL | BPF_K: 657 case BPF_ALU64 | BPF_MUL | BPF_K: 658 emit_imm(RV_REG_T1, imm, ctx); 659 emit(is64 ? rv_mul(rd, rd, RV_REG_T1) : 660 rv_mulw(rd, rd, RV_REG_T1), ctx); 661 if (!is64 && !aux->verifier_zext) 662 emit_zext_32(rd, ctx); 663 break; 664 case BPF_ALU | BPF_DIV | BPF_K: 665 case BPF_ALU64 | BPF_DIV | BPF_K: 666 emit_imm(RV_REG_T1, imm, ctx); 667 emit(is64 ? rv_divu(rd, rd, RV_REG_T1) : 668 rv_divuw(rd, rd, RV_REG_T1), ctx); 669 if (!is64 && !aux->verifier_zext) 670 emit_zext_32(rd, ctx); 671 break; 672 case BPF_ALU | BPF_MOD | BPF_K: 673 case BPF_ALU64 | BPF_MOD | BPF_K: 674 emit_imm(RV_REG_T1, imm, ctx); 675 emit(is64 ? rv_remu(rd, rd, RV_REG_T1) : 676 rv_remuw(rd, rd, RV_REG_T1), ctx); 677 if (!is64 && !aux->verifier_zext) 678 emit_zext_32(rd, ctx); 679 break; 680 case BPF_ALU | BPF_LSH | BPF_K: 681 case BPF_ALU64 | BPF_LSH | BPF_K: 682 emit(is64 ? rv_slli(rd, rd, imm) : rv_slliw(rd, rd, imm), ctx); 683 if (!is64) 684 emit_zext_32(rd, ctx); 685 break; 686 case BPF_ALU | BPF_RSH | BPF_K: 687 case BPF_ALU64 | BPF_RSH | BPF_K: 688 emit(is64 ? rv_srli(rd, rd, imm) : rv_srliw(rd, rd, imm), ctx); 689 if (!is64) 690 emit_zext_32(rd, ctx); 691 break; 692 case BPF_ALU | BPF_ARSH | BPF_K: 693 case BPF_ALU64 | BPF_ARSH | BPF_K: 694 emit(is64 ? rv_srai(rd, rd, imm) : rv_sraiw(rd, rd, imm), ctx); 695 if (!is64) 696 emit_zext_32(rd, ctx); 697 break; 698 699 /* JUMP off */ 700 case BPF_JMP | BPF_JA: 701 rvoff = rv_offset(i, off, ctx); 702 emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx); 703 break; 704 705 /* IF (dst COND src) JUMP off */ 706 case BPF_JMP | BPF_JEQ | BPF_X: 707 case BPF_JMP32 | BPF_JEQ | BPF_X: 708 case BPF_JMP | BPF_JGT | BPF_X: 709 case BPF_JMP32 | BPF_JGT | BPF_X: 710 case BPF_JMP | BPF_JLT | BPF_X: 711 case BPF_JMP32 | BPF_JLT | BPF_X: 712 case BPF_JMP | BPF_JGE | BPF_X: 713 case BPF_JMP32 | BPF_JGE | BPF_X: 714 case BPF_JMP | BPF_JLE | BPF_X: 715 case BPF_JMP32 | BPF_JLE | BPF_X: 716 case BPF_JMP | BPF_JNE | BPF_X: 717 case BPF_JMP32 | BPF_JNE | BPF_X: 718 case BPF_JMP | BPF_JSGT | BPF_X: 719 case BPF_JMP32 | BPF_JSGT | BPF_X: 720 case BPF_JMP | BPF_JSLT | BPF_X: 721 case BPF_JMP32 | BPF_JSLT | BPF_X: 722 case BPF_JMP | BPF_JSGE | BPF_X: 723 case BPF_JMP32 | BPF_JSGE | BPF_X: 724 case BPF_JMP | BPF_JSLE | BPF_X: 725 case BPF_JMP32 | BPF_JSLE | BPF_X: 726 case BPF_JMP | BPF_JSET | BPF_X: 727 case BPF_JMP32 | BPF_JSET | BPF_X: 728 rvoff = rv_offset(i, off, ctx); 729 if (!is64) { 730 s = ctx->ninsns; 731 if (is_signed_bpf_cond(BPF_OP(code))) 732 emit_sext_32_rd_rs(&rd, &rs, ctx); 733 else 734 emit_zext_32_rd_rs(&rd, &rs, ctx); 735 e = ctx->ninsns; 736 737 /* Adjust for extra insns */ 738 rvoff -= (e - s) << 2; 739 } 740 741 if (BPF_OP(code) == BPF_JSET) { 742 /* Adjust for and */ 743 rvoff -= 4; 744 emit(rv_and(RV_REG_T1, rd, rs), ctx); 745 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, 746 ctx); 747 } else { 748 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); 749 } 750 break; 751 752 /* IF (dst COND imm) JUMP off */ 753 case BPF_JMP | BPF_JEQ | BPF_K: 754 case BPF_JMP32 | BPF_JEQ | BPF_K: 755 case BPF_JMP | BPF_JGT | BPF_K: 756 case BPF_JMP32 | BPF_JGT | BPF_K: 757 case BPF_JMP | BPF_JLT | BPF_K: 758 case BPF_JMP32 | BPF_JLT | BPF_K: 759 case BPF_JMP | BPF_JGE | BPF_K: 760 case BPF_JMP32 | BPF_JGE | BPF_K: 761 case BPF_JMP | BPF_JLE | BPF_K: 762 case BPF_JMP32 | BPF_JLE | BPF_K: 763 case BPF_JMP | BPF_JNE | BPF_K: 764 case BPF_JMP32 | BPF_JNE | BPF_K: 765 case BPF_JMP | BPF_JSGT | BPF_K: 766 case BPF_JMP32 | BPF_JSGT | BPF_K: 767 case BPF_JMP | BPF_JSLT | BPF_K: 768 case BPF_JMP32 | BPF_JSLT | BPF_K: 769 case BPF_JMP | BPF_JSGE | BPF_K: 770 case BPF_JMP32 | BPF_JSGE | BPF_K: 771 case BPF_JMP | BPF_JSLE | BPF_K: 772 case BPF_JMP32 | BPF_JSLE | BPF_K: 773 case BPF_JMP | BPF_JSET | BPF_K: 774 case BPF_JMP32 | BPF_JSET | BPF_K: 775 rvoff = rv_offset(i, off, ctx); 776 s = ctx->ninsns; 777 emit_imm(RV_REG_T1, imm, ctx); 778 if (!is64) { 779 if (is_signed_bpf_cond(BPF_OP(code))) 780 emit_sext_32_rd(&rd, ctx); 781 else 782 emit_zext_32_rd_t1(&rd, ctx); 783 } 784 e = ctx->ninsns; 785 786 /* Adjust for extra insns */ 787 rvoff -= (e - s) << 2; 788 789 if (BPF_OP(code) == BPF_JSET) { 790 /* Adjust for and */ 791 rvoff -= 4; 792 emit(rv_and(RV_REG_T1, rd, RV_REG_T1), ctx); 793 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, 794 ctx); 795 } else { 796 emit_branch(BPF_OP(code), rd, RV_REG_T1, rvoff, ctx); 797 } 798 break; 799 800 /* function call */ 801 case BPF_JMP | BPF_CALL: 802 { 803 bool fixed; 804 int ret; 805 u64 addr; 806 807 mark_call(ctx); 808 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr, 809 &fixed); 810 if (ret < 0) 811 return ret; 812 ret = emit_call(fixed, addr, ctx); 813 if (ret) 814 return ret; 815 break; 816 } 817 /* tail call */ 818 case BPF_JMP | BPF_TAIL_CALL: 819 if (emit_bpf_tail_call(i, ctx)) 820 return -1; 821 break; 822 823 /* function return */ 824 case BPF_JMP | BPF_EXIT: 825 if (i == ctx->prog->len - 1) 826 break; 827 828 rvoff = epilogue_offset(ctx); 829 emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx); 830 break; 831 832 /* dst = imm64 */ 833 case BPF_LD | BPF_IMM | BPF_DW: 834 { 835 struct bpf_insn insn1 = insn[1]; 836 u64 imm64; 837 838 imm64 = (u64)insn1.imm << 32 | (u32)imm; 839 emit_imm(rd, imm64, ctx); 840 return 1; 841 } 842 843 /* LDX: dst = *(size *)(src + off) */ 844 case BPF_LDX | BPF_MEM | BPF_B: 845 if (is_12b_int(off)) { 846 emit(rv_lbu(rd, off, rs), ctx); 847 break; 848 } 849 850 emit_imm(RV_REG_T1, off, ctx); 851 emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx); 852 emit(rv_lbu(rd, 0, RV_REG_T1), ctx); 853 if (insn_is_zext(&insn[1])) 854 return 1; 855 break; 856 case BPF_LDX | BPF_MEM | BPF_H: 857 if (is_12b_int(off)) { 858 emit(rv_lhu(rd, off, rs), ctx); 859 break; 860 } 861 862 emit_imm(RV_REG_T1, off, ctx); 863 emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx); 864 emit(rv_lhu(rd, 0, RV_REG_T1), ctx); 865 if (insn_is_zext(&insn[1])) 866 return 1; 867 break; 868 case BPF_LDX | BPF_MEM | BPF_W: 869 if (is_12b_int(off)) { 870 emit(rv_lwu(rd, off, rs), ctx); 871 break; 872 } 873 874 emit_imm(RV_REG_T1, off, ctx); 875 emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx); 876 emit(rv_lwu(rd, 0, RV_REG_T1), ctx); 877 if (insn_is_zext(&insn[1])) 878 return 1; 879 break; 880 case BPF_LDX | BPF_MEM | BPF_DW: 881 if (is_12b_int(off)) { 882 emit(rv_ld(rd, off, rs), ctx); 883 break; 884 } 885 886 emit_imm(RV_REG_T1, off, ctx); 887 emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx); 888 emit(rv_ld(rd, 0, RV_REG_T1), ctx); 889 break; 890 891 /* ST: *(size *)(dst + off) = imm */ 892 case BPF_ST | BPF_MEM | BPF_B: 893 emit_imm(RV_REG_T1, imm, ctx); 894 if (is_12b_int(off)) { 895 emit(rv_sb(rd, off, RV_REG_T1), ctx); 896 break; 897 } 898 899 emit_imm(RV_REG_T2, off, ctx); 900 emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx); 901 emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx); 902 break; 903 904 case BPF_ST | BPF_MEM | BPF_H: 905 emit_imm(RV_REG_T1, imm, ctx); 906 if (is_12b_int(off)) { 907 emit(rv_sh(rd, off, RV_REG_T1), ctx); 908 break; 909 } 910 911 emit_imm(RV_REG_T2, off, ctx); 912 emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx); 913 emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx); 914 break; 915 case BPF_ST | BPF_MEM | BPF_W: 916 emit_imm(RV_REG_T1, imm, ctx); 917 if (is_12b_int(off)) { 918 emit(rv_sw(rd, off, RV_REG_T1), ctx); 919 break; 920 } 921 922 emit_imm(RV_REG_T2, off, ctx); 923 emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx); 924 emit(rv_sw(RV_REG_T2, 0, RV_REG_T1), ctx); 925 break; 926 case BPF_ST | BPF_MEM | BPF_DW: 927 emit_imm(RV_REG_T1, imm, ctx); 928 if (is_12b_int(off)) { 929 emit(rv_sd(rd, off, RV_REG_T1), ctx); 930 break; 931 } 932 933 emit_imm(RV_REG_T2, off, ctx); 934 emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx); 935 emit(rv_sd(RV_REG_T2, 0, RV_REG_T1), ctx); 936 break; 937 938 /* STX: *(size *)(dst + off) = src */ 939 case BPF_STX | BPF_MEM | BPF_B: 940 if (is_12b_int(off)) { 941 emit(rv_sb(rd, off, rs), ctx); 942 break; 943 } 944 945 emit_imm(RV_REG_T1, off, ctx); 946 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx); 947 emit(rv_sb(RV_REG_T1, 0, rs), ctx); 948 break; 949 case BPF_STX | BPF_MEM | BPF_H: 950 if (is_12b_int(off)) { 951 emit(rv_sh(rd, off, rs), ctx); 952 break; 953 } 954 955 emit_imm(RV_REG_T1, off, ctx); 956 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx); 957 emit(rv_sh(RV_REG_T1, 0, rs), ctx); 958 break; 959 case BPF_STX | BPF_MEM | BPF_W: 960 if (is_12b_int(off)) { 961 emit(rv_sw(rd, off, rs), ctx); 962 break; 963 } 964 965 emit_imm(RV_REG_T1, off, ctx); 966 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx); 967 emit(rv_sw(RV_REG_T1, 0, rs), ctx); 968 break; 969 case BPF_STX | BPF_MEM | BPF_DW: 970 if (is_12b_int(off)) { 971 emit(rv_sd(rd, off, rs), ctx); 972 break; 973 } 974 975 emit_imm(RV_REG_T1, off, ctx); 976 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx); 977 emit(rv_sd(RV_REG_T1, 0, rs), ctx); 978 break; 979 /* STX XADD: lock *(u32 *)(dst + off) += src */ 980 case BPF_STX | BPF_XADD | BPF_W: 981 /* STX XADD: lock *(u64 *)(dst + off) += src */ 982 case BPF_STX | BPF_XADD | BPF_DW: 983 if (off) { 984 if (is_12b_int(off)) { 985 emit(rv_addi(RV_REG_T1, rd, off), ctx); 986 } else { 987 emit_imm(RV_REG_T1, off, ctx); 988 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx); 989 } 990 991 rd = RV_REG_T1; 992 } 993 994 emit(BPF_SIZE(code) == BPF_W ? 995 rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0) : 996 rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0), ctx); 997 break; 998 default: 999 pr_err("bpf-jit: unknown opcode %02x\n", code); 1000 return -EINVAL; 1001 } 1002 1003 return 0; 1004 } 1005 1006 void bpf_jit_build_prologue(struct rv_jit_context *ctx) 1007 { 1008 int stack_adjust = 0, store_offset, bpf_stack_adjust; 1009 1010 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); 1011 if (bpf_stack_adjust) 1012 mark_fp(ctx); 1013 1014 if (seen_reg(RV_REG_RA, ctx)) 1015 stack_adjust += 8; 1016 stack_adjust += 8; /* RV_REG_FP */ 1017 if (seen_reg(RV_REG_S1, ctx)) 1018 stack_adjust += 8; 1019 if (seen_reg(RV_REG_S2, ctx)) 1020 stack_adjust += 8; 1021 if (seen_reg(RV_REG_S3, ctx)) 1022 stack_adjust += 8; 1023 if (seen_reg(RV_REG_S4, ctx)) 1024 stack_adjust += 8; 1025 if (seen_reg(RV_REG_S5, ctx)) 1026 stack_adjust += 8; 1027 if (seen_reg(RV_REG_S6, ctx)) 1028 stack_adjust += 8; 1029 1030 stack_adjust = round_up(stack_adjust, 16); 1031 stack_adjust += bpf_stack_adjust; 1032 1033 store_offset = stack_adjust - 8; 1034 1035 /* First instruction is always setting the tail-call-counter 1036 * (TCC) register. This instruction is skipped for tail calls. 1037 */ 1038 emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); 1039 1040 emit(rv_addi(RV_REG_SP, RV_REG_SP, -stack_adjust), ctx); 1041 1042 if (seen_reg(RV_REG_RA, ctx)) { 1043 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_RA), ctx); 1044 store_offset -= 8; 1045 } 1046 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_FP), ctx); 1047 store_offset -= 8; 1048 if (seen_reg(RV_REG_S1, ctx)) { 1049 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S1), ctx); 1050 store_offset -= 8; 1051 } 1052 if (seen_reg(RV_REG_S2, ctx)) { 1053 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S2), ctx); 1054 store_offset -= 8; 1055 } 1056 if (seen_reg(RV_REG_S3, ctx)) { 1057 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S3), ctx); 1058 store_offset -= 8; 1059 } 1060 if (seen_reg(RV_REG_S4, ctx)) { 1061 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S4), ctx); 1062 store_offset -= 8; 1063 } 1064 if (seen_reg(RV_REG_S5, ctx)) { 1065 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S5), ctx); 1066 store_offset -= 8; 1067 } 1068 if (seen_reg(RV_REG_S6, ctx)) { 1069 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S6), ctx); 1070 store_offset -= 8; 1071 } 1072 1073 emit(rv_addi(RV_REG_FP, RV_REG_SP, stack_adjust), ctx); 1074 1075 if (bpf_stack_adjust) 1076 emit(rv_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust), ctx); 1077 1078 /* Program contains calls and tail calls, so RV_REG_TCC need 1079 * to be saved across calls. 1080 */ 1081 if (seen_tail_call(ctx) && seen_call(ctx)) 1082 emit(rv_addi(RV_REG_TCC_SAVED, RV_REG_TCC, 0), ctx); 1083 1084 ctx->stack_size = stack_adjust; 1085 } 1086 1087 void bpf_jit_build_epilogue(struct rv_jit_context *ctx) 1088 { 1089 __build_epilogue(false, ctx); 1090 } 1091 1092 void *bpf_jit_alloc_exec(unsigned long size) 1093 { 1094 return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START, 1095 BPF_JIT_REGION_END, GFP_KERNEL, 1096 PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, 1097 __builtin_return_address(0)); 1098 } 1099 1100 void bpf_jit_free_exec(void *addr) 1101 { 1102 return vfree(addr); 1103 } 1104