1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Common functionality for RV32 and RV64 BPF JIT compilers 4 * 5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com> 6 * 7 */ 8 9 #ifndef _BPF_JIT_H 10 #define _BPF_JIT_H 11 12 #include <linux/bpf.h> 13 #include <linux/filter.h> 14 #include <asm/cacheflush.h> 15 16 static inline bool rvc_enabled(void) 17 { 18 return IS_ENABLED(CONFIG_RISCV_ISA_C); 19 } 20 21 enum { 22 RV_REG_ZERO = 0, /* The constant value 0 */ 23 RV_REG_RA = 1, /* Return address */ 24 RV_REG_SP = 2, /* Stack pointer */ 25 RV_REG_GP = 3, /* Global pointer */ 26 RV_REG_TP = 4, /* Thread pointer */ 27 RV_REG_T0 = 5, /* Temporaries */ 28 RV_REG_T1 = 6, 29 RV_REG_T2 = 7, 30 RV_REG_FP = 8, /* Saved register/frame pointer */ 31 RV_REG_S1 = 9, /* Saved register */ 32 RV_REG_A0 = 10, /* Function argument/return values */ 33 RV_REG_A1 = 11, /* Function arguments */ 34 RV_REG_A2 = 12, 35 RV_REG_A3 = 13, 36 RV_REG_A4 = 14, 37 RV_REG_A5 = 15, 38 RV_REG_A6 = 16, 39 RV_REG_A7 = 17, 40 RV_REG_S2 = 18, /* Saved registers */ 41 RV_REG_S3 = 19, 42 RV_REG_S4 = 20, 43 RV_REG_S5 = 21, 44 RV_REG_S6 = 22, 45 RV_REG_S7 = 23, 46 RV_REG_S8 = 24, 47 RV_REG_S9 = 25, 48 RV_REG_S10 = 26, 49 RV_REG_S11 = 27, 50 RV_REG_T3 = 28, /* Temporaries */ 51 RV_REG_T4 = 29, 52 RV_REG_T5 = 30, 53 RV_REG_T6 = 31, 54 }; 55 56 static inline bool is_creg(u8 reg) 57 { 58 return (1 << reg) & (BIT(RV_REG_FP) | 59 BIT(RV_REG_S1) | 60 BIT(RV_REG_A0) | 61 BIT(RV_REG_A1) | 62 BIT(RV_REG_A2) | 63 BIT(RV_REG_A3) | 64 BIT(RV_REG_A4) | 65 BIT(RV_REG_A5)); 66 } 67 68 struct rv_jit_context { 69 struct bpf_prog *prog; 70 u16 *insns; /* RV insns */ 71 int ninsns; 72 int prologue_len; 73 int epilogue_offset; 74 int *offset; /* BPF to RV */ 75 int nexentries; 76 unsigned long flags; 77 int stack_size; 78 }; 79 80 /* Convert from ninsns to bytes. */ 81 static inline int ninsns_rvoff(int ninsns) 82 { 83 return ninsns << 1; 84 } 85 86 struct rv_jit_data { 87 struct bpf_binary_header *header; 88 u8 *image; 89 struct rv_jit_context ctx; 90 }; 91 92 static inline void bpf_fill_ill_insns(void *area, unsigned int size) 93 { 94 memset(area, 0, size); 95 } 96 97 static inline void bpf_flush_icache(void *start, void *end) 98 { 99 flush_icache_range((unsigned long)start, (unsigned long)end); 100 } 101 102 /* Emit a 4-byte riscv instruction. */ 103 static inline void emit(const u32 insn, struct rv_jit_context *ctx) 104 { 105 if (ctx->insns) { 106 ctx->insns[ctx->ninsns] = insn; 107 ctx->insns[ctx->ninsns + 1] = (insn >> 16); 108 } 109 110 ctx->ninsns += 2; 111 } 112 113 /* Emit a 2-byte riscv compressed instruction. */ 114 static inline void emitc(const u16 insn, struct rv_jit_context *ctx) 115 { 116 BUILD_BUG_ON(!rvc_enabled()); 117 118 if (ctx->insns) 119 ctx->insns[ctx->ninsns] = insn; 120 121 ctx->ninsns++; 122 } 123 124 static inline int epilogue_offset(struct rv_jit_context *ctx) 125 { 126 int to = ctx->epilogue_offset, from = ctx->ninsns; 127 128 return ninsns_rvoff(to - from); 129 } 130 131 /* Return -1 or inverted cond. */ 132 static inline int invert_bpf_cond(u8 cond) 133 { 134 switch (cond) { 135 case BPF_JEQ: 136 return BPF_JNE; 137 case BPF_JGT: 138 return BPF_JLE; 139 case BPF_JLT: 140 return BPF_JGE; 141 case BPF_JGE: 142 return BPF_JLT; 143 case BPF_JLE: 144 return BPF_JGT; 145 case BPF_JNE: 146 return BPF_JEQ; 147 case BPF_JSGT: 148 return BPF_JSLE; 149 case BPF_JSLT: 150 return BPF_JSGE; 151 case BPF_JSGE: 152 return BPF_JSLT; 153 case BPF_JSLE: 154 return BPF_JSGT; 155 } 156 return -1; 157 } 158 159 static inline bool is_6b_int(long val) 160 { 161 return -(1L << 5) <= val && val < (1L << 5); 162 } 163 164 static inline bool is_7b_uint(unsigned long val) 165 { 166 return val < (1UL << 7); 167 } 168 169 static inline bool is_8b_uint(unsigned long val) 170 { 171 return val < (1UL << 8); 172 } 173 174 static inline bool is_9b_uint(unsigned long val) 175 { 176 return val < (1UL << 9); 177 } 178 179 static inline bool is_10b_int(long val) 180 { 181 return -(1L << 9) <= val && val < (1L << 9); 182 } 183 184 static inline bool is_10b_uint(unsigned long val) 185 { 186 return val < (1UL << 10); 187 } 188 189 static inline bool is_12b_int(long val) 190 { 191 return -(1L << 11) <= val && val < (1L << 11); 192 } 193 194 static inline int is_12b_check(int off, int insn) 195 { 196 if (!is_12b_int(off)) { 197 pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n", 198 insn, (int)off); 199 return -1; 200 } 201 return 0; 202 } 203 204 static inline bool is_13b_int(long val) 205 { 206 return -(1L << 12) <= val && val < (1L << 12); 207 } 208 209 static inline bool is_21b_int(long val) 210 { 211 return -(1L << 20) <= val && val < (1L << 20); 212 } 213 214 static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx) 215 { 216 int from, to; 217 218 off++; /* BPF branch is from PC+1, RV is from PC */ 219 from = (insn > 0) ? ctx->offset[insn - 1] : ctx->prologue_len; 220 to = (insn + off > 0) ? ctx->offset[insn + off - 1] : ctx->prologue_len; 221 return ninsns_rvoff(to - from); 222 } 223 224 /* Instruction formats. */ 225 226 static inline u32 rv_r_insn(u8 funct7, u8 rs2, u8 rs1, u8 funct3, u8 rd, 227 u8 opcode) 228 { 229 return (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 230 (rd << 7) | opcode; 231 } 232 233 static inline u32 rv_i_insn(u16 imm11_0, u8 rs1, u8 funct3, u8 rd, u8 opcode) 234 { 235 return (imm11_0 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) | 236 opcode; 237 } 238 239 static inline u32 rv_s_insn(u16 imm11_0, u8 rs2, u8 rs1, u8 funct3, u8 opcode) 240 { 241 u8 imm11_5 = imm11_0 >> 5, imm4_0 = imm11_0 & 0x1f; 242 243 return (imm11_5 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 244 (imm4_0 << 7) | opcode; 245 } 246 247 static inline u32 rv_b_insn(u16 imm12_1, u8 rs2, u8 rs1, u8 funct3, u8 opcode) 248 { 249 u8 imm12 = ((imm12_1 & 0x800) >> 5) | ((imm12_1 & 0x3f0) >> 4); 250 u8 imm4_1 = ((imm12_1 & 0xf) << 1) | ((imm12_1 & 0x400) >> 10); 251 252 return (imm12 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 253 (imm4_1 << 7) | opcode; 254 } 255 256 static inline u32 rv_u_insn(u32 imm31_12, u8 rd, u8 opcode) 257 { 258 return (imm31_12 << 12) | (rd << 7) | opcode; 259 } 260 261 static inline u32 rv_j_insn(u32 imm20_1, u8 rd, u8 opcode) 262 { 263 u32 imm; 264 265 imm = (imm20_1 & 0x80000) | ((imm20_1 & 0x3ff) << 9) | 266 ((imm20_1 & 0x400) >> 2) | ((imm20_1 & 0x7f800) >> 11); 267 268 return (imm << 12) | (rd << 7) | opcode; 269 } 270 271 static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1, 272 u8 funct3, u8 rd, u8 opcode) 273 { 274 u8 funct7 = (funct5 << 2) | (aq << 1) | rl; 275 276 return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode); 277 } 278 279 /* RISC-V compressed instruction formats. */ 280 281 static inline u16 rv_cr_insn(u8 funct4, u8 rd, u8 rs2, u8 op) 282 { 283 return (funct4 << 12) | (rd << 7) | (rs2 << 2) | op; 284 } 285 286 static inline u16 rv_ci_insn(u8 funct3, u32 imm6, u8 rd, u8 op) 287 { 288 u32 imm; 289 290 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2); 291 return (funct3 << 13) | (rd << 7) | op | imm; 292 } 293 294 static inline u16 rv_css_insn(u8 funct3, u32 uimm, u8 rs2, u8 op) 295 { 296 return (funct3 << 13) | (uimm << 7) | (rs2 << 2) | op; 297 } 298 299 static inline u16 rv_ciw_insn(u8 funct3, u32 uimm, u8 rd, u8 op) 300 { 301 return (funct3 << 13) | (uimm << 5) | ((rd & 0x7) << 2) | op; 302 } 303 304 static inline u16 rv_cl_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rd, 305 u8 op) 306 { 307 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) | 308 (imm_lo << 5) | ((rd & 0x7) << 2) | op; 309 } 310 311 static inline u16 rv_cs_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rs2, 312 u8 op) 313 { 314 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) | 315 (imm_lo << 5) | ((rs2 & 0x7) << 2) | op; 316 } 317 318 static inline u16 rv_ca_insn(u8 funct6, u8 rd, u8 funct2, u8 rs2, u8 op) 319 { 320 return (funct6 << 10) | ((rd & 0x7) << 7) | (funct2 << 5) | 321 ((rs2 & 0x7) << 2) | op; 322 } 323 324 static inline u16 rv_cb_insn(u8 funct3, u32 imm6, u8 funct2, u8 rd, u8 op) 325 { 326 u32 imm; 327 328 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2); 329 return (funct3 << 13) | (funct2 << 10) | ((rd & 0x7) << 7) | op | imm; 330 } 331 332 /* Instructions shared by both RV32 and RV64. */ 333 334 static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0) 335 { 336 return rv_i_insn(imm11_0, rs1, 0, rd, 0x13); 337 } 338 339 static inline u32 rv_andi(u8 rd, u8 rs1, u16 imm11_0) 340 { 341 return rv_i_insn(imm11_0, rs1, 7, rd, 0x13); 342 } 343 344 static inline u32 rv_ori(u8 rd, u8 rs1, u16 imm11_0) 345 { 346 return rv_i_insn(imm11_0, rs1, 6, rd, 0x13); 347 } 348 349 static inline u32 rv_xori(u8 rd, u8 rs1, u16 imm11_0) 350 { 351 return rv_i_insn(imm11_0, rs1, 4, rd, 0x13); 352 } 353 354 static inline u32 rv_slli(u8 rd, u8 rs1, u16 imm11_0) 355 { 356 return rv_i_insn(imm11_0, rs1, 1, rd, 0x13); 357 } 358 359 static inline u32 rv_srli(u8 rd, u8 rs1, u16 imm11_0) 360 { 361 return rv_i_insn(imm11_0, rs1, 5, rd, 0x13); 362 } 363 364 static inline u32 rv_srai(u8 rd, u8 rs1, u16 imm11_0) 365 { 366 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x13); 367 } 368 369 static inline u32 rv_lui(u8 rd, u32 imm31_12) 370 { 371 return rv_u_insn(imm31_12, rd, 0x37); 372 } 373 374 static inline u32 rv_auipc(u8 rd, u32 imm31_12) 375 { 376 return rv_u_insn(imm31_12, rd, 0x17); 377 } 378 379 static inline u32 rv_add(u8 rd, u8 rs1, u8 rs2) 380 { 381 return rv_r_insn(0, rs2, rs1, 0, rd, 0x33); 382 } 383 384 static inline u32 rv_sub(u8 rd, u8 rs1, u8 rs2) 385 { 386 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x33); 387 } 388 389 static inline u32 rv_sltu(u8 rd, u8 rs1, u8 rs2) 390 { 391 return rv_r_insn(0, rs2, rs1, 3, rd, 0x33); 392 } 393 394 static inline u32 rv_and(u8 rd, u8 rs1, u8 rs2) 395 { 396 return rv_r_insn(0, rs2, rs1, 7, rd, 0x33); 397 } 398 399 static inline u32 rv_or(u8 rd, u8 rs1, u8 rs2) 400 { 401 return rv_r_insn(0, rs2, rs1, 6, rd, 0x33); 402 } 403 404 static inline u32 rv_xor(u8 rd, u8 rs1, u8 rs2) 405 { 406 return rv_r_insn(0, rs2, rs1, 4, rd, 0x33); 407 } 408 409 static inline u32 rv_sll(u8 rd, u8 rs1, u8 rs2) 410 { 411 return rv_r_insn(0, rs2, rs1, 1, rd, 0x33); 412 } 413 414 static inline u32 rv_srl(u8 rd, u8 rs1, u8 rs2) 415 { 416 return rv_r_insn(0, rs2, rs1, 5, rd, 0x33); 417 } 418 419 static inline u32 rv_sra(u8 rd, u8 rs1, u8 rs2) 420 { 421 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x33); 422 } 423 424 static inline u32 rv_mul(u8 rd, u8 rs1, u8 rs2) 425 { 426 return rv_r_insn(1, rs2, rs1, 0, rd, 0x33); 427 } 428 429 static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2) 430 { 431 return rv_r_insn(1, rs2, rs1, 3, rd, 0x33); 432 } 433 434 static inline u32 rv_div(u8 rd, u8 rs1, u8 rs2) 435 { 436 return rv_r_insn(1, rs2, rs1, 4, rd, 0x33); 437 } 438 439 static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2) 440 { 441 return rv_r_insn(1, rs2, rs1, 5, rd, 0x33); 442 } 443 444 static inline u32 rv_rem(u8 rd, u8 rs1, u8 rs2) 445 { 446 return rv_r_insn(1, rs2, rs1, 6, rd, 0x33); 447 } 448 449 static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2) 450 { 451 return rv_r_insn(1, rs2, rs1, 7, rd, 0x33); 452 } 453 454 static inline u32 rv_jal(u8 rd, u32 imm20_1) 455 { 456 return rv_j_insn(imm20_1, rd, 0x6f); 457 } 458 459 static inline u32 rv_jalr(u8 rd, u8 rs1, u16 imm11_0) 460 { 461 return rv_i_insn(imm11_0, rs1, 0, rd, 0x67); 462 } 463 464 static inline u32 rv_beq(u8 rs1, u8 rs2, u16 imm12_1) 465 { 466 return rv_b_insn(imm12_1, rs2, rs1, 0, 0x63); 467 } 468 469 static inline u32 rv_bne(u8 rs1, u8 rs2, u16 imm12_1) 470 { 471 return rv_b_insn(imm12_1, rs2, rs1, 1, 0x63); 472 } 473 474 static inline u32 rv_bltu(u8 rs1, u8 rs2, u16 imm12_1) 475 { 476 return rv_b_insn(imm12_1, rs2, rs1, 6, 0x63); 477 } 478 479 static inline u32 rv_bgtu(u8 rs1, u8 rs2, u16 imm12_1) 480 { 481 return rv_bltu(rs2, rs1, imm12_1); 482 } 483 484 static inline u32 rv_bgeu(u8 rs1, u8 rs2, u16 imm12_1) 485 { 486 return rv_b_insn(imm12_1, rs2, rs1, 7, 0x63); 487 } 488 489 static inline u32 rv_bleu(u8 rs1, u8 rs2, u16 imm12_1) 490 { 491 return rv_bgeu(rs2, rs1, imm12_1); 492 } 493 494 static inline u32 rv_blt(u8 rs1, u8 rs2, u16 imm12_1) 495 { 496 return rv_b_insn(imm12_1, rs2, rs1, 4, 0x63); 497 } 498 499 static inline u32 rv_bgt(u8 rs1, u8 rs2, u16 imm12_1) 500 { 501 return rv_blt(rs2, rs1, imm12_1); 502 } 503 504 static inline u32 rv_bge(u8 rs1, u8 rs2, u16 imm12_1) 505 { 506 return rv_b_insn(imm12_1, rs2, rs1, 5, 0x63); 507 } 508 509 static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1) 510 { 511 return rv_bge(rs2, rs1, imm12_1); 512 } 513 514 static inline u32 rv_lb(u8 rd, u16 imm11_0, u8 rs1) 515 { 516 return rv_i_insn(imm11_0, rs1, 0, rd, 0x03); 517 } 518 519 static inline u32 rv_lh(u8 rd, u16 imm11_0, u8 rs1) 520 { 521 return rv_i_insn(imm11_0, rs1, 1, rd, 0x03); 522 } 523 524 static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1) 525 { 526 return rv_i_insn(imm11_0, rs1, 2, rd, 0x03); 527 } 528 529 static inline u32 rv_lbu(u8 rd, u16 imm11_0, u8 rs1) 530 { 531 return rv_i_insn(imm11_0, rs1, 4, rd, 0x03); 532 } 533 534 static inline u32 rv_lhu(u8 rd, u16 imm11_0, u8 rs1) 535 { 536 return rv_i_insn(imm11_0, rs1, 5, rd, 0x03); 537 } 538 539 static inline u32 rv_sb(u8 rs1, u16 imm11_0, u8 rs2) 540 { 541 return rv_s_insn(imm11_0, rs2, rs1, 0, 0x23); 542 } 543 544 static inline u32 rv_sh(u8 rs1, u16 imm11_0, u8 rs2) 545 { 546 return rv_s_insn(imm11_0, rs2, rs1, 1, 0x23); 547 } 548 549 static inline u32 rv_sw(u8 rs1, u16 imm11_0, u8 rs2) 550 { 551 return rv_s_insn(imm11_0, rs2, rs1, 2, 0x23); 552 } 553 554 static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 555 { 556 return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f); 557 } 558 559 static inline u32 rv_amoand_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 560 { 561 return rv_amo_insn(0xc, aq, rl, rs2, rs1, 2, rd, 0x2f); 562 } 563 564 static inline u32 rv_amoor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 565 { 566 return rv_amo_insn(0x8, aq, rl, rs2, rs1, 2, rd, 0x2f); 567 } 568 569 static inline u32 rv_amoxor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 570 { 571 return rv_amo_insn(0x4, aq, rl, rs2, rs1, 2, rd, 0x2f); 572 } 573 574 static inline u32 rv_amoswap_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 575 { 576 return rv_amo_insn(0x1, aq, rl, rs2, rs1, 2, rd, 0x2f); 577 } 578 579 static inline u32 rv_lr_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 580 { 581 return rv_amo_insn(0x2, aq, rl, rs2, rs1, 2, rd, 0x2f); 582 } 583 584 static inline u32 rv_sc_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 585 { 586 return rv_amo_insn(0x3, aq, rl, rs2, rs1, 2, rd, 0x2f); 587 } 588 589 static inline u32 rv_fence(u8 pred, u8 succ) 590 { 591 u16 imm11_0 = pred << 4 | succ; 592 593 return rv_i_insn(imm11_0, 0, 0, 0, 0xf); 594 } 595 596 static inline u32 rv_nop(void) 597 { 598 return rv_i_insn(0, 0, 0, 0, 0x13); 599 } 600 601 /* RVC instrutions. */ 602 603 static inline u16 rvc_addi4spn(u8 rd, u32 imm10) 604 { 605 u32 imm; 606 607 imm = ((imm10 & 0x30) << 2) | ((imm10 & 0x3c0) >> 4) | 608 ((imm10 & 0x4) >> 1) | ((imm10 & 0x8) >> 3); 609 return rv_ciw_insn(0x0, imm, rd, 0x0); 610 } 611 612 static inline u16 rvc_lw(u8 rd, u32 imm7, u8 rs1) 613 { 614 u32 imm_hi, imm_lo; 615 616 imm_hi = (imm7 & 0x38) >> 3; 617 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6); 618 return rv_cl_insn(0x2, imm_hi, rs1, imm_lo, rd, 0x0); 619 } 620 621 static inline u16 rvc_sw(u8 rs1, u32 imm7, u8 rs2) 622 { 623 u32 imm_hi, imm_lo; 624 625 imm_hi = (imm7 & 0x38) >> 3; 626 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6); 627 return rv_cs_insn(0x6, imm_hi, rs1, imm_lo, rs2, 0x0); 628 } 629 630 static inline u16 rvc_addi(u8 rd, u32 imm6) 631 { 632 return rv_ci_insn(0, imm6, rd, 0x1); 633 } 634 635 static inline u16 rvc_li(u8 rd, u32 imm6) 636 { 637 return rv_ci_insn(0x2, imm6, rd, 0x1); 638 } 639 640 static inline u16 rvc_addi16sp(u32 imm10) 641 { 642 u32 imm; 643 644 imm = ((imm10 & 0x200) >> 4) | (imm10 & 0x10) | ((imm10 & 0x40) >> 3) | 645 ((imm10 & 0x180) >> 6) | ((imm10 & 0x20) >> 5); 646 return rv_ci_insn(0x3, imm, RV_REG_SP, 0x1); 647 } 648 649 static inline u16 rvc_lui(u8 rd, u32 imm6) 650 { 651 return rv_ci_insn(0x3, imm6, rd, 0x1); 652 } 653 654 static inline u16 rvc_srli(u8 rd, u32 imm6) 655 { 656 return rv_cb_insn(0x4, imm6, 0, rd, 0x1); 657 } 658 659 static inline u16 rvc_srai(u8 rd, u32 imm6) 660 { 661 return rv_cb_insn(0x4, imm6, 0x1, rd, 0x1); 662 } 663 664 static inline u16 rvc_andi(u8 rd, u32 imm6) 665 { 666 return rv_cb_insn(0x4, imm6, 0x2, rd, 0x1); 667 } 668 669 static inline u16 rvc_sub(u8 rd, u8 rs) 670 { 671 return rv_ca_insn(0x23, rd, 0, rs, 0x1); 672 } 673 674 static inline u16 rvc_xor(u8 rd, u8 rs) 675 { 676 return rv_ca_insn(0x23, rd, 0x1, rs, 0x1); 677 } 678 679 static inline u16 rvc_or(u8 rd, u8 rs) 680 { 681 return rv_ca_insn(0x23, rd, 0x2, rs, 0x1); 682 } 683 684 static inline u16 rvc_and(u8 rd, u8 rs) 685 { 686 return rv_ca_insn(0x23, rd, 0x3, rs, 0x1); 687 } 688 689 static inline u16 rvc_slli(u8 rd, u32 imm6) 690 { 691 return rv_ci_insn(0, imm6, rd, 0x2); 692 } 693 694 static inline u16 rvc_lwsp(u8 rd, u32 imm8) 695 { 696 u32 imm; 697 698 imm = ((imm8 & 0xc0) >> 6) | (imm8 & 0x3c); 699 return rv_ci_insn(0x2, imm, rd, 0x2); 700 } 701 702 static inline u16 rvc_jr(u8 rs1) 703 { 704 return rv_cr_insn(0x8, rs1, RV_REG_ZERO, 0x2); 705 } 706 707 static inline u16 rvc_mv(u8 rd, u8 rs) 708 { 709 return rv_cr_insn(0x8, rd, rs, 0x2); 710 } 711 712 static inline u16 rvc_jalr(u8 rs1) 713 { 714 return rv_cr_insn(0x9, rs1, RV_REG_ZERO, 0x2); 715 } 716 717 static inline u16 rvc_add(u8 rd, u8 rs) 718 { 719 return rv_cr_insn(0x9, rd, rs, 0x2); 720 } 721 722 static inline u16 rvc_swsp(u32 imm8, u8 rs2) 723 { 724 u32 imm; 725 726 imm = (imm8 & 0x3c) | ((imm8 & 0xc0) >> 6); 727 return rv_css_insn(0x6, imm, rs2, 0x2); 728 } 729 730 /* 731 * RV64-only instructions. 732 * 733 * These instructions are not available on RV32. Wrap them below a #if to 734 * ensure that the RV32 JIT doesn't emit any of these instructions. 735 */ 736 737 #if __riscv_xlen == 64 738 739 static inline u32 rv_addiw(u8 rd, u8 rs1, u16 imm11_0) 740 { 741 return rv_i_insn(imm11_0, rs1, 0, rd, 0x1b); 742 } 743 744 static inline u32 rv_slliw(u8 rd, u8 rs1, u16 imm11_0) 745 { 746 return rv_i_insn(imm11_0, rs1, 1, rd, 0x1b); 747 } 748 749 static inline u32 rv_srliw(u8 rd, u8 rs1, u16 imm11_0) 750 { 751 return rv_i_insn(imm11_0, rs1, 5, rd, 0x1b); 752 } 753 754 static inline u32 rv_sraiw(u8 rd, u8 rs1, u16 imm11_0) 755 { 756 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x1b); 757 } 758 759 static inline u32 rv_addw(u8 rd, u8 rs1, u8 rs2) 760 { 761 return rv_r_insn(0, rs2, rs1, 0, rd, 0x3b); 762 } 763 764 static inline u32 rv_subw(u8 rd, u8 rs1, u8 rs2) 765 { 766 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x3b); 767 } 768 769 static inline u32 rv_sllw(u8 rd, u8 rs1, u8 rs2) 770 { 771 return rv_r_insn(0, rs2, rs1, 1, rd, 0x3b); 772 } 773 774 static inline u32 rv_srlw(u8 rd, u8 rs1, u8 rs2) 775 { 776 return rv_r_insn(0, rs2, rs1, 5, rd, 0x3b); 777 } 778 779 static inline u32 rv_sraw(u8 rd, u8 rs1, u8 rs2) 780 { 781 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x3b); 782 } 783 784 static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2) 785 { 786 return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b); 787 } 788 789 static inline u32 rv_divw(u8 rd, u8 rs1, u8 rs2) 790 { 791 return rv_r_insn(1, rs2, rs1, 4, rd, 0x3b); 792 } 793 794 static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2) 795 { 796 return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b); 797 } 798 799 static inline u32 rv_remw(u8 rd, u8 rs1, u8 rs2) 800 { 801 return rv_r_insn(1, rs2, rs1, 6, rd, 0x3b); 802 } 803 804 static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2) 805 { 806 return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b); 807 } 808 809 static inline u32 rv_ld(u8 rd, u16 imm11_0, u8 rs1) 810 { 811 return rv_i_insn(imm11_0, rs1, 3, rd, 0x03); 812 } 813 814 static inline u32 rv_lwu(u8 rd, u16 imm11_0, u8 rs1) 815 { 816 return rv_i_insn(imm11_0, rs1, 6, rd, 0x03); 817 } 818 819 static inline u32 rv_sd(u8 rs1, u16 imm11_0, u8 rs2) 820 { 821 return rv_s_insn(imm11_0, rs2, rs1, 3, 0x23); 822 } 823 824 static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 825 { 826 return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f); 827 } 828 829 static inline u32 rv_amoand_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 830 { 831 return rv_amo_insn(0xc, aq, rl, rs2, rs1, 3, rd, 0x2f); 832 } 833 834 static inline u32 rv_amoor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 835 { 836 return rv_amo_insn(0x8, aq, rl, rs2, rs1, 3, rd, 0x2f); 837 } 838 839 static inline u32 rv_amoxor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 840 { 841 return rv_amo_insn(0x4, aq, rl, rs2, rs1, 3, rd, 0x2f); 842 } 843 844 static inline u32 rv_amoswap_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 845 { 846 return rv_amo_insn(0x1, aq, rl, rs2, rs1, 3, rd, 0x2f); 847 } 848 849 static inline u32 rv_lr_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 850 { 851 return rv_amo_insn(0x2, aq, rl, rs2, rs1, 3, rd, 0x2f); 852 } 853 854 static inline u32 rv_sc_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 855 { 856 return rv_amo_insn(0x3, aq, rl, rs2, rs1, 3, rd, 0x2f); 857 } 858 859 /* RV64-only RVC instructions. */ 860 861 static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1) 862 { 863 u32 imm_hi, imm_lo; 864 865 imm_hi = (imm8 & 0x38) >> 3; 866 imm_lo = (imm8 & 0xc0) >> 6; 867 return rv_cl_insn(0x3, imm_hi, rs1, imm_lo, rd, 0x0); 868 } 869 870 static inline u16 rvc_sd(u8 rs1, u32 imm8, u8 rs2) 871 { 872 u32 imm_hi, imm_lo; 873 874 imm_hi = (imm8 & 0x38) >> 3; 875 imm_lo = (imm8 & 0xc0) >> 6; 876 return rv_cs_insn(0x7, imm_hi, rs1, imm_lo, rs2, 0x0); 877 } 878 879 static inline u16 rvc_subw(u8 rd, u8 rs) 880 { 881 return rv_ca_insn(0x27, rd, 0, rs, 0x1); 882 } 883 884 static inline u16 rvc_addiw(u8 rd, u32 imm6) 885 { 886 return rv_ci_insn(0x1, imm6, rd, 0x1); 887 } 888 889 static inline u16 rvc_ldsp(u8 rd, u32 imm9) 890 { 891 u32 imm; 892 893 imm = ((imm9 & 0x1c0) >> 6) | (imm9 & 0x38); 894 return rv_ci_insn(0x3, imm, rd, 0x2); 895 } 896 897 static inline u16 rvc_sdsp(u32 imm9, u8 rs2) 898 { 899 u32 imm; 900 901 imm = (imm9 & 0x38) | ((imm9 & 0x1c0) >> 6); 902 return rv_css_insn(0x7, imm, rs2, 0x2); 903 } 904 905 #endif /* __riscv_xlen == 64 */ 906 907 /* Helper functions that emit RVC instructions when possible. */ 908 909 static inline void emit_jalr(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 910 { 911 if (rvc_enabled() && rd == RV_REG_RA && rs && !imm) 912 emitc(rvc_jalr(rs), ctx); 913 else if (rvc_enabled() && !rd && rs && !imm) 914 emitc(rvc_jr(rs), ctx); 915 else 916 emit(rv_jalr(rd, rs, imm), ctx); 917 } 918 919 static inline void emit_mv(u8 rd, u8 rs, struct rv_jit_context *ctx) 920 { 921 if (rvc_enabled() && rd && rs) 922 emitc(rvc_mv(rd, rs), ctx); 923 else 924 emit(rv_addi(rd, rs, 0), ctx); 925 } 926 927 static inline void emit_add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 928 { 929 if (rvc_enabled() && rd && rd == rs1 && rs2) 930 emitc(rvc_add(rd, rs2), ctx); 931 else 932 emit(rv_add(rd, rs1, rs2), ctx); 933 } 934 935 static inline void emit_addi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 936 { 937 if (rvc_enabled() && rd == RV_REG_SP && rd == rs && is_10b_int(imm) && imm && !(imm & 0xf)) 938 emitc(rvc_addi16sp(imm), ctx); 939 else if (rvc_enabled() && is_creg(rd) && rs == RV_REG_SP && is_10b_uint(imm) && 940 !(imm & 0x3) && imm) 941 emitc(rvc_addi4spn(rd, imm), ctx); 942 else if (rvc_enabled() && rd && rd == rs && imm && is_6b_int(imm)) 943 emitc(rvc_addi(rd, imm), ctx); 944 else 945 emit(rv_addi(rd, rs, imm), ctx); 946 } 947 948 static inline void emit_li(u8 rd, s32 imm, struct rv_jit_context *ctx) 949 { 950 if (rvc_enabled() && rd && is_6b_int(imm)) 951 emitc(rvc_li(rd, imm), ctx); 952 else 953 emit(rv_addi(rd, RV_REG_ZERO, imm), ctx); 954 } 955 956 static inline void emit_lui(u8 rd, s32 imm, struct rv_jit_context *ctx) 957 { 958 if (rvc_enabled() && rd && rd != RV_REG_SP && is_6b_int(imm) && imm) 959 emitc(rvc_lui(rd, imm), ctx); 960 else 961 emit(rv_lui(rd, imm), ctx); 962 } 963 964 static inline void emit_slli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 965 { 966 if (rvc_enabled() && rd && rd == rs && imm && (u32)imm < __riscv_xlen) 967 emitc(rvc_slli(rd, imm), ctx); 968 else 969 emit(rv_slli(rd, rs, imm), ctx); 970 } 971 972 static inline void emit_andi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 973 { 974 if (rvc_enabled() && is_creg(rd) && rd == rs && is_6b_int(imm)) 975 emitc(rvc_andi(rd, imm), ctx); 976 else 977 emit(rv_andi(rd, rs, imm), ctx); 978 } 979 980 static inline void emit_srli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 981 { 982 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen) 983 emitc(rvc_srli(rd, imm), ctx); 984 else 985 emit(rv_srli(rd, rs, imm), ctx); 986 } 987 988 static inline void emit_srai(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 989 { 990 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen) 991 emitc(rvc_srai(rd, imm), ctx); 992 else 993 emit(rv_srai(rd, rs, imm), ctx); 994 } 995 996 static inline void emit_sub(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 997 { 998 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 999 emitc(rvc_sub(rd, rs2), ctx); 1000 else 1001 emit(rv_sub(rd, rs1, rs2), ctx); 1002 } 1003 1004 static inline void emit_or(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1005 { 1006 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1007 emitc(rvc_or(rd, rs2), ctx); 1008 else 1009 emit(rv_or(rd, rs1, rs2), ctx); 1010 } 1011 1012 static inline void emit_and(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1013 { 1014 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1015 emitc(rvc_and(rd, rs2), ctx); 1016 else 1017 emit(rv_and(rd, rs1, rs2), ctx); 1018 } 1019 1020 static inline void emit_xor(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1021 { 1022 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1023 emitc(rvc_xor(rd, rs2), ctx); 1024 else 1025 emit(rv_xor(rd, rs1, rs2), ctx); 1026 } 1027 1028 static inline void emit_lw(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx) 1029 { 1030 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_8b_uint(off) && !(off & 0x3)) 1031 emitc(rvc_lwsp(rd, off), ctx); 1032 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_7b_uint(off) && !(off & 0x3)) 1033 emitc(rvc_lw(rd, off, rs1), ctx); 1034 else 1035 emit(rv_lw(rd, off, rs1), ctx); 1036 } 1037 1038 static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) 1039 { 1040 if (rvc_enabled() && rs1 == RV_REG_SP && is_8b_uint(off) && !(off & 0x3)) 1041 emitc(rvc_swsp(off, rs2), ctx); 1042 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_7b_uint(off) && !(off & 0x3)) 1043 emitc(rvc_sw(rs1, off, rs2), ctx); 1044 else 1045 emit(rv_sw(rs1, off, rs2), ctx); 1046 } 1047 1048 /* RV64-only helper functions. */ 1049 #if __riscv_xlen == 64 1050 1051 static inline void emit_addiw(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 1052 { 1053 if (rvc_enabled() && rd && rd == rs && is_6b_int(imm)) 1054 emitc(rvc_addiw(rd, imm), ctx); 1055 else 1056 emit(rv_addiw(rd, rs, imm), ctx); 1057 } 1058 1059 static inline void emit_ld(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx) 1060 { 1061 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_9b_uint(off) && !(off & 0x7)) 1062 emitc(rvc_ldsp(rd, off), ctx); 1063 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_8b_uint(off) && !(off & 0x7)) 1064 emitc(rvc_ld(rd, off, rs1), ctx); 1065 else 1066 emit(rv_ld(rd, off, rs1), ctx); 1067 } 1068 1069 static inline void emit_sd(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) 1070 { 1071 if (rvc_enabled() && rs1 == RV_REG_SP && is_9b_uint(off) && !(off & 0x7)) 1072 emitc(rvc_sdsp(off, rs2), ctx); 1073 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_8b_uint(off) && !(off & 0x7)) 1074 emitc(rvc_sd(rs1, off, rs2), ctx); 1075 else 1076 emit(rv_sd(rs1, off, rs2), ctx); 1077 } 1078 1079 static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1080 { 1081 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1082 emitc(rvc_subw(rd, rs2), ctx); 1083 else 1084 emit(rv_subw(rd, rs1, rs2), ctx); 1085 } 1086 1087 #endif /* __riscv_xlen == 64 */ 1088 1089 void bpf_jit_build_prologue(struct rv_jit_context *ctx); 1090 void bpf_jit_build_epilogue(struct rv_jit_context *ctx); 1091 1092 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 1093 bool extra_pass); 1094 1095 #endif /* _BPF_JIT_H */ 1096