1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Common functionality for RV32 and RV64 BPF JIT compilers 4 * 5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com> 6 * 7 */ 8 9 #ifndef _BPF_JIT_H 10 #define _BPF_JIT_H 11 12 #include <linux/bpf.h> 13 #include <linux/filter.h> 14 #include <asm/cacheflush.h> 15 16 static inline bool rvc_enabled(void) 17 { 18 return IS_ENABLED(CONFIG_RISCV_ISA_C); 19 } 20 21 enum { 22 RV_REG_ZERO = 0, /* The constant value 0 */ 23 RV_REG_RA = 1, /* Return address */ 24 RV_REG_SP = 2, /* Stack pointer */ 25 RV_REG_GP = 3, /* Global pointer */ 26 RV_REG_TP = 4, /* Thread pointer */ 27 RV_REG_T0 = 5, /* Temporaries */ 28 RV_REG_T1 = 6, 29 RV_REG_T2 = 7, 30 RV_REG_FP = 8, /* Saved register/frame pointer */ 31 RV_REG_S1 = 9, /* Saved register */ 32 RV_REG_A0 = 10, /* Function argument/return values */ 33 RV_REG_A1 = 11, /* Function arguments */ 34 RV_REG_A2 = 12, 35 RV_REG_A3 = 13, 36 RV_REG_A4 = 14, 37 RV_REG_A5 = 15, 38 RV_REG_A6 = 16, 39 RV_REG_A7 = 17, 40 RV_REG_S2 = 18, /* Saved registers */ 41 RV_REG_S3 = 19, 42 RV_REG_S4 = 20, 43 RV_REG_S5 = 21, 44 RV_REG_S6 = 22, 45 RV_REG_S7 = 23, 46 RV_REG_S8 = 24, 47 RV_REG_S9 = 25, 48 RV_REG_S10 = 26, 49 RV_REG_S11 = 27, 50 RV_REG_T3 = 28, /* Temporaries */ 51 RV_REG_T4 = 29, 52 RV_REG_T5 = 30, 53 RV_REG_T6 = 31, 54 }; 55 56 static inline bool is_creg(u8 reg) 57 { 58 return (1 << reg) & (BIT(RV_REG_FP) | 59 BIT(RV_REG_S1) | 60 BIT(RV_REG_A0) | 61 BIT(RV_REG_A1) | 62 BIT(RV_REG_A2) | 63 BIT(RV_REG_A3) | 64 BIT(RV_REG_A4) | 65 BIT(RV_REG_A5)); 66 } 67 68 struct rv_jit_context { 69 struct bpf_prog *prog; 70 u16 *insns; /* RV insns */ 71 int ninsns; 72 int epilogue_offset; 73 int *offset; /* BPF to RV */ 74 int nexentries; 75 unsigned long flags; 76 int stack_size; 77 }; 78 79 /* Convert from ninsns to bytes. */ 80 static inline int ninsns_rvoff(int ninsns) 81 { 82 return ninsns << 1; 83 } 84 85 struct rv_jit_data { 86 struct bpf_binary_header *header; 87 u8 *image; 88 struct rv_jit_context ctx; 89 }; 90 91 static inline void bpf_fill_ill_insns(void *area, unsigned int size) 92 { 93 memset(area, 0, size); 94 } 95 96 static inline void bpf_flush_icache(void *start, void *end) 97 { 98 flush_icache_range((unsigned long)start, (unsigned long)end); 99 } 100 101 /* Emit a 4-byte riscv instruction. */ 102 static inline void emit(const u32 insn, struct rv_jit_context *ctx) 103 { 104 if (ctx->insns) { 105 ctx->insns[ctx->ninsns] = insn; 106 ctx->insns[ctx->ninsns + 1] = (insn >> 16); 107 } 108 109 ctx->ninsns += 2; 110 } 111 112 /* Emit a 2-byte riscv compressed instruction. */ 113 static inline void emitc(const u16 insn, struct rv_jit_context *ctx) 114 { 115 BUILD_BUG_ON(!rvc_enabled()); 116 117 if (ctx->insns) 118 ctx->insns[ctx->ninsns] = insn; 119 120 ctx->ninsns++; 121 } 122 123 static inline int epilogue_offset(struct rv_jit_context *ctx) 124 { 125 int to = ctx->epilogue_offset, from = ctx->ninsns; 126 127 return ninsns_rvoff(to - from); 128 } 129 130 /* Return -1 or inverted cond. */ 131 static inline int invert_bpf_cond(u8 cond) 132 { 133 switch (cond) { 134 case BPF_JEQ: 135 return BPF_JNE; 136 case BPF_JGT: 137 return BPF_JLE; 138 case BPF_JLT: 139 return BPF_JGE; 140 case BPF_JGE: 141 return BPF_JLT; 142 case BPF_JLE: 143 return BPF_JGT; 144 case BPF_JNE: 145 return BPF_JEQ; 146 case BPF_JSGT: 147 return BPF_JSLE; 148 case BPF_JSLT: 149 return BPF_JSGE; 150 case BPF_JSGE: 151 return BPF_JSLT; 152 case BPF_JSLE: 153 return BPF_JSGT; 154 } 155 return -1; 156 } 157 158 static inline bool is_6b_int(long val) 159 { 160 return -(1L << 5) <= val && val < (1L << 5); 161 } 162 163 static inline bool is_7b_uint(unsigned long val) 164 { 165 return val < (1UL << 7); 166 } 167 168 static inline bool is_8b_uint(unsigned long val) 169 { 170 return val < (1UL << 8); 171 } 172 173 static inline bool is_9b_uint(unsigned long val) 174 { 175 return val < (1UL << 9); 176 } 177 178 static inline bool is_10b_int(long val) 179 { 180 return -(1L << 9) <= val && val < (1L << 9); 181 } 182 183 static inline bool is_10b_uint(unsigned long val) 184 { 185 return val < (1UL << 10); 186 } 187 188 static inline bool is_12b_int(long val) 189 { 190 return -(1L << 11) <= val && val < (1L << 11); 191 } 192 193 static inline int is_12b_check(int off, int insn) 194 { 195 if (!is_12b_int(off)) { 196 pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n", 197 insn, (int)off); 198 return -1; 199 } 200 return 0; 201 } 202 203 static inline bool is_13b_int(long val) 204 { 205 return -(1L << 12) <= val && val < (1L << 12); 206 } 207 208 static inline bool is_21b_int(long val) 209 { 210 return -(1L << 20) <= val && val < (1L << 20); 211 } 212 213 static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx) 214 { 215 int from, to; 216 217 off++; /* BPF branch is from PC+1, RV is from PC */ 218 from = (insn > 0) ? ctx->offset[insn - 1] : 0; 219 to = (insn + off > 0) ? ctx->offset[insn + off - 1] : 0; 220 return ninsns_rvoff(to - from); 221 } 222 223 /* Instruction formats. */ 224 225 static inline u32 rv_r_insn(u8 funct7, u8 rs2, u8 rs1, u8 funct3, u8 rd, 226 u8 opcode) 227 { 228 return (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 229 (rd << 7) | opcode; 230 } 231 232 static inline u32 rv_i_insn(u16 imm11_0, u8 rs1, u8 funct3, u8 rd, u8 opcode) 233 { 234 return (imm11_0 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) | 235 opcode; 236 } 237 238 static inline u32 rv_s_insn(u16 imm11_0, u8 rs2, u8 rs1, u8 funct3, u8 opcode) 239 { 240 u8 imm11_5 = imm11_0 >> 5, imm4_0 = imm11_0 & 0x1f; 241 242 return (imm11_5 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 243 (imm4_0 << 7) | opcode; 244 } 245 246 static inline u32 rv_b_insn(u16 imm12_1, u8 rs2, u8 rs1, u8 funct3, u8 opcode) 247 { 248 u8 imm12 = ((imm12_1 & 0x800) >> 5) | ((imm12_1 & 0x3f0) >> 4); 249 u8 imm4_1 = ((imm12_1 & 0xf) << 1) | ((imm12_1 & 0x400) >> 10); 250 251 return (imm12 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 252 (imm4_1 << 7) | opcode; 253 } 254 255 static inline u32 rv_u_insn(u32 imm31_12, u8 rd, u8 opcode) 256 { 257 return (imm31_12 << 12) | (rd << 7) | opcode; 258 } 259 260 static inline u32 rv_j_insn(u32 imm20_1, u8 rd, u8 opcode) 261 { 262 u32 imm; 263 264 imm = (imm20_1 & 0x80000) | ((imm20_1 & 0x3ff) << 9) | 265 ((imm20_1 & 0x400) >> 2) | ((imm20_1 & 0x7f800) >> 11); 266 267 return (imm << 12) | (rd << 7) | opcode; 268 } 269 270 static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1, 271 u8 funct3, u8 rd, u8 opcode) 272 { 273 u8 funct7 = (funct5 << 2) | (aq << 1) | rl; 274 275 return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode); 276 } 277 278 /* RISC-V compressed instruction formats. */ 279 280 static inline u16 rv_cr_insn(u8 funct4, u8 rd, u8 rs2, u8 op) 281 { 282 return (funct4 << 12) | (rd << 7) | (rs2 << 2) | op; 283 } 284 285 static inline u16 rv_ci_insn(u8 funct3, u32 imm6, u8 rd, u8 op) 286 { 287 u32 imm; 288 289 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2); 290 return (funct3 << 13) | (rd << 7) | op | imm; 291 } 292 293 static inline u16 rv_css_insn(u8 funct3, u32 uimm, u8 rs2, u8 op) 294 { 295 return (funct3 << 13) | (uimm << 7) | (rs2 << 2) | op; 296 } 297 298 static inline u16 rv_ciw_insn(u8 funct3, u32 uimm, u8 rd, u8 op) 299 { 300 return (funct3 << 13) | (uimm << 5) | ((rd & 0x7) << 2) | op; 301 } 302 303 static inline u16 rv_cl_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rd, 304 u8 op) 305 { 306 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) | 307 (imm_lo << 5) | ((rd & 0x7) << 2) | op; 308 } 309 310 static inline u16 rv_cs_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rs2, 311 u8 op) 312 { 313 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) | 314 (imm_lo << 5) | ((rs2 & 0x7) << 2) | op; 315 } 316 317 static inline u16 rv_ca_insn(u8 funct6, u8 rd, u8 funct2, u8 rs2, u8 op) 318 { 319 return (funct6 << 10) | ((rd & 0x7) << 7) | (funct2 << 5) | 320 ((rs2 & 0x7) << 2) | op; 321 } 322 323 static inline u16 rv_cb_insn(u8 funct3, u32 imm6, u8 funct2, u8 rd, u8 op) 324 { 325 u32 imm; 326 327 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2); 328 return (funct3 << 13) | (funct2 << 10) | ((rd & 0x7) << 7) | op | imm; 329 } 330 331 /* Instructions shared by both RV32 and RV64. */ 332 333 static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0) 334 { 335 return rv_i_insn(imm11_0, rs1, 0, rd, 0x13); 336 } 337 338 static inline u32 rv_andi(u8 rd, u8 rs1, u16 imm11_0) 339 { 340 return rv_i_insn(imm11_0, rs1, 7, rd, 0x13); 341 } 342 343 static inline u32 rv_ori(u8 rd, u8 rs1, u16 imm11_0) 344 { 345 return rv_i_insn(imm11_0, rs1, 6, rd, 0x13); 346 } 347 348 static inline u32 rv_xori(u8 rd, u8 rs1, u16 imm11_0) 349 { 350 return rv_i_insn(imm11_0, rs1, 4, rd, 0x13); 351 } 352 353 static inline u32 rv_slli(u8 rd, u8 rs1, u16 imm11_0) 354 { 355 return rv_i_insn(imm11_0, rs1, 1, rd, 0x13); 356 } 357 358 static inline u32 rv_srli(u8 rd, u8 rs1, u16 imm11_0) 359 { 360 return rv_i_insn(imm11_0, rs1, 5, rd, 0x13); 361 } 362 363 static inline u32 rv_srai(u8 rd, u8 rs1, u16 imm11_0) 364 { 365 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x13); 366 } 367 368 static inline u32 rv_lui(u8 rd, u32 imm31_12) 369 { 370 return rv_u_insn(imm31_12, rd, 0x37); 371 } 372 373 static inline u32 rv_auipc(u8 rd, u32 imm31_12) 374 { 375 return rv_u_insn(imm31_12, rd, 0x17); 376 } 377 378 static inline u32 rv_add(u8 rd, u8 rs1, u8 rs2) 379 { 380 return rv_r_insn(0, rs2, rs1, 0, rd, 0x33); 381 } 382 383 static inline u32 rv_sub(u8 rd, u8 rs1, u8 rs2) 384 { 385 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x33); 386 } 387 388 static inline u32 rv_sltu(u8 rd, u8 rs1, u8 rs2) 389 { 390 return rv_r_insn(0, rs2, rs1, 3, rd, 0x33); 391 } 392 393 static inline u32 rv_and(u8 rd, u8 rs1, u8 rs2) 394 { 395 return rv_r_insn(0, rs2, rs1, 7, rd, 0x33); 396 } 397 398 static inline u32 rv_or(u8 rd, u8 rs1, u8 rs2) 399 { 400 return rv_r_insn(0, rs2, rs1, 6, rd, 0x33); 401 } 402 403 static inline u32 rv_xor(u8 rd, u8 rs1, u8 rs2) 404 { 405 return rv_r_insn(0, rs2, rs1, 4, rd, 0x33); 406 } 407 408 static inline u32 rv_sll(u8 rd, u8 rs1, u8 rs2) 409 { 410 return rv_r_insn(0, rs2, rs1, 1, rd, 0x33); 411 } 412 413 static inline u32 rv_srl(u8 rd, u8 rs1, u8 rs2) 414 { 415 return rv_r_insn(0, rs2, rs1, 5, rd, 0x33); 416 } 417 418 static inline u32 rv_sra(u8 rd, u8 rs1, u8 rs2) 419 { 420 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x33); 421 } 422 423 static inline u32 rv_mul(u8 rd, u8 rs1, u8 rs2) 424 { 425 return rv_r_insn(1, rs2, rs1, 0, rd, 0x33); 426 } 427 428 static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2) 429 { 430 return rv_r_insn(1, rs2, rs1, 3, rd, 0x33); 431 } 432 433 static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2) 434 { 435 return rv_r_insn(1, rs2, rs1, 5, rd, 0x33); 436 } 437 438 static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2) 439 { 440 return rv_r_insn(1, rs2, rs1, 7, rd, 0x33); 441 } 442 443 static inline u32 rv_jal(u8 rd, u32 imm20_1) 444 { 445 return rv_j_insn(imm20_1, rd, 0x6f); 446 } 447 448 static inline u32 rv_jalr(u8 rd, u8 rs1, u16 imm11_0) 449 { 450 return rv_i_insn(imm11_0, rs1, 0, rd, 0x67); 451 } 452 453 static inline u32 rv_beq(u8 rs1, u8 rs2, u16 imm12_1) 454 { 455 return rv_b_insn(imm12_1, rs2, rs1, 0, 0x63); 456 } 457 458 static inline u32 rv_bne(u8 rs1, u8 rs2, u16 imm12_1) 459 { 460 return rv_b_insn(imm12_1, rs2, rs1, 1, 0x63); 461 } 462 463 static inline u32 rv_bltu(u8 rs1, u8 rs2, u16 imm12_1) 464 { 465 return rv_b_insn(imm12_1, rs2, rs1, 6, 0x63); 466 } 467 468 static inline u32 rv_bgtu(u8 rs1, u8 rs2, u16 imm12_1) 469 { 470 return rv_bltu(rs2, rs1, imm12_1); 471 } 472 473 static inline u32 rv_bgeu(u8 rs1, u8 rs2, u16 imm12_1) 474 { 475 return rv_b_insn(imm12_1, rs2, rs1, 7, 0x63); 476 } 477 478 static inline u32 rv_bleu(u8 rs1, u8 rs2, u16 imm12_1) 479 { 480 return rv_bgeu(rs2, rs1, imm12_1); 481 } 482 483 static inline u32 rv_blt(u8 rs1, u8 rs2, u16 imm12_1) 484 { 485 return rv_b_insn(imm12_1, rs2, rs1, 4, 0x63); 486 } 487 488 static inline u32 rv_bgt(u8 rs1, u8 rs2, u16 imm12_1) 489 { 490 return rv_blt(rs2, rs1, imm12_1); 491 } 492 493 static inline u32 rv_bge(u8 rs1, u8 rs2, u16 imm12_1) 494 { 495 return rv_b_insn(imm12_1, rs2, rs1, 5, 0x63); 496 } 497 498 static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1) 499 { 500 return rv_bge(rs2, rs1, imm12_1); 501 } 502 503 static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1) 504 { 505 return rv_i_insn(imm11_0, rs1, 2, rd, 0x03); 506 } 507 508 static inline u32 rv_lbu(u8 rd, u16 imm11_0, u8 rs1) 509 { 510 return rv_i_insn(imm11_0, rs1, 4, rd, 0x03); 511 } 512 513 static inline u32 rv_lhu(u8 rd, u16 imm11_0, u8 rs1) 514 { 515 return rv_i_insn(imm11_0, rs1, 5, rd, 0x03); 516 } 517 518 static inline u32 rv_sb(u8 rs1, u16 imm11_0, u8 rs2) 519 { 520 return rv_s_insn(imm11_0, rs2, rs1, 0, 0x23); 521 } 522 523 static inline u32 rv_sh(u8 rs1, u16 imm11_0, u8 rs2) 524 { 525 return rv_s_insn(imm11_0, rs2, rs1, 1, 0x23); 526 } 527 528 static inline u32 rv_sw(u8 rs1, u16 imm11_0, u8 rs2) 529 { 530 return rv_s_insn(imm11_0, rs2, rs1, 2, 0x23); 531 } 532 533 static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 534 { 535 return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f); 536 } 537 538 static inline u32 rv_amoand_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 539 { 540 return rv_amo_insn(0xc, aq, rl, rs2, rs1, 2, rd, 0x2f); 541 } 542 543 static inline u32 rv_amoor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 544 { 545 return rv_amo_insn(0x8, aq, rl, rs2, rs1, 2, rd, 0x2f); 546 } 547 548 static inline u32 rv_amoxor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 549 { 550 return rv_amo_insn(0x4, aq, rl, rs2, rs1, 2, rd, 0x2f); 551 } 552 553 static inline u32 rv_amoswap_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 554 { 555 return rv_amo_insn(0x1, aq, rl, rs2, rs1, 2, rd, 0x2f); 556 } 557 558 static inline u32 rv_lr_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 559 { 560 return rv_amo_insn(0x2, aq, rl, rs2, rs1, 2, rd, 0x2f); 561 } 562 563 static inline u32 rv_sc_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 564 { 565 return rv_amo_insn(0x3, aq, rl, rs2, rs1, 2, rd, 0x2f); 566 } 567 568 static inline u32 rv_fence(u8 pred, u8 succ) 569 { 570 u16 imm11_0 = pred << 4 | succ; 571 572 return rv_i_insn(imm11_0, 0, 0, 0, 0xf); 573 } 574 575 /* RVC instrutions. */ 576 577 static inline u16 rvc_addi4spn(u8 rd, u32 imm10) 578 { 579 u32 imm; 580 581 imm = ((imm10 & 0x30) << 2) | ((imm10 & 0x3c0) >> 4) | 582 ((imm10 & 0x4) >> 1) | ((imm10 & 0x8) >> 3); 583 return rv_ciw_insn(0x0, imm, rd, 0x0); 584 } 585 586 static inline u16 rvc_lw(u8 rd, u32 imm7, u8 rs1) 587 { 588 u32 imm_hi, imm_lo; 589 590 imm_hi = (imm7 & 0x38) >> 3; 591 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6); 592 return rv_cl_insn(0x2, imm_hi, rs1, imm_lo, rd, 0x0); 593 } 594 595 static inline u16 rvc_sw(u8 rs1, u32 imm7, u8 rs2) 596 { 597 u32 imm_hi, imm_lo; 598 599 imm_hi = (imm7 & 0x38) >> 3; 600 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6); 601 return rv_cs_insn(0x6, imm_hi, rs1, imm_lo, rs2, 0x0); 602 } 603 604 static inline u16 rvc_addi(u8 rd, u32 imm6) 605 { 606 return rv_ci_insn(0, imm6, rd, 0x1); 607 } 608 609 static inline u16 rvc_li(u8 rd, u32 imm6) 610 { 611 return rv_ci_insn(0x2, imm6, rd, 0x1); 612 } 613 614 static inline u16 rvc_addi16sp(u32 imm10) 615 { 616 u32 imm; 617 618 imm = ((imm10 & 0x200) >> 4) | (imm10 & 0x10) | ((imm10 & 0x40) >> 3) | 619 ((imm10 & 0x180) >> 6) | ((imm10 & 0x20) >> 5); 620 return rv_ci_insn(0x3, imm, RV_REG_SP, 0x1); 621 } 622 623 static inline u16 rvc_lui(u8 rd, u32 imm6) 624 { 625 return rv_ci_insn(0x3, imm6, rd, 0x1); 626 } 627 628 static inline u16 rvc_srli(u8 rd, u32 imm6) 629 { 630 return rv_cb_insn(0x4, imm6, 0, rd, 0x1); 631 } 632 633 static inline u16 rvc_srai(u8 rd, u32 imm6) 634 { 635 return rv_cb_insn(0x4, imm6, 0x1, rd, 0x1); 636 } 637 638 static inline u16 rvc_andi(u8 rd, u32 imm6) 639 { 640 return rv_cb_insn(0x4, imm6, 0x2, rd, 0x1); 641 } 642 643 static inline u16 rvc_sub(u8 rd, u8 rs) 644 { 645 return rv_ca_insn(0x23, rd, 0, rs, 0x1); 646 } 647 648 static inline u16 rvc_xor(u8 rd, u8 rs) 649 { 650 return rv_ca_insn(0x23, rd, 0x1, rs, 0x1); 651 } 652 653 static inline u16 rvc_or(u8 rd, u8 rs) 654 { 655 return rv_ca_insn(0x23, rd, 0x2, rs, 0x1); 656 } 657 658 static inline u16 rvc_and(u8 rd, u8 rs) 659 { 660 return rv_ca_insn(0x23, rd, 0x3, rs, 0x1); 661 } 662 663 static inline u16 rvc_slli(u8 rd, u32 imm6) 664 { 665 return rv_ci_insn(0, imm6, rd, 0x2); 666 } 667 668 static inline u16 rvc_lwsp(u8 rd, u32 imm8) 669 { 670 u32 imm; 671 672 imm = ((imm8 & 0xc0) >> 6) | (imm8 & 0x3c); 673 return rv_ci_insn(0x2, imm, rd, 0x2); 674 } 675 676 static inline u16 rvc_jr(u8 rs1) 677 { 678 return rv_cr_insn(0x8, rs1, RV_REG_ZERO, 0x2); 679 } 680 681 static inline u16 rvc_mv(u8 rd, u8 rs) 682 { 683 return rv_cr_insn(0x8, rd, rs, 0x2); 684 } 685 686 static inline u16 rvc_jalr(u8 rs1) 687 { 688 return rv_cr_insn(0x9, rs1, RV_REG_ZERO, 0x2); 689 } 690 691 static inline u16 rvc_add(u8 rd, u8 rs) 692 { 693 return rv_cr_insn(0x9, rd, rs, 0x2); 694 } 695 696 static inline u16 rvc_swsp(u32 imm8, u8 rs2) 697 { 698 u32 imm; 699 700 imm = (imm8 & 0x3c) | ((imm8 & 0xc0) >> 6); 701 return rv_css_insn(0x6, imm, rs2, 0x2); 702 } 703 704 /* 705 * RV64-only instructions. 706 * 707 * These instructions are not available on RV32. Wrap them below a #if to 708 * ensure that the RV32 JIT doesn't emit any of these instructions. 709 */ 710 711 #if __riscv_xlen == 64 712 713 static inline u32 rv_addiw(u8 rd, u8 rs1, u16 imm11_0) 714 { 715 return rv_i_insn(imm11_0, rs1, 0, rd, 0x1b); 716 } 717 718 static inline u32 rv_slliw(u8 rd, u8 rs1, u16 imm11_0) 719 { 720 return rv_i_insn(imm11_0, rs1, 1, rd, 0x1b); 721 } 722 723 static inline u32 rv_srliw(u8 rd, u8 rs1, u16 imm11_0) 724 { 725 return rv_i_insn(imm11_0, rs1, 5, rd, 0x1b); 726 } 727 728 static inline u32 rv_sraiw(u8 rd, u8 rs1, u16 imm11_0) 729 { 730 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x1b); 731 } 732 733 static inline u32 rv_addw(u8 rd, u8 rs1, u8 rs2) 734 { 735 return rv_r_insn(0, rs2, rs1, 0, rd, 0x3b); 736 } 737 738 static inline u32 rv_subw(u8 rd, u8 rs1, u8 rs2) 739 { 740 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x3b); 741 } 742 743 static inline u32 rv_sllw(u8 rd, u8 rs1, u8 rs2) 744 { 745 return rv_r_insn(0, rs2, rs1, 1, rd, 0x3b); 746 } 747 748 static inline u32 rv_srlw(u8 rd, u8 rs1, u8 rs2) 749 { 750 return rv_r_insn(0, rs2, rs1, 5, rd, 0x3b); 751 } 752 753 static inline u32 rv_sraw(u8 rd, u8 rs1, u8 rs2) 754 { 755 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x3b); 756 } 757 758 static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2) 759 { 760 return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b); 761 } 762 763 static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2) 764 { 765 return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b); 766 } 767 768 static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2) 769 { 770 return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b); 771 } 772 773 static inline u32 rv_ld(u8 rd, u16 imm11_0, u8 rs1) 774 { 775 return rv_i_insn(imm11_0, rs1, 3, rd, 0x03); 776 } 777 778 static inline u32 rv_lwu(u8 rd, u16 imm11_0, u8 rs1) 779 { 780 return rv_i_insn(imm11_0, rs1, 6, rd, 0x03); 781 } 782 783 static inline u32 rv_sd(u8 rs1, u16 imm11_0, u8 rs2) 784 { 785 return rv_s_insn(imm11_0, rs2, rs1, 3, 0x23); 786 } 787 788 static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 789 { 790 return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f); 791 } 792 793 static inline u32 rv_amoand_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 794 { 795 return rv_amo_insn(0xc, aq, rl, rs2, rs1, 3, rd, 0x2f); 796 } 797 798 static inline u32 rv_amoor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 799 { 800 return rv_amo_insn(0x8, aq, rl, rs2, rs1, 3, rd, 0x2f); 801 } 802 803 static inline u32 rv_amoxor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 804 { 805 return rv_amo_insn(0x4, aq, rl, rs2, rs1, 3, rd, 0x2f); 806 } 807 808 static inline u32 rv_amoswap_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 809 { 810 return rv_amo_insn(0x1, aq, rl, rs2, rs1, 3, rd, 0x2f); 811 } 812 813 static inline u32 rv_lr_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 814 { 815 return rv_amo_insn(0x2, aq, rl, rs2, rs1, 3, rd, 0x2f); 816 } 817 818 static inline u32 rv_sc_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 819 { 820 return rv_amo_insn(0x3, aq, rl, rs2, rs1, 3, rd, 0x2f); 821 } 822 823 /* RV64-only RVC instructions. */ 824 825 static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1) 826 { 827 u32 imm_hi, imm_lo; 828 829 imm_hi = (imm8 & 0x38) >> 3; 830 imm_lo = (imm8 & 0xc0) >> 6; 831 return rv_cl_insn(0x3, imm_hi, rs1, imm_lo, rd, 0x0); 832 } 833 834 static inline u16 rvc_sd(u8 rs1, u32 imm8, u8 rs2) 835 { 836 u32 imm_hi, imm_lo; 837 838 imm_hi = (imm8 & 0x38) >> 3; 839 imm_lo = (imm8 & 0xc0) >> 6; 840 return rv_cs_insn(0x7, imm_hi, rs1, imm_lo, rs2, 0x0); 841 } 842 843 static inline u16 rvc_subw(u8 rd, u8 rs) 844 { 845 return rv_ca_insn(0x27, rd, 0, rs, 0x1); 846 } 847 848 static inline u16 rvc_addiw(u8 rd, u32 imm6) 849 { 850 return rv_ci_insn(0x1, imm6, rd, 0x1); 851 } 852 853 static inline u16 rvc_ldsp(u8 rd, u32 imm9) 854 { 855 u32 imm; 856 857 imm = ((imm9 & 0x1c0) >> 6) | (imm9 & 0x38); 858 return rv_ci_insn(0x3, imm, rd, 0x2); 859 } 860 861 static inline u16 rvc_sdsp(u32 imm9, u8 rs2) 862 { 863 u32 imm; 864 865 imm = (imm9 & 0x38) | ((imm9 & 0x1c0) >> 6); 866 return rv_css_insn(0x7, imm, rs2, 0x2); 867 } 868 869 #endif /* __riscv_xlen == 64 */ 870 871 /* Helper functions that emit RVC instructions when possible. */ 872 873 static inline void emit_jalr(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 874 { 875 if (rvc_enabled() && rd == RV_REG_RA && rs && !imm) 876 emitc(rvc_jalr(rs), ctx); 877 else if (rvc_enabled() && !rd && rs && !imm) 878 emitc(rvc_jr(rs), ctx); 879 else 880 emit(rv_jalr(rd, rs, imm), ctx); 881 } 882 883 static inline void emit_mv(u8 rd, u8 rs, struct rv_jit_context *ctx) 884 { 885 if (rvc_enabled() && rd && rs) 886 emitc(rvc_mv(rd, rs), ctx); 887 else 888 emit(rv_addi(rd, rs, 0), ctx); 889 } 890 891 static inline void emit_add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 892 { 893 if (rvc_enabled() && rd && rd == rs1 && rs2) 894 emitc(rvc_add(rd, rs2), ctx); 895 else 896 emit(rv_add(rd, rs1, rs2), ctx); 897 } 898 899 static inline void emit_addi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 900 { 901 if (rvc_enabled() && rd == RV_REG_SP && rd == rs && is_10b_int(imm) && imm && !(imm & 0xf)) 902 emitc(rvc_addi16sp(imm), ctx); 903 else if (rvc_enabled() && is_creg(rd) && rs == RV_REG_SP && is_10b_uint(imm) && 904 !(imm & 0x3) && imm) 905 emitc(rvc_addi4spn(rd, imm), ctx); 906 else if (rvc_enabled() && rd && rd == rs && imm && is_6b_int(imm)) 907 emitc(rvc_addi(rd, imm), ctx); 908 else 909 emit(rv_addi(rd, rs, imm), ctx); 910 } 911 912 static inline void emit_li(u8 rd, s32 imm, struct rv_jit_context *ctx) 913 { 914 if (rvc_enabled() && rd && is_6b_int(imm)) 915 emitc(rvc_li(rd, imm), ctx); 916 else 917 emit(rv_addi(rd, RV_REG_ZERO, imm), ctx); 918 } 919 920 static inline void emit_lui(u8 rd, s32 imm, struct rv_jit_context *ctx) 921 { 922 if (rvc_enabled() && rd && rd != RV_REG_SP && is_6b_int(imm) && imm) 923 emitc(rvc_lui(rd, imm), ctx); 924 else 925 emit(rv_lui(rd, imm), ctx); 926 } 927 928 static inline void emit_slli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 929 { 930 if (rvc_enabled() && rd && rd == rs && imm && (u32)imm < __riscv_xlen) 931 emitc(rvc_slli(rd, imm), ctx); 932 else 933 emit(rv_slli(rd, rs, imm), ctx); 934 } 935 936 static inline void emit_andi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 937 { 938 if (rvc_enabled() && is_creg(rd) && rd == rs && is_6b_int(imm)) 939 emitc(rvc_andi(rd, imm), ctx); 940 else 941 emit(rv_andi(rd, rs, imm), ctx); 942 } 943 944 static inline void emit_srli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 945 { 946 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen) 947 emitc(rvc_srli(rd, imm), ctx); 948 else 949 emit(rv_srli(rd, rs, imm), ctx); 950 } 951 952 static inline void emit_srai(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 953 { 954 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen) 955 emitc(rvc_srai(rd, imm), ctx); 956 else 957 emit(rv_srai(rd, rs, imm), ctx); 958 } 959 960 static inline void emit_sub(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 961 { 962 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 963 emitc(rvc_sub(rd, rs2), ctx); 964 else 965 emit(rv_sub(rd, rs1, rs2), ctx); 966 } 967 968 static inline void emit_or(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 969 { 970 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 971 emitc(rvc_or(rd, rs2), ctx); 972 else 973 emit(rv_or(rd, rs1, rs2), ctx); 974 } 975 976 static inline void emit_and(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 977 { 978 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 979 emitc(rvc_and(rd, rs2), ctx); 980 else 981 emit(rv_and(rd, rs1, rs2), ctx); 982 } 983 984 static inline void emit_xor(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 985 { 986 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 987 emitc(rvc_xor(rd, rs2), ctx); 988 else 989 emit(rv_xor(rd, rs1, rs2), ctx); 990 } 991 992 static inline void emit_lw(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx) 993 { 994 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_8b_uint(off) && !(off & 0x3)) 995 emitc(rvc_lwsp(rd, off), ctx); 996 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_7b_uint(off) && !(off & 0x3)) 997 emitc(rvc_lw(rd, off, rs1), ctx); 998 else 999 emit(rv_lw(rd, off, rs1), ctx); 1000 } 1001 1002 static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) 1003 { 1004 if (rvc_enabled() && rs1 == RV_REG_SP && is_8b_uint(off) && !(off & 0x3)) 1005 emitc(rvc_swsp(off, rs2), ctx); 1006 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_7b_uint(off) && !(off & 0x3)) 1007 emitc(rvc_sw(rs1, off, rs2), ctx); 1008 else 1009 emit(rv_sw(rs1, off, rs2), ctx); 1010 } 1011 1012 /* RV64-only helper functions. */ 1013 #if __riscv_xlen == 64 1014 1015 static inline void emit_addiw(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 1016 { 1017 if (rvc_enabled() && rd && rd == rs && is_6b_int(imm)) 1018 emitc(rvc_addiw(rd, imm), ctx); 1019 else 1020 emit(rv_addiw(rd, rs, imm), ctx); 1021 } 1022 1023 static inline void emit_ld(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx) 1024 { 1025 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_9b_uint(off) && !(off & 0x7)) 1026 emitc(rvc_ldsp(rd, off), ctx); 1027 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_8b_uint(off) && !(off & 0x7)) 1028 emitc(rvc_ld(rd, off, rs1), ctx); 1029 else 1030 emit(rv_ld(rd, off, rs1), ctx); 1031 } 1032 1033 static inline void emit_sd(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) 1034 { 1035 if (rvc_enabled() && rs1 == RV_REG_SP && is_9b_uint(off) && !(off & 0x7)) 1036 emitc(rvc_sdsp(off, rs2), ctx); 1037 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_8b_uint(off) && !(off & 0x7)) 1038 emitc(rvc_sd(rs1, off, rs2), ctx); 1039 else 1040 emit(rv_sd(rs1, off, rs2), ctx); 1041 } 1042 1043 static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1044 { 1045 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1046 emitc(rvc_subw(rd, rs2), ctx); 1047 else 1048 emit(rv_subw(rd, rs1, rs2), ctx); 1049 } 1050 1051 #endif /* __riscv_xlen == 64 */ 1052 1053 void bpf_jit_build_prologue(struct rv_jit_context *ctx); 1054 void bpf_jit_build_epilogue(struct rv_jit_context *ctx); 1055 1056 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 1057 bool extra_pass); 1058 1059 #endif /* _BPF_JIT_H */ 1060