1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Common functionality for RV32 and RV64 BPF JIT compilers 4 * 5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com> 6 * 7 */ 8 9 #ifndef _BPF_JIT_H 10 #define _BPF_JIT_H 11 12 #include <linux/bpf.h> 13 #include <linux/filter.h> 14 #include <asm/cacheflush.h> 15 16 static inline bool rvc_enabled(void) 17 { 18 return IS_ENABLED(CONFIG_RISCV_ISA_C); 19 } 20 21 enum { 22 RV_REG_ZERO = 0, /* The constant value 0 */ 23 RV_REG_RA = 1, /* Return address */ 24 RV_REG_SP = 2, /* Stack pointer */ 25 RV_REG_GP = 3, /* Global pointer */ 26 RV_REG_TP = 4, /* Thread pointer */ 27 RV_REG_T0 = 5, /* Temporaries */ 28 RV_REG_T1 = 6, 29 RV_REG_T2 = 7, 30 RV_REG_FP = 8, /* Saved register/frame pointer */ 31 RV_REG_S1 = 9, /* Saved register */ 32 RV_REG_A0 = 10, /* Function argument/return values */ 33 RV_REG_A1 = 11, /* Function arguments */ 34 RV_REG_A2 = 12, 35 RV_REG_A3 = 13, 36 RV_REG_A4 = 14, 37 RV_REG_A5 = 15, 38 RV_REG_A6 = 16, 39 RV_REG_A7 = 17, 40 RV_REG_S2 = 18, /* Saved registers */ 41 RV_REG_S3 = 19, 42 RV_REG_S4 = 20, 43 RV_REG_S5 = 21, 44 RV_REG_S6 = 22, 45 RV_REG_S7 = 23, 46 RV_REG_S8 = 24, 47 RV_REG_S9 = 25, 48 RV_REG_S10 = 26, 49 RV_REG_S11 = 27, 50 RV_REG_T3 = 28, /* Temporaries */ 51 RV_REG_T4 = 29, 52 RV_REG_T5 = 30, 53 RV_REG_T6 = 31, 54 }; 55 56 static inline bool is_creg(u8 reg) 57 { 58 return (1 << reg) & (BIT(RV_REG_FP) | 59 BIT(RV_REG_S1) | 60 BIT(RV_REG_A0) | 61 BIT(RV_REG_A1) | 62 BIT(RV_REG_A2) | 63 BIT(RV_REG_A3) | 64 BIT(RV_REG_A4) | 65 BIT(RV_REG_A5)); 66 } 67 68 struct rv_jit_context { 69 struct bpf_prog *prog; 70 u16 *insns; /* RV insns */ 71 int ninsns; 72 int epilogue_offset; 73 int *offset; /* BPF to RV */ 74 unsigned long flags; 75 int stack_size; 76 }; 77 78 /* Convert from ninsns to bytes. */ 79 static inline int ninsns_rvoff(int ninsns) 80 { 81 return ninsns << 1; 82 } 83 84 struct rv_jit_data { 85 struct bpf_binary_header *header; 86 u8 *image; 87 struct rv_jit_context ctx; 88 }; 89 90 static inline void bpf_fill_ill_insns(void *area, unsigned int size) 91 { 92 memset(area, 0, size); 93 } 94 95 static inline void bpf_flush_icache(void *start, void *end) 96 { 97 flush_icache_range((unsigned long)start, (unsigned long)end); 98 } 99 100 /* Emit a 4-byte riscv instruction. */ 101 static inline void emit(const u32 insn, struct rv_jit_context *ctx) 102 { 103 if (ctx->insns) { 104 ctx->insns[ctx->ninsns] = insn; 105 ctx->insns[ctx->ninsns + 1] = (insn >> 16); 106 } 107 108 ctx->ninsns += 2; 109 } 110 111 /* Emit a 2-byte riscv compressed instruction. */ 112 static inline void emitc(const u16 insn, struct rv_jit_context *ctx) 113 { 114 BUILD_BUG_ON(!rvc_enabled()); 115 116 if (ctx->insns) 117 ctx->insns[ctx->ninsns] = insn; 118 119 ctx->ninsns++; 120 } 121 122 static inline int epilogue_offset(struct rv_jit_context *ctx) 123 { 124 int to = ctx->epilogue_offset, from = ctx->ninsns; 125 126 return ninsns_rvoff(to - from); 127 } 128 129 /* Return -1 or inverted cond. */ 130 static inline int invert_bpf_cond(u8 cond) 131 { 132 switch (cond) { 133 case BPF_JEQ: 134 return BPF_JNE; 135 case BPF_JGT: 136 return BPF_JLE; 137 case BPF_JLT: 138 return BPF_JGE; 139 case BPF_JGE: 140 return BPF_JLT; 141 case BPF_JLE: 142 return BPF_JGT; 143 case BPF_JNE: 144 return BPF_JEQ; 145 case BPF_JSGT: 146 return BPF_JSLE; 147 case BPF_JSLT: 148 return BPF_JSGE; 149 case BPF_JSGE: 150 return BPF_JSLT; 151 case BPF_JSLE: 152 return BPF_JSGT; 153 } 154 return -1; 155 } 156 157 static inline bool is_6b_int(long val) 158 { 159 return -(1L << 5) <= val && val < (1L << 5); 160 } 161 162 static inline bool is_7b_uint(unsigned long val) 163 { 164 return val < (1UL << 7); 165 } 166 167 static inline bool is_8b_uint(unsigned long val) 168 { 169 return val < (1UL << 8); 170 } 171 172 static inline bool is_9b_uint(unsigned long val) 173 { 174 return val < (1UL << 9); 175 } 176 177 static inline bool is_10b_int(long val) 178 { 179 return -(1L << 9) <= val && val < (1L << 9); 180 } 181 182 static inline bool is_10b_uint(unsigned long val) 183 { 184 return val < (1UL << 10); 185 } 186 187 static inline bool is_12b_int(long val) 188 { 189 return -(1L << 11) <= val && val < (1L << 11); 190 } 191 192 static inline int is_12b_check(int off, int insn) 193 { 194 if (!is_12b_int(off)) { 195 pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n", 196 insn, (int)off); 197 return -1; 198 } 199 return 0; 200 } 201 202 static inline bool is_13b_int(long val) 203 { 204 return -(1L << 12) <= val && val < (1L << 12); 205 } 206 207 static inline bool is_21b_int(long val) 208 { 209 return -(1L << 20) <= val && val < (1L << 20); 210 } 211 212 static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx) 213 { 214 int from, to; 215 216 off++; /* BPF branch is from PC+1, RV is from PC */ 217 from = (insn > 0) ? ctx->offset[insn - 1] : 0; 218 to = (insn + off > 0) ? ctx->offset[insn + off - 1] : 0; 219 return ninsns_rvoff(to - from); 220 } 221 222 /* Instruction formats. */ 223 224 static inline u32 rv_r_insn(u8 funct7, u8 rs2, u8 rs1, u8 funct3, u8 rd, 225 u8 opcode) 226 { 227 return (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 228 (rd << 7) | opcode; 229 } 230 231 static inline u32 rv_i_insn(u16 imm11_0, u8 rs1, u8 funct3, u8 rd, u8 opcode) 232 { 233 return (imm11_0 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) | 234 opcode; 235 } 236 237 static inline u32 rv_s_insn(u16 imm11_0, u8 rs2, u8 rs1, u8 funct3, u8 opcode) 238 { 239 u8 imm11_5 = imm11_0 >> 5, imm4_0 = imm11_0 & 0x1f; 240 241 return (imm11_5 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 242 (imm4_0 << 7) | opcode; 243 } 244 245 static inline u32 rv_b_insn(u16 imm12_1, u8 rs2, u8 rs1, u8 funct3, u8 opcode) 246 { 247 u8 imm12 = ((imm12_1 & 0x800) >> 5) | ((imm12_1 & 0x3f0) >> 4); 248 u8 imm4_1 = ((imm12_1 & 0xf) << 1) | ((imm12_1 & 0x400) >> 10); 249 250 return (imm12 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 251 (imm4_1 << 7) | opcode; 252 } 253 254 static inline u32 rv_u_insn(u32 imm31_12, u8 rd, u8 opcode) 255 { 256 return (imm31_12 << 12) | (rd << 7) | opcode; 257 } 258 259 static inline u32 rv_j_insn(u32 imm20_1, u8 rd, u8 opcode) 260 { 261 u32 imm; 262 263 imm = (imm20_1 & 0x80000) | ((imm20_1 & 0x3ff) << 9) | 264 ((imm20_1 & 0x400) >> 2) | ((imm20_1 & 0x7f800) >> 11); 265 266 return (imm << 12) | (rd << 7) | opcode; 267 } 268 269 static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1, 270 u8 funct3, u8 rd, u8 opcode) 271 { 272 u8 funct7 = (funct5 << 2) | (aq << 1) | rl; 273 274 return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode); 275 } 276 277 /* RISC-V compressed instruction formats. */ 278 279 static inline u16 rv_cr_insn(u8 funct4, u8 rd, u8 rs2, u8 op) 280 { 281 return (funct4 << 12) | (rd << 7) | (rs2 << 2) | op; 282 } 283 284 static inline u16 rv_ci_insn(u8 funct3, u32 imm6, u8 rd, u8 op) 285 { 286 u32 imm; 287 288 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2); 289 return (funct3 << 13) | (rd << 7) | op | imm; 290 } 291 292 static inline u16 rv_css_insn(u8 funct3, u32 uimm, u8 rs2, u8 op) 293 { 294 return (funct3 << 13) | (uimm << 7) | (rs2 << 2) | op; 295 } 296 297 static inline u16 rv_ciw_insn(u8 funct3, u32 uimm, u8 rd, u8 op) 298 { 299 return (funct3 << 13) | (uimm << 5) | ((rd & 0x7) << 2) | op; 300 } 301 302 static inline u16 rv_cl_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rd, 303 u8 op) 304 { 305 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) | 306 (imm_lo << 5) | ((rd & 0x7) << 2) | op; 307 } 308 309 static inline u16 rv_cs_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rs2, 310 u8 op) 311 { 312 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) | 313 (imm_lo << 5) | ((rs2 & 0x7) << 2) | op; 314 } 315 316 static inline u16 rv_ca_insn(u8 funct6, u8 rd, u8 funct2, u8 rs2, u8 op) 317 { 318 return (funct6 << 10) | ((rd & 0x7) << 7) | (funct2 << 5) | 319 ((rs2 & 0x7) << 2) | op; 320 } 321 322 static inline u16 rv_cb_insn(u8 funct3, u32 imm6, u8 funct2, u8 rd, u8 op) 323 { 324 u32 imm; 325 326 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2); 327 return (funct3 << 13) | (funct2 << 10) | ((rd & 0x7) << 7) | op | imm; 328 } 329 330 /* Instructions shared by both RV32 and RV64. */ 331 332 static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0) 333 { 334 return rv_i_insn(imm11_0, rs1, 0, rd, 0x13); 335 } 336 337 static inline u32 rv_andi(u8 rd, u8 rs1, u16 imm11_0) 338 { 339 return rv_i_insn(imm11_0, rs1, 7, rd, 0x13); 340 } 341 342 static inline u32 rv_ori(u8 rd, u8 rs1, u16 imm11_0) 343 { 344 return rv_i_insn(imm11_0, rs1, 6, rd, 0x13); 345 } 346 347 static inline u32 rv_xori(u8 rd, u8 rs1, u16 imm11_0) 348 { 349 return rv_i_insn(imm11_0, rs1, 4, rd, 0x13); 350 } 351 352 static inline u32 rv_slli(u8 rd, u8 rs1, u16 imm11_0) 353 { 354 return rv_i_insn(imm11_0, rs1, 1, rd, 0x13); 355 } 356 357 static inline u32 rv_srli(u8 rd, u8 rs1, u16 imm11_0) 358 { 359 return rv_i_insn(imm11_0, rs1, 5, rd, 0x13); 360 } 361 362 static inline u32 rv_srai(u8 rd, u8 rs1, u16 imm11_0) 363 { 364 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x13); 365 } 366 367 static inline u32 rv_lui(u8 rd, u32 imm31_12) 368 { 369 return rv_u_insn(imm31_12, rd, 0x37); 370 } 371 372 static inline u32 rv_auipc(u8 rd, u32 imm31_12) 373 { 374 return rv_u_insn(imm31_12, rd, 0x17); 375 } 376 377 static inline u32 rv_add(u8 rd, u8 rs1, u8 rs2) 378 { 379 return rv_r_insn(0, rs2, rs1, 0, rd, 0x33); 380 } 381 382 static inline u32 rv_sub(u8 rd, u8 rs1, u8 rs2) 383 { 384 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x33); 385 } 386 387 static inline u32 rv_sltu(u8 rd, u8 rs1, u8 rs2) 388 { 389 return rv_r_insn(0, rs2, rs1, 3, rd, 0x33); 390 } 391 392 static inline u32 rv_and(u8 rd, u8 rs1, u8 rs2) 393 { 394 return rv_r_insn(0, rs2, rs1, 7, rd, 0x33); 395 } 396 397 static inline u32 rv_or(u8 rd, u8 rs1, u8 rs2) 398 { 399 return rv_r_insn(0, rs2, rs1, 6, rd, 0x33); 400 } 401 402 static inline u32 rv_xor(u8 rd, u8 rs1, u8 rs2) 403 { 404 return rv_r_insn(0, rs2, rs1, 4, rd, 0x33); 405 } 406 407 static inline u32 rv_sll(u8 rd, u8 rs1, u8 rs2) 408 { 409 return rv_r_insn(0, rs2, rs1, 1, rd, 0x33); 410 } 411 412 static inline u32 rv_srl(u8 rd, u8 rs1, u8 rs2) 413 { 414 return rv_r_insn(0, rs2, rs1, 5, rd, 0x33); 415 } 416 417 static inline u32 rv_sra(u8 rd, u8 rs1, u8 rs2) 418 { 419 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x33); 420 } 421 422 static inline u32 rv_mul(u8 rd, u8 rs1, u8 rs2) 423 { 424 return rv_r_insn(1, rs2, rs1, 0, rd, 0x33); 425 } 426 427 static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2) 428 { 429 return rv_r_insn(1, rs2, rs1, 3, rd, 0x33); 430 } 431 432 static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2) 433 { 434 return rv_r_insn(1, rs2, rs1, 5, rd, 0x33); 435 } 436 437 static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2) 438 { 439 return rv_r_insn(1, rs2, rs1, 7, rd, 0x33); 440 } 441 442 static inline u32 rv_jal(u8 rd, u32 imm20_1) 443 { 444 return rv_j_insn(imm20_1, rd, 0x6f); 445 } 446 447 static inline u32 rv_jalr(u8 rd, u8 rs1, u16 imm11_0) 448 { 449 return rv_i_insn(imm11_0, rs1, 0, rd, 0x67); 450 } 451 452 static inline u32 rv_beq(u8 rs1, u8 rs2, u16 imm12_1) 453 { 454 return rv_b_insn(imm12_1, rs2, rs1, 0, 0x63); 455 } 456 457 static inline u32 rv_bne(u8 rs1, u8 rs2, u16 imm12_1) 458 { 459 return rv_b_insn(imm12_1, rs2, rs1, 1, 0x63); 460 } 461 462 static inline u32 rv_bltu(u8 rs1, u8 rs2, u16 imm12_1) 463 { 464 return rv_b_insn(imm12_1, rs2, rs1, 6, 0x63); 465 } 466 467 static inline u32 rv_bgtu(u8 rs1, u8 rs2, u16 imm12_1) 468 { 469 return rv_bltu(rs2, rs1, imm12_1); 470 } 471 472 static inline u32 rv_bgeu(u8 rs1, u8 rs2, u16 imm12_1) 473 { 474 return rv_b_insn(imm12_1, rs2, rs1, 7, 0x63); 475 } 476 477 static inline u32 rv_bleu(u8 rs1, u8 rs2, u16 imm12_1) 478 { 479 return rv_bgeu(rs2, rs1, imm12_1); 480 } 481 482 static inline u32 rv_blt(u8 rs1, u8 rs2, u16 imm12_1) 483 { 484 return rv_b_insn(imm12_1, rs2, rs1, 4, 0x63); 485 } 486 487 static inline u32 rv_bgt(u8 rs1, u8 rs2, u16 imm12_1) 488 { 489 return rv_blt(rs2, rs1, imm12_1); 490 } 491 492 static inline u32 rv_bge(u8 rs1, u8 rs2, u16 imm12_1) 493 { 494 return rv_b_insn(imm12_1, rs2, rs1, 5, 0x63); 495 } 496 497 static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1) 498 { 499 return rv_bge(rs2, rs1, imm12_1); 500 } 501 502 static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1) 503 { 504 return rv_i_insn(imm11_0, rs1, 2, rd, 0x03); 505 } 506 507 static inline u32 rv_lbu(u8 rd, u16 imm11_0, u8 rs1) 508 { 509 return rv_i_insn(imm11_0, rs1, 4, rd, 0x03); 510 } 511 512 static inline u32 rv_lhu(u8 rd, u16 imm11_0, u8 rs1) 513 { 514 return rv_i_insn(imm11_0, rs1, 5, rd, 0x03); 515 } 516 517 static inline u32 rv_sb(u8 rs1, u16 imm11_0, u8 rs2) 518 { 519 return rv_s_insn(imm11_0, rs2, rs1, 0, 0x23); 520 } 521 522 static inline u32 rv_sh(u8 rs1, u16 imm11_0, u8 rs2) 523 { 524 return rv_s_insn(imm11_0, rs2, rs1, 1, 0x23); 525 } 526 527 static inline u32 rv_sw(u8 rs1, u16 imm11_0, u8 rs2) 528 { 529 return rv_s_insn(imm11_0, rs2, rs1, 2, 0x23); 530 } 531 532 static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 533 { 534 return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f); 535 } 536 537 /* RVC instrutions. */ 538 539 static inline u16 rvc_addi4spn(u8 rd, u32 imm10) 540 { 541 u32 imm; 542 543 imm = ((imm10 & 0x30) << 2) | ((imm10 & 0x3c0) >> 4) | 544 ((imm10 & 0x4) >> 1) | ((imm10 & 0x8) >> 3); 545 return rv_ciw_insn(0x0, imm, rd, 0x0); 546 } 547 548 static inline u16 rvc_lw(u8 rd, u32 imm7, u8 rs1) 549 { 550 u32 imm_hi, imm_lo; 551 552 imm_hi = (imm7 & 0x38) >> 3; 553 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6); 554 return rv_cl_insn(0x2, imm_hi, rs1, imm_lo, rd, 0x0); 555 } 556 557 static inline u16 rvc_sw(u8 rs1, u32 imm7, u8 rs2) 558 { 559 u32 imm_hi, imm_lo; 560 561 imm_hi = (imm7 & 0x38) >> 3; 562 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6); 563 return rv_cs_insn(0x6, imm_hi, rs1, imm_lo, rs2, 0x0); 564 } 565 566 static inline u16 rvc_addi(u8 rd, u32 imm6) 567 { 568 return rv_ci_insn(0, imm6, rd, 0x1); 569 } 570 571 static inline u16 rvc_li(u8 rd, u32 imm6) 572 { 573 return rv_ci_insn(0x2, imm6, rd, 0x1); 574 } 575 576 static inline u16 rvc_addi16sp(u32 imm10) 577 { 578 u32 imm; 579 580 imm = ((imm10 & 0x200) >> 4) | (imm10 & 0x10) | ((imm10 & 0x40) >> 3) | 581 ((imm10 & 0x180) >> 6) | ((imm10 & 0x20) >> 5); 582 return rv_ci_insn(0x3, imm, RV_REG_SP, 0x1); 583 } 584 585 static inline u16 rvc_lui(u8 rd, u32 imm6) 586 { 587 return rv_ci_insn(0x3, imm6, rd, 0x1); 588 } 589 590 static inline u16 rvc_srli(u8 rd, u32 imm6) 591 { 592 return rv_cb_insn(0x4, imm6, 0, rd, 0x1); 593 } 594 595 static inline u16 rvc_srai(u8 rd, u32 imm6) 596 { 597 return rv_cb_insn(0x4, imm6, 0x1, rd, 0x1); 598 } 599 600 static inline u16 rvc_andi(u8 rd, u32 imm6) 601 { 602 return rv_cb_insn(0x4, imm6, 0x2, rd, 0x1); 603 } 604 605 static inline u16 rvc_sub(u8 rd, u8 rs) 606 { 607 return rv_ca_insn(0x23, rd, 0, rs, 0x1); 608 } 609 610 static inline u16 rvc_xor(u8 rd, u8 rs) 611 { 612 return rv_ca_insn(0x23, rd, 0x1, rs, 0x1); 613 } 614 615 static inline u16 rvc_or(u8 rd, u8 rs) 616 { 617 return rv_ca_insn(0x23, rd, 0x2, rs, 0x1); 618 } 619 620 static inline u16 rvc_and(u8 rd, u8 rs) 621 { 622 return rv_ca_insn(0x23, rd, 0x3, rs, 0x1); 623 } 624 625 static inline u16 rvc_slli(u8 rd, u32 imm6) 626 { 627 return rv_ci_insn(0, imm6, rd, 0x2); 628 } 629 630 static inline u16 rvc_lwsp(u8 rd, u32 imm8) 631 { 632 u32 imm; 633 634 imm = ((imm8 & 0xc0) >> 6) | (imm8 & 0x3c); 635 return rv_ci_insn(0x2, imm, rd, 0x2); 636 } 637 638 static inline u16 rvc_jr(u8 rs1) 639 { 640 return rv_cr_insn(0x8, rs1, RV_REG_ZERO, 0x2); 641 } 642 643 static inline u16 rvc_mv(u8 rd, u8 rs) 644 { 645 return rv_cr_insn(0x8, rd, rs, 0x2); 646 } 647 648 static inline u16 rvc_jalr(u8 rs1) 649 { 650 return rv_cr_insn(0x9, rs1, RV_REG_ZERO, 0x2); 651 } 652 653 static inline u16 rvc_add(u8 rd, u8 rs) 654 { 655 return rv_cr_insn(0x9, rd, rs, 0x2); 656 } 657 658 static inline u16 rvc_swsp(u32 imm8, u8 rs2) 659 { 660 u32 imm; 661 662 imm = (imm8 & 0x3c) | ((imm8 & 0xc0) >> 6); 663 return rv_css_insn(0x6, imm, rs2, 0x2); 664 } 665 666 /* 667 * RV64-only instructions. 668 * 669 * These instructions are not available on RV32. Wrap them below a #if to 670 * ensure that the RV32 JIT doesn't emit any of these instructions. 671 */ 672 673 #if __riscv_xlen == 64 674 675 static inline u32 rv_addiw(u8 rd, u8 rs1, u16 imm11_0) 676 { 677 return rv_i_insn(imm11_0, rs1, 0, rd, 0x1b); 678 } 679 680 static inline u32 rv_slliw(u8 rd, u8 rs1, u16 imm11_0) 681 { 682 return rv_i_insn(imm11_0, rs1, 1, rd, 0x1b); 683 } 684 685 static inline u32 rv_srliw(u8 rd, u8 rs1, u16 imm11_0) 686 { 687 return rv_i_insn(imm11_0, rs1, 5, rd, 0x1b); 688 } 689 690 static inline u32 rv_sraiw(u8 rd, u8 rs1, u16 imm11_0) 691 { 692 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x1b); 693 } 694 695 static inline u32 rv_addw(u8 rd, u8 rs1, u8 rs2) 696 { 697 return rv_r_insn(0, rs2, rs1, 0, rd, 0x3b); 698 } 699 700 static inline u32 rv_subw(u8 rd, u8 rs1, u8 rs2) 701 { 702 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x3b); 703 } 704 705 static inline u32 rv_sllw(u8 rd, u8 rs1, u8 rs2) 706 { 707 return rv_r_insn(0, rs2, rs1, 1, rd, 0x3b); 708 } 709 710 static inline u32 rv_srlw(u8 rd, u8 rs1, u8 rs2) 711 { 712 return rv_r_insn(0, rs2, rs1, 5, rd, 0x3b); 713 } 714 715 static inline u32 rv_sraw(u8 rd, u8 rs1, u8 rs2) 716 { 717 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x3b); 718 } 719 720 static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2) 721 { 722 return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b); 723 } 724 725 static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2) 726 { 727 return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b); 728 } 729 730 static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2) 731 { 732 return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b); 733 } 734 735 static inline u32 rv_ld(u8 rd, u16 imm11_0, u8 rs1) 736 { 737 return rv_i_insn(imm11_0, rs1, 3, rd, 0x03); 738 } 739 740 static inline u32 rv_lwu(u8 rd, u16 imm11_0, u8 rs1) 741 { 742 return rv_i_insn(imm11_0, rs1, 6, rd, 0x03); 743 } 744 745 static inline u32 rv_sd(u8 rs1, u16 imm11_0, u8 rs2) 746 { 747 return rv_s_insn(imm11_0, rs2, rs1, 3, 0x23); 748 } 749 750 static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 751 { 752 return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f); 753 } 754 755 /* RV64-only RVC instructions. */ 756 757 static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1) 758 { 759 u32 imm_hi, imm_lo; 760 761 imm_hi = (imm8 & 0x38) >> 3; 762 imm_lo = (imm8 & 0xc0) >> 6; 763 return rv_cl_insn(0x3, imm_hi, rs1, imm_lo, rd, 0x0); 764 } 765 766 static inline u16 rvc_sd(u8 rs1, u32 imm8, u8 rs2) 767 { 768 u32 imm_hi, imm_lo; 769 770 imm_hi = (imm8 & 0x38) >> 3; 771 imm_lo = (imm8 & 0xc0) >> 6; 772 return rv_cs_insn(0x7, imm_hi, rs1, imm_lo, rs2, 0x0); 773 } 774 775 static inline u16 rvc_subw(u8 rd, u8 rs) 776 { 777 return rv_ca_insn(0x27, rd, 0, rs, 0x1); 778 } 779 780 static inline u16 rvc_addiw(u8 rd, u32 imm6) 781 { 782 return rv_ci_insn(0x1, imm6, rd, 0x1); 783 } 784 785 static inline u16 rvc_ldsp(u8 rd, u32 imm9) 786 { 787 u32 imm; 788 789 imm = ((imm9 & 0x1c0) >> 6) | (imm9 & 0x38); 790 return rv_ci_insn(0x3, imm, rd, 0x2); 791 } 792 793 static inline u16 rvc_sdsp(u32 imm9, u8 rs2) 794 { 795 u32 imm; 796 797 imm = (imm9 & 0x38) | ((imm9 & 0x1c0) >> 6); 798 return rv_css_insn(0x7, imm, rs2, 0x2); 799 } 800 801 #endif /* __riscv_xlen == 64 */ 802 803 /* Helper functions that emit RVC instructions when possible. */ 804 805 static inline void emit_jalr(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 806 { 807 if (rvc_enabled() && rd == RV_REG_RA && rs && !imm) 808 emitc(rvc_jalr(rs), ctx); 809 else if (rvc_enabled() && !rd && rs && !imm) 810 emitc(rvc_jr(rs), ctx); 811 else 812 emit(rv_jalr(rd, rs, imm), ctx); 813 } 814 815 static inline void emit_mv(u8 rd, u8 rs, struct rv_jit_context *ctx) 816 { 817 if (rvc_enabled() && rd && rs) 818 emitc(rvc_mv(rd, rs), ctx); 819 else 820 emit(rv_addi(rd, rs, 0), ctx); 821 } 822 823 static inline void emit_add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 824 { 825 if (rvc_enabled() && rd && rd == rs1 && rs2) 826 emitc(rvc_add(rd, rs2), ctx); 827 else 828 emit(rv_add(rd, rs1, rs2), ctx); 829 } 830 831 static inline void emit_addi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 832 { 833 if (rvc_enabled() && rd == RV_REG_SP && rd == rs && is_10b_int(imm) && imm && !(imm & 0xf)) 834 emitc(rvc_addi16sp(imm), ctx); 835 else if (rvc_enabled() && is_creg(rd) && rs == RV_REG_SP && is_10b_uint(imm) && 836 !(imm & 0x3) && imm) 837 emitc(rvc_addi4spn(rd, imm), ctx); 838 else if (rvc_enabled() && rd && rd == rs && imm && is_6b_int(imm)) 839 emitc(rvc_addi(rd, imm), ctx); 840 else 841 emit(rv_addi(rd, rs, imm), ctx); 842 } 843 844 static inline void emit_li(u8 rd, s32 imm, struct rv_jit_context *ctx) 845 { 846 if (rvc_enabled() && rd && is_6b_int(imm)) 847 emitc(rvc_li(rd, imm), ctx); 848 else 849 emit(rv_addi(rd, RV_REG_ZERO, imm), ctx); 850 } 851 852 static inline void emit_lui(u8 rd, s32 imm, struct rv_jit_context *ctx) 853 { 854 if (rvc_enabled() && rd && rd != RV_REG_SP && is_6b_int(imm) && imm) 855 emitc(rvc_lui(rd, imm), ctx); 856 else 857 emit(rv_lui(rd, imm), ctx); 858 } 859 860 static inline void emit_slli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 861 { 862 if (rvc_enabled() && rd && rd == rs && imm && (u32)imm < __riscv_xlen) 863 emitc(rvc_slli(rd, imm), ctx); 864 else 865 emit(rv_slli(rd, rs, imm), ctx); 866 } 867 868 static inline void emit_andi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 869 { 870 if (rvc_enabled() && is_creg(rd) && rd == rs && is_6b_int(imm)) 871 emitc(rvc_andi(rd, imm), ctx); 872 else 873 emit(rv_andi(rd, rs, imm), ctx); 874 } 875 876 static inline void emit_srli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 877 { 878 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen) 879 emitc(rvc_srli(rd, imm), ctx); 880 else 881 emit(rv_srli(rd, rs, imm), ctx); 882 } 883 884 static inline void emit_srai(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 885 { 886 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen) 887 emitc(rvc_srai(rd, imm), ctx); 888 else 889 emit(rv_srai(rd, rs, imm), ctx); 890 } 891 892 static inline void emit_sub(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 893 { 894 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 895 emitc(rvc_sub(rd, rs2), ctx); 896 else 897 emit(rv_sub(rd, rs1, rs2), ctx); 898 } 899 900 static inline void emit_or(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 901 { 902 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 903 emitc(rvc_or(rd, rs2), ctx); 904 else 905 emit(rv_or(rd, rs1, rs2), ctx); 906 } 907 908 static inline void emit_and(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 909 { 910 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 911 emitc(rvc_and(rd, rs2), ctx); 912 else 913 emit(rv_and(rd, rs1, rs2), ctx); 914 } 915 916 static inline void emit_xor(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 917 { 918 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 919 emitc(rvc_xor(rd, rs2), ctx); 920 else 921 emit(rv_xor(rd, rs1, rs2), ctx); 922 } 923 924 static inline void emit_lw(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx) 925 { 926 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_8b_uint(off) && !(off & 0x3)) 927 emitc(rvc_lwsp(rd, off), ctx); 928 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_7b_uint(off) && !(off & 0x3)) 929 emitc(rvc_lw(rd, off, rs1), ctx); 930 else 931 emit(rv_lw(rd, off, rs1), ctx); 932 } 933 934 static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) 935 { 936 if (rvc_enabled() && rs1 == RV_REG_SP && is_8b_uint(off) && !(off & 0x3)) 937 emitc(rvc_swsp(off, rs2), ctx); 938 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_7b_uint(off) && !(off & 0x3)) 939 emitc(rvc_sw(rs1, off, rs2), ctx); 940 else 941 emit(rv_sw(rs1, off, rs2), ctx); 942 } 943 944 /* RV64-only helper functions. */ 945 #if __riscv_xlen == 64 946 947 static inline void emit_addiw(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 948 { 949 if (rvc_enabled() && rd && rd == rs && is_6b_int(imm)) 950 emitc(rvc_addiw(rd, imm), ctx); 951 else 952 emit(rv_addiw(rd, rs, imm), ctx); 953 } 954 955 static inline void emit_ld(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx) 956 { 957 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_9b_uint(off) && !(off & 0x7)) 958 emitc(rvc_ldsp(rd, off), ctx); 959 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_8b_uint(off) && !(off & 0x7)) 960 emitc(rvc_ld(rd, off, rs1), ctx); 961 else 962 emit(rv_ld(rd, off, rs1), ctx); 963 } 964 965 static inline void emit_sd(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) 966 { 967 if (rvc_enabled() && rs1 == RV_REG_SP && is_9b_uint(off) && !(off & 0x7)) 968 emitc(rvc_sdsp(off, rs2), ctx); 969 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_8b_uint(off) && !(off & 0x7)) 970 emitc(rvc_sd(rs1, off, rs2), ctx); 971 else 972 emit(rv_sd(rs1, off, rs2), ctx); 973 } 974 975 static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 976 { 977 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 978 emitc(rvc_subw(rd, rs2), ctx); 979 else 980 emit(rv_subw(rd, rs1, rs2), ctx); 981 } 982 983 #endif /* __riscv_xlen == 64 */ 984 985 void bpf_jit_build_prologue(struct rv_jit_context *ctx); 986 void bpf_jit_build_epilogue(struct rv_jit_context *ctx); 987 988 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 989 bool extra_pass); 990 991 #endif /* _BPF_JIT_H */ 992