1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Common functionality for RV32 and RV64 BPF JIT compilers 4 * 5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com> 6 * 7 */ 8 9 #ifndef _BPF_JIT_H 10 #define _BPF_JIT_H 11 12 #include <linux/bpf.h> 13 #include <linux/filter.h> 14 #include <asm/cacheflush.h> 15 16 enum { 17 RV_REG_ZERO = 0, /* The constant value 0 */ 18 RV_REG_RA = 1, /* Return address */ 19 RV_REG_SP = 2, /* Stack pointer */ 20 RV_REG_GP = 3, /* Global pointer */ 21 RV_REG_TP = 4, /* Thread pointer */ 22 RV_REG_T0 = 5, /* Temporaries */ 23 RV_REG_T1 = 6, 24 RV_REG_T2 = 7, 25 RV_REG_FP = 8, /* Saved register/frame pointer */ 26 RV_REG_S1 = 9, /* Saved register */ 27 RV_REG_A0 = 10, /* Function argument/return values */ 28 RV_REG_A1 = 11, /* Function arguments */ 29 RV_REG_A2 = 12, 30 RV_REG_A3 = 13, 31 RV_REG_A4 = 14, 32 RV_REG_A5 = 15, 33 RV_REG_A6 = 16, 34 RV_REG_A7 = 17, 35 RV_REG_S2 = 18, /* Saved registers */ 36 RV_REG_S3 = 19, 37 RV_REG_S4 = 20, 38 RV_REG_S5 = 21, 39 RV_REG_S6 = 22, 40 RV_REG_S7 = 23, 41 RV_REG_S8 = 24, 42 RV_REG_S9 = 25, 43 RV_REG_S10 = 26, 44 RV_REG_S11 = 27, 45 RV_REG_T3 = 28, /* Temporaries */ 46 RV_REG_T4 = 29, 47 RV_REG_T5 = 30, 48 RV_REG_T6 = 31, 49 }; 50 51 struct rv_jit_context { 52 struct bpf_prog *prog; 53 u32 *insns; /* RV insns */ 54 int ninsns; 55 int epilogue_offset; 56 int *offset; /* BPF to RV */ 57 unsigned long flags; 58 int stack_size; 59 }; 60 61 struct rv_jit_data { 62 struct bpf_binary_header *header; 63 u8 *image; 64 struct rv_jit_context ctx; 65 }; 66 67 static inline void bpf_fill_ill_insns(void *area, unsigned int size) 68 { 69 memset(area, 0, size); 70 } 71 72 static inline void bpf_flush_icache(void *start, void *end) 73 { 74 flush_icache_range((unsigned long)start, (unsigned long)end); 75 } 76 77 static inline void emit(const u32 insn, struct rv_jit_context *ctx) 78 { 79 if (ctx->insns) 80 ctx->insns[ctx->ninsns] = insn; 81 82 ctx->ninsns++; 83 } 84 85 static inline int epilogue_offset(struct rv_jit_context *ctx) 86 { 87 int to = ctx->epilogue_offset, from = ctx->ninsns; 88 89 return (to - from) << 2; 90 } 91 92 /* Return -1 or inverted cond. */ 93 static inline int invert_bpf_cond(u8 cond) 94 { 95 switch (cond) { 96 case BPF_JEQ: 97 return BPF_JNE; 98 case BPF_JGT: 99 return BPF_JLE; 100 case BPF_JLT: 101 return BPF_JGE; 102 case BPF_JGE: 103 return BPF_JLT; 104 case BPF_JLE: 105 return BPF_JGT; 106 case BPF_JNE: 107 return BPF_JEQ; 108 case BPF_JSGT: 109 return BPF_JSLE; 110 case BPF_JSLT: 111 return BPF_JSGE; 112 case BPF_JSGE: 113 return BPF_JSLT; 114 case BPF_JSLE: 115 return BPF_JSGT; 116 } 117 return -1; 118 } 119 120 static inline bool is_12b_int(long val) 121 { 122 return -(1L << 11) <= val && val < (1L << 11); 123 } 124 125 static inline int is_12b_check(int off, int insn) 126 { 127 if (!is_12b_int(off)) { 128 pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n", 129 insn, (int)off); 130 return -1; 131 } 132 return 0; 133 } 134 135 static inline bool is_13b_int(long val) 136 { 137 return -(1L << 12) <= val && val < (1L << 12); 138 } 139 140 static inline bool is_21b_int(long val) 141 { 142 return -(1L << 20) <= val && val < (1L << 20); 143 } 144 145 static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx) 146 { 147 int from, to; 148 149 off++; /* BPF branch is from PC+1, RV is from PC */ 150 from = (insn > 0) ? ctx->offset[insn - 1] : 0; 151 to = (insn + off > 0) ? ctx->offset[insn + off - 1] : 0; 152 return (to - from) << 2; 153 } 154 155 /* Instruction formats. */ 156 157 static inline u32 rv_r_insn(u8 funct7, u8 rs2, u8 rs1, u8 funct3, u8 rd, 158 u8 opcode) 159 { 160 return (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 161 (rd << 7) | opcode; 162 } 163 164 static inline u32 rv_i_insn(u16 imm11_0, u8 rs1, u8 funct3, u8 rd, u8 opcode) 165 { 166 return (imm11_0 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) | 167 opcode; 168 } 169 170 static inline u32 rv_s_insn(u16 imm11_0, u8 rs2, u8 rs1, u8 funct3, u8 opcode) 171 { 172 u8 imm11_5 = imm11_0 >> 5, imm4_0 = imm11_0 & 0x1f; 173 174 return (imm11_5 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 175 (imm4_0 << 7) | opcode; 176 } 177 178 static inline u32 rv_b_insn(u16 imm12_1, u8 rs2, u8 rs1, u8 funct3, u8 opcode) 179 { 180 u8 imm12 = ((imm12_1 & 0x800) >> 5) | ((imm12_1 & 0x3f0) >> 4); 181 u8 imm4_1 = ((imm12_1 & 0xf) << 1) | ((imm12_1 & 0x400) >> 10); 182 183 return (imm12 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 184 (imm4_1 << 7) | opcode; 185 } 186 187 static inline u32 rv_u_insn(u32 imm31_12, u8 rd, u8 opcode) 188 { 189 return (imm31_12 << 12) | (rd << 7) | opcode; 190 } 191 192 static inline u32 rv_j_insn(u32 imm20_1, u8 rd, u8 opcode) 193 { 194 u32 imm; 195 196 imm = (imm20_1 & 0x80000) | ((imm20_1 & 0x3ff) << 9) | 197 ((imm20_1 & 0x400) >> 2) | ((imm20_1 & 0x7f800) >> 11); 198 199 return (imm << 12) | (rd << 7) | opcode; 200 } 201 202 static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1, 203 u8 funct3, u8 rd, u8 opcode) 204 { 205 u8 funct7 = (funct5 << 2) | (aq << 1) | rl; 206 207 return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode); 208 } 209 210 /* Instructions shared by both RV32 and RV64. */ 211 212 static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0) 213 { 214 return rv_i_insn(imm11_0, rs1, 0, rd, 0x13); 215 } 216 217 static inline u32 rv_andi(u8 rd, u8 rs1, u16 imm11_0) 218 { 219 return rv_i_insn(imm11_0, rs1, 7, rd, 0x13); 220 } 221 222 static inline u32 rv_ori(u8 rd, u8 rs1, u16 imm11_0) 223 { 224 return rv_i_insn(imm11_0, rs1, 6, rd, 0x13); 225 } 226 227 static inline u32 rv_xori(u8 rd, u8 rs1, u16 imm11_0) 228 { 229 return rv_i_insn(imm11_0, rs1, 4, rd, 0x13); 230 } 231 232 static inline u32 rv_slli(u8 rd, u8 rs1, u16 imm11_0) 233 { 234 return rv_i_insn(imm11_0, rs1, 1, rd, 0x13); 235 } 236 237 static inline u32 rv_srli(u8 rd, u8 rs1, u16 imm11_0) 238 { 239 return rv_i_insn(imm11_0, rs1, 5, rd, 0x13); 240 } 241 242 static inline u32 rv_srai(u8 rd, u8 rs1, u16 imm11_0) 243 { 244 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x13); 245 } 246 247 static inline u32 rv_lui(u8 rd, u32 imm31_12) 248 { 249 return rv_u_insn(imm31_12, rd, 0x37); 250 } 251 252 static inline u32 rv_auipc(u8 rd, u32 imm31_12) 253 { 254 return rv_u_insn(imm31_12, rd, 0x17); 255 } 256 257 static inline u32 rv_add(u8 rd, u8 rs1, u8 rs2) 258 { 259 return rv_r_insn(0, rs2, rs1, 0, rd, 0x33); 260 } 261 262 static inline u32 rv_sub(u8 rd, u8 rs1, u8 rs2) 263 { 264 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x33); 265 } 266 267 static inline u32 rv_sltu(u8 rd, u8 rs1, u8 rs2) 268 { 269 return rv_r_insn(0, rs2, rs1, 3, rd, 0x33); 270 } 271 272 static inline u32 rv_and(u8 rd, u8 rs1, u8 rs2) 273 { 274 return rv_r_insn(0, rs2, rs1, 7, rd, 0x33); 275 } 276 277 static inline u32 rv_or(u8 rd, u8 rs1, u8 rs2) 278 { 279 return rv_r_insn(0, rs2, rs1, 6, rd, 0x33); 280 } 281 282 static inline u32 rv_xor(u8 rd, u8 rs1, u8 rs2) 283 { 284 return rv_r_insn(0, rs2, rs1, 4, rd, 0x33); 285 } 286 287 static inline u32 rv_sll(u8 rd, u8 rs1, u8 rs2) 288 { 289 return rv_r_insn(0, rs2, rs1, 1, rd, 0x33); 290 } 291 292 static inline u32 rv_srl(u8 rd, u8 rs1, u8 rs2) 293 { 294 return rv_r_insn(0, rs2, rs1, 5, rd, 0x33); 295 } 296 297 static inline u32 rv_sra(u8 rd, u8 rs1, u8 rs2) 298 { 299 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x33); 300 } 301 302 static inline u32 rv_mul(u8 rd, u8 rs1, u8 rs2) 303 { 304 return rv_r_insn(1, rs2, rs1, 0, rd, 0x33); 305 } 306 307 static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2) 308 { 309 return rv_r_insn(1, rs2, rs1, 3, rd, 0x33); 310 } 311 312 static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2) 313 { 314 return rv_r_insn(1, rs2, rs1, 5, rd, 0x33); 315 } 316 317 static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2) 318 { 319 return rv_r_insn(1, rs2, rs1, 7, rd, 0x33); 320 } 321 322 static inline u32 rv_jal(u8 rd, u32 imm20_1) 323 { 324 return rv_j_insn(imm20_1, rd, 0x6f); 325 } 326 327 static inline u32 rv_jalr(u8 rd, u8 rs1, u16 imm11_0) 328 { 329 return rv_i_insn(imm11_0, rs1, 0, rd, 0x67); 330 } 331 332 static inline u32 rv_beq(u8 rs1, u8 rs2, u16 imm12_1) 333 { 334 return rv_b_insn(imm12_1, rs2, rs1, 0, 0x63); 335 } 336 337 static inline u32 rv_bne(u8 rs1, u8 rs2, u16 imm12_1) 338 { 339 return rv_b_insn(imm12_1, rs2, rs1, 1, 0x63); 340 } 341 342 static inline u32 rv_bltu(u8 rs1, u8 rs2, u16 imm12_1) 343 { 344 return rv_b_insn(imm12_1, rs2, rs1, 6, 0x63); 345 } 346 347 static inline u32 rv_bgtu(u8 rs1, u8 rs2, u16 imm12_1) 348 { 349 return rv_bltu(rs2, rs1, imm12_1); 350 } 351 352 static inline u32 rv_bgeu(u8 rs1, u8 rs2, u16 imm12_1) 353 { 354 return rv_b_insn(imm12_1, rs2, rs1, 7, 0x63); 355 } 356 357 static inline u32 rv_bleu(u8 rs1, u8 rs2, u16 imm12_1) 358 { 359 return rv_bgeu(rs2, rs1, imm12_1); 360 } 361 362 static inline u32 rv_blt(u8 rs1, u8 rs2, u16 imm12_1) 363 { 364 return rv_b_insn(imm12_1, rs2, rs1, 4, 0x63); 365 } 366 367 static inline u32 rv_bgt(u8 rs1, u8 rs2, u16 imm12_1) 368 { 369 return rv_blt(rs2, rs1, imm12_1); 370 } 371 372 static inline u32 rv_bge(u8 rs1, u8 rs2, u16 imm12_1) 373 { 374 return rv_b_insn(imm12_1, rs2, rs1, 5, 0x63); 375 } 376 377 static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1) 378 { 379 return rv_bge(rs2, rs1, imm12_1); 380 } 381 382 static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1) 383 { 384 return rv_i_insn(imm11_0, rs1, 2, rd, 0x03); 385 } 386 387 static inline u32 rv_lbu(u8 rd, u16 imm11_0, u8 rs1) 388 { 389 return rv_i_insn(imm11_0, rs1, 4, rd, 0x03); 390 } 391 392 static inline u32 rv_lhu(u8 rd, u16 imm11_0, u8 rs1) 393 { 394 return rv_i_insn(imm11_0, rs1, 5, rd, 0x03); 395 } 396 397 static inline u32 rv_sb(u8 rs1, u16 imm11_0, u8 rs2) 398 { 399 return rv_s_insn(imm11_0, rs2, rs1, 0, 0x23); 400 } 401 402 static inline u32 rv_sh(u8 rs1, u16 imm11_0, u8 rs2) 403 { 404 return rv_s_insn(imm11_0, rs2, rs1, 1, 0x23); 405 } 406 407 static inline u32 rv_sw(u8 rs1, u16 imm11_0, u8 rs2) 408 { 409 return rv_s_insn(imm11_0, rs2, rs1, 2, 0x23); 410 } 411 412 static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 413 { 414 return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f); 415 } 416 417 /* 418 * RV64-only instructions. 419 * 420 * These instructions are not available on RV32. Wrap them below a #if to 421 * ensure that the RV32 JIT doesn't emit any of these instructions. 422 */ 423 424 #if __riscv_xlen == 64 425 426 static inline u32 rv_addiw(u8 rd, u8 rs1, u16 imm11_0) 427 { 428 return rv_i_insn(imm11_0, rs1, 0, rd, 0x1b); 429 } 430 431 static inline u32 rv_slliw(u8 rd, u8 rs1, u16 imm11_0) 432 { 433 return rv_i_insn(imm11_0, rs1, 1, rd, 0x1b); 434 } 435 436 static inline u32 rv_srliw(u8 rd, u8 rs1, u16 imm11_0) 437 { 438 return rv_i_insn(imm11_0, rs1, 5, rd, 0x1b); 439 } 440 441 static inline u32 rv_sraiw(u8 rd, u8 rs1, u16 imm11_0) 442 { 443 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x1b); 444 } 445 446 static inline u32 rv_addw(u8 rd, u8 rs1, u8 rs2) 447 { 448 return rv_r_insn(0, rs2, rs1, 0, rd, 0x3b); 449 } 450 451 static inline u32 rv_subw(u8 rd, u8 rs1, u8 rs2) 452 { 453 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x3b); 454 } 455 456 static inline u32 rv_sllw(u8 rd, u8 rs1, u8 rs2) 457 { 458 return rv_r_insn(0, rs2, rs1, 1, rd, 0x3b); 459 } 460 461 static inline u32 rv_srlw(u8 rd, u8 rs1, u8 rs2) 462 { 463 return rv_r_insn(0, rs2, rs1, 5, rd, 0x3b); 464 } 465 466 static inline u32 rv_sraw(u8 rd, u8 rs1, u8 rs2) 467 { 468 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x3b); 469 } 470 471 static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2) 472 { 473 return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b); 474 } 475 476 static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2) 477 { 478 return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b); 479 } 480 481 static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2) 482 { 483 return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b); 484 } 485 486 static inline u32 rv_ld(u8 rd, u16 imm11_0, u8 rs1) 487 { 488 return rv_i_insn(imm11_0, rs1, 3, rd, 0x03); 489 } 490 491 static inline u32 rv_lwu(u8 rd, u16 imm11_0, u8 rs1) 492 { 493 return rv_i_insn(imm11_0, rs1, 6, rd, 0x03); 494 } 495 496 static inline u32 rv_sd(u8 rs1, u16 imm11_0, u8 rs2) 497 { 498 return rv_s_insn(imm11_0, rs2, rs1, 3, 0x23); 499 } 500 501 static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 502 { 503 return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f); 504 } 505 506 #endif /* __riscv_xlen == 64 */ 507 508 void bpf_jit_build_prologue(struct rv_jit_context *ctx); 509 void bpf_jit_build_epilogue(struct rv_jit_context *ctx); 510 511 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 512 bool extra_pass); 513 514 #endif /* _BPF_JIT_H */ 515