1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Common functionality for PARISC32 and PARISC64 BPF JIT compilers 4 * 5 * Copyright (c) 2023 Helge Deller <deller@gmx.de> 6 * 7 */ 8 9 #ifndef _BPF_JIT_H 10 #define _BPF_JIT_H 11 12 #include <linux/bpf.h> 13 #include <linux/filter.h> 14 #include <asm/cacheflush.h> 15 16 #define HPPA_JIT_DEBUG 0 17 #define HPPA_JIT_REBOOT 0 18 #define HPPA_JIT_DUMP 0 19 20 #define OPTIMIZE_HPPA 1 /* enable some asm optimizations */ 21 // echo 1 > /proc/sys/net/core/bpf_jit_enable 22 23 #define HPPA_R(nr) nr /* use HPPA register #nr */ 24 25 enum { 26 HPPA_REG_ZERO = 0, /* The constant value 0 */ 27 HPPA_REG_R1 = 1, /* used for addil */ 28 HPPA_REG_RP = 2, /* Return address */ 29 30 HPPA_REG_ARG7 = 19, /* ARG4-7 used in 64-bit ABI */ 31 HPPA_REG_ARG6 = 20, 32 HPPA_REG_ARG5 = 21, 33 HPPA_REG_ARG4 = 22, 34 35 HPPA_REG_ARG3 = 23, /* ARG0-3 in 32- and 64-bit ABI */ 36 HPPA_REG_ARG2 = 24, 37 HPPA_REG_ARG1 = 25, 38 HPPA_REG_ARG0 = 26, 39 HPPA_REG_GP = 27, /* Global pointer */ 40 HPPA_REG_RET0 = 28, /* Return value, HI in 32-bit */ 41 HPPA_REG_RET1 = 29, /* Return value, LOW in 32-bit */ 42 HPPA_REG_SP = 30, /* Stack pointer */ 43 HPPA_REG_R31 = 31, 44 45 #ifdef CONFIG_64BIT 46 HPPA_REG_TCC = 3, 47 HPPA_REG_TCC_SAVED = 4, 48 HPPA_REG_TCC_IN_INIT = HPPA_REG_R31, 49 #else 50 HPPA_REG_TCC = 18, 51 HPPA_REG_TCC_SAVED = 17, 52 HPPA_REG_TCC_IN_INIT = HPPA_REG_R31, 53 #endif 54 55 HPPA_REG_T0 = HPPA_REG_R1, /* Temporaries */ 56 HPPA_REG_T1 = HPPA_REG_R31, 57 HPPA_REG_T2 = HPPA_REG_ARG4, 58 #ifndef CONFIG_64BIT 59 HPPA_REG_T3 = HPPA_REG_ARG5, /* not used in 64-bit */ 60 HPPA_REG_T4 = HPPA_REG_ARG6, 61 HPPA_REG_T5 = HPPA_REG_ARG7, 62 #endif 63 }; 64 65 struct hppa_jit_context { 66 struct bpf_prog *prog; 67 u32 *insns; /* HPPA insns */ 68 int ninsns; 69 int reg_seen_collect; 70 int reg_seen; 71 int body_len; 72 int epilogue_offset; 73 int prologue_len; 74 int *offset; /* BPF to HPPA */ 75 }; 76 77 #define REG_SET_SEEN(ctx, nr) { if (ctx->reg_seen_collect) ctx->reg_seen |= BIT(nr); } 78 #define REG_SET_SEEN_ALL(ctx) { if (ctx->reg_seen_collect) ctx->reg_seen = -1; } 79 #define REG_FORCE_SEEN(ctx, nr) { ctx->reg_seen |= BIT(nr); } 80 #define REG_WAS_SEEN(ctx, nr) (ctx->reg_seen & BIT(nr)) 81 #define REG_ALL_SEEN(ctx) (ctx->reg_seen == -1) 82 83 #define HPPA_INSN_SIZE 4 /* bytes per HPPA asm instruction */ 84 #define REG_SIZE REG_SZ /* bytes per native "long" word */ 85 86 /* subtract hppa displacement on branches which is .+8 */ 87 #define HPPA_BRANCH_DISPLACEMENT 2 /* instructions */ 88 89 /* asm statement indicator to execute delay slot */ 90 #define EXEC_NEXT_INSTR 0 91 #define NOP_NEXT_INSTR 1 92 93 #define im11(val) (((u32)(val)) & 0x07ff) 94 95 #define hppa_ldil(addr, reg) \ 96 hppa_t5_insn(0x08, reg, ((u32)(addr)) >> 11) /* ldil im21,reg */ 97 #define hppa_addil(addr, reg) \ 98 hppa_t5_insn(0x0a, reg, ((u32)(addr)) >> 11) /* addil im21,reg -> result in gr1 */ 99 #define hppa_ldo(im14, reg, target) \ 100 hppa_t1_insn(0x0d, reg, target, im14) /* ldo val14(reg),target */ 101 #define hppa_ldi(im14, reg) \ 102 hppa_ldo(im14, HPPA_REG_ZERO, reg) /* ldi val14,reg */ 103 #define hppa_or(reg1, reg2, target) \ 104 hppa_t6_insn(0x02, reg2, reg1, 0, 0, 0x09, target) /* or reg1,reg2,target */ 105 #define hppa_or_cond(reg1, reg2, cond, f, target) \ 106 hppa_t6_insn(0x02, reg2, reg1, cond, f, 0x09, target) 107 #define hppa_and(reg1, reg2, target) \ 108 hppa_t6_insn(0x02, reg2, reg1, 0, 0, 0x08, target) /* and reg1,reg2,target */ 109 #define hppa_and_cond(reg1, reg2, cond, f, target) \ 110 hppa_t6_insn(0x02, reg2, reg1, cond, f, 0x08, target) 111 #define hppa_xor(reg1, reg2, target) \ 112 hppa_t6_insn(0x02, reg2, reg1, 0, 0, 0x0a, target) /* xor reg1,reg2,target */ 113 #define hppa_add(reg1, reg2, target) \ 114 hppa_t6_insn(0x02, reg2, reg1, 0, 0, 0x18, target) /* add reg1,reg2,target */ 115 #define hppa_addc(reg1, reg2, target) \ 116 hppa_t6_insn(0x02, reg2, reg1, 0, 0, 0x1c, target) /* add,c reg1,reg2,target */ 117 #define hppa_sub(reg1, reg2, target) \ 118 hppa_t6_insn(0x02, reg2, reg1, 0, 0, 0x10, target) /* sub reg1,reg2,target */ 119 #define hppa_subb(reg1, reg2, target) \ 120 hppa_t6_insn(0x02, reg2, reg1, 0, 0, 0x14, target) /* sub,b reg1,reg2,target */ 121 #define hppa_nop() \ 122 hppa_or(0,0,0) /* nop: or 0,0,0 */ 123 #define hppa_addi(val11, reg, target) \ 124 hppa_t7_insn(0x2d, reg, target, val11) /* addi im11,reg,target */ 125 #define hppa_subi(val11, reg, target) \ 126 hppa_t7_insn(0x25, reg, target, val11) /* subi im11,reg,target */ 127 #define hppa_copy(reg, target) \ 128 hppa_or(reg, HPPA_REG_ZERO, target) /* copy reg,target */ 129 #define hppa_ldw(val14, reg, target) \ 130 hppa_t1_insn(0x12, reg, target, val14) /* ldw im14(reg),target */ 131 #define hppa_ldb(val14, reg, target) \ 132 hppa_t1_insn(0x10, reg, target, val14) /* ldb im14(reg),target */ 133 #define hppa_ldh(val14, reg, target) \ 134 hppa_t1_insn(0x11, reg, target, val14) /* ldh im14(reg),target */ 135 #define hppa_stw(reg, val14, base) \ 136 hppa_t1_insn(0x1a, base, reg, val14) /* stw reg,im14(base) */ 137 #define hppa_stb(reg, val14, base) \ 138 hppa_t1_insn(0x18, base, reg, val14) /* stb reg,im14(base) */ 139 #define hppa_sth(reg, val14, base) \ 140 hppa_t1_insn(0x19, base, reg, val14) /* sth reg,im14(base) */ 141 #define hppa_stwma(reg, val14, base) \ 142 hppa_t1_insn(0x1b, base, reg, val14) /* stw,ma reg,im14(base) */ 143 #define hppa_bv(reg, base, nop) \ 144 hppa_t11_insn(0x3a, base, reg, 0x06, 0, nop) /* bv(,n) reg(base) */ 145 #define hppa_be(offset, base) \ 146 hppa_t12_insn(0x38, base, offset, 0x00, 1) /* be,n offset(0,base) */ 147 #define hppa_be_l(offset, base, nop) \ 148 hppa_t12_insn(0x39, base, offset, 0x00, nop) /* ble(,nop) offset(0,base) */ 149 #define hppa_mtctl(reg, cr) \ 150 hppa_t21_insn(0x00, cr, reg, 0xc2, 0) /* mtctl reg,cr */ 151 #define hppa_mtsar(reg) \ 152 hppa_mtctl(reg, 11) /* mtsar reg */ 153 #define hppa_zdep(r, p, len, target) \ 154 hppa_t10_insn(0x35, target, r, 0, 2, p, len) /* zdep r,a,b,t */ 155 #define hppa_shl(r, len, target) \ 156 hppa_zdep(r, len, len, lo(rd)) 157 #define hppa_depwz(r, p, len, target) \ 158 hppa_t10_insn(0x35, target, r, 0, 3, 31-(p), 32-(len)) /* depw,z r,p,len,ret1 */ 159 #define hppa_depwz_sar(reg, target) \ 160 hppa_t1_insn(0x35, target, reg, 0) /* depw,z reg,sar,32,target */ 161 #define hppa_shrpw_sar(reg, target) \ 162 hppa_t10_insn(0x34, reg, 0, 0, 0, 0, target) /* shrpw r0,reg,sar,target */ 163 #define hppa_shrpw(r1, r2, p, target) \ 164 hppa_t10_insn(0x34, r2, r1, 0, 2, 31-(p), target) /* shrpw r1,r2,p,target */ 165 #define hppa_shd(r1, r2, p, target) \ 166 hppa_t10_insn(0x34, r2, r1, 0, 2, 31-(p), target) /* shrpw r1,r2,p,tarfer */ 167 #define hppa_extrws_sar(reg, target) \ 168 hppa_t10_insn(0x34, reg, target, 0, 5, 0, 0) /* extrw,s reg,sar,32,ret0 */ 169 #define hppa_extrws(reg, p, len, target) \ 170 hppa_t10_insn(0x34, reg, target, 0, 7, p, len) /* extrw,s reg,p,len,target */ 171 #define hppa_extru(r, p, len, target) \ 172 hppa_t10_insn(0x34, r, target, 0, 6, p, 32-(len)) 173 #define hppa_shr(r, len, target) \ 174 hppa_extru(r, 31-(len), 32-(len), target) 175 #define hppa_bl(imm17, rp) \ 176 hppa_t12_insn(0x3a, rp, imm17, 0x00, 1) /* bl,n target_addr,rp */ 177 #define hppa_sh2add(r1, r2, target) \ 178 hppa_t6_insn(0x02, r2, r1, 0, 0, 0x1a, target) /* sh2add r1,r2,target */ 179 180 #define hppa_combt(r1, r2, target_addr, condition, nop) \ 181 hppa_t11_insn(IS_ENABLED(CONFIG_64BIT) ? 0x27 : 0x20, \ 182 r2, r1, condition, target_addr, nop) /* combt,cond,n r1,r2,addr */ 183 #define hppa_beq(r1, r2, target_addr) \ 184 hppa_combt(r1, r2, target_addr, 1, NOP_NEXT_INSTR) 185 #define hppa_blt(r1, r2, target_addr) \ 186 hppa_combt(r1, r2, target_addr, 2, NOP_NEXT_INSTR) 187 #define hppa_ble(r1, r2, target_addr) \ 188 hppa_combt(r1, r2, target_addr, 3, NOP_NEXT_INSTR) 189 #define hppa_bltu(r1, r2, target_addr) \ 190 hppa_combt(r1, r2, target_addr, 4, NOP_NEXT_INSTR) 191 #define hppa_bleu(r1, r2, target_addr) \ 192 hppa_combt(r1, r2, target_addr, 5, NOP_NEXT_INSTR) 193 194 #define hppa_combf(r1, r2, target_addr, condition, nop) \ 195 hppa_t11_insn(IS_ENABLED(CONFIG_64BIT) ? 0x2f : 0x22, \ 196 r2, r1, condition, target_addr, nop) /* combf,cond,n r1,r2,addr */ 197 #define hppa_bne(r1, r2, target_addr) \ 198 hppa_combf(r1, r2, target_addr, 1, NOP_NEXT_INSTR) 199 #define hppa_bge(r1, r2, target_addr) \ 200 hppa_combf(r1, r2, target_addr, 2, NOP_NEXT_INSTR) 201 #define hppa_bgt(r1, r2, target_addr) \ 202 hppa_combf(r1, r2, target_addr, 3, NOP_NEXT_INSTR) 203 #define hppa_bgeu(r1, r2, target_addr) \ 204 hppa_combf(r1, r2, target_addr, 4, NOP_NEXT_INSTR) 205 #define hppa_bgtu(r1, r2, target_addr) \ 206 hppa_combf(r1, r2, target_addr, 5, NOP_NEXT_INSTR) 207 208 /* 64-bit instructions */ 209 #ifdef CONFIG_64BIT 210 #define hppa64_ldd_reg(reg, b, target) \ 211 hppa_t10_insn(0x03, b, reg, 0, 0, 3<<1, target) 212 #define hppa64_ldd_im5(im5, b, target) \ 213 hppa_t10_insn(0x03, b, low_sign_unext(im5,5), 0, 1<<2, 3<<1, target) 214 #define hppa64_ldd_im16(im16, b, target) \ 215 hppa_t10_insn(0x14, b, target, 0, 0, 0, 0) | re_assemble_16(im16) 216 #define hppa64_std_im5(src, im5, b) \ 217 hppa_t10_insn(0x03, b, src, 0, 1<<2, 0xB<<1, low_sign_unext(im5,5)) 218 #define hppa64_std_im16(src, im16, b) \ 219 hppa_t10_insn(0x1c, b, src, 0, 0, 0, 0) | re_assemble_16(im16) 220 #define hppa64_bl_long(offs22) \ 221 hppa_t12_L_insn(0x3a, offs22, 1) 222 #define hppa64_mtsarcm(reg) \ 223 hppa_t21_insn(0x00, 11, reg, 0xc6, 0) 224 #define hppa64_shrpd_sar(reg, target) \ 225 hppa_t10_insn(0x34, reg, 0, 0, 0, 1<<4, target) 226 #define hppa64_shladd(r1, sa, r2, target) \ 227 hppa_t6_insn(0x02, r2, r1, 0, 0, 1<<4|1<<3|sa, target) 228 #define hppa64_depdz_sar(reg, target) \ 229 hppa_t21_insn(0x35, target, reg, 3<<3, 0) 230 #define hppa_extrd_sar(reg, target, se) \ 231 hppa_t10_insn(0x34, reg, target, 0, 0, 0, 0) | 2<<11 | (se&1)<<10 | 1<<9 | 1<<8 232 #define hppa64_bve_l_rp(base) \ 233 (0x3a << 26) | (base << 21) | 0xf000 234 #define hppa64_permh_3210(r, target) \ 235 (0x3e << 26) | (r << 21) | (r << 16) | (target) | 0x00006900 236 #define hppa64_hshl(r, sa, target) \ 237 (0x3e << 26) | (0 << 21) | (r << 16) | (sa << 6) | (target) | 0x00008800 238 #define hppa64_hshr_u(r, sa, target) \ 239 (0x3e << 26) | (r << 21) | (0 << 16) | (sa << 6) | (target) | 0x0000c800 240 #endif 241 242 struct hppa_jit_data { 243 struct bpf_binary_header *header; 244 u8 *image; 245 struct hppa_jit_context ctx; 246 }; 247 248 static inline void bpf_fill_ill_insns(void *area, unsigned int size) 249 { 250 memset(area, 0, size); 251 } 252 253 static inline void bpf_flush_icache(void *start, void *end) 254 { 255 flush_icache_range((unsigned long)start, (unsigned long)end); 256 } 257 258 /* Emit a 4-byte HPPA instruction. */ 259 static inline void emit(const u32 insn, struct hppa_jit_context *ctx) 260 { 261 if (ctx->insns) { 262 ctx->insns[ctx->ninsns] = insn; 263 } 264 265 ctx->ninsns++; 266 } 267 268 static inline int epilogue_offset(struct hppa_jit_context *ctx) 269 { 270 int to = ctx->epilogue_offset, from = ctx->ninsns; 271 272 return (to - from); 273 } 274 275 /* Return -1 or inverted cond. */ 276 static inline int invert_bpf_cond(u8 cond) 277 { 278 switch (cond) { 279 case BPF_JEQ: 280 return BPF_JNE; 281 case BPF_JGT: 282 return BPF_JLE; 283 case BPF_JLT: 284 return BPF_JGE; 285 case BPF_JGE: 286 return BPF_JLT; 287 case BPF_JLE: 288 return BPF_JGT; 289 case BPF_JNE: 290 return BPF_JEQ; 291 case BPF_JSGT: 292 return BPF_JSLE; 293 case BPF_JSLT: 294 return BPF_JSGE; 295 case BPF_JSGE: 296 return BPF_JSLT; 297 case BPF_JSLE: 298 return BPF_JSGT; 299 } 300 return -1; 301 } 302 303 304 static inline signed long hppa_offset(int insn, int off, struct hppa_jit_context *ctx) 305 { 306 signed long from, to; 307 308 off++; /* BPF branch is from PC+1 */ 309 from = (insn > 0) ? ctx->offset[insn - 1] : 0; 310 to = (insn + off > 0) ? ctx->offset[insn + off - 1] : 0; 311 return (to - from); 312 } 313 314 /* does the signed value fits into a given number of bits ? */ 315 static inline int check_bits_int(signed long val, int bits) 316 { 317 return ((val >= 0) && ((val >> bits) == 0)) || 318 ((val < 0) && (((~((u32)val)) >> (bits-1)) == 0)); 319 } 320 321 /* can the signed value be used in relative code ? */ 322 static inline int relative_bits_ok(signed long val, int bits) 323 { 324 return ((val >= 0) && (val < (1UL << (bits-1)))) || /* XXX */ 325 ((val < 0) && (((~((unsigned long)val)) >> (bits-1)) == 0) 326 && (val & (1UL << (bits-1)))); 327 } 328 329 /* can the signed value be used in relative branches ? */ 330 static inline int relative_branch_ok(signed long val, int bits) 331 { 332 return ((val >= 0) && (val < (1UL << (bits-2)))) || /* XXX */ 333 ((val < 0) && (((~((unsigned long)val)) < (1UL << (bits-2)))) 334 && (val & (1UL << (bits-1)))); 335 } 336 337 338 #define is_5b_int(val) check_bits_int(val, 5) 339 340 static inline unsigned sign_unext(unsigned x, unsigned len) 341 { 342 unsigned len_ones; 343 344 len_ones = (1 << len) - 1; 345 return x & len_ones; 346 } 347 348 static inline unsigned low_sign_unext(unsigned x, unsigned len) 349 { 350 unsigned temp; 351 unsigned sign; 352 353 sign = (x >> (len-1)) & 1; 354 temp = sign_unext (x, len-1); 355 return (temp << 1) | sign; 356 } 357 358 static inline unsigned re_assemble_12(unsigned as12) 359 { 360 return (( (as12 & 0x800) >> 11) 361 | ((as12 & 0x400) >> (10 - 2)) 362 | ((as12 & 0x3ff) << (1 + 2))); 363 } 364 365 static inline unsigned re_assemble_14(unsigned as14) 366 { 367 return (( (as14 & 0x1fff) << 1) 368 | ((as14 & 0x2000) >> 13)); 369 } 370 371 #ifdef CONFIG_64BIT 372 static inline unsigned re_assemble_16(unsigned as16) 373 { 374 unsigned s, t; 375 376 /* Unusual 16-bit encoding, for wide mode only. */ 377 t = (as16 << 1) & 0xffff; 378 s = (as16 & 0x8000); 379 return (t ^ s ^ (s >> 1)) | (s >> 15); 380 } 381 #endif 382 383 static inline unsigned re_assemble_17(unsigned as17) 384 { 385 return (( (as17 & 0x10000) >> 16) 386 | ((as17 & 0x0f800) << (16 - 11)) 387 | ((as17 & 0x00400) >> (10 - 2)) 388 | ((as17 & 0x003ff) << (1 + 2))); 389 } 390 391 static inline unsigned re_assemble_21(unsigned as21) 392 { 393 return (( (as21 & 0x100000) >> 20) 394 | ((as21 & 0x0ffe00) >> 8) 395 | ((as21 & 0x000180) << 7) 396 | ((as21 & 0x00007c) << 14) 397 | ((as21 & 0x000003) << 12)); 398 } 399 400 static inline unsigned re_assemble_22(unsigned as22) 401 { 402 return (( (as22 & 0x200000) >> 21) 403 | ((as22 & 0x1f0000) << (21 - 16)) 404 | ((as22 & 0x00f800) << (16 - 11)) 405 | ((as22 & 0x000400) >> (10 - 2)) 406 | ((as22 & 0x0003ff) << (1 + 2))); 407 } 408 409 /* Various HPPA instruction formats. */ 410 /* see https://parisc.wiki.kernel.org/images-parisc/6/68/Pa11_acd.pdf, appendix C */ 411 412 static inline u32 hppa_t1_insn(u8 opcode, u8 b, u8 r, s16 im14) 413 { 414 return ((opcode << 26) | (b << 21) | (r << 16) | re_assemble_14(im14)); 415 } 416 417 static inline u32 hppa_t5_insn(u8 opcode, u8 tr, u32 val21) 418 { 419 return ((opcode << 26) | (tr << 21) | re_assemble_21(val21)); 420 } 421 422 static inline u32 hppa_t6_insn(u8 opcode, u8 r2, u8 r1, u8 c, u8 f, u8 ext6, u16 t) 423 { 424 return ((opcode << 26) | (r2 << 21) | (r1 << 16) | (c << 13) | (f << 12) | 425 (ext6 << 6) | t); 426 } 427 428 /* 7. Arithmetic immediate */ 429 static inline u32 hppa_t7_insn(u8 opcode, u8 r, u8 t, u32 im11) 430 { 431 return ((opcode << 26) | (r << 21) | (t << 16) | low_sign_unext(im11, 11)); 432 } 433 434 /* 10. Shift instructions */ 435 static inline u32 hppa_t10_insn(u8 opcode, u8 r2, u8 r1, u8 c, u8 ext3, u8 cp, u8 t) 436 { 437 return ((opcode << 26) | (r2 << 21) | (r1 << 16) | (c << 13) | 438 (ext3 << 10) | (cp << 5) | t); 439 } 440 441 /* 11. Conditional branch instructions */ 442 static inline u32 hppa_t11_insn(u8 opcode, u8 r2, u8 r1, u8 c, u32 w, u8 nop) 443 { 444 u32 ra = re_assemble_12(w); 445 // ra = low_sign_unext(w,11) | (w & (1<<10) 446 return ((opcode << 26) | (r2 << 21) | (r1 << 16) | (c << 13) | (nop << 1) | ra); 447 } 448 449 /* 12. Branch instructions */ 450 static inline u32 hppa_t12_insn(u8 opcode, u8 rp, u32 w, u8 ext3, u8 nop) 451 { 452 return ((opcode << 26) | (rp << 21) | (ext3 << 13) | (nop << 1) | re_assemble_17(w)); 453 } 454 455 static inline u32 hppa_t12_L_insn(u8 opcode, u32 w, u8 nop) 456 { 457 return ((opcode << 26) | (0x05 << 13) | (nop << 1) | re_assemble_22(w)); 458 } 459 460 /* 21. Move to control register */ 461 static inline u32 hppa_t21_insn(u8 opcode, u8 r2, u8 r1, u8 ext8, u8 t) 462 { 463 return ((opcode << 26) | (r2 << 21) | (r1 << 16) | (ext8 << 5) | t); 464 } 465 466 /* Helper functions called by jit code on HPPA32 and HPPA64. */ 467 468 u64 hppa_div64(u64 div, u64 divisor); 469 u64 hppa_div64_rem(u64 div, u64 divisor); 470 471 /* Helper functions that emit HPPA instructions when possible. */ 472 473 void bpf_jit_build_prologue(struct hppa_jit_context *ctx); 474 void bpf_jit_build_epilogue(struct hppa_jit_context *ctx); 475 476 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx, 477 bool extra_pass); 478 479 #endif /* _BPF_JIT_H */ 480