1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Just-In-Time compiler for eBPF bytecode on 32-bit and 64-bit MIPS. 4 * 5 * Copyright (c) 2021 Anyfi Networks AB. 6 * Author: Johan Almbladh <johan.almbladh@gmail.com> 7 * 8 * Based on code and ideas from 9 * Copyright (c) 2017 Cavium, Inc. 10 * Copyright (c) 2017 Shubham Bansal <illusionist.neo@gmail.com> 11 * Copyright (c) 2011 Mircea Gherzan <mgherzan@gmail.com> 12 */ 13 14 #ifndef _BPF_JIT_COMP_H 15 #define _BPF_JIT_COMP_H 16 17 /* MIPS registers */ 18 #define MIPS_R_ZERO 0 /* Const zero */ 19 #define MIPS_R_AT 1 /* Asm temp */ 20 #define MIPS_R_V0 2 /* Result */ 21 #define MIPS_R_V1 3 /* Result */ 22 #define MIPS_R_A0 4 /* Argument */ 23 #define MIPS_R_A1 5 /* Argument */ 24 #define MIPS_R_A2 6 /* Argument */ 25 #define MIPS_R_A3 7 /* Argument */ 26 #define MIPS_R_A4 8 /* Arg (n64) */ 27 #define MIPS_R_A5 9 /* Arg (n64) */ 28 #define MIPS_R_A6 10 /* Arg (n64) */ 29 #define MIPS_R_A7 11 /* Arg (n64) */ 30 #define MIPS_R_T0 8 /* Temp (o32) */ 31 #define MIPS_R_T1 9 /* Temp (o32) */ 32 #define MIPS_R_T2 10 /* Temp (o32) */ 33 #define MIPS_R_T3 11 /* Temp (o32) */ 34 #define MIPS_R_T4 12 /* Temporary */ 35 #define MIPS_R_T5 13 /* Temporary */ 36 #define MIPS_R_T6 14 /* Temporary */ 37 #define MIPS_R_T7 15 /* Temporary */ 38 #define MIPS_R_S0 16 /* Saved */ 39 #define MIPS_R_S1 17 /* Saved */ 40 #define MIPS_R_S2 18 /* Saved */ 41 #define MIPS_R_S3 19 /* Saved */ 42 #define MIPS_R_S4 20 /* Saved */ 43 #define MIPS_R_S5 21 /* Saved */ 44 #define MIPS_R_S6 22 /* Saved */ 45 #define MIPS_R_S7 23 /* Saved */ 46 #define MIPS_R_T8 24 /* Temporary */ 47 #define MIPS_R_T9 25 /* Temporary */ 48 /* MIPS_R_K0 26 Reserved */ 49 /* MIPS_R_K1 27 Reserved */ 50 #define MIPS_R_GP 28 /* Global ptr */ 51 #define MIPS_R_SP 29 /* Stack ptr */ 52 #define MIPS_R_FP 30 /* Frame ptr */ 53 #define MIPS_R_RA 31 /* Return */ 54 55 /* 56 * Jump address mask for immediate jumps. The four most significant bits 57 * must be equal to PC. 58 */ 59 #define MIPS_JMP_MASK 0x0fffffffUL 60 61 /* Maximum number of iterations in offset table computation */ 62 #define JIT_MAX_ITERATIONS 8 63 64 /* 65 * Jump pseudo-instructions used internally 66 * for branch conversion and branch optimization. 67 */ 68 #define JIT_JNSET 0xe0 69 #define JIT_JNOP 0xf0 70 71 /* Descriptor flag for PC-relative branch conversion */ 72 #define JIT_DESC_CONVERT BIT(31) 73 74 /* JIT context for an eBPF program */ 75 struct jit_context { 76 struct bpf_prog *program; /* The eBPF program being JITed */ 77 u32 *descriptors; /* eBPF to JITed CPU insn descriptors */ 78 u32 *target; /* JITed code buffer */ 79 u32 bpf_index; /* Index of current BPF program insn */ 80 u32 jit_index; /* Index of current JIT target insn */ 81 u32 changes; /* Number of PC-relative branch conv */ 82 u32 accessed; /* Bit mask of read eBPF registers */ 83 u32 clobbered; /* Bit mask of modified CPU registers */ 84 u32 stack_size; /* Total allocated stack size in bytes */ 85 u32 saved_size; /* Size of callee-saved registers */ 86 u32 stack_used; /* Stack size used for function calls */ 87 }; 88 89 /* Emit the instruction if the JIT memory space has been allocated */ 90 #define __emit(ctx, func, ...) \ 91 do { \ 92 if ((ctx)->target != NULL) { \ 93 u32 *p = &(ctx)->target[ctx->jit_index]; \ 94 uasm_i_##func(&p, ##__VA_ARGS__); \ 95 } \ 96 (ctx)->jit_index++; \ 97 } while (0) 98 #define emit(...) __emit(__VA_ARGS__) 99 100 /* Workaround for R10000 ll/sc errata */ 101 #ifdef CONFIG_WAR_R10000_LLSC 102 #define LLSC_beqz beqzl 103 #else 104 #define LLSC_beqz beqz 105 #endif 106 107 /* Workaround for Loongson-3 ll/sc errata */ 108 #ifdef CONFIG_CPU_LOONGSON3_WORKAROUNDS 109 #define LLSC_sync(ctx) emit(ctx, sync, 0) 110 #define LLSC_offset 4 111 #else 112 #define LLSC_sync(ctx) 113 #define LLSC_offset 0 114 #endif 115 116 /* Workaround for Loongson-2F jump errata */ 117 #ifdef CONFIG_CPU_JUMP_WORKAROUNDS 118 #define JALR_MASK 0xffffffffcfffffffULL 119 #else 120 #define JALR_MASK (~0ULL) 121 #endif 122 123 /* 124 * Mark a BPF register as accessed, it needs to be 125 * initialized by the program if expected, e.g. FP. 126 */ 127 static inline void access_reg(struct jit_context *ctx, u8 reg) 128 { 129 ctx->accessed |= BIT(reg); 130 } 131 132 /* 133 * Mark a CPU register as clobbered, it needs to be 134 * saved/restored by the program if callee-saved. 135 */ 136 static inline void clobber_reg(struct jit_context *ctx, u8 reg) 137 { 138 ctx->clobbered |= BIT(reg); 139 } 140 141 /* 142 * Push registers on the stack, starting at a given depth from the stack 143 * pointer and increasing. The next depth to be written is returned. 144 */ 145 int push_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth); 146 147 /* 148 * Pop registers from the stack, starting at a given depth from the stack 149 * pointer and increasing. The next depth to be read is returned. 150 */ 151 int pop_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth); 152 153 /* Compute the 28-bit jump target address from a BPF program location */ 154 int get_target(struct jit_context *ctx, u32 loc); 155 156 /* Compute the PC-relative offset to relative BPF program offset */ 157 int get_offset(const struct jit_context *ctx, int off); 158 159 /* dst = imm (32-bit) */ 160 void emit_mov_i(struct jit_context *ctx, u8 dst, s32 imm); 161 162 /* dst = src (32-bit) */ 163 void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src); 164 165 /* Validate ALU/ALU64 immediate range */ 166 bool valid_alu_i(u8 op, s32 imm); 167 168 /* Rewrite ALU/ALU64 immediate operation */ 169 bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val); 170 171 /* ALU immediate operation (32-bit) */ 172 void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op); 173 174 /* ALU register operation (32-bit) */ 175 void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op); 176 177 /* Atomic read-modify-write (32-bit) */ 178 void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code); 179 180 /* Atomic compare-and-exchange (32-bit) */ 181 void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off); 182 183 /* Swap bytes and truncate a register word or half word */ 184 void emit_bswap_r(struct jit_context *ctx, u8 dst, u32 width); 185 186 /* Validate JMP/JMP32 immediate range */ 187 bool valid_jmp_i(u8 op, s32 imm); 188 189 /* Prepare a PC-relative jump operation with immediate conditional */ 190 void setup_jmp_i(struct jit_context *ctx, s32 imm, u8 width, 191 u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off); 192 193 /* Prepare a PC-relative jump operation with register conditional */ 194 void setup_jmp_r(struct jit_context *ctx, bool same_reg, 195 u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off); 196 197 /* Finish a PC-relative jump operation */ 198 int finish_jmp(struct jit_context *ctx, u8 jit_op, s16 bpf_off); 199 200 /* Conditional JMP/JMP32 immediate */ 201 void emit_jmp_i(struct jit_context *ctx, u8 dst, s32 imm, s32 off, u8 op); 202 203 /* Conditional JMP/JMP32 register */ 204 void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op); 205 206 /* Jump always */ 207 int emit_ja(struct jit_context *ctx, s16 off); 208 209 /* Jump to epilogue */ 210 int emit_exit(struct jit_context *ctx); 211 212 /* 213 * Build program prologue to set up the stack and registers. 214 * This function is implemented separately for 32-bit and 64-bit JITs. 215 */ 216 void build_prologue(struct jit_context *ctx); 217 218 /* 219 * Build the program epilogue to restore the stack and registers. 220 * This function is implemented separately for 32-bit and 64-bit JITs. 221 */ 222 void build_epilogue(struct jit_context *ctx, int dest_reg); 223 224 /* 225 * Convert an eBPF instruction to native instruction, i.e 226 * JITs an eBPF instruction. 227 * Returns : 228 * 0 - Successfully JITed an 8-byte eBPF instruction 229 * >0 - Successfully JITed a 16-byte eBPF instruction 230 * <0 - Failed to JIT. 231 * This function is implemented separately for 32-bit and 64-bit JITs. 232 */ 233 int build_insn(const struct bpf_insn *insn, struct jit_context *ctx); 234 235 #endif /* _BPF_JIT_COMP_H */ 236