1ddecdfceSMircea Gherzan /* 2ddecdfceSMircea Gherzan * Just-In-Time compiler for BPF filters on 32bit ARM 3ddecdfceSMircea Gherzan * 4ddecdfceSMircea Gherzan * Copyright (c) 2011 Mircea Gherzan <mgherzan@gmail.com> 5ddecdfceSMircea Gherzan * 6ddecdfceSMircea Gherzan * This program is free software; you can redistribute it and/or modify it 7ddecdfceSMircea Gherzan * under the terms of the GNU General Public License as published by the 8ddecdfceSMircea Gherzan * Free Software Foundation; version 2 of the License. 9ddecdfceSMircea Gherzan */ 10ddecdfceSMircea Gherzan 11ddecdfceSMircea Gherzan #include <linux/bitops.h> 12ddecdfceSMircea Gherzan #include <linux/compiler.h> 13ddecdfceSMircea Gherzan #include <linux/errno.h> 14ddecdfceSMircea Gherzan #include <linux/filter.h> 15ddecdfceSMircea Gherzan #include <linux/moduleloader.h> 16ddecdfceSMircea Gherzan #include <linux/netdevice.h> 17ddecdfceSMircea Gherzan #include <linux/string.h> 18ddecdfceSMircea Gherzan #include <linux/slab.h> 19bf0098f2SDaniel Borkmann #include <linux/if_vlan.h> 20ddecdfceSMircea Gherzan #include <asm/cacheflush.h> 21ddecdfceSMircea Gherzan #include <asm/hwcap.h> 223460743eSBen Dooks #include <asm/opcodes.h> 23ddecdfceSMircea Gherzan 24ddecdfceSMircea Gherzan #include "bpf_jit_32.h" 25ddecdfceSMircea Gherzan 26ddecdfceSMircea Gherzan /* 27ddecdfceSMircea Gherzan * ABI: 28ddecdfceSMircea Gherzan * 29ddecdfceSMircea Gherzan * r0 scratch register 30ddecdfceSMircea Gherzan * r4 BPF register A 31ddecdfceSMircea Gherzan * r5 BPF register X 32ddecdfceSMircea Gherzan * r6 pointer to the skb 33ddecdfceSMircea Gherzan * r7 skb->data 34ddecdfceSMircea Gherzan * r8 skb_headlen(skb) 35ddecdfceSMircea Gherzan */ 36ddecdfceSMircea Gherzan 37ddecdfceSMircea Gherzan #define r_scratch ARM_R0 38ddecdfceSMircea Gherzan /* r1-r3 are (also) used for the unaligned loads on the non-ARMv7 slowpath */ 39ddecdfceSMircea Gherzan #define r_off ARM_R1 40ddecdfceSMircea Gherzan #define r_A ARM_R4 41ddecdfceSMircea Gherzan #define r_X ARM_R5 42ddecdfceSMircea Gherzan #define r_skb ARM_R6 43ddecdfceSMircea Gherzan #define r_skb_data ARM_R7 44ddecdfceSMircea Gherzan #define r_skb_hl ARM_R8 45ddecdfceSMircea Gherzan 46ddecdfceSMircea Gherzan #define SCRATCH_SP_OFFSET 0 47fe15f3f1SSchichan Nicolas #define SCRATCH_OFF(k) (SCRATCH_SP_OFFSET + 4 * (k)) 48ddecdfceSMircea Gherzan 49ddecdfceSMircea Gherzan #define SEEN_MEM ((1 << BPF_MEMWORDS) - 1) 50ddecdfceSMircea Gherzan #define SEEN_MEM_WORD(k) (1 << (k)) 51ddecdfceSMircea Gherzan #define SEEN_X (1 << BPF_MEMWORDS) 52ddecdfceSMircea Gherzan #define SEEN_CALL (1 << (BPF_MEMWORDS + 1)) 53ddecdfceSMircea Gherzan #define SEEN_SKB (1 << (BPF_MEMWORDS + 2)) 54ddecdfceSMircea Gherzan #define SEEN_DATA (1 << (BPF_MEMWORDS + 3)) 55ddecdfceSMircea Gherzan 56ddecdfceSMircea Gherzan #define FLAG_NEED_X_RESET (1 << 0) 57ddecdfceSMircea Gherzan 58ddecdfceSMircea Gherzan struct jit_ctx { 597ae457c1SAlexei Starovoitov const struct bpf_prog *skf; 60ddecdfceSMircea Gherzan unsigned idx; 61ddecdfceSMircea Gherzan unsigned prologue_bytes; 62ddecdfceSMircea Gherzan int ret0_fp_idx; 63ddecdfceSMircea Gherzan u32 seen; 64ddecdfceSMircea Gherzan u32 flags; 65ddecdfceSMircea Gherzan u32 *offsets; 66ddecdfceSMircea Gherzan u32 *target; 67ddecdfceSMircea Gherzan #if __LINUX_ARM_ARCH__ < 7 68ddecdfceSMircea Gherzan u16 epilogue_bytes; 69ddecdfceSMircea Gherzan u16 imm_count; 70ddecdfceSMircea Gherzan u32 *imms; 71ddecdfceSMircea Gherzan #endif 72ddecdfceSMircea Gherzan }; 73ddecdfceSMircea Gherzan 74ddecdfceSMircea Gherzan int bpf_jit_enable __read_mostly; 75ddecdfceSMircea Gherzan 76ddecdfceSMircea Gherzan static u64 jit_get_skb_b(struct sk_buff *skb, unsigned offset) 77ddecdfceSMircea Gherzan { 78ddecdfceSMircea Gherzan u8 ret; 79ddecdfceSMircea Gherzan int err; 80ddecdfceSMircea Gherzan 81ddecdfceSMircea Gherzan err = skb_copy_bits(skb, offset, &ret, 1); 82ddecdfceSMircea Gherzan 83ddecdfceSMircea Gherzan return (u64)err << 32 | ret; 84ddecdfceSMircea Gherzan } 85ddecdfceSMircea Gherzan 86ddecdfceSMircea Gherzan static u64 jit_get_skb_h(struct sk_buff *skb, unsigned offset) 87ddecdfceSMircea Gherzan { 88ddecdfceSMircea Gherzan u16 ret; 89ddecdfceSMircea Gherzan int err; 90ddecdfceSMircea Gherzan 91ddecdfceSMircea Gherzan err = skb_copy_bits(skb, offset, &ret, 2); 92ddecdfceSMircea Gherzan 93ddecdfceSMircea Gherzan return (u64)err << 32 | ntohs(ret); 94ddecdfceSMircea Gherzan } 95ddecdfceSMircea Gherzan 96ddecdfceSMircea Gherzan static u64 jit_get_skb_w(struct sk_buff *skb, unsigned offset) 97ddecdfceSMircea Gherzan { 98ddecdfceSMircea Gherzan u32 ret; 99ddecdfceSMircea Gherzan int err; 100ddecdfceSMircea Gherzan 101ddecdfceSMircea Gherzan err = skb_copy_bits(skb, offset, &ret, 4); 102ddecdfceSMircea Gherzan 103ddecdfceSMircea Gherzan return (u64)err << 32 | ntohl(ret); 104ddecdfceSMircea Gherzan } 105ddecdfceSMircea Gherzan 106ddecdfceSMircea Gherzan /* 107ddecdfceSMircea Gherzan * Wrapper that handles both OABI and EABI and assures Thumb2 interworking 108ddecdfceSMircea Gherzan * (where the assembly routines like __aeabi_uidiv could cause problems). 109ddecdfceSMircea Gherzan */ 110ddecdfceSMircea Gherzan static u32 jit_udiv(u32 dividend, u32 divisor) 111ddecdfceSMircea Gherzan { 112ddecdfceSMircea Gherzan return dividend / divisor; 113ddecdfceSMircea Gherzan } 114ddecdfceSMircea Gherzan 115ddecdfceSMircea Gherzan static inline void _emit(int cond, u32 inst, struct jit_ctx *ctx) 116ddecdfceSMircea Gherzan { 1173460743eSBen Dooks inst |= (cond << 28); 1183460743eSBen Dooks inst = __opcode_to_mem_arm(inst); 1193460743eSBen Dooks 120ddecdfceSMircea Gherzan if (ctx->target != NULL) 1213460743eSBen Dooks ctx->target[ctx->idx] = inst; 122ddecdfceSMircea Gherzan 123ddecdfceSMircea Gherzan ctx->idx++; 124ddecdfceSMircea Gherzan } 125ddecdfceSMircea Gherzan 126ddecdfceSMircea Gherzan /* 127ddecdfceSMircea Gherzan * Emit an instruction that will be executed unconditionally. 128ddecdfceSMircea Gherzan */ 129ddecdfceSMircea Gherzan static inline void emit(u32 inst, struct jit_ctx *ctx) 130ddecdfceSMircea Gherzan { 131ddecdfceSMircea Gherzan _emit(ARM_COND_AL, inst, ctx); 132ddecdfceSMircea Gherzan } 133ddecdfceSMircea Gherzan 134ddecdfceSMircea Gherzan static u16 saved_regs(struct jit_ctx *ctx) 135ddecdfceSMircea Gherzan { 136ddecdfceSMircea Gherzan u16 ret = 0; 137ddecdfceSMircea Gherzan 138ddecdfceSMircea Gherzan if ((ctx->skf->len > 1) || 13934805931SDaniel Borkmann (ctx->skf->insns[0].code == (BPF_RET | BPF_A))) 140ddecdfceSMircea Gherzan ret |= 1 << r_A; 141ddecdfceSMircea Gherzan 142ddecdfceSMircea Gherzan #ifdef CONFIG_FRAME_POINTER 143ddecdfceSMircea Gherzan ret |= (1 << ARM_FP) | (1 << ARM_IP) | (1 << ARM_LR) | (1 << ARM_PC); 144ddecdfceSMircea Gherzan #else 145ddecdfceSMircea Gherzan if (ctx->seen & SEEN_CALL) 146ddecdfceSMircea Gherzan ret |= 1 << ARM_LR; 147ddecdfceSMircea Gherzan #endif 148ddecdfceSMircea Gherzan if (ctx->seen & (SEEN_DATA | SEEN_SKB)) 149ddecdfceSMircea Gherzan ret |= 1 << r_skb; 150ddecdfceSMircea Gherzan if (ctx->seen & SEEN_DATA) 151ddecdfceSMircea Gherzan ret |= (1 << r_skb_data) | (1 << r_skb_hl); 152ddecdfceSMircea Gherzan if (ctx->seen & SEEN_X) 153ddecdfceSMircea Gherzan ret |= 1 << r_X; 154ddecdfceSMircea Gherzan 155ddecdfceSMircea Gherzan return ret; 156ddecdfceSMircea Gherzan } 157ddecdfceSMircea Gherzan 158ddecdfceSMircea Gherzan static inline int mem_words_used(struct jit_ctx *ctx) 159ddecdfceSMircea Gherzan { 160ddecdfceSMircea Gherzan /* yes, we do waste some stack space IF there are "holes" in the set" */ 161ddecdfceSMircea Gherzan return fls(ctx->seen & SEEN_MEM); 162ddecdfceSMircea Gherzan } 163ddecdfceSMircea Gherzan 164ddecdfceSMircea Gherzan static inline bool is_load_to_a(u16 inst) 165ddecdfceSMircea Gherzan { 166ddecdfceSMircea Gherzan switch (inst) { 16734805931SDaniel Borkmann case BPF_LD | BPF_W | BPF_LEN: 16834805931SDaniel Borkmann case BPF_LD | BPF_W | BPF_ABS: 16934805931SDaniel Borkmann case BPF_LD | BPF_H | BPF_ABS: 17034805931SDaniel Borkmann case BPF_LD | BPF_B | BPF_ABS: 171ddecdfceSMircea Gherzan return true; 172ddecdfceSMircea Gherzan default: 173ddecdfceSMircea Gherzan return false; 174ddecdfceSMircea Gherzan } 175ddecdfceSMircea Gherzan } 176ddecdfceSMircea Gherzan 177ddecdfceSMircea Gherzan static void build_prologue(struct jit_ctx *ctx) 178ddecdfceSMircea Gherzan { 179ddecdfceSMircea Gherzan u16 reg_set = saved_regs(ctx); 180ddecdfceSMircea Gherzan u16 first_inst = ctx->skf->insns[0].code; 181ddecdfceSMircea Gherzan u16 off; 182ddecdfceSMircea Gherzan 183ddecdfceSMircea Gherzan #ifdef CONFIG_FRAME_POINTER 184ddecdfceSMircea Gherzan emit(ARM_MOV_R(ARM_IP, ARM_SP), ctx); 185ddecdfceSMircea Gherzan emit(ARM_PUSH(reg_set), ctx); 186ddecdfceSMircea Gherzan emit(ARM_SUB_I(ARM_FP, ARM_IP, 4), ctx); 187ddecdfceSMircea Gherzan #else 188ddecdfceSMircea Gherzan if (reg_set) 189ddecdfceSMircea Gherzan emit(ARM_PUSH(reg_set), ctx); 190ddecdfceSMircea Gherzan #endif 191ddecdfceSMircea Gherzan 192ddecdfceSMircea Gherzan if (ctx->seen & (SEEN_DATA | SEEN_SKB)) 193ddecdfceSMircea Gherzan emit(ARM_MOV_R(r_skb, ARM_R0), ctx); 194ddecdfceSMircea Gherzan 195ddecdfceSMircea Gherzan if (ctx->seen & SEEN_DATA) { 196ddecdfceSMircea Gherzan off = offsetof(struct sk_buff, data); 197ddecdfceSMircea Gherzan emit(ARM_LDR_I(r_skb_data, r_skb, off), ctx); 198ddecdfceSMircea Gherzan /* headlen = len - data_len */ 199ddecdfceSMircea Gherzan off = offsetof(struct sk_buff, len); 200ddecdfceSMircea Gherzan emit(ARM_LDR_I(r_skb_hl, r_skb, off), ctx); 201ddecdfceSMircea Gherzan off = offsetof(struct sk_buff, data_len); 202ddecdfceSMircea Gherzan emit(ARM_LDR_I(r_scratch, r_skb, off), ctx); 203ddecdfceSMircea Gherzan emit(ARM_SUB_R(r_skb_hl, r_skb_hl, r_scratch), ctx); 204ddecdfceSMircea Gherzan } 205ddecdfceSMircea Gherzan 206ddecdfceSMircea Gherzan if (ctx->flags & FLAG_NEED_X_RESET) 207ddecdfceSMircea Gherzan emit(ARM_MOV_I(r_X, 0), ctx); 208ddecdfceSMircea Gherzan 209ddecdfceSMircea Gherzan /* do not leak kernel data to userspace */ 21034805931SDaniel Borkmann if ((first_inst != (BPF_RET | BPF_K)) && !(is_load_to_a(first_inst))) 211ddecdfceSMircea Gherzan emit(ARM_MOV_I(r_A, 0), ctx); 212ddecdfceSMircea Gherzan 213ddecdfceSMircea Gherzan /* stack space for the BPF_MEM words */ 214ddecdfceSMircea Gherzan if (ctx->seen & SEEN_MEM) 215ddecdfceSMircea Gherzan emit(ARM_SUB_I(ARM_SP, ARM_SP, mem_words_used(ctx) * 4), ctx); 216ddecdfceSMircea Gherzan } 217ddecdfceSMircea Gherzan 218ddecdfceSMircea Gherzan static void build_epilogue(struct jit_ctx *ctx) 219ddecdfceSMircea Gherzan { 220ddecdfceSMircea Gherzan u16 reg_set = saved_regs(ctx); 221ddecdfceSMircea Gherzan 222ddecdfceSMircea Gherzan if (ctx->seen & SEEN_MEM) 223ddecdfceSMircea Gherzan emit(ARM_ADD_I(ARM_SP, ARM_SP, mem_words_used(ctx) * 4), ctx); 224ddecdfceSMircea Gherzan 225ddecdfceSMircea Gherzan reg_set &= ~(1 << ARM_LR); 226ddecdfceSMircea Gherzan 227ddecdfceSMircea Gherzan #ifdef CONFIG_FRAME_POINTER 228ddecdfceSMircea Gherzan /* the first instruction of the prologue was: mov ip, sp */ 229ddecdfceSMircea Gherzan reg_set &= ~(1 << ARM_IP); 230ddecdfceSMircea Gherzan reg_set |= (1 << ARM_SP); 231ddecdfceSMircea Gherzan emit(ARM_LDM(ARM_SP, reg_set), ctx); 232ddecdfceSMircea Gherzan #else 233ddecdfceSMircea Gherzan if (reg_set) { 234ddecdfceSMircea Gherzan if (ctx->seen & SEEN_CALL) 235ddecdfceSMircea Gherzan reg_set |= 1 << ARM_PC; 236ddecdfceSMircea Gherzan emit(ARM_POP(reg_set), ctx); 237ddecdfceSMircea Gherzan } 238ddecdfceSMircea Gherzan 239ddecdfceSMircea Gherzan if (!(ctx->seen & SEEN_CALL)) 240ddecdfceSMircea Gherzan emit(ARM_BX(ARM_LR), ctx); 241ddecdfceSMircea Gherzan #endif 242ddecdfceSMircea Gherzan } 243ddecdfceSMircea Gherzan 244ddecdfceSMircea Gherzan static int16_t imm8m(u32 x) 245ddecdfceSMircea Gherzan { 246ddecdfceSMircea Gherzan u32 rot; 247ddecdfceSMircea Gherzan 248ddecdfceSMircea Gherzan for (rot = 0; rot < 16; rot++) 249ddecdfceSMircea Gherzan if ((x & ~ror32(0xff, 2 * rot)) == 0) 250ddecdfceSMircea Gherzan return rol32(x, 2 * rot) | (rot << 8); 251ddecdfceSMircea Gherzan 252ddecdfceSMircea Gherzan return -1; 253ddecdfceSMircea Gherzan } 254ddecdfceSMircea Gherzan 255ddecdfceSMircea Gherzan #if __LINUX_ARM_ARCH__ < 7 256ddecdfceSMircea Gherzan 257ddecdfceSMircea Gherzan static u16 imm_offset(u32 k, struct jit_ctx *ctx) 258ddecdfceSMircea Gherzan { 259ddecdfceSMircea Gherzan unsigned i = 0, offset; 260ddecdfceSMircea Gherzan u16 imm; 261ddecdfceSMircea Gherzan 262ddecdfceSMircea Gherzan /* on the "fake" run we just count them (duplicates included) */ 263ddecdfceSMircea Gherzan if (ctx->target == NULL) { 264ddecdfceSMircea Gherzan ctx->imm_count++; 265ddecdfceSMircea Gherzan return 0; 266ddecdfceSMircea Gherzan } 267ddecdfceSMircea Gherzan 268ddecdfceSMircea Gherzan while ((i < ctx->imm_count) && ctx->imms[i]) { 269ddecdfceSMircea Gherzan if (ctx->imms[i] == k) 270ddecdfceSMircea Gherzan break; 271ddecdfceSMircea Gherzan i++; 272ddecdfceSMircea Gherzan } 273ddecdfceSMircea Gherzan 274ddecdfceSMircea Gherzan if (ctx->imms[i] == 0) 275ddecdfceSMircea Gherzan ctx->imms[i] = k; 276ddecdfceSMircea Gherzan 277ddecdfceSMircea Gherzan /* constants go just after the epilogue */ 278ddecdfceSMircea Gherzan offset = ctx->offsets[ctx->skf->len]; 279ddecdfceSMircea Gherzan offset += ctx->prologue_bytes; 280ddecdfceSMircea Gherzan offset += ctx->epilogue_bytes; 281ddecdfceSMircea Gherzan offset += i * 4; 282ddecdfceSMircea Gherzan 283ddecdfceSMircea Gherzan ctx->target[offset / 4] = k; 284ddecdfceSMircea Gherzan 285ddecdfceSMircea Gherzan /* PC in ARM mode == address of the instruction + 8 */ 286ddecdfceSMircea Gherzan imm = offset - (8 + ctx->idx * 4); 287ddecdfceSMircea Gherzan 288ddecdfceSMircea Gherzan return imm; 289ddecdfceSMircea Gherzan } 290ddecdfceSMircea Gherzan 291ddecdfceSMircea Gherzan #endif /* __LINUX_ARM_ARCH__ */ 292ddecdfceSMircea Gherzan 293ddecdfceSMircea Gherzan /* 294ddecdfceSMircea Gherzan * Move an immediate that's not an imm8m to a core register. 295ddecdfceSMircea Gherzan */ 296ddecdfceSMircea Gherzan static inline void emit_mov_i_no8m(int rd, u32 val, struct jit_ctx *ctx) 297ddecdfceSMircea Gherzan { 298ddecdfceSMircea Gherzan #if __LINUX_ARM_ARCH__ < 7 299ddecdfceSMircea Gherzan emit(ARM_LDR_I(rd, ARM_PC, imm_offset(val, ctx)), ctx); 300ddecdfceSMircea Gherzan #else 301ddecdfceSMircea Gherzan emit(ARM_MOVW(rd, val & 0xffff), ctx); 302ddecdfceSMircea Gherzan if (val > 0xffff) 303ddecdfceSMircea Gherzan emit(ARM_MOVT(rd, val >> 16), ctx); 304ddecdfceSMircea Gherzan #endif 305ddecdfceSMircea Gherzan } 306ddecdfceSMircea Gherzan 307ddecdfceSMircea Gherzan static inline void emit_mov_i(int rd, u32 val, struct jit_ctx *ctx) 308ddecdfceSMircea Gherzan { 309ddecdfceSMircea Gherzan int imm12 = imm8m(val); 310ddecdfceSMircea Gherzan 311ddecdfceSMircea Gherzan if (imm12 >= 0) 312ddecdfceSMircea Gherzan emit(ARM_MOV_I(rd, imm12), ctx); 313ddecdfceSMircea Gherzan else 314ddecdfceSMircea Gherzan emit_mov_i_no8m(rd, val, ctx); 315ddecdfceSMircea Gherzan } 316ddecdfceSMircea Gherzan 317ddecdfceSMircea Gherzan #if __LINUX_ARM_ARCH__ < 6 318ddecdfceSMircea Gherzan 319ddecdfceSMircea Gherzan static void emit_load_be32(u8 cond, u8 r_res, u8 r_addr, struct jit_ctx *ctx) 320ddecdfceSMircea Gherzan { 321ddecdfceSMircea Gherzan _emit(cond, ARM_LDRB_I(ARM_R3, r_addr, 1), ctx); 322ddecdfceSMircea Gherzan _emit(cond, ARM_LDRB_I(ARM_R1, r_addr, 0), ctx); 323ddecdfceSMircea Gherzan _emit(cond, ARM_LDRB_I(ARM_R2, r_addr, 3), ctx); 324ddecdfceSMircea Gherzan _emit(cond, ARM_LSL_I(ARM_R3, ARM_R3, 16), ctx); 325ddecdfceSMircea Gherzan _emit(cond, ARM_LDRB_I(ARM_R0, r_addr, 2), ctx); 326ddecdfceSMircea Gherzan _emit(cond, ARM_ORR_S(ARM_R3, ARM_R3, ARM_R1, SRTYPE_LSL, 24), ctx); 327ddecdfceSMircea Gherzan _emit(cond, ARM_ORR_R(ARM_R3, ARM_R3, ARM_R2), ctx); 328ddecdfceSMircea Gherzan _emit(cond, ARM_ORR_S(r_res, ARM_R3, ARM_R0, SRTYPE_LSL, 8), ctx); 329ddecdfceSMircea Gherzan } 330ddecdfceSMircea Gherzan 331ddecdfceSMircea Gherzan static void emit_load_be16(u8 cond, u8 r_res, u8 r_addr, struct jit_ctx *ctx) 332ddecdfceSMircea Gherzan { 333ddecdfceSMircea Gherzan _emit(cond, ARM_LDRB_I(ARM_R1, r_addr, 0), ctx); 334ddecdfceSMircea Gherzan _emit(cond, ARM_LDRB_I(ARM_R2, r_addr, 1), ctx); 335ddecdfceSMircea Gherzan _emit(cond, ARM_ORR_S(r_res, ARM_R2, ARM_R1, SRTYPE_LSL, 8), ctx); 336ddecdfceSMircea Gherzan } 337ddecdfceSMircea Gherzan 338ddecdfceSMircea Gherzan static inline void emit_swap16(u8 r_dst, u8 r_src, struct jit_ctx *ctx) 339ddecdfceSMircea Gherzan { 340462738f4SNicolas Schichan /* r_dst = (r_src << 8) | (r_src >> 8) */ 341462738f4SNicolas Schichan emit(ARM_LSL_I(ARM_R1, r_src, 8), ctx); 342462738f4SNicolas Schichan emit(ARM_ORR_S(r_dst, ARM_R1, r_src, SRTYPE_LSR, 8), ctx); 343462738f4SNicolas Schichan 344462738f4SNicolas Schichan /* 345462738f4SNicolas Schichan * we need to mask out the bits set in r_dst[23:16] due to 346462738f4SNicolas Schichan * the first shift instruction. 347462738f4SNicolas Schichan * 348462738f4SNicolas Schichan * note that 0x8ff is the encoded immediate 0x00ff0000. 349462738f4SNicolas Schichan */ 350462738f4SNicolas Schichan emit(ARM_BIC_I(r_dst, r_dst, 0x8ff), ctx); 351ddecdfceSMircea Gherzan } 352ddecdfceSMircea Gherzan 353ddecdfceSMircea Gherzan #else /* ARMv6+ */ 354ddecdfceSMircea Gherzan 355ddecdfceSMircea Gherzan static void emit_load_be32(u8 cond, u8 r_res, u8 r_addr, struct jit_ctx *ctx) 356ddecdfceSMircea Gherzan { 357ddecdfceSMircea Gherzan _emit(cond, ARM_LDR_I(r_res, r_addr, 0), ctx); 358ddecdfceSMircea Gherzan #ifdef __LITTLE_ENDIAN 359ddecdfceSMircea Gherzan _emit(cond, ARM_REV(r_res, r_res), ctx); 360ddecdfceSMircea Gherzan #endif 361ddecdfceSMircea Gherzan } 362ddecdfceSMircea Gherzan 363ddecdfceSMircea Gherzan static void emit_load_be16(u8 cond, u8 r_res, u8 r_addr, struct jit_ctx *ctx) 364ddecdfceSMircea Gherzan { 365ddecdfceSMircea Gherzan _emit(cond, ARM_LDRH_I(r_res, r_addr, 0), ctx); 366ddecdfceSMircea Gherzan #ifdef __LITTLE_ENDIAN 367ddecdfceSMircea Gherzan _emit(cond, ARM_REV16(r_res, r_res), ctx); 368ddecdfceSMircea Gherzan #endif 369ddecdfceSMircea Gherzan } 370ddecdfceSMircea Gherzan 371ddecdfceSMircea Gherzan static inline void emit_swap16(u8 r_dst __maybe_unused, 372ddecdfceSMircea Gherzan u8 r_src __maybe_unused, 373ddecdfceSMircea Gherzan struct jit_ctx *ctx __maybe_unused) 374ddecdfceSMircea Gherzan { 375ddecdfceSMircea Gherzan #ifdef __LITTLE_ENDIAN 376ddecdfceSMircea Gherzan emit(ARM_REV16(r_dst, r_src), ctx); 377ddecdfceSMircea Gherzan #endif 378ddecdfceSMircea Gherzan } 379ddecdfceSMircea Gherzan 380ddecdfceSMircea Gherzan #endif /* __LINUX_ARM_ARCH__ < 6 */ 381ddecdfceSMircea Gherzan 382ddecdfceSMircea Gherzan 383ddecdfceSMircea Gherzan /* Compute the immediate value for a PC-relative branch. */ 384ddecdfceSMircea Gherzan static inline u32 b_imm(unsigned tgt, struct jit_ctx *ctx) 385ddecdfceSMircea Gherzan { 386ddecdfceSMircea Gherzan u32 imm; 387ddecdfceSMircea Gherzan 388ddecdfceSMircea Gherzan if (ctx->target == NULL) 389ddecdfceSMircea Gherzan return 0; 390ddecdfceSMircea Gherzan /* 391ddecdfceSMircea Gherzan * BPF allows only forward jumps and the offset of the target is 392ddecdfceSMircea Gherzan * still the one computed during the first pass. 393ddecdfceSMircea Gherzan */ 394ddecdfceSMircea Gherzan imm = ctx->offsets[tgt] + ctx->prologue_bytes - (ctx->idx * 4 + 8); 395ddecdfceSMircea Gherzan 396ddecdfceSMircea Gherzan return imm >> 2; 397ddecdfceSMircea Gherzan } 398ddecdfceSMircea Gherzan 399ddecdfceSMircea Gherzan #define OP_IMM3(op, r1, r2, imm_val, ctx) \ 400ddecdfceSMircea Gherzan do { \ 401ddecdfceSMircea Gherzan imm12 = imm8m(imm_val); \ 402ddecdfceSMircea Gherzan if (imm12 < 0) { \ 403ddecdfceSMircea Gherzan emit_mov_i_no8m(r_scratch, imm_val, ctx); \ 404ddecdfceSMircea Gherzan emit(op ## _R((r1), (r2), r_scratch), ctx); \ 405ddecdfceSMircea Gherzan } else { \ 406ddecdfceSMircea Gherzan emit(op ## _I((r1), (r2), imm12), ctx); \ 407ddecdfceSMircea Gherzan } \ 408ddecdfceSMircea Gherzan } while (0) 409ddecdfceSMircea Gherzan 410ddecdfceSMircea Gherzan static inline void emit_err_ret(u8 cond, struct jit_ctx *ctx) 411ddecdfceSMircea Gherzan { 412ddecdfceSMircea Gherzan if (ctx->ret0_fp_idx >= 0) { 413ddecdfceSMircea Gherzan _emit(cond, ARM_B(b_imm(ctx->ret0_fp_idx, ctx)), ctx); 414ddecdfceSMircea Gherzan /* NOP to keep the size constant between passes */ 415ddecdfceSMircea Gherzan emit(ARM_MOV_R(ARM_R0, ARM_R0), ctx); 416ddecdfceSMircea Gherzan } else { 417ddecdfceSMircea Gherzan _emit(cond, ARM_MOV_I(ARM_R0, 0), ctx); 418ddecdfceSMircea Gherzan _emit(cond, ARM_B(b_imm(ctx->skf->len, ctx)), ctx); 419ddecdfceSMircea Gherzan } 420ddecdfceSMircea Gherzan } 421ddecdfceSMircea Gherzan 422ddecdfceSMircea Gherzan static inline void emit_blx_r(u8 tgt_reg, struct jit_ctx *ctx) 423ddecdfceSMircea Gherzan { 424ddecdfceSMircea Gherzan #if __LINUX_ARM_ARCH__ < 5 425ddecdfceSMircea Gherzan emit(ARM_MOV_R(ARM_LR, ARM_PC), ctx); 426ddecdfceSMircea Gherzan 427ddecdfceSMircea Gherzan if (elf_hwcap & HWCAP_THUMB) 428ddecdfceSMircea Gherzan emit(ARM_BX(tgt_reg), ctx); 429ddecdfceSMircea Gherzan else 430ddecdfceSMircea Gherzan emit(ARM_MOV_R(ARM_PC, tgt_reg), ctx); 431ddecdfceSMircea Gherzan #else 432ddecdfceSMircea Gherzan emit(ARM_BLX_R(tgt_reg), ctx); 433ddecdfceSMircea Gherzan #endif 434ddecdfceSMircea Gherzan } 435ddecdfceSMircea Gherzan 436ddecdfceSMircea Gherzan static inline void emit_udiv(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx) 437ddecdfceSMircea Gherzan { 438ddecdfceSMircea Gherzan #if __LINUX_ARM_ARCH__ == 7 439ddecdfceSMircea Gherzan if (elf_hwcap & HWCAP_IDIVA) { 440ddecdfceSMircea Gherzan emit(ARM_UDIV(rd, rm, rn), ctx); 441ddecdfceSMircea Gherzan return; 442ddecdfceSMircea Gherzan } 443ddecdfceSMircea Gherzan #endif 444ddecdfceSMircea Gherzan if (rm != ARM_R0) 445ddecdfceSMircea Gherzan emit(ARM_MOV_R(ARM_R0, rm), ctx); 446ddecdfceSMircea Gherzan if (rn != ARM_R1) 447ddecdfceSMircea Gherzan emit(ARM_MOV_R(ARM_R1, rn), ctx); 448ddecdfceSMircea Gherzan 449ddecdfceSMircea Gherzan ctx->seen |= SEEN_CALL; 450ddecdfceSMircea Gherzan emit_mov_i(ARM_R3, (u32)jit_udiv, ctx); 451ddecdfceSMircea Gherzan emit_blx_r(ARM_R3, ctx); 452ddecdfceSMircea Gherzan 453ddecdfceSMircea Gherzan if (rd != ARM_R0) 454ddecdfceSMircea Gherzan emit(ARM_MOV_R(rd, ARM_R0), ctx); 455ddecdfceSMircea Gherzan } 456ddecdfceSMircea Gherzan 457ddecdfceSMircea Gherzan static inline void update_on_xread(struct jit_ctx *ctx) 458ddecdfceSMircea Gherzan { 459ddecdfceSMircea Gherzan if (!(ctx->seen & SEEN_X)) 460ddecdfceSMircea Gherzan ctx->flags |= FLAG_NEED_X_RESET; 461ddecdfceSMircea Gherzan 462ddecdfceSMircea Gherzan ctx->seen |= SEEN_X; 463ddecdfceSMircea Gherzan } 464ddecdfceSMircea Gherzan 465ddecdfceSMircea Gherzan static int build_body(struct jit_ctx *ctx) 466ddecdfceSMircea Gherzan { 467ddecdfceSMircea Gherzan void *load_func[] = {jit_get_skb_b, jit_get_skb_h, jit_get_skb_w}; 4687ae457c1SAlexei Starovoitov const struct bpf_prog *prog = ctx->skf; 469ddecdfceSMircea Gherzan const struct sock_filter *inst; 470ddecdfceSMircea Gherzan unsigned i, load_order, off, condt; 471ddecdfceSMircea Gherzan int imm12; 472ddecdfceSMircea Gherzan u32 k; 473ddecdfceSMircea Gherzan 474ddecdfceSMircea Gherzan for (i = 0; i < prog->len; i++) { 47534805931SDaniel Borkmann u16 code; 47634805931SDaniel Borkmann 477ddecdfceSMircea Gherzan inst = &(prog->insns[i]); 478ddecdfceSMircea Gherzan /* K as an immediate value operand */ 479ddecdfceSMircea Gherzan k = inst->k; 48034805931SDaniel Borkmann code = bpf_anc_helper(inst); 481ddecdfceSMircea Gherzan 482ddecdfceSMircea Gherzan /* compute offsets only in the fake pass */ 483ddecdfceSMircea Gherzan if (ctx->target == NULL) 484ddecdfceSMircea Gherzan ctx->offsets[i] = ctx->idx * 4; 485ddecdfceSMircea Gherzan 48634805931SDaniel Borkmann switch (code) { 48734805931SDaniel Borkmann case BPF_LD | BPF_IMM: 488ddecdfceSMircea Gherzan emit_mov_i(r_A, k, ctx); 489ddecdfceSMircea Gherzan break; 49034805931SDaniel Borkmann case BPF_LD | BPF_W | BPF_LEN: 491ddecdfceSMircea Gherzan ctx->seen |= SEEN_SKB; 492ddecdfceSMircea Gherzan BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4); 493ddecdfceSMircea Gherzan emit(ARM_LDR_I(r_A, r_skb, 494ddecdfceSMircea Gherzan offsetof(struct sk_buff, len)), ctx); 495ddecdfceSMircea Gherzan break; 49634805931SDaniel Borkmann case BPF_LD | BPF_MEM: 497ddecdfceSMircea Gherzan /* A = scratch[k] */ 498ddecdfceSMircea Gherzan ctx->seen |= SEEN_MEM_WORD(k); 499ddecdfceSMircea Gherzan emit(ARM_LDR_I(r_A, ARM_SP, SCRATCH_OFF(k)), ctx); 500ddecdfceSMircea Gherzan break; 50134805931SDaniel Borkmann case BPF_LD | BPF_W | BPF_ABS: 502ddecdfceSMircea Gherzan load_order = 2; 503ddecdfceSMircea Gherzan goto load; 50434805931SDaniel Borkmann case BPF_LD | BPF_H | BPF_ABS: 505ddecdfceSMircea Gherzan load_order = 1; 506ddecdfceSMircea Gherzan goto load; 50734805931SDaniel Borkmann case BPF_LD | BPF_B | BPF_ABS: 508ddecdfceSMircea Gherzan load_order = 0; 509ddecdfceSMircea Gherzan load: 510ddecdfceSMircea Gherzan /* the interpreter will deal with the negative K */ 511ddecdfceSMircea Gherzan if ((int)k < 0) 512ddecdfceSMircea Gherzan return -ENOTSUPP; 513ddecdfceSMircea Gherzan emit_mov_i(r_off, k, ctx); 514ddecdfceSMircea Gherzan load_common: 515ddecdfceSMircea Gherzan ctx->seen |= SEEN_DATA | SEEN_CALL; 516ddecdfceSMircea Gherzan 517ddecdfceSMircea Gherzan if (load_order > 0) { 518ddecdfceSMircea Gherzan emit(ARM_SUB_I(r_scratch, r_skb_hl, 519ddecdfceSMircea Gherzan 1 << load_order), ctx); 520ddecdfceSMircea Gherzan emit(ARM_CMP_R(r_scratch, r_off), ctx); 521ddecdfceSMircea Gherzan condt = ARM_COND_HS; 522ddecdfceSMircea Gherzan } else { 523ddecdfceSMircea Gherzan emit(ARM_CMP_R(r_skb_hl, r_off), ctx); 524ddecdfceSMircea Gherzan condt = ARM_COND_HI; 525ddecdfceSMircea Gherzan } 526ddecdfceSMircea Gherzan 527ddecdfceSMircea Gherzan _emit(condt, ARM_ADD_R(r_scratch, r_off, r_skb_data), 528ddecdfceSMircea Gherzan ctx); 529ddecdfceSMircea Gherzan 530ddecdfceSMircea Gherzan if (load_order == 0) 531ddecdfceSMircea Gherzan _emit(condt, ARM_LDRB_I(r_A, r_scratch, 0), 532ddecdfceSMircea Gherzan ctx); 533ddecdfceSMircea Gherzan else if (load_order == 1) 534ddecdfceSMircea Gherzan emit_load_be16(condt, r_A, r_scratch, ctx); 535ddecdfceSMircea Gherzan else if (load_order == 2) 536ddecdfceSMircea Gherzan emit_load_be32(condt, r_A, r_scratch, ctx); 537ddecdfceSMircea Gherzan 538ddecdfceSMircea Gherzan _emit(condt, ARM_B(b_imm(i + 1, ctx)), ctx); 539ddecdfceSMircea Gherzan 540ddecdfceSMircea Gherzan /* the slowpath */ 541ddecdfceSMircea Gherzan emit_mov_i(ARM_R3, (u32)load_func[load_order], ctx); 542ddecdfceSMircea Gherzan emit(ARM_MOV_R(ARM_R0, r_skb), ctx); 543ddecdfceSMircea Gherzan /* the offset is already in R1 */ 544ddecdfceSMircea Gherzan emit_blx_r(ARM_R3, ctx); 545ddecdfceSMircea Gherzan /* check the result of skb_copy_bits */ 546ddecdfceSMircea Gherzan emit(ARM_CMP_I(ARM_R1, 0), ctx); 547ddecdfceSMircea Gherzan emit_err_ret(ARM_COND_NE, ctx); 548ddecdfceSMircea Gherzan emit(ARM_MOV_R(r_A, ARM_R0), ctx); 549ddecdfceSMircea Gherzan break; 55034805931SDaniel Borkmann case BPF_LD | BPF_W | BPF_IND: 551ddecdfceSMircea Gherzan load_order = 2; 552ddecdfceSMircea Gherzan goto load_ind; 55334805931SDaniel Borkmann case BPF_LD | BPF_H | BPF_IND: 554ddecdfceSMircea Gherzan load_order = 1; 555ddecdfceSMircea Gherzan goto load_ind; 55634805931SDaniel Borkmann case BPF_LD | BPF_B | BPF_IND: 557ddecdfceSMircea Gherzan load_order = 0; 558ddecdfceSMircea Gherzan load_ind: 559ddecdfceSMircea Gherzan OP_IMM3(ARM_ADD, r_off, r_X, k, ctx); 560ddecdfceSMircea Gherzan goto load_common; 56134805931SDaniel Borkmann case BPF_LDX | BPF_IMM: 562ddecdfceSMircea Gherzan ctx->seen |= SEEN_X; 563ddecdfceSMircea Gherzan emit_mov_i(r_X, k, ctx); 564ddecdfceSMircea Gherzan break; 56534805931SDaniel Borkmann case BPF_LDX | BPF_W | BPF_LEN: 566ddecdfceSMircea Gherzan ctx->seen |= SEEN_X | SEEN_SKB; 567ddecdfceSMircea Gherzan emit(ARM_LDR_I(r_X, r_skb, 568ddecdfceSMircea Gherzan offsetof(struct sk_buff, len)), ctx); 569ddecdfceSMircea Gherzan break; 57034805931SDaniel Borkmann case BPF_LDX | BPF_MEM: 571ddecdfceSMircea Gherzan ctx->seen |= SEEN_X | SEEN_MEM_WORD(k); 572ddecdfceSMircea Gherzan emit(ARM_LDR_I(r_X, ARM_SP, SCRATCH_OFF(k)), ctx); 573ddecdfceSMircea Gherzan break; 57434805931SDaniel Borkmann case BPF_LDX | BPF_B | BPF_MSH: 575ddecdfceSMircea Gherzan /* x = ((*(frame + k)) & 0xf) << 2; */ 576ddecdfceSMircea Gherzan ctx->seen |= SEEN_X | SEEN_DATA | SEEN_CALL; 577ddecdfceSMircea Gherzan /* the interpreter should deal with the negative K */ 57845549a68SChen Gang if ((int)k < 0) 579ddecdfceSMircea Gherzan return -1; 580ddecdfceSMircea Gherzan /* offset in r1: we might have to take the slow path */ 581ddecdfceSMircea Gherzan emit_mov_i(r_off, k, ctx); 582ddecdfceSMircea Gherzan emit(ARM_CMP_R(r_skb_hl, r_off), ctx); 583ddecdfceSMircea Gherzan 584ddecdfceSMircea Gherzan /* load in r0: common with the slowpath */ 585ddecdfceSMircea Gherzan _emit(ARM_COND_HI, ARM_LDRB_R(ARM_R0, r_skb_data, 586ddecdfceSMircea Gherzan ARM_R1), ctx); 587ddecdfceSMircea Gherzan /* 588ddecdfceSMircea Gherzan * emit_mov_i() might generate one or two instructions, 589ddecdfceSMircea Gherzan * the same holds for emit_blx_r() 590ddecdfceSMircea Gherzan */ 591ddecdfceSMircea Gherzan _emit(ARM_COND_HI, ARM_B(b_imm(i + 1, ctx) - 2), ctx); 592ddecdfceSMircea Gherzan 593ddecdfceSMircea Gherzan emit(ARM_MOV_R(ARM_R0, r_skb), ctx); 594ddecdfceSMircea Gherzan /* r_off is r1 */ 595ddecdfceSMircea Gherzan emit_mov_i(ARM_R3, (u32)jit_get_skb_b, ctx); 596ddecdfceSMircea Gherzan emit_blx_r(ARM_R3, ctx); 597ddecdfceSMircea Gherzan /* check the return value of skb_copy_bits */ 598ddecdfceSMircea Gherzan emit(ARM_CMP_I(ARM_R1, 0), ctx); 599ddecdfceSMircea Gherzan emit_err_ret(ARM_COND_NE, ctx); 600ddecdfceSMircea Gherzan 601ddecdfceSMircea Gherzan emit(ARM_AND_I(r_X, ARM_R0, 0x00f), ctx); 602ddecdfceSMircea Gherzan emit(ARM_LSL_I(r_X, r_X, 2), ctx); 603ddecdfceSMircea Gherzan break; 60434805931SDaniel Borkmann case BPF_ST: 605ddecdfceSMircea Gherzan ctx->seen |= SEEN_MEM_WORD(k); 606ddecdfceSMircea Gherzan emit(ARM_STR_I(r_A, ARM_SP, SCRATCH_OFF(k)), ctx); 607ddecdfceSMircea Gherzan break; 60834805931SDaniel Borkmann case BPF_STX: 609ddecdfceSMircea Gherzan update_on_xread(ctx); 610ddecdfceSMircea Gherzan ctx->seen |= SEEN_MEM_WORD(k); 611ddecdfceSMircea Gherzan emit(ARM_STR_I(r_X, ARM_SP, SCRATCH_OFF(k)), ctx); 612ddecdfceSMircea Gherzan break; 61334805931SDaniel Borkmann case BPF_ALU | BPF_ADD | BPF_K: 614ddecdfceSMircea Gherzan /* A += K */ 615ddecdfceSMircea Gherzan OP_IMM3(ARM_ADD, r_A, r_A, k, ctx); 616ddecdfceSMircea Gherzan break; 61734805931SDaniel Borkmann case BPF_ALU | BPF_ADD | BPF_X: 618ddecdfceSMircea Gherzan update_on_xread(ctx); 619ddecdfceSMircea Gherzan emit(ARM_ADD_R(r_A, r_A, r_X), ctx); 620ddecdfceSMircea Gherzan break; 62134805931SDaniel Borkmann case BPF_ALU | BPF_SUB | BPF_K: 622ddecdfceSMircea Gherzan /* A -= K */ 623ddecdfceSMircea Gherzan OP_IMM3(ARM_SUB, r_A, r_A, k, ctx); 624ddecdfceSMircea Gherzan break; 62534805931SDaniel Borkmann case BPF_ALU | BPF_SUB | BPF_X: 626ddecdfceSMircea Gherzan update_on_xread(ctx); 627ddecdfceSMircea Gherzan emit(ARM_SUB_R(r_A, r_A, r_X), ctx); 628ddecdfceSMircea Gherzan break; 62934805931SDaniel Borkmann case BPF_ALU | BPF_MUL | BPF_K: 630ddecdfceSMircea Gherzan /* A *= K */ 631ddecdfceSMircea Gherzan emit_mov_i(r_scratch, k, ctx); 632ddecdfceSMircea Gherzan emit(ARM_MUL(r_A, r_A, r_scratch), ctx); 633ddecdfceSMircea Gherzan break; 63434805931SDaniel Borkmann case BPF_ALU | BPF_MUL | BPF_X: 635ddecdfceSMircea Gherzan update_on_xread(ctx); 636ddecdfceSMircea Gherzan emit(ARM_MUL(r_A, r_A, r_X), ctx); 637ddecdfceSMircea Gherzan break; 63834805931SDaniel Borkmann case BPF_ALU | BPF_DIV | BPF_K: 639aee636c4SEric Dumazet if (k == 1) 640aee636c4SEric Dumazet break; 641ddecdfceSMircea Gherzan emit_mov_i(r_scratch, k, ctx); 642aee636c4SEric Dumazet emit_udiv(r_A, r_A, r_scratch, ctx); 643ddecdfceSMircea Gherzan break; 64434805931SDaniel Borkmann case BPF_ALU | BPF_DIV | BPF_X: 645ddecdfceSMircea Gherzan update_on_xread(ctx); 646ddecdfceSMircea Gherzan emit(ARM_CMP_I(r_X, 0), ctx); 647ddecdfceSMircea Gherzan emit_err_ret(ARM_COND_EQ, ctx); 648ddecdfceSMircea Gherzan emit_udiv(r_A, r_A, r_X, ctx); 649ddecdfceSMircea Gherzan break; 65034805931SDaniel Borkmann case BPF_ALU | BPF_OR | BPF_K: 651ddecdfceSMircea Gherzan /* A |= K */ 652ddecdfceSMircea Gherzan OP_IMM3(ARM_ORR, r_A, r_A, k, ctx); 653ddecdfceSMircea Gherzan break; 65434805931SDaniel Borkmann case BPF_ALU | BPF_OR | BPF_X: 655ddecdfceSMircea Gherzan update_on_xread(ctx); 656ddecdfceSMircea Gherzan emit(ARM_ORR_R(r_A, r_A, r_X), ctx); 657ddecdfceSMircea Gherzan break; 65834805931SDaniel Borkmann case BPF_ALU | BPF_XOR | BPF_K: 6593cbe2041SDaniel Borkmann /* A ^= K; */ 6603cbe2041SDaniel Borkmann OP_IMM3(ARM_EOR, r_A, r_A, k, ctx); 6613cbe2041SDaniel Borkmann break; 66234805931SDaniel Borkmann case BPF_ANC | SKF_AD_ALU_XOR_X: 66334805931SDaniel Borkmann case BPF_ALU | BPF_XOR | BPF_X: 6643cbe2041SDaniel Borkmann /* A ^= X */ 6653cbe2041SDaniel Borkmann update_on_xread(ctx); 6663cbe2041SDaniel Borkmann emit(ARM_EOR_R(r_A, r_A, r_X), ctx); 6673cbe2041SDaniel Borkmann break; 66834805931SDaniel Borkmann case BPF_ALU | BPF_AND | BPF_K: 669ddecdfceSMircea Gherzan /* A &= K */ 670ddecdfceSMircea Gherzan OP_IMM3(ARM_AND, r_A, r_A, k, ctx); 671ddecdfceSMircea Gherzan break; 67234805931SDaniel Borkmann case BPF_ALU | BPF_AND | BPF_X: 673ddecdfceSMircea Gherzan update_on_xread(ctx); 674ddecdfceSMircea Gherzan emit(ARM_AND_R(r_A, r_A, r_X), ctx); 675ddecdfceSMircea Gherzan break; 67634805931SDaniel Borkmann case BPF_ALU | BPF_LSH | BPF_K: 677ddecdfceSMircea Gherzan if (unlikely(k > 31)) 678ddecdfceSMircea Gherzan return -1; 679ddecdfceSMircea Gherzan emit(ARM_LSL_I(r_A, r_A, k), ctx); 680ddecdfceSMircea Gherzan break; 68134805931SDaniel Borkmann case BPF_ALU | BPF_LSH | BPF_X: 682ddecdfceSMircea Gherzan update_on_xread(ctx); 683ddecdfceSMircea Gherzan emit(ARM_LSL_R(r_A, r_A, r_X), ctx); 684ddecdfceSMircea Gherzan break; 68534805931SDaniel Borkmann case BPF_ALU | BPF_RSH | BPF_K: 686ddecdfceSMircea Gherzan if (unlikely(k > 31)) 687ddecdfceSMircea Gherzan return -1; 688ddecdfceSMircea Gherzan emit(ARM_LSR_I(r_A, r_A, k), ctx); 689ddecdfceSMircea Gherzan break; 69034805931SDaniel Borkmann case BPF_ALU | BPF_RSH | BPF_X: 691ddecdfceSMircea Gherzan update_on_xread(ctx); 692ddecdfceSMircea Gherzan emit(ARM_LSR_R(r_A, r_A, r_X), ctx); 693ddecdfceSMircea Gherzan break; 69434805931SDaniel Borkmann case BPF_ALU | BPF_NEG: 695ddecdfceSMircea Gherzan /* A = -A */ 696ddecdfceSMircea Gherzan emit(ARM_RSB_I(r_A, r_A, 0), ctx); 697ddecdfceSMircea Gherzan break; 69834805931SDaniel Borkmann case BPF_JMP | BPF_JA: 699ddecdfceSMircea Gherzan /* pc += K */ 700ddecdfceSMircea Gherzan emit(ARM_B(b_imm(i + k + 1, ctx)), ctx); 701ddecdfceSMircea Gherzan break; 70234805931SDaniel Borkmann case BPF_JMP | BPF_JEQ | BPF_K: 703ddecdfceSMircea Gherzan /* pc += (A == K) ? pc->jt : pc->jf */ 704ddecdfceSMircea Gherzan condt = ARM_COND_EQ; 705ddecdfceSMircea Gherzan goto cmp_imm; 70634805931SDaniel Borkmann case BPF_JMP | BPF_JGT | BPF_K: 707ddecdfceSMircea Gherzan /* pc += (A > K) ? pc->jt : pc->jf */ 708ddecdfceSMircea Gherzan condt = ARM_COND_HI; 709ddecdfceSMircea Gherzan goto cmp_imm; 71034805931SDaniel Borkmann case BPF_JMP | BPF_JGE | BPF_K: 711ddecdfceSMircea Gherzan /* pc += (A >= K) ? pc->jt : pc->jf */ 712ddecdfceSMircea Gherzan condt = ARM_COND_HS; 713ddecdfceSMircea Gherzan cmp_imm: 714ddecdfceSMircea Gherzan imm12 = imm8m(k); 715ddecdfceSMircea Gherzan if (imm12 < 0) { 716ddecdfceSMircea Gherzan emit_mov_i_no8m(r_scratch, k, ctx); 717ddecdfceSMircea Gherzan emit(ARM_CMP_R(r_A, r_scratch), ctx); 718ddecdfceSMircea Gherzan } else { 719ddecdfceSMircea Gherzan emit(ARM_CMP_I(r_A, imm12), ctx); 720ddecdfceSMircea Gherzan } 721ddecdfceSMircea Gherzan cond_jump: 722ddecdfceSMircea Gherzan if (inst->jt) 723ddecdfceSMircea Gherzan _emit(condt, ARM_B(b_imm(i + inst->jt + 1, 724ddecdfceSMircea Gherzan ctx)), ctx); 725ddecdfceSMircea Gherzan if (inst->jf) 726ddecdfceSMircea Gherzan _emit(condt ^ 1, ARM_B(b_imm(i + inst->jf + 1, 727ddecdfceSMircea Gherzan ctx)), ctx); 728ddecdfceSMircea Gherzan break; 72934805931SDaniel Borkmann case BPF_JMP | BPF_JEQ | BPF_X: 730ddecdfceSMircea Gherzan /* pc += (A == X) ? pc->jt : pc->jf */ 731ddecdfceSMircea Gherzan condt = ARM_COND_EQ; 732ddecdfceSMircea Gherzan goto cmp_x; 73334805931SDaniel Borkmann case BPF_JMP | BPF_JGT | BPF_X: 734ddecdfceSMircea Gherzan /* pc += (A > X) ? pc->jt : pc->jf */ 735ddecdfceSMircea Gherzan condt = ARM_COND_HI; 736ddecdfceSMircea Gherzan goto cmp_x; 73734805931SDaniel Borkmann case BPF_JMP | BPF_JGE | BPF_X: 738ddecdfceSMircea Gherzan /* pc += (A >= X) ? pc->jt : pc->jf */ 739ddecdfceSMircea Gherzan condt = ARM_COND_CS; 740ddecdfceSMircea Gherzan cmp_x: 741ddecdfceSMircea Gherzan update_on_xread(ctx); 742ddecdfceSMircea Gherzan emit(ARM_CMP_R(r_A, r_X), ctx); 743ddecdfceSMircea Gherzan goto cond_jump; 74434805931SDaniel Borkmann case BPF_JMP | BPF_JSET | BPF_K: 745ddecdfceSMircea Gherzan /* pc += (A & K) ? pc->jt : pc->jf */ 746ddecdfceSMircea Gherzan condt = ARM_COND_NE; 747ddecdfceSMircea Gherzan /* not set iff all zeroes iff Z==1 iff EQ */ 748ddecdfceSMircea Gherzan 749ddecdfceSMircea Gherzan imm12 = imm8m(k); 750ddecdfceSMircea Gherzan if (imm12 < 0) { 751ddecdfceSMircea Gherzan emit_mov_i_no8m(r_scratch, k, ctx); 752ddecdfceSMircea Gherzan emit(ARM_TST_R(r_A, r_scratch), ctx); 753ddecdfceSMircea Gherzan } else { 754ddecdfceSMircea Gherzan emit(ARM_TST_I(r_A, imm12), ctx); 755ddecdfceSMircea Gherzan } 756ddecdfceSMircea Gherzan goto cond_jump; 75734805931SDaniel Borkmann case BPF_JMP | BPF_JSET | BPF_X: 758ddecdfceSMircea Gherzan /* pc += (A & X) ? pc->jt : pc->jf */ 759ddecdfceSMircea Gherzan update_on_xread(ctx); 760ddecdfceSMircea Gherzan condt = ARM_COND_NE; 761ddecdfceSMircea Gherzan emit(ARM_TST_R(r_A, r_X), ctx); 762ddecdfceSMircea Gherzan goto cond_jump; 76334805931SDaniel Borkmann case BPF_RET | BPF_A: 764ddecdfceSMircea Gherzan emit(ARM_MOV_R(ARM_R0, r_A), ctx); 765ddecdfceSMircea Gherzan goto b_epilogue; 76634805931SDaniel Borkmann case BPF_RET | BPF_K: 767ddecdfceSMircea Gherzan if ((k == 0) && (ctx->ret0_fp_idx < 0)) 768ddecdfceSMircea Gherzan ctx->ret0_fp_idx = i; 769ddecdfceSMircea Gherzan emit_mov_i(ARM_R0, k, ctx); 770ddecdfceSMircea Gherzan b_epilogue: 771ddecdfceSMircea Gherzan if (i != ctx->skf->len - 1) 772ddecdfceSMircea Gherzan emit(ARM_B(b_imm(prog->len, ctx)), ctx); 773ddecdfceSMircea Gherzan break; 77434805931SDaniel Borkmann case BPF_MISC | BPF_TAX: 775ddecdfceSMircea Gherzan /* X = A */ 776ddecdfceSMircea Gherzan ctx->seen |= SEEN_X; 777ddecdfceSMircea Gherzan emit(ARM_MOV_R(r_X, r_A), ctx); 778ddecdfceSMircea Gherzan break; 77934805931SDaniel Borkmann case BPF_MISC | BPF_TXA: 780ddecdfceSMircea Gherzan /* A = X */ 781ddecdfceSMircea Gherzan update_on_xread(ctx); 782ddecdfceSMircea Gherzan emit(ARM_MOV_R(r_A, r_X), ctx); 783ddecdfceSMircea Gherzan break; 78434805931SDaniel Borkmann case BPF_ANC | SKF_AD_PROTOCOL: 785ddecdfceSMircea Gherzan /* A = ntohs(skb->protocol) */ 786ddecdfceSMircea Gherzan ctx->seen |= SEEN_SKB; 787ddecdfceSMircea Gherzan BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, 788ddecdfceSMircea Gherzan protocol) != 2); 789ddecdfceSMircea Gherzan off = offsetof(struct sk_buff, protocol); 790ddecdfceSMircea Gherzan emit(ARM_LDRH_I(r_scratch, r_skb, off), ctx); 791ddecdfceSMircea Gherzan emit_swap16(r_A, r_scratch, ctx); 792ddecdfceSMircea Gherzan break; 79334805931SDaniel Borkmann case BPF_ANC | SKF_AD_CPU: 794ddecdfceSMircea Gherzan /* r_scratch = current_thread_info() */ 795ddecdfceSMircea Gherzan OP_IMM3(ARM_BIC, r_scratch, ARM_SP, THREAD_SIZE - 1, ctx); 796ddecdfceSMircea Gherzan /* A = current_thread_info()->cpu */ 797ddecdfceSMircea Gherzan BUILD_BUG_ON(FIELD_SIZEOF(struct thread_info, cpu) != 4); 798ddecdfceSMircea Gherzan off = offsetof(struct thread_info, cpu); 799ddecdfceSMircea Gherzan emit(ARM_LDR_I(r_A, r_scratch, off), ctx); 800ddecdfceSMircea Gherzan break; 80134805931SDaniel Borkmann case BPF_ANC | SKF_AD_IFINDEX: 802ddecdfceSMircea Gherzan /* A = skb->dev->ifindex */ 803ddecdfceSMircea Gherzan ctx->seen |= SEEN_SKB; 804ddecdfceSMircea Gherzan off = offsetof(struct sk_buff, dev); 805ddecdfceSMircea Gherzan emit(ARM_LDR_I(r_scratch, r_skb, off), ctx); 806ddecdfceSMircea Gherzan 807ddecdfceSMircea Gherzan emit(ARM_CMP_I(r_scratch, 0), ctx); 808ddecdfceSMircea Gherzan emit_err_ret(ARM_COND_EQ, ctx); 809ddecdfceSMircea Gherzan 810ddecdfceSMircea Gherzan BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, 811ddecdfceSMircea Gherzan ifindex) != 4); 812ddecdfceSMircea Gherzan off = offsetof(struct net_device, ifindex); 813ddecdfceSMircea Gherzan emit(ARM_LDR_I(r_A, r_scratch, off), ctx); 814ddecdfceSMircea Gherzan break; 81534805931SDaniel Borkmann case BPF_ANC | SKF_AD_MARK: 816ddecdfceSMircea Gherzan ctx->seen |= SEEN_SKB; 817ddecdfceSMircea Gherzan BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4); 818ddecdfceSMircea Gherzan off = offsetof(struct sk_buff, mark); 819ddecdfceSMircea Gherzan emit(ARM_LDR_I(r_A, r_skb, off), ctx); 820ddecdfceSMircea Gherzan break; 82134805931SDaniel Borkmann case BPF_ANC | SKF_AD_RXHASH: 822ddecdfceSMircea Gherzan ctx->seen |= SEEN_SKB; 82361b905daSTom Herbert BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4); 82461b905daSTom Herbert off = offsetof(struct sk_buff, hash); 825ddecdfceSMircea Gherzan emit(ARM_LDR_I(r_A, r_skb, off), ctx); 826ddecdfceSMircea Gherzan break; 82734805931SDaniel Borkmann case BPF_ANC | SKF_AD_VLAN_TAG: 82834805931SDaniel Borkmann case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT: 829bf0098f2SDaniel Borkmann ctx->seen |= SEEN_SKB; 830bf0098f2SDaniel Borkmann BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2); 831bf0098f2SDaniel Borkmann off = offsetof(struct sk_buff, vlan_tci); 832bf0098f2SDaniel Borkmann emit(ARM_LDRH_I(r_A, r_skb, off), ctx); 83334805931SDaniel Borkmann if (code == (BPF_ANC | SKF_AD_VLAN_TAG)) 834bf0098f2SDaniel Borkmann OP_IMM3(ARM_AND, r_A, r_A, VLAN_VID_MASK, ctx); 835bf0098f2SDaniel Borkmann else 836bf0098f2SDaniel Borkmann OP_IMM3(ARM_AND, r_A, r_A, VLAN_TAG_PRESENT, ctx); 837bf0098f2SDaniel Borkmann break; 83834805931SDaniel Borkmann case BPF_ANC | SKF_AD_QUEUE: 839ddecdfceSMircea Gherzan ctx->seen |= SEEN_SKB; 840ddecdfceSMircea Gherzan BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, 841ddecdfceSMircea Gherzan queue_mapping) != 2); 842ddecdfceSMircea Gherzan BUILD_BUG_ON(offsetof(struct sk_buff, 843ddecdfceSMircea Gherzan queue_mapping) > 0xff); 844ddecdfceSMircea Gherzan off = offsetof(struct sk_buff, queue_mapping); 845ddecdfceSMircea Gherzan emit(ARM_LDRH_I(r_A, r_skb, off), ctx); 846ddecdfceSMircea Gherzan break; 847ddecdfceSMircea Gherzan default: 848ddecdfceSMircea Gherzan return -1; 849ddecdfceSMircea Gherzan } 850ddecdfceSMircea Gherzan } 851ddecdfceSMircea Gherzan 852ddecdfceSMircea Gherzan /* compute offsets only during the first pass */ 853ddecdfceSMircea Gherzan if (ctx->target == NULL) 854ddecdfceSMircea Gherzan ctx->offsets[i] = ctx->idx * 4; 855ddecdfceSMircea Gherzan 856ddecdfceSMircea Gherzan return 0; 857ddecdfceSMircea Gherzan } 858ddecdfceSMircea Gherzan 859ddecdfceSMircea Gherzan 8607ae457c1SAlexei Starovoitov void bpf_jit_compile(struct bpf_prog *fp) 861ddecdfceSMircea Gherzan { 862ddecdfceSMircea Gherzan struct jit_ctx ctx; 863ddecdfceSMircea Gherzan unsigned tmp_idx; 864ddecdfceSMircea Gherzan unsigned alloc_size; 865ddecdfceSMircea Gherzan 866ddecdfceSMircea Gherzan if (!bpf_jit_enable) 867ddecdfceSMircea Gherzan return; 868ddecdfceSMircea Gherzan 869ddecdfceSMircea Gherzan memset(&ctx, 0, sizeof(ctx)); 870ddecdfceSMircea Gherzan ctx.skf = fp; 871ddecdfceSMircea Gherzan ctx.ret0_fp_idx = -1; 872ddecdfceSMircea Gherzan 87389c2e009SSchichan Nicolas ctx.offsets = kzalloc(4 * (ctx.skf->len + 1), GFP_KERNEL); 874ddecdfceSMircea Gherzan if (ctx.offsets == NULL) 875ddecdfceSMircea Gherzan return; 876ddecdfceSMircea Gherzan 877ddecdfceSMircea Gherzan /* fake pass to fill in the ctx->seen */ 878ddecdfceSMircea Gherzan if (unlikely(build_body(&ctx))) 879ddecdfceSMircea Gherzan goto out; 880ddecdfceSMircea Gherzan 881ddecdfceSMircea Gherzan tmp_idx = ctx.idx; 882ddecdfceSMircea Gherzan build_prologue(&ctx); 883ddecdfceSMircea Gherzan ctx.prologue_bytes = (ctx.idx - tmp_idx) * 4; 884ddecdfceSMircea Gherzan 885ddecdfceSMircea Gherzan #if __LINUX_ARM_ARCH__ < 7 886ddecdfceSMircea Gherzan tmp_idx = ctx.idx; 887ddecdfceSMircea Gherzan build_epilogue(&ctx); 888ddecdfceSMircea Gherzan ctx.epilogue_bytes = (ctx.idx - tmp_idx) * 4; 889ddecdfceSMircea Gherzan 890ddecdfceSMircea Gherzan ctx.idx += ctx.imm_count; 891ddecdfceSMircea Gherzan if (ctx.imm_count) { 89289c2e009SSchichan Nicolas ctx.imms = kzalloc(4 * ctx.imm_count, GFP_KERNEL); 893ddecdfceSMircea Gherzan if (ctx.imms == NULL) 894ddecdfceSMircea Gherzan goto out; 895ddecdfceSMircea Gherzan } 896ddecdfceSMircea Gherzan #else 897ddecdfceSMircea Gherzan /* there's nothing after the epilogue on ARMv7 */ 898ddecdfceSMircea Gherzan build_epilogue(&ctx); 899ddecdfceSMircea Gherzan #endif 900ddecdfceSMircea Gherzan 901ddecdfceSMircea Gherzan alloc_size = 4 * ctx.idx; 902aafc787eSDaniel Borkmann ctx.target = module_alloc(alloc_size); 903ddecdfceSMircea Gherzan if (unlikely(ctx.target == NULL)) 904ddecdfceSMircea Gherzan goto out; 905ddecdfceSMircea Gherzan 906ddecdfceSMircea Gherzan ctx.idx = 0; 907ddecdfceSMircea Gherzan build_prologue(&ctx); 908ddecdfceSMircea Gherzan build_body(&ctx); 909ddecdfceSMircea Gherzan build_epilogue(&ctx); 910ddecdfceSMircea Gherzan 911ddecdfceSMircea Gherzan flush_icache_range((u32)ctx.target, (u32)(ctx.target + ctx.idx)); 912ddecdfceSMircea Gherzan 913ddecdfceSMircea Gherzan #if __LINUX_ARM_ARCH__ < 7 914ddecdfceSMircea Gherzan if (ctx.imm_count) 915ddecdfceSMircea Gherzan kfree(ctx.imms); 916ddecdfceSMircea Gherzan #endif 917ddecdfceSMircea Gherzan 918ddecdfceSMircea Gherzan if (bpf_jit_enable > 1) 91979617801SDaniel Borkmann /* there are 2 passes here */ 92079617801SDaniel Borkmann bpf_jit_dump(fp->len, alloc_size, 2, ctx.target); 921ddecdfceSMircea Gherzan 922ddecdfceSMircea Gherzan fp->bpf_func = (void *)ctx.target; 923f8bbbfc3SDaniel Borkmann fp->jited = 1; 924ddecdfceSMircea Gherzan out: 925ddecdfceSMircea Gherzan kfree(ctx.offsets); 926ddecdfceSMircea Gherzan return; 927ddecdfceSMircea Gherzan } 928ddecdfceSMircea Gherzan 9297ae457c1SAlexei Starovoitov void bpf_jit_free(struct bpf_prog *fp) 930ddecdfceSMircea Gherzan { 931f8bbbfc3SDaniel Borkmann if (fp->jited) 932aafc787eSDaniel Borkmann module_free(NULL, fp->bpf_func); 93360a3b225SDaniel Borkmann 93460a3b225SDaniel Borkmann bpf_prog_unlock_free(fp); 935ddecdfceSMircea Gherzan } 936