10a14842fSEric Dumazet /* bpf_jit_comp.c : BPF JIT compiler 20a14842fSEric Dumazet * 33b58908aSEric Dumazet * Copyright (C) 2011-2013 Eric Dumazet (eric.dumazet@gmail.com) 40a14842fSEric Dumazet * 50a14842fSEric Dumazet * This program is free software; you can redistribute it and/or 60a14842fSEric Dumazet * modify it under the terms of the GNU General Public License 70a14842fSEric Dumazet * as published by the Free Software Foundation; version 2 80a14842fSEric Dumazet * of the License. 90a14842fSEric Dumazet */ 100a14842fSEric Dumazet #include <linux/moduleloader.h> 110a14842fSEric Dumazet #include <asm/cacheflush.h> 120a14842fSEric Dumazet #include <linux/netdevice.h> 130a14842fSEric Dumazet #include <linux/filter.h> 14855ddb56SEric Dumazet #include <linux/if_vlan.h> 150a14842fSEric Dumazet 160a14842fSEric Dumazet /* 170a14842fSEric Dumazet * Conventions : 180a14842fSEric Dumazet * EAX : BPF A accumulator 190a14842fSEric Dumazet * EBX : BPF X accumulator 200a14842fSEric Dumazet * RDI : pointer to skb (first argument given to JIT function) 210a14842fSEric Dumazet * RBP : frame pointer (even if CONFIG_FRAME_POINTER=n) 220a14842fSEric Dumazet * ECX,EDX,ESI : scratch registers 230a14842fSEric Dumazet * r9d : skb->len - skb->data_len (headlen) 240a14842fSEric Dumazet * r8 : skb->data 250a14842fSEric Dumazet * -8(RBP) : saved RBX value 260a14842fSEric Dumazet * -16(RBP)..-80(RBP) : BPF_MEMWORDS values 270a14842fSEric Dumazet */ 280a14842fSEric Dumazet int bpf_jit_enable __read_mostly; 290a14842fSEric Dumazet 300a14842fSEric Dumazet /* 310a14842fSEric Dumazet * assembly code in arch/x86/net/bpf_jit.S 320a14842fSEric Dumazet */ 330a14842fSEric Dumazet extern u8 sk_load_word[], sk_load_half[], sk_load_byte[], sk_load_byte_msh[]; 34a998d434SJan Seiffert extern u8 sk_load_word_positive_offset[], sk_load_half_positive_offset[]; 35a998d434SJan Seiffert extern u8 sk_load_byte_positive_offset[], sk_load_byte_msh_positive_offset[]; 36a998d434SJan Seiffert extern u8 sk_load_word_negative_offset[], sk_load_half_negative_offset[]; 37a998d434SJan Seiffert extern u8 sk_load_byte_negative_offset[], sk_load_byte_msh_negative_offset[]; 380a14842fSEric Dumazet 390a14842fSEric Dumazet static inline u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) 400a14842fSEric Dumazet { 410a14842fSEric Dumazet if (len == 1) 420a14842fSEric Dumazet *ptr = bytes; 430a14842fSEric Dumazet else if (len == 2) 440a14842fSEric Dumazet *(u16 *)ptr = bytes; 450a14842fSEric Dumazet else { 460a14842fSEric Dumazet *(u32 *)ptr = bytes; 470a14842fSEric Dumazet barrier(); 480a14842fSEric Dumazet } 490a14842fSEric Dumazet return ptr + len; 500a14842fSEric Dumazet } 510a14842fSEric Dumazet 520a14842fSEric Dumazet #define EMIT(bytes, len) do { prog = emit_code(prog, bytes, len); } while (0) 530a14842fSEric Dumazet 540a14842fSEric Dumazet #define EMIT1(b1) EMIT(b1, 1) 550a14842fSEric Dumazet #define EMIT2(b1, b2) EMIT((b1) + ((b2) << 8), 2) 560a14842fSEric Dumazet #define EMIT3(b1, b2, b3) EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3) 570a14842fSEric Dumazet #define EMIT4(b1, b2, b3, b4) EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4) 580a14842fSEric Dumazet #define EMIT1_off32(b1, off) do { EMIT1(b1); EMIT(off, 4);} while (0) 590a14842fSEric Dumazet 600a14842fSEric Dumazet #define CLEAR_A() EMIT2(0x31, 0xc0) /* xor %eax,%eax */ 610a14842fSEric Dumazet #define CLEAR_X() EMIT2(0x31, 0xdb) /* xor %ebx,%ebx */ 620a14842fSEric Dumazet 630a14842fSEric Dumazet static inline bool is_imm8(int value) 640a14842fSEric Dumazet { 650a14842fSEric Dumazet return value <= 127 && value >= -128; 660a14842fSEric Dumazet } 670a14842fSEric Dumazet 680a14842fSEric Dumazet static inline bool is_near(int offset) 690a14842fSEric Dumazet { 700a14842fSEric Dumazet return offset <= 127 && offset >= -128; 710a14842fSEric Dumazet } 720a14842fSEric Dumazet 730a14842fSEric Dumazet #define EMIT_JMP(offset) \ 740a14842fSEric Dumazet do { \ 750a14842fSEric Dumazet if (offset) { \ 760a14842fSEric Dumazet if (is_near(offset)) \ 770a14842fSEric Dumazet EMIT2(0xeb, offset); /* jmp .+off8 */ \ 780a14842fSEric Dumazet else \ 790a14842fSEric Dumazet EMIT1_off32(0xe9, offset); /* jmp .+off32 */ \ 800a14842fSEric Dumazet } \ 810a14842fSEric Dumazet } while (0) 820a14842fSEric Dumazet 830a14842fSEric Dumazet /* list of x86 cond jumps opcodes (. + s8) 840a14842fSEric Dumazet * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32) 850a14842fSEric Dumazet */ 860a14842fSEric Dumazet #define X86_JB 0x72 870a14842fSEric Dumazet #define X86_JAE 0x73 880a14842fSEric Dumazet #define X86_JE 0x74 890a14842fSEric Dumazet #define X86_JNE 0x75 900a14842fSEric Dumazet #define X86_JBE 0x76 910a14842fSEric Dumazet #define X86_JA 0x77 920a14842fSEric Dumazet 930a14842fSEric Dumazet #define EMIT_COND_JMP(op, offset) \ 940a14842fSEric Dumazet do { \ 950a14842fSEric Dumazet if (is_near(offset)) \ 960a14842fSEric Dumazet EMIT2(op, offset); /* jxx .+off8 */ \ 970a14842fSEric Dumazet else { \ 980a14842fSEric Dumazet EMIT2(0x0f, op + 0x10); \ 990a14842fSEric Dumazet EMIT(offset, 4); /* jxx .+off32 */ \ 1000a14842fSEric Dumazet } \ 1010a14842fSEric Dumazet } while (0) 1020a14842fSEric Dumazet 1030a14842fSEric Dumazet #define COND_SEL(CODE, TOP, FOP) \ 1040a14842fSEric Dumazet case CODE: \ 1050a14842fSEric Dumazet t_op = TOP; \ 1060a14842fSEric Dumazet f_op = FOP; \ 1070a14842fSEric Dumazet goto cond_branch 1080a14842fSEric Dumazet 1090a14842fSEric Dumazet 1100a14842fSEric Dumazet #define SEEN_DATAREF 1 /* might call external helpers */ 1110a14842fSEric Dumazet #define SEEN_XREG 2 /* ebx is used */ 1120a14842fSEric Dumazet #define SEEN_MEM 4 /* use mem[] for temporary storage */ 1130a14842fSEric Dumazet 1140a14842fSEric Dumazet static inline void bpf_flush_icache(void *start, void *end) 1150a14842fSEric Dumazet { 1160a14842fSEric Dumazet mm_segment_t old_fs = get_fs(); 1170a14842fSEric Dumazet 1180a14842fSEric Dumazet set_fs(KERNEL_DS); 1190a14842fSEric Dumazet smp_wmb(); 1200a14842fSEric Dumazet flush_icache_range((unsigned long)start, (unsigned long)end); 1210a14842fSEric Dumazet set_fs(old_fs); 1220a14842fSEric Dumazet } 1230a14842fSEric Dumazet 124a998d434SJan Seiffert #define CHOOSE_LOAD_FUNC(K, func) \ 125a998d434SJan Seiffert ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset) 1260a14842fSEric Dumazet 1273b58908aSEric Dumazet /* Helper to find the offset of pkt_type in sk_buff 1283b58908aSEric Dumazet * We want to make sure its still a 3bit field starting at a byte boundary. 1293b58908aSEric Dumazet */ 1303b58908aSEric Dumazet #define PKT_TYPE_MAX 7 1313b58908aSEric Dumazet static int pkt_type_offset(void) 1323b58908aSEric Dumazet { 1333b58908aSEric Dumazet struct sk_buff skb_probe = { 1343b58908aSEric Dumazet .pkt_type = ~0, 1353b58908aSEric Dumazet }; 1363b58908aSEric Dumazet char *ct = (char *)&skb_probe; 1373b58908aSEric Dumazet unsigned int off; 1383b58908aSEric Dumazet 1393b58908aSEric Dumazet for (off = 0; off < sizeof(struct sk_buff); off++) { 1403b58908aSEric Dumazet if (ct[off] == PKT_TYPE_MAX) 1413b58908aSEric Dumazet return off; 1423b58908aSEric Dumazet } 1433b58908aSEric Dumazet pr_err_once("Please fix pkt_type_offset(), as pkt_type couldn't be found\n"); 1443b58908aSEric Dumazet return -1; 1453b58908aSEric Dumazet } 1463b58908aSEric Dumazet 1470a14842fSEric Dumazet void bpf_jit_compile(struct sk_filter *fp) 1480a14842fSEric Dumazet { 1490a14842fSEric Dumazet u8 temp[64]; 1500a14842fSEric Dumazet u8 *prog; 1510a14842fSEric Dumazet unsigned int proglen, oldproglen = 0; 1520a14842fSEric Dumazet int ilen, i; 1530a14842fSEric Dumazet int t_offset, f_offset; 1540a14842fSEric Dumazet u8 t_op, f_op, seen = 0, pass; 1550a14842fSEric Dumazet u8 *image = NULL; 1560a14842fSEric Dumazet u8 *func; 1570a14842fSEric Dumazet int pc_ret0 = -1; /* bpf index of first RET #0 instruction (if any) */ 1580a14842fSEric Dumazet unsigned int cleanup_addr; /* epilogue code offset */ 1590a14842fSEric Dumazet unsigned int *addrs; 1600a14842fSEric Dumazet const struct sock_filter *filter = fp->insns; 1610a14842fSEric Dumazet int flen = fp->len; 1620a14842fSEric Dumazet 1630a14842fSEric Dumazet if (!bpf_jit_enable) 1640a14842fSEric Dumazet return; 1650a14842fSEric Dumazet 1660a14842fSEric Dumazet addrs = kmalloc(flen * sizeof(*addrs), GFP_KERNEL); 1670a14842fSEric Dumazet if (addrs == NULL) 1680a14842fSEric Dumazet return; 1690a14842fSEric Dumazet 1700a14842fSEric Dumazet /* Before first pass, make a rough estimation of addrs[] 1710a14842fSEric Dumazet * each bpf instruction is translated to less than 64 bytes 1720a14842fSEric Dumazet */ 1730a14842fSEric Dumazet for (proglen = 0, i = 0; i < flen; i++) { 1740a14842fSEric Dumazet proglen += 64; 1750a14842fSEric Dumazet addrs[i] = proglen; 1760a14842fSEric Dumazet } 1770a14842fSEric Dumazet cleanup_addr = proglen; /* epilogue address */ 1780a14842fSEric Dumazet 1790a14842fSEric Dumazet for (pass = 0; pass < 10; pass++) { 180d00a9dd2SEric Dumazet u8 seen_or_pass0 = (pass == 0) ? (SEEN_XREG | SEEN_DATAREF | SEEN_MEM) : seen; 1810a14842fSEric Dumazet /* no prologue/epilogue for trivial filters (RET something) */ 1820a14842fSEric Dumazet proglen = 0; 1830a14842fSEric Dumazet prog = temp; 1840a14842fSEric Dumazet 185d00a9dd2SEric Dumazet if (seen_or_pass0) { 1860a14842fSEric Dumazet EMIT4(0x55, 0x48, 0x89, 0xe5); /* push %rbp; mov %rsp,%rbp */ 1870a14842fSEric Dumazet EMIT4(0x48, 0x83, 0xec, 96); /* subq $96,%rsp */ 1880a14842fSEric Dumazet /* note : must save %rbx in case bpf_error is hit */ 189d00a9dd2SEric Dumazet if (seen_or_pass0 & (SEEN_XREG | SEEN_DATAREF)) 1900a14842fSEric Dumazet EMIT4(0x48, 0x89, 0x5d, 0xf8); /* mov %rbx, -8(%rbp) */ 191d00a9dd2SEric Dumazet if (seen_or_pass0 & SEEN_XREG) 1920a14842fSEric Dumazet CLEAR_X(); /* make sure we dont leek kernel memory */ 1930a14842fSEric Dumazet 1940a14842fSEric Dumazet /* 1950a14842fSEric Dumazet * If this filter needs to access skb data, 1960a14842fSEric Dumazet * loads r9 and r8 with : 1970a14842fSEric Dumazet * r9 = skb->len - skb->data_len 1980a14842fSEric Dumazet * r8 = skb->data 1990a14842fSEric Dumazet */ 200d00a9dd2SEric Dumazet if (seen_or_pass0 & SEEN_DATAREF) { 2010a14842fSEric Dumazet if (offsetof(struct sk_buff, len) <= 127) 2020a14842fSEric Dumazet /* mov off8(%rdi),%r9d */ 2030a14842fSEric Dumazet EMIT4(0x44, 0x8b, 0x4f, offsetof(struct sk_buff, len)); 2040a14842fSEric Dumazet else { 2050a14842fSEric Dumazet /* mov off32(%rdi),%r9d */ 2060a14842fSEric Dumazet EMIT3(0x44, 0x8b, 0x8f); 2070a14842fSEric Dumazet EMIT(offsetof(struct sk_buff, len), 4); 2080a14842fSEric Dumazet } 2090a14842fSEric Dumazet if (is_imm8(offsetof(struct sk_buff, data_len))) 2100a14842fSEric Dumazet /* sub off8(%rdi),%r9d */ 2110a14842fSEric Dumazet EMIT4(0x44, 0x2b, 0x4f, offsetof(struct sk_buff, data_len)); 2120a14842fSEric Dumazet else { 2130a14842fSEric Dumazet EMIT3(0x44, 0x2b, 0x8f); 2140a14842fSEric Dumazet EMIT(offsetof(struct sk_buff, data_len), 4); 2150a14842fSEric Dumazet } 2160a14842fSEric Dumazet 2170a14842fSEric Dumazet if (is_imm8(offsetof(struct sk_buff, data))) 2180a14842fSEric Dumazet /* mov off8(%rdi),%r8 */ 2190a14842fSEric Dumazet EMIT4(0x4c, 0x8b, 0x47, offsetof(struct sk_buff, data)); 2200a14842fSEric Dumazet else { 2210a14842fSEric Dumazet /* mov off32(%rdi),%r8 */ 2220a14842fSEric Dumazet EMIT3(0x4c, 0x8b, 0x87); 2230a14842fSEric Dumazet EMIT(offsetof(struct sk_buff, data), 4); 2240a14842fSEric Dumazet } 2250a14842fSEric Dumazet } 2260a14842fSEric Dumazet } 2270a14842fSEric Dumazet 2280a14842fSEric Dumazet switch (filter[0].code) { 2290a14842fSEric Dumazet case BPF_S_RET_K: 2300a14842fSEric Dumazet case BPF_S_LD_W_LEN: 2310a14842fSEric Dumazet case BPF_S_ANC_PROTOCOL: 2320a14842fSEric Dumazet case BPF_S_ANC_IFINDEX: 2330a14842fSEric Dumazet case BPF_S_ANC_MARK: 2340a14842fSEric Dumazet case BPF_S_ANC_RXHASH: 2350a14842fSEric Dumazet case BPF_S_ANC_CPU: 236855ddb56SEric Dumazet case BPF_S_ANC_VLAN_TAG: 237855ddb56SEric Dumazet case BPF_S_ANC_VLAN_TAG_PRESENT: 2380a14842fSEric Dumazet case BPF_S_ANC_QUEUE: 2393b58908aSEric Dumazet case BPF_S_ANC_PKTTYPE: 2400a14842fSEric Dumazet case BPF_S_LD_W_ABS: 2410a14842fSEric Dumazet case BPF_S_LD_H_ABS: 2420a14842fSEric Dumazet case BPF_S_LD_B_ABS: 2430a14842fSEric Dumazet /* first instruction sets A register (or is RET 'constant') */ 2440a14842fSEric Dumazet break; 2450a14842fSEric Dumazet default: 2460a14842fSEric Dumazet /* make sure we dont leak kernel information to user */ 2470a14842fSEric Dumazet CLEAR_A(); /* A = 0 */ 2480a14842fSEric Dumazet } 2490a14842fSEric Dumazet 2500a14842fSEric Dumazet for (i = 0; i < flen; i++) { 2510a14842fSEric Dumazet unsigned int K = filter[i].k; 2520a14842fSEric Dumazet 2530a14842fSEric Dumazet switch (filter[i].code) { 2540a14842fSEric Dumazet case BPF_S_ALU_ADD_X: /* A += X; */ 2550a14842fSEric Dumazet seen |= SEEN_XREG; 2560a14842fSEric Dumazet EMIT2(0x01, 0xd8); /* add %ebx,%eax */ 2570a14842fSEric Dumazet break; 2580a14842fSEric Dumazet case BPF_S_ALU_ADD_K: /* A += K; */ 2590a14842fSEric Dumazet if (!K) 2600a14842fSEric Dumazet break; 2610a14842fSEric Dumazet if (is_imm8(K)) 2620a14842fSEric Dumazet EMIT3(0x83, 0xc0, K); /* add imm8,%eax */ 2630a14842fSEric Dumazet else 2640a14842fSEric Dumazet EMIT1_off32(0x05, K); /* add imm32,%eax */ 2650a14842fSEric Dumazet break; 2660a14842fSEric Dumazet case BPF_S_ALU_SUB_X: /* A -= X; */ 2670a14842fSEric Dumazet seen |= SEEN_XREG; 2680a14842fSEric Dumazet EMIT2(0x29, 0xd8); /* sub %ebx,%eax */ 2690a14842fSEric Dumazet break; 2700a14842fSEric Dumazet case BPF_S_ALU_SUB_K: /* A -= K */ 2710a14842fSEric Dumazet if (!K) 2720a14842fSEric Dumazet break; 2730a14842fSEric Dumazet if (is_imm8(K)) 2740a14842fSEric Dumazet EMIT3(0x83, 0xe8, K); /* sub imm8,%eax */ 2750a14842fSEric Dumazet else 2760a14842fSEric Dumazet EMIT1_off32(0x2d, K); /* sub imm32,%eax */ 2770a14842fSEric Dumazet break; 2780a14842fSEric Dumazet case BPF_S_ALU_MUL_X: /* A *= X; */ 2790a14842fSEric Dumazet seen |= SEEN_XREG; 2800a14842fSEric Dumazet EMIT3(0x0f, 0xaf, 0xc3); /* imul %ebx,%eax */ 2810a14842fSEric Dumazet break; 2820a14842fSEric Dumazet case BPF_S_ALU_MUL_K: /* A *= K */ 2830a14842fSEric Dumazet if (is_imm8(K)) 2840a14842fSEric Dumazet EMIT3(0x6b, 0xc0, K); /* imul imm8,%eax,%eax */ 2850a14842fSEric Dumazet else { 2860a14842fSEric Dumazet EMIT2(0x69, 0xc0); /* imul imm32,%eax */ 2870a14842fSEric Dumazet EMIT(K, 4); 2880a14842fSEric Dumazet } 2890a14842fSEric Dumazet break; 2900a14842fSEric Dumazet case BPF_S_ALU_DIV_X: /* A /= X; */ 2910a14842fSEric Dumazet seen |= SEEN_XREG; 2920a14842fSEric Dumazet EMIT2(0x85, 0xdb); /* test %ebx,%ebx */ 293d00a9dd2SEric Dumazet if (pc_ret0 > 0) { 294d00a9dd2SEric Dumazet /* addrs[pc_ret0 - 1] is start address of target 295d00a9dd2SEric Dumazet * (addrs[i] - 4) is the address following this jmp 296d00a9dd2SEric Dumazet * ("xor %edx,%edx; div %ebx" being 4 bytes long) 297d00a9dd2SEric Dumazet */ 298d00a9dd2SEric Dumazet EMIT_COND_JMP(X86_JE, addrs[pc_ret0 - 1] - 299d00a9dd2SEric Dumazet (addrs[i] - 4)); 300d00a9dd2SEric Dumazet } else { 3010a14842fSEric Dumazet EMIT_COND_JMP(X86_JNE, 2 + 5); 3020a14842fSEric Dumazet CLEAR_A(); 3030a14842fSEric Dumazet EMIT1_off32(0xe9, cleanup_addr - (addrs[i] - 4)); /* jmp .+off32 */ 3040a14842fSEric Dumazet } 3050a14842fSEric Dumazet EMIT4(0x31, 0xd2, 0xf7, 0xf3); /* xor %edx,%edx; div %ebx */ 3060a14842fSEric Dumazet break; 307280050ccSEric Dumazet case BPF_S_ALU_MOD_X: /* A %= X; */ 308280050ccSEric Dumazet seen |= SEEN_XREG; 309280050ccSEric Dumazet EMIT2(0x85, 0xdb); /* test %ebx,%ebx */ 310280050ccSEric Dumazet if (pc_ret0 > 0) { 311280050ccSEric Dumazet /* addrs[pc_ret0 - 1] is start address of target 312280050ccSEric Dumazet * (addrs[i] - 6) is the address following this jmp 313280050ccSEric Dumazet * ("xor %edx,%edx; div %ebx;mov %edx,%eax" being 6 bytes long) 314280050ccSEric Dumazet */ 315280050ccSEric Dumazet EMIT_COND_JMP(X86_JE, addrs[pc_ret0 - 1] - 316280050ccSEric Dumazet (addrs[i] - 6)); 317280050ccSEric Dumazet } else { 318280050ccSEric Dumazet EMIT_COND_JMP(X86_JNE, 2 + 5); 319280050ccSEric Dumazet CLEAR_A(); 320280050ccSEric Dumazet EMIT1_off32(0xe9, cleanup_addr - (addrs[i] - 6)); /* jmp .+off32 */ 321280050ccSEric Dumazet } 322280050ccSEric Dumazet EMIT2(0x31, 0xd2); /* xor %edx,%edx */ 323280050ccSEric Dumazet EMIT2(0xf7, 0xf3); /* div %ebx */ 324280050ccSEric Dumazet EMIT2(0x89, 0xd0); /* mov %edx,%eax */ 325280050ccSEric Dumazet break; 326280050ccSEric Dumazet case BPF_S_ALU_MOD_K: /* A %= K; */ 327280050ccSEric Dumazet EMIT2(0x31, 0xd2); /* xor %edx,%edx */ 328280050ccSEric Dumazet EMIT1(0xb9);EMIT(K, 4); /* mov imm32,%ecx */ 329280050ccSEric Dumazet EMIT2(0xf7, 0xf1); /* div %ecx */ 330280050ccSEric Dumazet EMIT2(0x89, 0xd0); /* mov %edx,%eax */ 331280050ccSEric Dumazet break; 3320a14842fSEric Dumazet case BPF_S_ALU_DIV_K: /* A = reciprocal_divide(A, K); */ 3330a14842fSEric Dumazet EMIT3(0x48, 0x69, 0xc0); /* imul imm32,%rax,%rax */ 3340a14842fSEric Dumazet EMIT(K, 4); 3350a14842fSEric Dumazet EMIT4(0x48, 0xc1, 0xe8, 0x20); /* shr $0x20,%rax */ 3360a14842fSEric Dumazet break; 3370a14842fSEric Dumazet case BPF_S_ALU_AND_X: 3380a14842fSEric Dumazet seen |= SEEN_XREG; 3390a14842fSEric Dumazet EMIT2(0x21, 0xd8); /* and %ebx,%eax */ 3400a14842fSEric Dumazet break; 3410a14842fSEric Dumazet case BPF_S_ALU_AND_K: 3420a14842fSEric Dumazet if (K >= 0xFFFFFF00) { 3430a14842fSEric Dumazet EMIT2(0x24, K & 0xFF); /* and imm8,%al */ 3440a14842fSEric Dumazet } else if (K >= 0xFFFF0000) { 3450a14842fSEric Dumazet EMIT2(0x66, 0x25); /* and imm16,%ax */ 3461d24fb36Szhuangfeiran@ict.ac.cn EMIT(K, 2); 3470a14842fSEric Dumazet } else { 3480a14842fSEric Dumazet EMIT1_off32(0x25, K); /* and imm32,%eax */ 3490a14842fSEric Dumazet } 3500a14842fSEric Dumazet break; 3510a14842fSEric Dumazet case BPF_S_ALU_OR_X: 3520a14842fSEric Dumazet seen |= SEEN_XREG; 3530a14842fSEric Dumazet EMIT2(0x09, 0xd8); /* or %ebx,%eax */ 3540a14842fSEric Dumazet break; 3550a14842fSEric Dumazet case BPF_S_ALU_OR_K: 3560a14842fSEric Dumazet if (is_imm8(K)) 3570a14842fSEric Dumazet EMIT3(0x83, 0xc8, K); /* or imm8,%eax */ 3580a14842fSEric Dumazet else 3590a14842fSEric Dumazet EMIT1_off32(0x0d, K); /* or imm32,%eax */ 3600a14842fSEric Dumazet break; 3614bfaddf1SEric Dumazet case BPF_S_ANC_ALU_XOR_X: /* A ^= X; */ 36282c93fccSDaniel Borkmann case BPF_S_ALU_XOR_X: 3634bfaddf1SEric Dumazet seen |= SEEN_XREG; 3644bfaddf1SEric Dumazet EMIT2(0x31, 0xd8); /* xor %ebx,%eax */ 3654bfaddf1SEric Dumazet break; 36682c93fccSDaniel Borkmann case BPF_S_ALU_XOR_K: /* A ^= K; */ 36782c93fccSDaniel Borkmann if (K == 0) 36882c93fccSDaniel Borkmann break; 36982c93fccSDaniel Borkmann if (is_imm8(K)) 37082c93fccSDaniel Borkmann EMIT3(0x83, 0xf0, K); /* xor imm8,%eax */ 37182c93fccSDaniel Borkmann else 37282c93fccSDaniel Borkmann EMIT1_off32(0x35, K); /* xor imm32,%eax */ 37382c93fccSDaniel Borkmann break; 3740a14842fSEric Dumazet case BPF_S_ALU_LSH_X: /* A <<= X; */ 3750a14842fSEric Dumazet seen |= SEEN_XREG; 3760a14842fSEric Dumazet EMIT4(0x89, 0xd9, 0xd3, 0xe0); /* mov %ebx,%ecx; shl %cl,%eax */ 3770a14842fSEric Dumazet break; 3780a14842fSEric Dumazet case BPF_S_ALU_LSH_K: 3790a14842fSEric Dumazet if (K == 0) 3800a14842fSEric Dumazet break; 3810a14842fSEric Dumazet else if (K == 1) 3820a14842fSEric Dumazet EMIT2(0xd1, 0xe0); /* shl %eax */ 3830a14842fSEric Dumazet else 3840a14842fSEric Dumazet EMIT3(0xc1, 0xe0, K); 3850a14842fSEric Dumazet break; 3860a14842fSEric Dumazet case BPF_S_ALU_RSH_X: /* A >>= X; */ 3870a14842fSEric Dumazet seen |= SEEN_XREG; 3880a14842fSEric Dumazet EMIT4(0x89, 0xd9, 0xd3, 0xe8); /* mov %ebx,%ecx; shr %cl,%eax */ 3890a14842fSEric Dumazet break; 3900a14842fSEric Dumazet case BPF_S_ALU_RSH_K: /* A >>= K; */ 3910a14842fSEric Dumazet if (K == 0) 3920a14842fSEric Dumazet break; 3930a14842fSEric Dumazet else if (K == 1) 3940a14842fSEric Dumazet EMIT2(0xd1, 0xe8); /* shr %eax */ 3950a14842fSEric Dumazet else 3960a14842fSEric Dumazet EMIT3(0xc1, 0xe8, K); 3970a14842fSEric Dumazet break; 3980a14842fSEric Dumazet case BPF_S_ALU_NEG: 3990a14842fSEric Dumazet EMIT2(0xf7, 0xd8); /* neg %eax */ 4000a14842fSEric Dumazet break; 4010a14842fSEric Dumazet case BPF_S_RET_K: 4020a14842fSEric Dumazet if (!K) { 4030a14842fSEric Dumazet if (pc_ret0 == -1) 4040a14842fSEric Dumazet pc_ret0 = i; 4050a14842fSEric Dumazet CLEAR_A(); 4060a14842fSEric Dumazet } else { 4070a14842fSEric Dumazet EMIT1_off32(0xb8, K); /* mov $imm32,%eax */ 4080a14842fSEric Dumazet } 4090a14842fSEric Dumazet /* fallinto */ 4100a14842fSEric Dumazet case BPF_S_RET_A: 411d00a9dd2SEric Dumazet if (seen_or_pass0) { 4120a14842fSEric Dumazet if (i != flen - 1) { 4130a14842fSEric Dumazet EMIT_JMP(cleanup_addr - addrs[i]); 4140a14842fSEric Dumazet break; 4150a14842fSEric Dumazet } 416d00a9dd2SEric Dumazet if (seen_or_pass0 & SEEN_XREG) 4170a14842fSEric Dumazet EMIT4(0x48, 0x8b, 0x5d, 0xf8); /* mov -8(%rbp),%rbx */ 4180a14842fSEric Dumazet EMIT1(0xc9); /* leaveq */ 4190a14842fSEric Dumazet } 4200a14842fSEric Dumazet EMIT1(0xc3); /* ret */ 4210a14842fSEric Dumazet break; 4220a14842fSEric Dumazet case BPF_S_MISC_TAX: /* X = A */ 4230a14842fSEric Dumazet seen |= SEEN_XREG; 4240a14842fSEric Dumazet EMIT2(0x89, 0xc3); /* mov %eax,%ebx */ 4250a14842fSEric Dumazet break; 4260a14842fSEric Dumazet case BPF_S_MISC_TXA: /* A = X */ 4270a14842fSEric Dumazet seen |= SEEN_XREG; 4280a14842fSEric Dumazet EMIT2(0x89, 0xd8); /* mov %ebx,%eax */ 4290a14842fSEric Dumazet break; 4300a14842fSEric Dumazet case BPF_S_LD_IMM: /* A = K */ 4310a14842fSEric Dumazet if (!K) 4320a14842fSEric Dumazet CLEAR_A(); 4330a14842fSEric Dumazet else 4340a14842fSEric Dumazet EMIT1_off32(0xb8, K); /* mov $imm32,%eax */ 4350a14842fSEric Dumazet break; 4360a14842fSEric Dumazet case BPF_S_LDX_IMM: /* X = K */ 4370a14842fSEric Dumazet seen |= SEEN_XREG; 4380a14842fSEric Dumazet if (!K) 4390a14842fSEric Dumazet CLEAR_X(); 4400a14842fSEric Dumazet else 4410a14842fSEric Dumazet EMIT1_off32(0xbb, K); /* mov $imm32,%ebx */ 4420a14842fSEric Dumazet break; 4430a14842fSEric Dumazet case BPF_S_LD_MEM: /* A = mem[K] : mov off8(%rbp),%eax */ 4440a14842fSEric Dumazet seen |= SEEN_MEM; 4450a14842fSEric Dumazet EMIT3(0x8b, 0x45, 0xf0 - K*4); 4460a14842fSEric Dumazet break; 4470a14842fSEric Dumazet case BPF_S_LDX_MEM: /* X = mem[K] : mov off8(%rbp),%ebx */ 4480a14842fSEric Dumazet seen |= SEEN_XREG | SEEN_MEM; 4490a14842fSEric Dumazet EMIT3(0x8b, 0x5d, 0xf0 - K*4); 4500a14842fSEric Dumazet break; 4510a14842fSEric Dumazet case BPF_S_ST: /* mem[K] = A : mov %eax,off8(%rbp) */ 4520a14842fSEric Dumazet seen |= SEEN_MEM; 4530a14842fSEric Dumazet EMIT3(0x89, 0x45, 0xf0 - K*4); 4540a14842fSEric Dumazet break; 4550a14842fSEric Dumazet case BPF_S_STX: /* mem[K] = X : mov %ebx,off8(%rbp) */ 4560a14842fSEric Dumazet seen |= SEEN_XREG | SEEN_MEM; 4570a14842fSEric Dumazet EMIT3(0x89, 0x5d, 0xf0 - K*4); 4580a14842fSEric Dumazet break; 4590a14842fSEric Dumazet case BPF_S_LD_W_LEN: /* A = skb->len; */ 4600a14842fSEric Dumazet BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4); 4610a14842fSEric Dumazet if (is_imm8(offsetof(struct sk_buff, len))) 4620a14842fSEric Dumazet /* mov off8(%rdi),%eax */ 4630a14842fSEric Dumazet EMIT3(0x8b, 0x47, offsetof(struct sk_buff, len)); 4640a14842fSEric Dumazet else { 4650a14842fSEric Dumazet EMIT2(0x8b, 0x87); 4660a14842fSEric Dumazet EMIT(offsetof(struct sk_buff, len), 4); 4670a14842fSEric Dumazet } 4680a14842fSEric Dumazet break; 4690a14842fSEric Dumazet case BPF_S_LDX_W_LEN: /* X = skb->len; */ 4700a14842fSEric Dumazet seen |= SEEN_XREG; 4710a14842fSEric Dumazet if (is_imm8(offsetof(struct sk_buff, len))) 4720a14842fSEric Dumazet /* mov off8(%rdi),%ebx */ 4730a14842fSEric Dumazet EMIT3(0x8b, 0x5f, offsetof(struct sk_buff, len)); 4740a14842fSEric Dumazet else { 4750a14842fSEric Dumazet EMIT2(0x8b, 0x9f); 4760a14842fSEric Dumazet EMIT(offsetof(struct sk_buff, len), 4); 4770a14842fSEric Dumazet } 4780a14842fSEric Dumazet break; 4790a14842fSEric Dumazet case BPF_S_ANC_PROTOCOL: /* A = ntohs(skb->protocol); */ 4800a14842fSEric Dumazet BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2); 4810a14842fSEric Dumazet if (is_imm8(offsetof(struct sk_buff, protocol))) { 4820a14842fSEric Dumazet /* movzwl off8(%rdi),%eax */ 4830a14842fSEric Dumazet EMIT4(0x0f, 0xb7, 0x47, offsetof(struct sk_buff, protocol)); 4840a14842fSEric Dumazet } else { 4850a14842fSEric Dumazet EMIT3(0x0f, 0xb7, 0x87); /* movzwl off32(%rdi),%eax */ 4860a14842fSEric Dumazet EMIT(offsetof(struct sk_buff, protocol), 4); 4870a14842fSEric Dumazet } 4880a14842fSEric Dumazet EMIT2(0x86, 0xc4); /* ntohs() : xchg %al,%ah */ 4890a14842fSEric Dumazet break; 4900a14842fSEric Dumazet case BPF_S_ANC_IFINDEX: 4910a14842fSEric Dumazet if (is_imm8(offsetof(struct sk_buff, dev))) { 4920a14842fSEric Dumazet /* movq off8(%rdi),%rax */ 4930a14842fSEric Dumazet EMIT4(0x48, 0x8b, 0x47, offsetof(struct sk_buff, dev)); 4940a14842fSEric Dumazet } else { 4950a14842fSEric Dumazet EMIT3(0x48, 0x8b, 0x87); /* movq off32(%rdi),%rax */ 4960a14842fSEric Dumazet EMIT(offsetof(struct sk_buff, dev), 4); 4970a14842fSEric Dumazet } 4980a14842fSEric Dumazet EMIT3(0x48, 0x85, 0xc0); /* test %rax,%rax */ 4990a14842fSEric Dumazet EMIT_COND_JMP(X86_JE, cleanup_addr - (addrs[i] - 6)); 5000a14842fSEric Dumazet BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4); 5010a14842fSEric Dumazet EMIT2(0x8b, 0x80); /* mov off32(%rax),%eax */ 5020a14842fSEric Dumazet EMIT(offsetof(struct net_device, ifindex), 4); 5030a14842fSEric Dumazet break; 5040a14842fSEric Dumazet case BPF_S_ANC_MARK: 5050a14842fSEric Dumazet BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4); 5060a14842fSEric Dumazet if (is_imm8(offsetof(struct sk_buff, mark))) { 5070a14842fSEric Dumazet /* mov off8(%rdi),%eax */ 5080a14842fSEric Dumazet EMIT3(0x8b, 0x47, offsetof(struct sk_buff, mark)); 5090a14842fSEric Dumazet } else { 5100a14842fSEric Dumazet EMIT2(0x8b, 0x87); 5110a14842fSEric Dumazet EMIT(offsetof(struct sk_buff, mark), 4); 5120a14842fSEric Dumazet } 5130a14842fSEric Dumazet break; 5140a14842fSEric Dumazet case BPF_S_ANC_RXHASH: 5150a14842fSEric Dumazet BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, rxhash) != 4); 5160a14842fSEric Dumazet if (is_imm8(offsetof(struct sk_buff, rxhash))) { 5170a14842fSEric Dumazet /* mov off8(%rdi),%eax */ 5180a14842fSEric Dumazet EMIT3(0x8b, 0x47, offsetof(struct sk_buff, rxhash)); 5190a14842fSEric Dumazet } else { 5200a14842fSEric Dumazet EMIT2(0x8b, 0x87); 5210a14842fSEric Dumazet EMIT(offsetof(struct sk_buff, rxhash), 4); 5220a14842fSEric Dumazet } 5230a14842fSEric Dumazet break; 5240a14842fSEric Dumazet case BPF_S_ANC_QUEUE: 5250a14842fSEric Dumazet BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2); 5260a14842fSEric Dumazet if (is_imm8(offsetof(struct sk_buff, queue_mapping))) { 5270a14842fSEric Dumazet /* movzwl off8(%rdi),%eax */ 5280a14842fSEric Dumazet EMIT4(0x0f, 0xb7, 0x47, offsetof(struct sk_buff, queue_mapping)); 5290a14842fSEric Dumazet } else { 5300a14842fSEric Dumazet EMIT3(0x0f, 0xb7, 0x87); /* movzwl off32(%rdi),%eax */ 5310a14842fSEric Dumazet EMIT(offsetof(struct sk_buff, queue_mapping), 4); 5320a14842fSEric Dumazet } 5330a14842fSEric Dumazet break; 5340a14842fSEric Dumazet case BPF_S_ANC_CPU: 5350a14842fSEric Dumazet #ifdef CONFIG_SMP 5360a14842fSEric Dumazet EMIT4(0x65, 0x8b, 0x04, 0x25); /* mov %gs:off32,%eax */ 5370a14842fSEric Dumazet EMIT((u32)(unsigned long)&cpu_number, 4); /* A = smp_processor_id(); */ 5380a14842fSEric Dumazet #else 5390a14842fSEric Dumazet CLEAR_A(); 5400a14842fSEric Dumazet #endif 5410a14842fSEric Dumazet break; 542855ddb56SEric Dumazet case BPF_S_ANC_VLAN_TAG: 543855ddb56SEric Dumazet case BPF_S_ANC_VLAN_TAG_PRESENT: 544855ddb56SEric Dumazet BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2); 545855ddb56SEric Dumazet if (is_imm8(offsetof(struct sk_buff, vlan_tci))) { 546855ddb56SEric Dumazet /* movzwl off8(%rdi),%eax */ 547855ddb56SEric Dumazet EMIT4(0x0f, 0xb7, 0x47, offsetof(struct sk_buff, vlan_tci)); 548855ddb56SEric Dumazet } else { 549855ddb56SEric Dumazet EMIT3(0x0f, 0xb7, 0x87); /* movzwl off32(%rdi),%eax */ 550855ddb56SEric Dumazet EMIT(offsetof(struct sk_buff, vlan_tci), 4); 551855ddb56SEric Dumazet } 552855ddb56SEric Dumazet BUILD_BUG_ON(VLAN_TAG_PRESENT != 0x1000); 553855ddb56SEric Dumazet if (filter[i].code == BPF_S_ANC_VLAN_TAG) { 554855ddb56SEric Dumazet EMIT3(0x80, 0xe4, 0xef); /* and $0xef,%ah */ 555855ddb56SEric Dumazet } else { 556855ddb56SEric Dumazet EMIT3(0xc1, 0xe8, 0x0c); /* shr $0xc,%eax */ 557855ddb56SEric Dumazet EMIT3(0x83, 0xe0, 0x01); /* and $0x1,%eax */ 558855ddb56SEric Dumazet } 559855ddb56SEric Dumazet break; 5603b58908aSEric Dumazet case BPF_S_ANC_PKTTYPE: 5613b58908aSEric Dumazet { 5623b58908aSEric Dumazet int off = pkt_type_offset(); 5633b58908aSEric Dumazet 5643b58908aSEric Dumazet if (off < 0) 5653b58908aSEric Dumazet goto out; 5663b58908aSEric Dumazet if (is_imm8(off)) { 5673b58908aSEric Dumazet /* movzbl off8(%rdi),%eax */ 5683b58908aSEric Dumazet EMIT4(0x0f, 0xb6, 0x47, off); 5693b58908aSEric Dumazet } else { 5703b58908aSEric Dumazet /* movbl off32(%rdi),%eax */ 5713b58908aSEric Dumazet EMIT3(0x0f, 0xb6, 0x87); 5723b58908aSEric Dumazet EMIT(off, 4); 5733b58908aSEric Dumazet } 5743b58908aSEric Dumazet EMIT3(0x83, 0xe0, PKT_TYPE_MAX); /* and $0x7,%eax */ 5753b58908aSEric Dumazet break; 5763b58908aSEric Dumazet } 5770a14842fSEric Dumazet case BPF_S_LD_W_ABS: 578a998d434SJan Seiffert func = CHOOSE_LOAD_FUNC(K, sk_load_word); 5790a14842fSEric Dumazet common_load: seen |= SEEN_DATAREF; 5800a14842fSEric Dumazet t_offset = func - (image + addrs[i]); 5810a14842fSEric Dumazet EMIT1_off32(0xbe, K); /* mov imm32,%esi */ 5820a14842fSEric Dumazet EMIT1_off32(0xe8, t_offset); /* call */ 5830a14842fSEric Dumazet break; 5840a14842fSEric Dumazet case BPF_S_LD_H_ABS: 585a998d434SJan Seiffert func = CHOOSE_LOAD_FUNC(K, sk_load_half); 5860a14842fSEric Dumazet goto common_load; 5870a14842fSEric Dumazet case BPF_S_LD_B_ABS: 588a998d434SJan Seiffert func = CHOOSE_LOAD_FUNC(K, sk_load_byte); 5890a14842fSEric Dumazet goto common_load; 5900a14842fSEric Dumazet case BPF_S_LDX_B_MSH: 591a998d434SJan Seiffert func = CHOOSE_LOAD_FUNC(K, sk_load_byte_msh); 5920a14842fSEric Dumazet seen |= SEEN_DATAREF | SEEN_XREG; 593a998d434SJan Seiffert t_offset = func - (image + addrs[i]); 5940a14842fSEric Dumazet EMIT1_off32(0xbe, K); /* mov imm32,%esi */ 5950a14842fSEric Dumazet EMIT1_off32(0xe8, t_offset); /* call sk_load_byte_msh */ 5960a14842fSEric Dumazet break; 5970a14842fSEric Dumazet case BPF_S_LD_W_IND: 598a998d434SJan Seiffert func = sk_load_word; 5990a14842fSEric Dumazet common_load_ind: seen |= SEEN_DATAREF | SEEN_XREG; 6000a14842fSEric Dumazet t_offset = func - (image + addrs[i]); 601a998d434SJan Seiffert if (K) { 602a998d434SJan Seiffert if (is_imm8(K)) { 603a998d434SJan Seiffert EMIT3(0x8d, 0x73, K); /* lea imm8(%rbx), %esi */ 604a998d434SJan Seiffert } else { 605a998d434SJan Seiffert EMIT2(0x8d, 0xb3); /* lea imm32(%rbx),%esi */ 606a998d434SJan Seiffert EMIT(K, 4); 607a998d434SJan Seiffert } 608a998d434SJan Seiffert } else { 609a998d434SJan Seiffert EMIT2(0x89,0xde); /* mov %ebx,%esi */ 610a998d434SJan Seiffert } 6110a14842fSEric Dumazet EMIT1_off32(0xe8, t_offset); /* call sk_load_xxx_ind */ 6120a14842fSEric Dumazet break; 6130a14842fSEric Dumazet case BPF_S_LD_H_IND: 614a998d434SJan Seiffert func = sk_load_half; 6150a14842fSEric Dumazet goto common_load_ind; 6160a14842fSEric Dumazet case BPF_S_LD_B_IND: 617a998d434SJan Seiffert func = sk_load_byte; 6180a14842fSEric Dumazet goto common_load_ind; 6190a14842fSEric Dumazet case BPF_S_JMP_JA: 6200a14842fSEric Dumazet t_offset = addrs[i + K] - addrs[i]; 6210a14842fSEric Dumazet EMIT_JMP(t_offset); 6220a14842fSEric Dumazet break; 6230a14842fSEric Dumazet COND_SEL(BPF_S_JMP_JGT_K, X86_JA, X86_JBE); 6240a14842fSEric Dumazet COND_SEL(BPF_S_JMP_JGE_K, X86_JAE, X86_JB); 6250a14842fSEric Dumazet COND_SEL(BPF_S_JMP_JEQ_K, X86_JE, X86_JNE); 6260a14842fSEric Dumazet COND_SEL(BPF_S_JMP_JSET_K,X86_JNE, X86_JE); 6270a14842fSEric Dumazet COND_SEL(BPF_S_JMP_JGT_X, X86_JA, X86_JBE); 6280a14842fSEric Dumazet COND_SEL(BPF_S_JMP_JGE_X, X86_JAE, X86_JB); 6290a14842fSEric Dumazet COND_SEL(BPF_S_JMP_JEQ_X, X86_JE, X86_JNE); 6300a14842fSEric Dumazet COND_SEL(BPF_S_JMP_JSET_X,X86_JNE, X86_JE); 6310a14842fSEric Dumazet 6320a14842fSEric Dumazet cond_branch: f_offset = addrs[i + filter[i].jf] - addrs[i]; 6330a14842fSEric Dumazet t_offset = addrs[i + filter[i].jt] - addrs[i]; 6340a14842fSEric Dumazet 6350a14842fSEric Dumazet /* same targets, can avoid doing the test :) */ 6360a14842fSEric Dumazet if (filter[i].jt == filter[i].jf) { 6370a14842fSEric Dumazet EMIT_JMP(t_offset); 6380a14842fSEric Dumazet break; 6390a14842fSEric Dumazet } 6400a14842fSEric Dumazet 6410a14842fSEric Dumazet switch (filter[i].code) { 6420a14842fSEric Dumazet case BPF_S_JMP_JGT_X: 6430a14842fSEric Dumazet case BPF_S_JMP_JGE_X: 6440a14842fSEric Dumazet case BPF_S_JMP_JEQ_X: 6450a14842fSEric Dumazet seen |= SEEN_XREG; 6460a14842fSEric Dumazet EMIT2(0x39, 0xd8); /* cmp %ebx,%eax */ 6470a14842fSEric Dumazet break; 6480a14842fSEric Dumazet case BPF_S_JMP_JSET_X: 6490a14842fSEric Dumazet seen |= SEEN_XREG; 6500a14842fSEric Dumazet EMIT2(0x85, 0xd8); /* test %ebx,%eax */ 6510a14842fSEric Dumazet break; 6520a14842fSEric Dumazet case BPF_S_JMP_JEQ_K: 6530a14842fSEric Dumazet if (K == 0) { 6540a14842fSEric Dumazet EMIT2(0x85, 0xc0); /* test %eax,%eax */ 6550a14842fSEric Dumazet break; 6560a14842fSEric Dumazet } 6570a14842fSEric Dumazet case BPF_S_JMP_JGT_K: 6580a14842fSEric Dumazet case BPF_S_JMP_JGE_K: 6590a14842fSEric Dumazet if (K <= 127) 6600a14842fSEric Dumazet EMIT3(0x83, 0xf8, K); /* cmp imm8,%eax */ 6610a14842fSEric Dumazet else 6620a14842fSEric Dumazet EMIT1_off32(0x3d, K); /* cmp imm32,%eax */ 6630a14842fSEric Dumazet break; 6640a14842fSEric Dumazet case BPF_S_JMP_JSET_K: 6650a14842fSEric Dumazet if (K <= 0xFF) 6660a14842fSEric Dumazet EMIT2(0xa8, K); /* test imm8,%al */ 6670a14842fSEric Dumazet else if (!(K & 0xFFFF00FF)) 6680a14842fSEric Dumazet EMIT3(0xf6, 0xc4, K >> 8); /* test imm8,%ah */ 6690a14842fSEric Dumazet else if (K <= 0xFFFF) { 6700a14842fSEric Dumazet EMIT2(0x66, 0xa9); /* test imm16,%ax */ 6710a14842fSEric Dumazet EMIT(K, 2); 6720a14842fSEric Dumazet } else { 6730a14842fSEric Dumazet EMIT1_off32(0xa9, K); /* test imm32,%eax */ 6740a14842fSEric Dumazet } 6750a14842fSEric Dumazet break; 6760a14842fSEric Dumazet } 6770a14842fSEric Dumazet if (filter[i].jt != 0) { 678a03ffcf8SMarkus Kötter if (filter[i].jf && f_offset) 679a03ffcf8SMarkus Kötter t_offset += is_near(f_offset) ? 2 : 5; 6800a14842fSEric Dumazet EMIT_COND_JMP(t_op, t_offset); 6810a14842fSEric Dumazet if (filter[i].jf) 6820a14842fSEric Dumazet EMIT_JMP(f_offset); 6830a14842fSEric Dumazet break; 6840a14842fSEric Dumazet } 6850a14842fSEric Dumazet EMIT_COND_JMP(f_op, f_offset); 6860a14842fSEric Dumazet break; 6870a14842fSEric Dumazet default: 6880a14842fSEric Dumazet /* hmm, too complex filter, give up with jit compiler */ 6890a14842fSEric Dumazet goto out; 6900a14842fSEric Dumazet } 6910a14842fSEric Dumazet ilen = prog - temp; 6920a14842fSEric Dumazet if (image) { 6930a14842fSEric Dumazet if (unlikely(proglen + ilen > oldproglen)) { 6940a14842fSEric Dumazet pr_err("bpb_jit_compile fatal error\n"); 6950a14842fSEric Dumazet kfree(addrs); 6960a14842fSEric Dumazet module_free(NULL, image); 6970a14842fSEric Dumazet return; 6980a14842fSEric Dumazet } 6990a14842fSEric Dumazet memcpy(image + proglen, temp, ilen); 7000a14842fSEric Dumazet } 7010a14842fSEric Dumazet proglen += ilen; 7020a14842fSEric Dumazet addrs[i] = proglen; 7030a14842fSEric Dumazet prog = temp; 7040a14842fSEric Dumazet } 7050a14842fSEric Dumazet /* last bpf instruction is always a RET : 7060a14842fSEric Dumazet * use it to give the cleanup instruction(s) addr 7070a14842fSEric Dumazet */ 7080a14842fSEric Dumazet cleanup_addr = proglen - 1; /* ret */ 709d00a9dd2SEric Dumazet if (seen_or_pass0) 7100a14842fSEric Dumazet cleanup_addr -= 1; /* leaveq */ 711d00a9dd2SEric Dumazet if (seen_or_pass0 & SEEN_XREG) 7120a14842fSEric Dumazet cleanup_addr -= 4; /* mov -8(%rbp),%rbx */ 7130a14842fSEric Dumazet 7140a14842fSEric Dumazet if (image) { 715d00a9dd2SEric Dumazet if (proglen != oldproglen) 716d00a9dd2SEric Dumazet pr_err("bpb_jit_compile proglen=%u != oldproglen=%u\n", proglen, oldproglen); 7170a14842fSEric Dumazet break; 7180a14842fSEric Dumazet } 7190a14842fSEric Dumazet if (proglen == oldproglen) { 7200a14842fSEric Dumazet image = module_alloc(max_t(unsigned int, 7210a14842fSEric Dumazet proglen, 7220a14842fSEric Dumazet sizeof(struct work_struct))); 7230a14842fSEric Dumazet if (!image) 7240a14842fSEric Dumazet goto out; 7250a14842fSEric Dumazet } 7260a14842fSEric Dumazet oldproglen = proglen; 7270a14842fSEric Dumazet } 72879617801SDaniel Borkmann 7290a14842fSEric Dumazet if (bpf_jit_enable > 1) 73079617801SDaniel Borkmann bpf_jit_dump(flen, proglen, pass, image); 7310a14842fSEric Dumazet 7320a14842fSEric Dumazet if (image) { 7330a14842fSEric Dumazet bpf_flush_icache(image, image + proglen); 7340a14842fSEric Dumazet fp->bpf_func = (void *)image; 7350a14842fSEric Dumazet } 7360a14842fSEric Dumazet out: 7370a14842fSEric Dumazet kfree(addrs); 7380a14842fSEric Dumazet return; 7390a14842fSEric Dumazet } 7400a14842fSEric Dumazet 7410a14842fSEric Dumazet static void jit_free_defer(struct work_struct *arg) 7420a14842fSEric Dumazet { 7430a14842fSEric Dumazet module_free(NULL, arg); 7440a14842fSEric Dumazet } 7450a14842fSEric Dumazet 7460a14842fSEric Dumazet /* run from softirq, we must use a work_struct to call 7470a14842fSEric Dumazet * module_free() from process context 7480a14842fSEric Dumazet */ 7490a14842fSEric Dumazet void bpf_jit_free(struct sk_filter *fp) 7500a14842fSEric Dumazet { 7510a14842fSEric Dumazet if (fp->bpf_func != sk_run_filter) { 7520a14842fSEric Dumazet struct work_struct *work = (struct work_struct *)fp->bpf_func; 7530a14842fSEric Dumazet 7540a14842fSEric Dumazet INIT_WORK(work, jit_free_defer); 7550a14842fSEric Dumazet schedule_work(work); 7560a14842fSEric Dumazet } 7570a14842fSEric Dumazet } 758