1 /* This test is a demo of using get_socket_uid and get_socket_cookie
2  * helper function to do per socket based network traffic monitoring.
3  * It requires iptables version higher then 1.6.1. to load pinned eBPF
4  * program into the xt_bpf match.
5  *
6  * TEST:
7  * ./run_cookie_uid_helper_example.sh -option
8  * option:
9  *	-t: do traffic monitoring test, the program will continuously
10  * print out network traffic happens after program started A sample
11  * output is shown below:
12  *
13  * cookie: 877, uid: 0x3e8, Pakcet Count: 20, Bytes Count: 11058
14  * cookie: 132, uid: 0x0, Pakcet Count: 2, Bytes Count: 286
15  * cookie: 812, uid: 0x3e8, Pakcet Count: 3, Bytes Count: 1726
16  * cookie: 802, uid: 0x3e8, Pakcet Count: 2, Bytes Count: 104
17  * cookie: 877, uid: 0x3e8, Pakcet Count: 20, Bytes Count: 11058
18  * cookie: 831, uid: 0x3e8, Pakcet Count: 2, Bytes Count: 104
19  * cookie: 0, uid: 0x0, Pakcet Count: 6, Bytes Count: 712
20  * cookie: 880, uid: 0xfffe, Pakcet Count: 1, Bytes Count: 70
21  *
22  *	-s: do getsockopt SO_COOKIE test, the program will set up a pair of
23  * UDP sockets and send packets between them. And read out the traffic data
24  * directly from the ebpf map based on the socket cookie.
25  *
26  * Clean up: if using shell script, the script file will delete the iptables
27  * rule and unmount the bpf program when exit. Else the iptables rule need
28  * to be deleted by hand, see run_cookie_uid_helper_example.sh for detail.
29  */
30 
31 #define _GNU_SOURCE
32 
33 #define offsetof(type, member)	__builtin_offsetof(type, member)
34 #define ARRAY_SIZE(x) (sizeof(x) / sizeof(*(x)))
35 
36 #include <arpa/inet.h>
37 #include <errno.h>
38 #include <error.h>
39 #include <limits.h>
40 #include <linux/bpf.h>
41 #include <linux/if_ether.h>
42 #include <net/if.h>
43 #include <signal.h>
44 #include <stdbool.h>
45 #include <stdint.h>
46 #include <stdio.h>
47 #include <stdlib.h>
48 #include <string.h>
49 #include <sys/socket.h>
50 #include <sys/stat.h>
51 #include <sys/types.h>
52 #include <unistd.h>
53 #include <bpf/bpf.h>
54 #include "bpf_insn.h"
55 
56 #define PORT 8888
57 
58 struct stats {
59 	uint32_t uid;
60 	uint64_t packets;
61 	uint64_t bytes;
62 };
63 
64 static int map_fd, prog_fd;
65 
66 static bool test_finish;
67 
68 static void maps_create(void)
69 {
70 	map_fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(uint32_t),
71 				sizeof(struct stats), 100, 0);
72 	if (map_fd < 0)
73 		error(1, errno, "map create failed!\n");
74 }
75 
76 static void prog_load(void)
77 {
78 	static char log_buf[1 << 16];
79 
80 	struct bpf_insn prog[] = {
81 		/*
82 		 * Save sk_buff for future usage. value stored in R6 to R10 will
83 		 * not be reset after a bpf helper function call.
84 		 */
85 		BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
86 		/*
87 		 * pc1: BPF_FUNC_get_socket_cookie takes one parameter,
88 		 * R1: sk_buff
89 		 */
90 		BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
91 				BPF_FUNC_get_socket_cookie),
92 		/* pc2-4: save &socketCookie to r7 for future usage*/
93 		BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -8),
94 		BPF_MOV64_REG(BPF_REG_7, BPF_REG_10),
95 		BPF_ALU64_IMM(BPF_ADD, BPF_REG_7, -8),
96 		/*
97 		 * pc5-8: set up the registers for BPF_FUNC_map_lookup_elem,
98 		 * it takes two parameters (R1: map_fd,  R2: &socket_cookie)
99 		 */
100 		BPF_LD_MAP_FD(BPF_REG_1, map_fd),
101 		BPF_MOV64_REG(BPF_REG_2, BPF_REG_7),
102 		BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
103 				BPF_FUNC_map_lookup_elem),
104 		/*
105 		 * pc9. if r0 != 0x0, go to pc+14, since we have the cookie
106 		 * stored already
107 		 * Otherwise do pc10-22 to setup a new data entry.
108 		 */
109 		BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 14),
110 		BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
111 		BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
112 				BPF_FUNC_get_socket_uid),
113 		/*
114 		 * Place a struct stats in the R10 stack and sequentially
115 		 * place the member value into the memory. Packets value
116 		 * is set by directly place a IMM value 1 into the stack.
117 		 */
118 		BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0,
119 			    -32 + (__s16)offsetof(struct stats, uid)),
120 		BPF_ST_MEM(BPF_DW, BPF_REG_10,
121 			   -32 + (__s16)offsetof(struct stats, packets), 1),
122 		/*
123 		 * __sk_buff is a special struct used for eBPF program to
124 		 * directly access some sk_buff field.
125 		 */
126 		BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_6,
127 				offsetof(struct __sk_buff, len)),
128 		BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1,
129 			    -32 + (__s16)offsetof(struct stats, bytes)),
130 		/*
131 		 * add new map entry using BPF_FUNC_map_update_elem, it takes
132 		 * 4 parameters (R1: map_fd, R2: &socket_cookie, R3: &stats,
133 		 * R4: flags)
134 		 */
135 		BPF_LD_MAP_FD(BPF_REG_1, map_fd),
136 		BPF_MOV64_REG(BPF_REG_2, BPF_REG_7),
137 		BPF_MOV64_REG(BPF_REG_3, BPF_REG_10),
138 		BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, -32),
139 		BPF_MOV64_IMM(BPF_REG_4, 0),
140 		BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
141 				BPF_FUNC_map_update_elem),
142 		BPF_JMP_IMM(BPF_JA, 0, 0, 5),
143 		/*
144 		 * pc24-30 update the packet info to a exist data entry, it can
145 		 * be done by directly write to pointers instead of using
146 		 * BPF_FUNC_map_update_elem helper function
147 		 */
148 		BPF_MOV64_REG(BPF_REG_9, BPF_REG_0),
149 		BPF_MOV64_IMM(BPF_REG_1, 1),
150 		BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_9, BPF_REG_1,
151 			      offsetof(struct stats, packets)),
152 		BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_6,
153 				offsetof(struct __sk_buff, len)),
154 		BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_9, BPF_REG_1,
155 			      offsetof(struct stats, bytes)),
156 		BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6,
157 				offsetof(struct __sk_buff, len)),
158 		BPF_EXIT_INSN(),
159 	};
160 	prog_fd = bpf_load_program(BPF_PROG_TYPE_SOCKET_FILTER, prog,
161 					ARRAY_SIZE(prog), "GPL", 0,
162 					log_buf, sizeof(log_buf));
163 	if (prog_fd < 0)
164 		error(1, errno, "failed to load prog\n%s\n", log_buf);
165 }
166 
167 static void prog_attach_iptables(char *file)
168 {
169 	int ret;
170 	char rules[100];
171 
172 	if (bpf_obj_pin(prog_fd, file))
173 		error(1, errno, "bpf_obj_pin");
174 	if (strlen(file) > 50) {
175 		printf("file path too long: %s\n", file);
176 		exit(1);
177 	}
178 	sprintf(rules, "iptables -A OUTPUT -m bpf --object-pinned %s -j ACCEPT",
179 		file);
180 	ret = system(rules);
181 	if (ret < 0) {
182 		printf("iptables rule update failed: %d/n", WEXITSTATUS(ret));
183 		exit(1);
184 	}
185 }
186 
187 static void print_table(void)
188 {
189 	struct stats curEntry;
190 	uint32_t curN = UINT32_MAX;
191 	uint32_t nextN;
192 	int res;
193 
194 	while (bpf_map_get_next_key(map_fd, &curN, &nextN) > -1) {
195 		curN = nextN;
196 		res = bpf_map_lookup_elem(map_fd, &curN, &curEntry);
197 		if (res < 0) {
198 			error(1, errno, "fail to get entry value of Key: %u\n",
199 				curN);
200 		} else {
201 			printf("cookie: %u, uid: 0x%x, Packet Count: %lu,"
202 				" Bytes Count: %lu\n", curN, curEntry.uid,
203 				curEntry.packets, curEntry.bytes);
204 		}
205 	}
206 }
207 
208 static void udp_client(void)
209 {
210 	struct sockaddr_in si_other = {0};
211 	struct sockaddr_in si_me = {0};
212 	struct stats dataEntry;
213 	int s_rcv, s_send, i, recv_len;
214 	char message = 'a';
215 	char buf;
216 	uint64_t cookie;
217 	int res;
218 	socklen_t cookie_len = sizeof(cookie);
219 	socklen_t slen = sizeof(si_other);
220 
221 	s_rcv = socket(PF_INET, SOCK_DGRAM, 0);
222 	if (s_rcv < 0)
223 		error(1, errno, "rcv socket creat failed!\n");
224 	si_other.sin_family = AF_INET;
225 	si_other.sin_port = htons(PORT);
226 	if (inet_aton("127.0.0.1", &si_other.sin_addr) == 0)
227 		error(1, errno, "inet_aton\n");
228 	if (bind(s_rcv, (struct sockaddr *)&si_other, sizeof(si_other)) == -1)
229 		error(1, errno, "bind\n");
230 	s_send = socket(PF_INET, SOCK_DGRAM, 0);
231 	if (s_send < 0)
232 		error(1, errno, "send socket creat failed!\n");
233 	res = getsockopt(s_send, SOL_SOCKET, SO_COOKIE, &cookie, &cookie_len);
234 	if (res < 0)
235 		printf("get cookie failed: %s\n", strerror(errno));
236 	res = bpf_map_lookup_elem(map_fd, &cookie, &dataEntry);
237 	if (res != -1)
238 		error(1, errno, "socket stat found while flow not active\n");
239 	for (i = 0; i < 10; i++) {
240 		res = sendto(s_send, &message, sizeof(message), 0,
241 			     (struct sockaddr *)&si_other, slen);
242 		if (res == -1)
243 			error(1, errno, "send\n");
244 		if (res != sizeof(message))
245 			error(1, 0, "%uB != %luB\n", res, sizeof(message));
246 		recv_len = recvfrom(s_rcv, &buf, sizeof(buf), 0,
247 			     (struct sockaddr *)&si_me, &slen);
248 		if (recv_len < 0)
249 			error(1, errno, "receive\n");
250 		res = memcmp(&(si_other.sin_addr), &(si_me.sin_addr),
251 			   sizeof(si_me.sin_addr));
252 		if (res != 0)
253 			error(1, EFAULT, "sender addr error: %d\n", res);
254 		printf("Message received: %c\n", buf);
255 		res = bpf_map_lookup_elem(map_fd, &cookie, &dataEntry);
256 		if (res < 0)
257 			error(1, errno, "lookup sk stat failed, cookie: %lu\n",
258 			      cookie);
259 		printf("cookie: %lu, uid: 0x%x, Packet Count: %lu,"
260 			" Bytes Count: %lu\n\n", cookie, dataEntry.uid,
261 			dataEntry.packets, dataEntry.bytes);
262 	}
263 	close(s_send);
264 	close(s_rcv);
265 }
266 
267 static int usage(void)
268 {
269 	printf("Usage: ./run_cookie_uid_helper_example.sh"
270 		" bpfObjName -option\n"
271 		"	-t	traffic monitor test\n"
272 		"	-s	getsockopt cookie test\n");
273 	return 1;
274 }
275 
276 static void finish(int ret)
277 {
278 	test_finish = true;
279 }
280 
281 int main(int argc, char *argv[])
282 {
283 	int opt;
284 	bool cfg_test_traffic = false;
285 	bool cfg_test_cookie = false;
286 
287 	if (argc != 3)
288 		return usage();
289 	while ((opt = getopt(argc, argv, "ts")) != -1) {
290 		switch (opt) {
291 		case 't':
292 			cfg_test_traffic = true;
293 			break;
294 		case 's':
295 			cfg_test_cookie = true;
296 			break;
297 
298 		default:
299 			printf("unknown option %c\n", opt);
300 			usage();
301 			return -1;
302 		}
303 	}
304 	maps_create();
305 	prog_load();
306 	prog_attach_iptables(argv[2]);
307 	if (cfg_test_traffic) {
308 		if (signal(SIGINT, finish) == SIG_ERR)
309 			error(1, errno, "register SIGINT handler failed");
310 		if (signal(SIGTERM, finish) == SIG_ERR)
311 			error(1, errno, "register SIGTERM handler failed");
312 		while (!test_finish) {
313 			print_table();
314 			printf("\n");
315 			sleep(1);
316 		}
317 	} else if (cfg_test_cookie) {
318 		udp_client();
319 	}
320 	close(prog_fd);
321 	close(map_fd);
322 	return 0;
323 }
324