1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2019 Facebook
3 
4 #include <stdint.h>
5 #include <stddef.h>
6 #include <stdbool.h>
7 #include <linux/bpf.h>
8 #include <linux/ptrace.h>
9 #include <linux/sched.h>
10 #include <linux/types.h>
11 #include <bpf/bpf_helpers.h>
12 
13 typedef uint32_t pid_t;
14 struct task_struct {};
15 
16 #define TASK_COMM_LEN 16
17 #define PERF_MAX_STACK_DEPTH 127
18 
19 #define STROBE_TYPE_INVALID 0
20 #define STROBE_TYPE_INT 1
21 #define STROBE_TYPE_STR 2
22 #define STROBE_TYPE_MAP 3
23 
24 #define STACK_TABLE_EPOCH_SHIFT 20
25 #define STROBE_MAX_STR_LEN 1
26 #define STROBE_MAX_CFGS 32
27 #define STROBE_MAX_PAYLOAD						\
28 	(STROBE_MAX_STRS * STROBE_MAX_STR_LEN +				\
29 	STROBE_MAX_MAPS * (1 + STROBE_MAX_MAP_ENTRIES * 2) * STROBE_MAX_STR_LEN)
30 
31 struct strobe_value_header {
32 	/*
33 	 * meaning depends on type:
34 	 * 1. int: 0, if value not set, 1 otherwise
35 	 * 2. str: 1 always, whether value is set or not is determined by ptr
36 	 * 3. map: 1 always, pointer points to additional struct with number
37 	 *    of entries (up to STROBE_MAX_MAP_ENTRIES)
38 	 */
39 	uint16_t len;
40 	/*
41 	 * _reserved might be used for some future fields/flags, but we always
42 	 * want to keep strobe_value_header to be 8 bytes, so BPF can read 16
43 	 * bytes in one go and get both header and value
44 	 */
45 	uint8_t _reserved[6];
46 };
47 
48 /*
49  * strobe_value_generic is used from BPF probe only, but needs to be a union
50  * of strobe_value_int/strobe_value_str/strobe_value_map
51  */
52 struct strobe_value_generic {
53 	struct strobe_value_header header;
54 	union {
55 		int64_t val;
56 		void *ptr;
57 	};
58 };
59 
60 struct strobe_value_int {
61 	struct strobe_value_header header;
62 	int64_t value;
63 };
64 
65 struct strobe_value_str {
66 	struct strobe_value_header header;
67 	const char* value;
68 };
69 
70 struct strobe_value_map {
71 	struct strobe_value_header header;
72 	const struct strobe_map_raw* value;
73 };
74 
75 struct strobe_map_entry {
76 	const char* key;
77 	const char* val;
78 };
79 
80 /*
81  * Map of C-string key/value pairs with fixed maximum capacity. Each map has
82  * corresponding int64 ID, which application can use (or ignore) in whatever
83  * way appropriate. Map is "write-only", there is no way to get data out of
84  * map. Map is intended to be used to provide metadata for profilers and is
85  * not to be used for internal in-app communication. All methods are
86  * thread-safe.
87  */
88 struct strobe_map_raw {
89 	/*
90 	 * general purpose unique ID that's up to application to decide
91 	 * whether and how to use; for request metadata use case id is unique
92 	 * request ID that's used to match metadata with stack traces on
93 	 * Strobelight backend side
94 	 */
95 	int64_t id;
96 	/* number of used entries in map */
97 	int64_t cnt;
98 	/*
99 	 * having volatile doesn't change anything on BPF side, but clang
100 	 * emits warnings for passing `volatile const char *` into
101 	 * bpf_probe_read_user_str that expects just `const char *`
102 	 */
103 	const char* tag;
104 	/*
105 	 * key/value entries, each consisting of 2 pointers to key and value
106 	 * C strings
107 	 */
108 	struct strobe_map_entry entries[STROBE_MAX_MAP_ENTRIES];
109 };
110 
111 /* Following values define supported values of TLS mode */
112 #define TLS_NOT_SET -1
113 #define TLS_LOCAL_EXEC 0
114 #define TLS_IMM_EXEC 1
115 #define TLS_GENERAL_DYN 2
116 
117 /*
118  * structure that universally represents TLS location (both for static
119  * executables and shared libraries)
120  */
121 struct strobe_value_loc {
122 	/*
123 	 * tls_mode defines what TLS mode was used for particular metavariable:
124 	 * - -1 (TLS_NOT_SET) - no metavariable;
125 	 * - 0 (TLS_LOCAL_EXEC) - Local Executable mode;
126 	 * - 1 (TLS_IMM_EXEC) - Immediate Executable mode;
127 	 * - 2 (TLS_GENERAL_DYN) - General Dynamic mode;
128 	 * Local Dynamic mode is not yet supported, because never seen in
129 	 * practice.  Mode defines how offset field is interpreted. See
130 	 * calc_location() in below for details.
131 	 */
132 	int64_t tls_mode;
133 	/*
134 	 * TLS_LOCAL_EXEC: offset from thread pointer (fs:0 for x86-64,
135 	 * tpidr_el0 for aarch64).
136 	 * TLS_IMM_EXEC: absolute address of GOT entry containing offset
137 	 * from thread pointer;
138 	 * TLS_GENERAL_DYN: absolute address of double GOT entry
139 	 * containing tls_index_t struct;
140 	 */
141 	int64_t offset;
142 };
143 
144 struct strobemeta_cfg {
145 	int64_t req_meta_idx;
146 	struct strobe_value_loc int_locs[STROBE_MAX_INTS];
147 	struct strobe_value_loc str_locs[STROBE_MAX_STRS];
148 	struct strobe_value_loc map_locs[STROBE_MAX_MAPS];
149 };
150 
151 struct strobe_map_descr {
152 	uint64_t id;
153 	int16_t tag_len;
154 	/*
155 	 * cnt <0 - map value isn't set;
156 	 * 0 - map has id set, but no key/value entries
157 	 */
158 	int16_t cnt;
159 	/*
160 	 * both key_lens[i] and val_lens[i] should be >0 for present key/value
161 	 * entry
162 	 */
163 	uint16_t key_lens[STROBE_MAX_MAP_ENTRIES];
164 	uint16_t val_lens[STROBE_MAX_MAP_ENTRIES];
165 };
166 
167 struct strobemeta_payload {
168 	/* req_id has valid request ID, if req_meta_valid == 1 */
169 	int64_t req_id;
170 	uint8_t req_meta_valid;
171 	/*
172 	 * mask has Nth bit set to 1, if Nth metavar was present and
173 	 * successfully read
174 	 */
175 	uint64_t int_vals_set_mask;
176 	int64_t int_vals[STROBE_MAX_INTS];
177 	/* len is >0 for present values */
178 	uint16_t str_lens[STROBE_MAX_STRS];
179 	/* if map_descrs[i].cnt == -1, metavar is not present/set */
180 	struct strobe_map_descr map_descrs[STROBE_MAX_MAPS];
181 	/*
182 	 * payload has compactly packed values of str and map variables in the
183 	 * form: strval1\0strval2\0map1key1\0map1val1\0map2key1\0map2val1\0
184 	 * (and so on); str_lens[i], key_lens[i] and val_lens[i] determines
185 	 * value length
186 	 */
187 	char payload[STROBE_MAX_PAYLOAD];
188 };
189 
190 struct strobelight_bpf_sample {
191 	uint64_t ktime;
192 	char comm[TASK_COMM_LEN];
193 	pid_t pid;
194 	int user_stack_id;
195 	int kernel_stack_id;
196 	int has_meta;
197 	struct strobemeta_payload metadata;
198 	/*
199 	 * makes it possible to pass (<real payload size> + 1) as data size to
200 	 * perf_submit() to avoid perf_submit's paranoia about passing zero as
201 	 * size, as it deduces that <real payload size> might be
202 	 * **theoretically** zero
203 	 */
204 	char dummy_safeguard;
205 };
206 
207 struct {
208 	__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
209 	__uint(max_entries, 32);
210 	__uint(key_size, sizeof(int));
211 	__uint(value_size, sizeof(int));
212 } samples SEC(".maps");
213 
214 struct {
215 	__uint(type, BPF_MAP_TYPE_STACK_TRACE);
216 	__uint(max_entries, 16);
217 	__uint(key_size, sizeof(uint32_t));
218 	__uint(value_size, sizeof(uint64_t) * PERF_MAX_STACK_DEPTH);
219 } stacks_0 SEC(".maps");
220 
221 struct {
222 	__uint(type, BPF_MAP_TYPE_STACK_TRACE);
223 	__uint(max_entries, 16);
224 	__uint(key_size, sizeof(uint32_t));
225 	__uint(value_size, sizeof(uint64_t) * PERF_MAX_STACK_DEPTH);
226 } stacks_1 SEC(".maps");
227 
228 struct {
229 	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
230 	__uint(max_entries, 1);
231 	__type(key, uint32_t);
232 	__type(value, struct strobelight_bpf_sample);
233 } sample_heap SEC(".maps");
234 
235 struct {
236 	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
237 	__uint(max_entries, STROBE_MAX_CFGS);
238 	__type(key, pid_t);
239 	__type(value, struct strobemeta_cfg);
240 } strobemeta_cfgs SEC(".maps");
241 
242 /* Type for the dtv.  */
243 /* https://github.com/lattera/glibc/blob/master/nptl/sysdeps/x86_64/tls.h#L34 */
244 typedef union dtv {
245 	size_t counter;
246 	struct {
247 		void* val;
248 		bool is_static;
249 	} pointer;
250 } dtv_t;
251 
252 /* Partial definition for tcbhead_t */
253 /* https://github.com/bminor/glibc/blob/master/sysdeps/x86_64/nptl/tls.h#L42 */
254 struct tcbhead {
255 	void* tcb;
256 	dtv_t* dtv;
257 };
258 
259 /*
260  * TLS module/offset information for shared library case.
261  * For x86-64, this is mapped onto two entries in GOT.
262  * For aarch64, this is pointed to by second GOT entry.
263  */
264 struct tls_index {
265 	uint64_t module;
266 	uint64_t offset;
267 };
268 
269 #ifdef SUBPROGS
270 __noinline
271 #else
272 __always_inline
273 #endif
274 static void *calc_location(struct strobe_value_loc *loc, void *tls_base)
275 {
276 	/*
277 	 * tls_mode value is:
278 	 * - -1 (TLS_NOT_SET), if no metavar is present;
279 	 * - 0 (TLS_LOCAL_EXEC), if metavar uses Local Executable mode of TLS
280 	 * (offset from fs:0 for x86-64 or tpidr_el0 for aarch64);
281 	 * - 1 (TLS_IMM_EXEC), if metavar uses Immediate Executable mode of TLS;
282 	 * - 2 (TLS_GENERAL_DYN), if metavar uses General Dynamic mode of TLS;
283 	 * This schema allows to use something like:
284 	 * (tls_mode + 1) * (tls_base + offset)
285 	 * to get NULL for "no metavar" location, or correct pointer for local
286 	 * executable mode without doing extra ifs.
287 	 */
288 	if (loc->tls_mode <= TLS_LOCAL_EXEC) {
289 		/* static executable is simple, we just have offset from
290 		 * tls_base */
291 		void *addr = tls_base + loc->offset;
292 		/* multiply by (tls_mode + 1) to get NULL, if we have no
293 		 * metavar in this slot */
294 		return (void *)((loc->tls_mode + 1) * (int64_t)addr);
295 	}
296 	/*
297 	 * Other modes are more complicated, we need to jump through few hoops.
298 	 *
299 	 * For immediate executable mode (currently supported only for aarch64):
300 	 *  - loc->offset is pointing to a GOT entry containing fixed offset
301 	 *  relative to tls_base;
302 	 *
303 	 * For general dynamic mode:
304 	 *  - loc->offset is pointing to a beginning of double GOT entries;
305 	 *  - (for aarch64 only) second entry points to tls_index_t struct;
306 	 *  - (for x86-64 only) two GOT entries are already tls_index_t;
307 	 *  - tls_index_t->module is used to find start of TLS section in
308 	 *  which variable resides;
309 	 *  - tls_index_t->offset provides offset within that TLS section,
310 	 *  pointing to value of variable.
311 	 */
312 	struct tls_index tls_index;
313 	dtv_t *dtv;
314 	void *tls_ptr;
315 
316 	bpf_probe_read_user(&tls_index, sizeof(struct tls_index),
317 			    (void *)loc->offset);
318 	/* valid module index is always positive */
319 	if (tls_index.module > 0) {
320 		/* dtv = ((struct tcbhead *)tls_base)->dtv[tls_index.module] */
321 		bpf_probe_read_user(&dtv, sizeof(dtv),
322 				    &((struct tcbhead *)tls_base)->dtv);
323 		dtv += tls_index.module;
324 	} else {
325 		dtv = NULL;
326 	}
327 	bpf_probe_read_user(&tls_ptr, sizeof(void *), dtv);
328 	/* if pointer has (void *)-1 value, then TLS wasn't initialized yet */
329 	return tls_ptr && tls_ptr != (void *)-1
330 		? tls_ptr + tls_index.offset
331 		: NULL;
332 }
333 
334 #ifdef SUBPROGS
335 __noinline
336 #else
337 __always_inline
338 #endif
339 static void read_int_var(struct strobemeta_cfg *cfg,
340 			 size_t idx, void *tls_base,
341 			 struct strobe_value_generic *value,
342 			 struct strobemeta_payload *data)
343 {
344 	void *location = calc_location(&cfg->int_locs[idx], tls_base);
345 	if (!location)
346 		return;
347 
348 	bpf_probe_read_user(value, sizeof(struct strobe_value_generic), location);
349 	data->int_vals[idx] = value->val;
350 	if (value->header.len)
351 		data->int_vals_set_mask |= (1 << idx);
352 }
353 
354 static __always_inline uint64_t read_str_var(struct strobemeta_cfg *cfg,
355 					     size_t idx, void *tls_base,
356 					     struct strobe_value_generic *value,
357 					     struct strobemeta_payload *data,
358 					     void *payload)
359 {
360 	void *location;
361 	uint64_t len;
362 
363 	data->str_lens[idx] = 0;
364 	location = calc_location(&cfg->str_locs[idx], tls_base);
365 	if (!location)
366 		return 0;
367 
368 	bpf_probe_read_user(value, sizeof(struct strobe_value_generic), location);
369 	len = bpf_probe_read_user_str(payload, STROBE_MAX_STR_LEN, value->ptr);
370 	/*
371 	 * if bpf_probe_read_user_str returns error (<0), due to casting to
372 	 * unsinged int, it will become big number, so next check is
373 	 * sufficient to check for errors AND prove to BPF verifier, that
374 	 * bpf_probe_read_user_str won't return anything bigger than
375 	 * STROBE_MAX_STR_LEN
376 	 */
377 	if (len > STROBE_MAX_STR_LEN)
378 		return 0;
379 
380 	data->str_lens[idx] = len;
381 	return len;
382 }
383 
384 static __always_inline void *read_map_var(struct strobemeta_cfg *cfg,
385 					  size_t idx, void *tls_base,
386 					  struct strobe_value_generic *value,
387 					  struct strobemeta_payload *data,
388 					  void *payload)
389 {
390 	struct strobe_map_descr* descr = &data->map_descrs[idx];
391 	struct strobe_map_raw map;
392 	void *location;
393 	uint64_t len;
394 
395 	descr->tag_len = 0; /* presume no tag is set */
396 	descr->cnt = -1; /* presume no value is set */
397 
398 	location = calc_location(&cfg->map_locs[idx], tls_base);
399 	if (!location)
400 		return payload;
401 
402 	bpf_probe_read_user(value, sizeof(struct strobe_value_generic), location);
403 	if (bpf_probe_read_user(&map, sizeof(struct strobe_map_raw), value->ptr))
404 		return payload;
405 
406 	descr->id = map.id;
407 	descr->cnt = map.cnt;
408 	if (cfg->req_meta_idx == idx) {
409 		data->req_id = map.id;
410 		data->req_meta_valid = 1;
411 	}
412 
413 	len = bpf_probe_read_user_str(payload, STROBE_MAX_STR_LEN, map.tag);
414 	if (len <= STROBE_MAX_STR_LEN) {
415 		descr->tag_len = len;
416 		payload += len;
417 	}
418 
419 #ifdef NO_UNROLL
420 #pragma clang loop unroll(disable)
421 #else
422 #pragma unroll
423 #endif
424 	for (int i = 0; i < STROBE_MAX_MAP_ENTRIES; ++i) {
425 		if (i >= map.cnt)
426 			break;
427 
428 		descr->key_lens[i] = 0;
429 		len = bpf_probe_read_user_str(payload, STROBE_MAX_STR_LEN,
430 					      map.entries[i].key);
431 		if (len <= STROBE_MAX_STR_LEN) {
432 			descr->key_lens[i] = len;
433 			payload += len;
434 		}
435 		descr->val_lens[i] = 0;
436 		len = bpf_probe_read_user_str(payload, STROBE_MAX_STR_LEN,
437 					      map.entries[i].val);
438 		if (len <= STROBE_MAX_STR_LEN) {
439 			descr->val_lens[i] = len;
440 			payload += len;
441 		}
442 	}
443 
444 	return payload;
445 }
446 
447 #ifdef USE_BPF_LOOP
448 enum read_type {
449 	READ_INT_VAR,
450 	READ_MAP_VAR,
451 	READ_STR_VAR,
452 };
453 
454 struct read_var_ctx {
455 	struct strobemeta_payload *data;
456 	void *tls_base;
457 	struct strobemeta_cfg *cfg;
458 	void *payload;
459 	/* value gets mutated */
460 	struct strobe_value_generic *value;
461 	enum read_type type;
462 };
463 
464 static int read_var_callback(__u32 index, struct read_var_ctx *ctx)
465 {
466 	switch (ctx->type) {
467 	case READ_INT_VAR:
468 		if (index >= STROBE_MAX_INTS)
469 			return 1;
470 		read_int_var(ctx->cfg, index, ctx->tls_base, ctx->value, ctx->data);
471 		break;
472 	case READ_MAP_VAR:
473 		if (index >= STROBE_MAX_MAPS)
474 			return 1;
475 		ctx->payload = read_map_var(ctx->cfg, index, ctx->tls_base,
476 					    ctx->value, ctx->data, ctx->payload);
477 		break;
478 	case READ_STR_VAR:
479 		if (index >= STROBE_MAX_STRS)
480 			return 1;
481 		ctx->payload += read_str_var(ctx->cfg, index, ctx->tls_base,
482 					     ctx->value, ctx->data, ctx->payload);
483 		break;
484 	}
485 	return 0;
486 }
487 #endif /* USE_BPF_LOOP */
488 
489 /*
490  * read_strobe_meta returns NULL, if no metadata was read; otherwise returns
491  * pointer to *right after* payload ends
492  */
493 #ifdef SUBPROGS
494 __noinline
495 #else
496 __always_inline
497 #endif
498 static void *read_strobe_meta(struct task_struct *task,
499 			      struct strobemeta_payload *data)
500 {
501 	pid_t pid = bpf_get_current_pid_tgid() >> 32;
502 	struct strobe_value_generic value = {0};
503 	struct strobemeta_cfg *cfg;
504 	void *tls_base, *payload;
505 
506 	cfg = bpf_map_lookup_elem(&strobemeta_cfgs, &pid);
507 	if (!cfg)
508 		return NULL;
509 
510 	data->int_vals_set_mask = 0;
511 	data->req_meta_valid = 0;
512 	payload = data->payload;
513 	/*
514 	 * we don't have struct task_struct definition, it should be:
515 	 * tls_base = (void *)task->thread.fsbase;
516 	 */
517 	tls_base = (void *)task;
518 
519 #ifdef USE_BPF_LOOP
520 	struct read_var_ctx ctx = {
521 		.cfg = cfg,
522 		.tls_base = tls_base,
523 		.value = &value,
524 		.data = data,
525 		.payload = payload,
526 	};
527 	int err;
528 
529 	ctx.type = READ_INT_VAR;
530 	err = bpf_loop(STROBE_MAX_INTS, read_var_callback, &ctx, 0);
531 	if (err != STROBE_MAX_INTS)
532 		return NULL;
533 
534 	ctx.type = READ_STR_VAR;
535 	err = bpf_loop(STROBE_MAX_STRS, read_var_callback, &ctx, 0);
536 	if (err != STROBE_MAX_STRS)
537 		return NULL;
538 
539 	ctx.type = READ_MAP_VAR;
540 	err = bpf_loop(STROBE_MAX_MAPS, read_var_callback, &ctx, 0);
541 	if (err != STROBE_MAX_MAPS)
542 		return NULL;
543 #else
544 #ifdef NO_UNROLL
545 #pragma clang loop unroll(disable)
546 #else
547 #pragma unroll
548 #endif /* NO_UNROLL */
549 	for (int i = 0; i < STROBE_MAX_INTS; ++i) {
550 		read_int_var(cfg, i, tls_base, &value, data);
551 	}
552 #ifdef NO_UNROLL
553 #pragma clang loop unroll(disable)
554 #else
555 #pragma unroll
556 #endif /* NO_UNROLL */
557 	for (int i = 0; i < STROBE_MAX_STRS; ++i) {
558 		payload += read_str_var(cfg, i, tls_base, &value, data, payload);
559 	}
560 #ifdef NO_UNROLL
561 #pragma clang loop unroll(disable)
562 #else
563 #pragma unroll
564 #endif /* NO_UNROLL */
565 	for (int i = 0; i < STROBE_MAX_MAPS; ++i) {
566 		payload = read_map_var(cfg, i, tls_base, &value, data, payload);
567 	}
568 #endif /* USE_BPF_LOOP */
569 
570 	/*
571 	 * return pointer right after end of payload, so it's possible to
572 	 * calculate exact amount of useful data that needs to be sent
573 	 */
574 	return payload;
575 }
576 
577 SEC("raw_tracepoint/kfree_skb")
578 int on_event(struct pt_regs *ctx) {
579 	pid_t pid =  bpf_get_current_pid_tgid() >> 32;
580 	struct strobelight_bpf_sample* sample;
581 	struct task_struct *task;
582 	uint32_t zero = 0;
583 	uint64_t ktime_ns;
584 	void *sample_end;
585 
586 	sample = bpf_map_lookup_elem(&sample_heap, &zero);
587 	if (!sample)
588 		return 0; /* this will never happen */
589 
590 	sample->pid = pid;
591 	bpf_get_current_comm(&sample->comm, TASK_COMM_LEN);
592 	ktime_ns = bpf_ktime_get_ns();
593 	sample->ktime = ktime_ns;
594 
595 	task = (struct task_struct *)bpf_get_current_task();
596 	sample_end = read_strobe_meta(task, &sample->metadata);
597 	sample->has_meta = sample_end != NULL;
598 	sample_end = sample_end ? : &sample->metadata;
599 
600 	if ((ktime_ns >> STACK_TABLE_EPOCH_SHIFT) & 1) {
601 		sample->kernel_stack_id = bpf_get_stackid(ctx, &stacks_1, 0);
602 		sample->user_stack_id = bpf_get_stackid(ctx, &stacks_1, BPF_F_USER_STACK);
603 	} else {
604 		sample->kernel_stack_id = bpf_get_stackid(ctx, &stacks_0, 0);
605 		sample->user_stack_id = bpf_get_stackid(ctx, &stacks_0, BPF_F_USER_STACK);
606 	}
607 
608 	uint64_t sample_size = sample_end - (void *)sample;
609 	/* should always be true */
610 	if (sample_size < sizeof(struct strobelight_bpf_sample))
611 		bpf_perf_event_output(ctx, &samples, 0, sample, 1 + sample_size);
612 	return 0;
613 }
614 
615 char _license[] SEC("license") = "GPL";
616