1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Common code for probe-based Dynamic events.
4 *
5 * This code was copied from kernel/trace/trace_kprobe.c written by
6 * Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
7 *
8 * Updates to make this generic:
9 * Copyright (C) IBM Corporation, 2010-2011
10 * Author: Srikar Dronamraju
11 */
12 #define pr_fmt(fmt) "trace_probe: " fmt
13
14 #include <linux/bpf.h>
15 #include "trace_btf.h"
16
17 #include "trace_probe.h"
18
19 #undef C
20 #define C(a, b) b
21
22 static const char *trace_probe_err_text[] = { ERRORS };
23
24 static const char *reserved_field_names[] = {
25 "common_type",
26 "common_flags",
27 "common_preempt_count",
28 "common_pid",
29 "common_tgid",
30 FIELD_STRING_IP,
31 FIELD_STRING_RETIP,
32 FIELD_STRING_FUNC,
33 };
34
35 /* Printing in basic type function template */
36 #define DEFINE_BASIC_PRINT_TYPE_FUNC(tname, type, fmt) \
37 int PRINT_TYPE_FUNC_NAME(tname)(struct trace_seq *s, void *data, void *ent)\
38 { \
39 trace_seq_printf(s, fmt, *(type *)data); \
40 return !trace_seq_has_overflowed(s); \
41 } \
42 const char PRINT_TYPE_FMT_NAME(tname)[] = fmt;
43
44 DEFINE_BASIC_PRINT_TYPE_FUNC(u8, u8, "%u")
45 DEFINE_BASIC_PRINT_TYPE_FUNC(u16, u16, "%u")
46 DEFINE_BASIC_PRINT_TYPE_FUNC(u32, u32, "%u")
47 DEFINE_BASIC_PRINT_TYPE_FUNC(u64, u64, "%Lu")
48 DEFINE_BASIC_PRINT_TYPE_FUNC(s8, s8, "%d")
49 DEFINE_BASIC_PRINT_TYPE_FUNC(s16, s16, "%d")
50 DEFINE_BASIC_PRINT_TYPE_FUNC(s32, s32, "%d")
51 DEFINE_BASIC_PRINT_TYPE_FUNC(s64, s64, "%Ld")
52 DEFINE_BASIC_PRINT_TYPE_FUNC(x8, u8, "0x%x")
53 DEFINE_BASIC_PRINT_TYPE_FUNC(x16, u16, "0x%x")
54 DEFINE_BASIC_PRINT_TYPE_FUNC(x32, u32, "0x%x")
55 DEFINE_BASIC_PRINT_TYPE_FUNC(x64, u64, "0x%Lx")
56 DEFINE_BASIC_PRINT_TYPE_FUNC(char, u8, "'%c'")
57
PRINT_TYPE_FUNC_NAME(symbol)58 int PRINT_TYPE_FUNC_NAME(symbol)(struct trace_seq *s, void *data, void *ent)
59 {
60 trace_seq_printf(s, "%pS", (void *)*(unsigned long *)data);
61 return !trace_seq_has_overflowed(s);
62 }
63 const char PRINT_TYPE_FMT_NAME(symbol)[] = "%pS";
64
65 /* Print type function for string type */
PRINT_TYPE_FUNC_NAME(string)66 int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, void *data, void *ent)
67 {
68 int len = *(u32 *)data >> 16;
69
70 if (!len)
71 trace_seq_puts(s, FAULT_STRING);
72 else
73 trace_seq_printf(s, "\"%s\"",
74 (const char *)get_loc_data(data, ent));
75 return !trace_seq_has_overflowed(s);
76 }
77
78 const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\"";
79
80 /* Fetch type information table */
81 static const struct fetch_type probe_fetch_types[] = {
82 /* Special types */
83 __ASSIGN_FETCH_TYPE("string", string, string, sizeof(u32), 1, 1,
84 "__data_loc char[]"),
85 __ASSIGN_FETCH_TYPE("ustring", string, string, sizeof(u32), 1, 1,
86 "__data_loc char[]"),
87 __ASSIGN_FETCH_TYPE("symstr", string, string, sizeof(u32), 1, 1,
88 "__data_loc char[]"),
89 /* Basic types */
90 ASSIGN_FETCH_TYPE(u8, u8, 0),
91 ASSIGN_FETCH_TYPE(u16, u16, 0),
92 ASSIGN_FETCH_TYPE(u32, u32, 0),
93 ASSIGN_FETCH_TYPE(u64, u64, 0),
94 ASSIGN_FETCH_TYPE(s8, u8, 1),
95 ASSIGN_FETCH_TYPE(s16, u16, 1),
96 ASSIGN_FETCH_TYPE(s32, u32, 1),
97 ASSIGN_FETCH_TYPE(s64, u64, 1),
98 ASSIGN_FETCH_TYPE_ALIAS(x8, u8, u8, 0),
99 ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
100 ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
101 ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
102 ASSIGN_FETCH_TYPE_ALIAS(char, u8, u8, 0),
103 ASSIGN_FETCH_TYPE_ALIAS(symbol, ADDR_FETCH_TYPE, ADDR_FETCH_TYPE, 0),
104
105 ASSIGN_FETCH_TYPE_END
106 };
107
find_fetch_type(const char * type,unsigned long flags)108 static const struct fetch_type *find_fetch_type(const char *type, unsigned long flags)
109 {
110 int i;
111
112 /* Reject the symbol/symstr for uprobes */
113 if (type && (flags & TPARG_FL_USER) &&
114 (!strcmp(type, "symbol") || !strcmp(type, "symstr")))
115 return NULL;
116
117 if (!type)
118 type = DEFAULT_FETCH_TYPE_STR;
119
120 /* Special case: bitfield */
121 if (*type == 'b') {
122 unsigned long bs;
123
124 type = strchr(type, '/');
125 if (!type)
126 goto fail;
127
128 type++;
129 if (kstrtoul(type, 0, &bs))
130 goto fail;
131
132 switch (bs) {
133 case 8:
134 return find_fetch_type("u8", flags);
135 case 16:
136 return find_fetch_type("u16", flags);
137 case 32:
138 return find_fetch_type("u32", flags);
139 case 64:
140 return find_fetch_type("u64", flags);
141 default:
142 goto fail;
143 }
144 }
145
146 for (i = 0; probe_fetch_types[i].name; i++) {
147 if (strcmp(type, probe_fetch_types[i].name) == 0)
148 return &probe_fetch_types[i];
149 }
150
151 fail:
152 return NULL;
153 }
154
155 static struct trace_probe_log trace_probe_log;
156
trace_probe_log_init(const char * subsystem,int argc,const char ** argv)157 void trace_probe_log_init(const char *subsystem, int argc, const char **argv)
158 {
159 trace_probe_log.subsystem = subsystem;
160 trace_probe_log.argc = argc;
161 trace_probe_log.argv = argv;
162 trace_probe_log.index = 0;
163 }
164
trace_probe_log_clear(void)165 void trace_probe_log_clear(void)
166 {
167 memset(&trace_probe_log, 0, sizeof(trace_probe_log));
168 }
169
trace_probe_log_set_index(int index)170 void trace_probe_log_set_index(int index)
171 {
172 trace_probe_log.index = index;
173 }
174
__trace_probe_log_err(int offset,int err_type)175 void __trace_probe_log_err(int offset, int err_type)
176 {
177 char *command, *p;
178 int i, len = 0, pos = 0;
179
180 if (!trace_probe_log.argv)
181 return;
182
183 /* Recalculate the length and allocate buffer */
184 for (i = 0; i < trace_probe_log.argc; i++) {
185 if (i == trace_probe_log.index)
186 pos = len;
187 len += strlen(trace_probe_log.argv[i]) + 1;
188 }
189 command = kzalloc(len, GFP_KERNEL);
190 if (!command)
191 return;
192
193 if (trace_probe_log.index >= trace_probe_log.argc) {
194 /**
195 * Set the error position is next to the last arg + space.
196 * Note that len includes the terminal null and the cursor
197 * appears at pos + 1.
198 */
199 pos = len;
200 offset = 0;
201 }
202
203 /* And make a command string from argv array */
204 p = command;
205 for (i = 0; i < trace_probe_log.argc; i++) {
206 len = strlen(trace_probe_log.argv[i]);
207 strcpy(p, trace_probe_log.argv[i]);
208 p[len] = ' ';
209 p += len + 1;
210 }
211 *(p - 1) = '\0';
212
213 tracing_log_err(NULL, trace_probe_log.subsystem, command,
214 trace_probe_err_text, err_type, pos + offset);
215
216 kfree(command);
217 }
218
219 /* Split symbol and offset. */
traceprobe_split_symbol_offset(char * symbol,long * offset)220 int traceprobe_split_symbol_offset(char *symbol, long *offset)
221 {
222 char *tmp;
223 int ret;
224
225 if (!offset)
226 return -EINVAL;
227
228 tmp = strpbrk(symbol, "+-");
229 if (tmp) {
230 ret = kstrtol(tmp, 0, offset);
231 if (ret)
232 return ret;
233 *tmp = '\0';
234 } else
235 *offset = 0;
236
237 return 0;
238 }
239
240 /* @buf must has MAX_EVENT_NAME_LEN size */
traceprobe_parse_event_name(const char ** pevent,const char ** pgroup,char * buf,int offset)241 int traceprobe_parse_event_name(const char **pevent, const char **pgroup,
242 char *buf, int offset)
243 {
244 const char *slash, *event = *pevent;
245 int len;
246
247 slash = strchr(event, '/');
248 if (!slash)
249 slash = strchr(event, '.');
250
251 if (slash) {
252 if (slash == event) {
253 trace_probe_log_err(offset, NO_GROUP_NAME);
254 return -EINVAL;
255 }
256 if (slash - event + 1 > MAX_EVENT_NAME_LEN) {
257 trace_probe_log_err(offset, GROUP_TOO_LONG);
258 return -EINVAL;
259 }
260 strscpy(buf, event, slash - event + 1);
261 if (!is_good_system_name(buf)) {
262 trace_probe_log_err(offset, BAD_GROUP_NAME);
263 return -EINVAL;
264 }
265 *pgroup = buf;
266 *pevent = slash + 1;
267 offset += slash - event + 1;
268 event = *pevent;
269 }
270 len = strlen(event);
271 if (len == 0) {
272 if (slash) {
273 *pevent = NULL;
274 return 0;
275 }
276 trace_probe_log_err(offset, NO_EVENT_NAME);
277 return -EINVAL;
278 } else if (len > MAX_EVENT_NAME_LEN) {
279 trace_probe_log_err(offset, EVENT_TOO_LONG);
280 return -EINVAL;
281 }
282 if (!is_good_name(event)) {
283 trace_probe_log_err(offset, BAD_EVENT_NAME);
284 return -EINVAL;
285 }
286 return 0;
287 }
288
parse_trace_event_arg(char * arg,struct fetch_insn * code,struct traceprobe_parse_context * ctx)289 static int parse_trace_event_arg(char *arg, struct fetch_insn *code,
290 struct traceprobe_parse_context *ctx)
291 {
292 struct ftrace_event_field *field;
293 struct list_head *head;
294
295 head = trace_get_fields(ctx->event);
296 list_for_each_entry(field, head, link) {
297 if (!strcmp(arg, field->name)) {
298 code->op = FETCH_OP_TP_ARG;
299 code->data = field;
300 return 0;
301 }
302 }
303 return -ENOENT;
304 }
305
306 #ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
307
btf_type_int(const struct btf_type * t)308 static u32 btf_type_int(const struct btf_type *t)
309 {
310 return *(u32 *)(t + 1);
311 }
312
btf_type_is_char_ptr(struct btf * btf,const struct btf_type * type)313 static bool btf_type_is_char_ptr(struct btf *btf, const struct btf_type *type)
314 {
315 const struct btf_type *real_type;
316 u32 intdata;
317 s32 tid;
318
319 real_type = btf_type_skip_modifiers(btf, type->type, &tid);
320 if (!real_type)
321 return false;
322
323 if (BTF_INFO_KIND(real_type->info) != BTF_KIND_INT)
324 return false;
325
326 intdata = btf_type_int(real_type);
327 return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
328 && BTF_INT_BITS(intdata) == 8;
329 }
330
btf_type_is_char_array(struct btf * btf,const struct btf_type * type)331 static bool btf_type_is_char_array(struct btf *btf, const struct btf_type *type)
332 {
333 const struct btf_type *real_type;
334 const struct btf_array *array;
335 u32 intdata;
336 s32 tid;
337
338 if (BTF_INFO_KIND(type->info) != BTF_KIND_ARRAY)
339 return false;
340
341 array = (const struct btf_array *)(type + 1);
342
343 real_type = btf_type_skip_modifiers(btf, array->type, &tid);
344
345 intdata = btf_type_int(real_type);
346 return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
347 && BTF_INT_BITS(intdata) == 8;
348 }
349
check_prepare_btf_string_fetch(char * typename,struct fetch_insn ** pcode,struct traceprobe_parse_context * ctx)350 static int check_prepare_btf_string_fetch(char *typename,
351 struct fetch_insn **pcode,
352 struct traceprobe_parse_context *ctx)
353 {
354 struct btf *btf = ctx->btf;
355
356 if (!btf || !ctx->last_type)
357 return 0;
358
359 /* char [] does not need any change. */
360 if (btf_type_is_char_array(btf, ctx->last_type))
361 return 0;
362
363 /* char * requires dereference the pointer. */
364 if (btf_type_is_char_ptr(btf, ctx->last_type)) {
365 struct fetch_insn *code = *pcode + 1;
366
367 if (code->op == FETCH_OP_END) {
368 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
369 return -E2BIG;
370 }
371 if (typename[0] == 'u')
372 code->op = FETCH_OP_UDEREF;
373 else
374 code->op = FETCH_OP_DEREF;
375 code->offset = 0;
376 *pcode = code;
377 return 0;
378 }
379 /* Other types are not available for string */
380 trace_probe_log_err(ctx->offset, BAD_TYPE4STR);
381 return -EINVAL;
382 }
383
fetch_type_from_btf_type(struct btf * btf,const struct btf_type * type,struct traceprobe_parse_context * ctx)384 static const char *fetch_type_from_btf_type(struct btf *btf,
385 const struct btf_type *type,
386 struct traceprobe_parse_context *ctx)
387 {
388 u32 intdata;
389
390 /* TODO: const char * could be converted as a string */
391 switch (BTF_INFO_KIND(type->info)) {
392 case BTF_KIND_ENUM:
393 /* enum is "int", so convert to "s32" */
394 return "s32";
395 case BTF_KIND_ENUM64:
396 return "s64";
397 case BTF_KIND_PTR:
398 /* pointer will be converted to "x??" */
399 if (IS_ENABLED(CONFIG_64BIT))
400 return "x64";
401 else
402 return "x32";
403 case BTF_KIND_INT:
404 intdata = btf_type_int(type);
405 if (BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED) {
406 switch (BTF_INT_BITS(intdata)) {
407 case 8:
408 return "s8";
409 case 16:
410 return "s16";
411 case 32:
412 return "s32";
413 case 64:
414 return "s64";
415 }
416 } else { /* unsigned */
417 switch (BTF_INT_BITS(intdata)) {
418 case 8:
419 return "u8";
420 case 16:
421 return "u16";
422 case 32:
423 return "u32";
424 case 64:
425 return "u64";
426 }
427 /* bitfield, size is encoded in the type */
428 ctx->last_bitsize = BTF_INT_BITS(intdata);
429 ctx->last_bitoffs += BTF_INT_OFFSET(intdata);
430 return "u64";
431 }
432 }
433 /* TODO: support other types */
434
435 return NULL;
436 }
437
query_btf_context(struct traceprobe_parse_context * ctx)438 static int query_btf_context(struct traceprobe_parse_context *ctx)
439 {
440 const struct btf_param *param;
441 const struct btf_type *type;
442 struct btf *btf;
443 s32 nr;
444
445 if (ctx->btf)
446 return 0;
447
448 if (!ctx->funcname)
449 return -EINVAL;
450
451 type = btf_find_func_proto(ctx->funcname, &btf);
452 if (!type)
453 return -ENOENT;
454
455 ctx->btf = btf;
456 ctx->proto = type;
457
458 /* ctx->params is optional, since func(void) will not have params. */
459 nr = 0;
460 param = btf_get_func_param(type, &nr);
461 if (!IS_ERR_OR_NULL(param)) {
462 /* Hide the first 'data' argument of tracepoint */
463 if (ctx->flags & TPARG_FL_TPOINT) {
464 nr--;
465 param++;
466 }
467 }
468
469 if (nr > 0) {
470 ctx->nr_params = nr;
471 ctx->params = param;
472 } else {
473 ctx->nr_params = 0;
474 ctx->params = NULL;
475 }
476
477 return 0;
478 }
479
clear_btf_context(struct traceprobe_parse_context * ctx)480 static void clear_btf_context(struct traceprobe_parse_context *ctx)
481 {
482 if (ctx->btf) {
483 btf_put(ctx->btf);
484 ctx->btf = NULL;
485 ctx->proto = NULL;
486 ctx->params = NULL;
487 ctx->nr_params = 0;
488 }
489 }
490
491 /* Return 1 if the field separater is arrow operator ('->') */
split_next_field(char * varname,char ** next_field,struct traceprobe_parse_context * ctx)492 static int split_next_field(char *varname, char **next_field,
493 struct traceprobe_parse_context *ctx)
494 {
495 char *field;
496 int ret = 0;
497
498 field = strpbrk(varname, ".-");
499 if (field) {
500 if (field[0] == '-' && field[1] == '>') {
501 field[0] = '\0';
502 field += 2;
503 ret = 1;
504 } else if (field[0] == '.') {
505 field[0] = '\0';
506 field += 1;
507 } else {
508 trace_probe_log_err(ctx->offset + field - varname, BAD_HYPHEN);
509 return -EINVAL;
510 }
511 *next_field = field;
512 }
513
514 return ret;
515 }
516
517 /*
518 * Parse the field of data structure. The @type must be a pointer type
519 * pointing the target data structure type.
520 */
parse_btf_field(char * fieldname,const struct btf_type * type,struct fetch_insn ** pcode,struct fetch_insn * end,struct traceprobe_parse_context * ctx)521 static int parse_btf_field(char *fieldname, const struct btf_type *type,
522 struct fetch_insn **pcode, struct fetch_insn *end,
523 struct traceprobe_parse_context *ctx)
524 {
525 struct fetch_insn *code = *pcode;
526 const struct btf_member *field;
527 u32 bitoffs, anon_offs;
528 char *next;
529 int is_ptr;
530 s32 tid;
531
532 do {
533 /* Outer loop for solving arrow operator ('->') */
534 if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) {
535 trace_probe_log_err(ctx->offset, NO_PTR_STRCT);
536 return -EINVAL;
537 }
538 /* Convert a struct pointer type to a struct type */
539 type = btf_type_skip_modifiers(ctx->btf, type->type, &tid);
540 if (!type) {
541 trace_probe_log_err(ctx->offset, BAD_BTF_TID);
542 return -EINVAL;
543 }
544
545 bitoffs = 0;
546 do {
547 /* Inner loop for solving dot operator ('.') */
548 next = NULL;
549 is_ptr = split_next_field(fieldname, &next, ctx);
550 if (is_ptr < 0)
551 return is_ptr;
552
553 anon_offs = 0;
554 field = btf_find_struct_member(ctx->btf, type, fieldname,
555 &anon_offs);
556 if (IS_ERR(field)) {
557 trace_probe_log_err(ctx->offset, BAD_BTF_TID);
558 return PTR_ERR(field);
559 }
560 if (!field) {
561 trace_probe_log_err(ctx->offset, NO_BTF_FIELD);
562 return -ENOENT;
563 }
564 /* Add anonymous structure/union offset */
565 bitoffs += anon_offs;
566
567 /* Accumulate the bit-offsets of the dot-connected fields */
568 if (btf_type_kflag(type)) {
569 bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset);
570 ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset);
571 } else {
572 bitoffs += field->offset;
573 ctx->last_bitsize = 0;
574 }
575
576 type = btf_type_skip_modifiers(ctx->btf, field->type, &tid);
577 if (!type) {
578 trace_probe_log_err(ctx->offset, BAD_BTF_TID);
579 return -EINVAL;
580 }
581
582 ctx->offset += next - fieldname;
583 fieldname = next;
584 } while (!is_ptr && fieldname);
585
586 if (++code == end) {
587 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
588 return -EINVAL;
589 }
590 code->op = FETCH_OP_DEREF; /* TODO: user deref support */
591 code->offset = bitoffs / 8;
592 *pcode = code;
593
594 ctx->last_bitoffs = bitoffs % 8;
595 ctx->last_type = type;
596 } while (fieldname);
597
598 return 0;
599 }
600
parse_btf_arg(char * varname,struct fetch_insn ** pcode,struct fetch_insn * end,struct traceprobe_parse_context * ctx)601 static int parse_btf_arg(char *varname,
602 struct fetch_insn **pcode, struct fetch_insn *end,
603 struct traceprobe_parse_context *ctx)
604 {
605 struct fetch_insn *code = *pcode;
606 const struct btf_param *params;
607 const struct btf_type *type;
608 char *field = NULL;
609 int i, is_ptr, ret;
610 u32 tid;
611
612 if (WARN_ON_ONCE(!ctx->funcname))
613 return -EINVAL;
614
615 is_ptr = split_next_field(varname, &field, ctx);
616 if (is_ptr < 0)
617 return is_ptr;
618 if (!is_ptr && field) {
619 /* dot-connected field on an argument is not supported. */
620 trace_probe_log_err(ctx->offset + field - varname,
621 NOSUP_DAT_ARG);
622 return -EOPNOTSUPP;
623 }
624
625 if (ctx->flags & TPARG_FL_RETURN) {
626 if (strcmp(varname, "$retval") != 0) {
627 trace_probe_log_err(ctx->offset, NO_BTFARG);
628 return -ENOENT;
629 }
630 code->op = FETCH_OP_RETVAL;
631 /* Check whether the function return type is not void */
632 if (query_btf_context(ctx) == 0) {
633 if (ctx->proto->type == 0) {
634 trace_probe_log_err(ctx->offset, NO_RETVAL);
635 return -ENOENT;
636 }
637 tid = ctx->proto->type;
638 goto found;
639 }
640 if (field) {
641 trace_probe_log_err(ctx->offset + field - varname,
642 NO_BTF_ENTRY);
643 return -ENOENT;
644 }
645 return 0;
646 }
647
648 if (!ctx->btf) {
649 ret = query_btf_context(ctx);
650 if (ret < 0 || ctx->nr_params == 0) {
651 trace_probe_log_err(ctx->offset, NO_BTF_ENTRY);
652 return PTR_ERR(params);
653 }
654 }
655 params = ctx->params;
656
657 for (i = 0; i < ctx->nr_params; i++) {
658 const char *name = btf_name_by_offset(ctx->btf, params[i].name_off);
659
660 if (name && !strcmp(name, varname)) {
661 code->op = FETCH_OP_ARG;
662 if (ctx->flags & TPARG_FL_TPOINT)
663 code->param = i + 1;
664 else
665 code->param = i;
666 tid = params[i].type;
667 goto found;
668 }
669 }
670 trace_probe_log_err(ctx->offset, NO_BTFARG);
671 return -ENOENT;
672
673 found:
674 type = btf_type_skip_modifiers(ctx->btf, tid, &tid);
675 if (!type) {
676 trace_probe_log_err(ctx->offset, BAD_BTF_TID);
677 return -EINVAL;
678 }
679 /* Initialize the last type information */
680 ctx->last_type = type;
681 ctx->last_bitoffs = 0;
682 ctx->last_bitsize = 0;
683 if (field) {
684 ctx->offset += field - varname;
685 return parse_btf_field(field, type, pcode, end, ctx);
686 }
687 return 0;
688 }
689
find_fetch_type_from_btf_type(struct traceprobe_parse_context * ctx)690 static const struct fetch_type *find_fetch_type_from_btf_type(
691 struct traceprobe_parse_context *ctx)
692 {
693 struct btf *btf = ctx->btf;
694 const char *typestr = NULL;
695
696 if (btf && ctx->last_type)
697 typestr = fetch_type_from_btf_type(btf, ctx->last_type, ctx);
698
699 return find_fetch_type(typestr, ctx->flags);
700 }
701
parse_btf_bitfield(struct fetch_insn ** pcode,struct traceprobe_parse_context * ctx)702 static int parse_btf_bitfield(struct fetch_insn **pcode,
703 struct traceprobe_parse_context *ctx)
704 {
705 struct fetch_insn *code = *pcode;
706
707 if ((ctx->last_bitsize % 8 == 0) && ctx->last_bitoffs == 0)
708 return 0;
709
710 code++;
711 if (code->op != FETCH_OP_NOP) {
712 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
713 return -EINVAL;
714 }
715 *pcode = code;
716
717 code->op = FETCH_OP_MOD_BF;
718 code->lshift = 64 - (ctx->last_bitsize + ctx->last_bitoffs);
719 code->rshift = 64 - ctx->last_bitsize;
720 code->basesize = 64 / 8;
721 return 0;
722 }
723
724 #else
clear_btf_context(struct traceprobe_parse_context * ctx)725 static void clear_btf_context(struct traceprobe_parse_context *ctx)
726 {
727 ctx->btf = NULL;
728 }
729
query_btf_context(struct traceprobe_parse_context * ctx)730 static int query_btf_context(struct traceprobe_parse_context *ctx)
731 {
732 return -EOPNOTSUPP;
733 }
734
parse_btf_arg(char * varname,struct fetch_insn ** pcode,struct fetch_insn * end,struct traceprobe_parse_context * ctx)735 static int parse_btf_arg(char *varname,
736 struct fetch_insn **pcode, struct fetch_insn *end,
737 struct traceprobe_parse_context *ctx)
738 {
739 trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
740 return -EOPNOTSUPP;
741 }
742
parse_btf_bitfield(struct fetch_insn ** pcode,struct traceprobe_parse_context * ctx)743 static int parse_btf_bitfield(struct fetch_insn **pcode,
744 struct traceprobe_parse_context *ctx)
745 {
746 trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
747 return -EOPNOTSUPP;
748 }
749
750 #define find_fetch_type_from_btf_type(ctx) \
751 find_fetch_type(NULL, ctx->flags)
752
check_prepare_btf_string_fetch(char * typename,struct fetch_insn ** pcode,struct traceprobe_parse_context * ctx)753 static int check_prepare_btf_string_fetch(char *typename,
754 struct fetch_insn **pcode,
755 struct traceprobe_parse_context *ctx)
756 {
757 return 0;
758 }
759
760 #endif
761
762 #define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
763
764 /* Parse $vars. @orig_arg points '$', which syncs to @ctx->offset */
parse_probe_vars(char * orig_arg,const struct fetch_type * t,struct fetch_insn ** pcode,struct fetch_insn * end,struct traceprobe_parse_context * ctx)765 static int parse_probe_vars(char *orig_arg, const struct fetch_type *t,
766 struct fetch_insn **pcode,
767 struct fetch_insn *end,
768 struct traceprobe_parse_context *ctx)
769 {
770 struct fetch_insn *code = *pcode;
771 int err = TP_ERR_BAD_VAR;
772 char *arg = orig_arg + 1;
773 unsigned long param;
774 int ret = 0;
775 int len;
776
777 if (ctx->flags & TPARG_FL_TEVENT) {
778 if (code->data)
779 return -EFAULT;
780 ret = parse_trace_event_arg(arg, code, ctx);
781 if (!ret)
782 return 0;
783 if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
784 code->op = FETCH_OP_COMM;
785 return 0;
786 }
787 /* backward compatibility */
788 ctx->offset = 0;
789 goto inval;
790 }
791
792 if (str_has_prefix(arg, "retval")) {
793 if (!(ctx->flags & TPARG_FL_RETURN)) {
794 err = TP_ERR_RETVAL_ON_PROBE;
795 goto inval;
796 }
797 if (!(ctx->flags & TPARG_FL_KERNEL) ||
798 !IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) {
799 code->op = FETCH_OP_RETVAL;
800 return 0;
801 }
802 return parse_btf_arg(orig_arg, pcode, end, ctx);
803 }
804
805 len = str_has_prefix(arg, "stack");
806 if (len) {
807
808 if (arg[len] == '\0') {
809 code->op = FETCH_OP_STACKP;
810 return 0;
811 }
812
813 if (isdigit(arg[len])) {
814 ret = kstrtoul(arg + len, 10, ¶m);
815 if (ret)
816 goto inval;
817
818 if ((ctx->flags & TPARG_FL_KERNEL) &&
819 param > PARAM_MAX_STACK) {
820 err = TP_ERR_BAD_STACK_NUM;
821 goto inval;
822 }
823 code->op = FETCH_OP_STACK;
824 code->param = (unsigned int)param;
825 return 0;
826 }
827 goto inval;
828 }
829
830 if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
831 code->op = FETCH_OP_COMM;
832 return 0;
833 }
834
835 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
836 len = str_has_prefix(arg, "arg");
837 if (len && tparg_is_function_entry(ctx->flags)) {
838 ret = kstrtoul(arg + len, 10, ¶m);
839 if (ret)
840 goto inval;
841
842 if (!param || param > PARAM_MAX_STACK) {
843 err = TP_ERR_BAD_ARG_NUM;
844 goto inval;
845 }
846
847 code->op = FETCH_OP_ARG;
848 code->param = (unsigned int)param - 1;
849 /*
850 * The tracepoint probe will probe a stub function, and the
851 * first parameter of the stub is a dummy and should be ignored.
852 */
853 if (ctx->flags & TPARG_FL_TPOINT)
854 code->param++;
855 return 0;
856 }
857 #endif
858
859 inval:
860 __trace_probe_log_err(ctx->offset, err);
861 return -EINVAL;
862 }
863
str_to_immediate(char * str,unsigned long * imm)864 static int str_to_immediate(char *str, unsigned long *imm)
865 {
866 if (isdigit(str[0]))
867 return kstrtoul(str, 0, imm);
868 else if (str[0] == '-')
869 return kstrtol(str, 0, (long *)imm);
870 else if (str[0] == '+')
871 return kstrtol(str + 1, 0, (long *)imm);
872 return -EINVAL;
873 }
874
__parse_imm_string(char * str,char ** pbuf,int offs)875 static int __parse_imm_string(char *str, char **pbuf, int offs)
876 {
877 size_t len = strlen(str);
878
879 if (str[len - 1] != '"') {
880 trace_probe_log_err(offs + len, IMMSTR_NO_CLOSE);
881 return -EINVAL;
882 }
883 *pbuf = kstrndup(str, len - 1, GFP_KERNEL);
884 if (!*pbuf)
885 return -ENOMEM;
886 return 0;
887 }
888
889 /* Recursive argument parser */
890 static int
parse_probe_arg(char * arg,const struct fetch_type * type,struct fetch_insn ** pcode,struct fetch_insn * end,struct traceprobe_parse_context * ctx)891 parse_probe_arg(char *arg, const struct fetch_type *type,
892 struct fetch_insn **pcode, struct fetch_insn *end,
893 struct traceprobe_parse_context *ctx)
894 {
895 struct fetch_insn *code = *pcode;
896 unsigned long param;
897 int deref = FETCH_OP_DEREF;
898 long offset = 0;
899 char *tmp;
900 int ret = 0;
901
902 switch (arg[0]) {
903 case '$':
904 ret = parse_probe_vars(arg, type, pcode, end, ctx);
905 break;
906
907 case '%': /* named register */
908 if (ctx->flags & (TPARG_FL_TEVENT | TPARG_FL_FPROBE)) {
909 /* eprobe and fprobe do not handle registers */
910 trace_probe_log_err(ctx->offset, BAD_VAR);
911 break;
912 }
913 ret = regs_query_register_offset(arg + 1);
914 if (ret >= 0) {
915 code->op = FETCH_OP_REG;
916 code->param = (unsigned int)ret;
917 ret = 0;
918 } else
919 trace_probe_log_err(ctx->offset, BAD_REG_NAME);
920 break;
921
922 case '@': /* memory, file-offset or symbol */
923 if (isdigit(arg[1])) {
924 ret = kstrtoul(arg + 1, 0, ¶m);
925 if (ret) {
926 trace_probe_log_err(ctx->offset, BAD_MEM_ADDR);
927 break;
928 }
929 /* load address */
930 code->op = FETCH_OP_IMM;
931 code->immediate = param;
932 } else if (arg[1] == '+') {
933 /* kprobes don't support file offsets */
934 if (ctx->flags & TPARG_FL_KERNEL) {
935 trace_probe_log_err(ctx->offset, FILE_ON_KPROBE);
936 return -EINVAL;
937 }
938 ret = kstrtol(arg + 2, 0, &offset);
939 if (ret) {
940 trace_probe_log_err(ctx->offset, BAD_FILE_OFFS);
941 break;
942 }
943
944 code->op = FETCH_OP_FOFFS;
945 code->immediate = (unsigned long)offset; // imm64?
946 } else {
947 /* uprobes don't support symbols */
948 if (!(ctx->flags & TPARG_FL_KERNEL)) {
949 trace_probe_log_err(ctx->offset, SYM_ON_UPROBE);
950 return -EINVAL;
951 }
952 /* Preserve symbol for updating */
953 code->op = FETCH_NOP_SYMBOL;
954 code->data = kstrdup(arg + 1, GFP_KERNEL);
955 if (!code->data)
956 return -ENOMEM;
957 if (++code == end) {
958 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
959 return -EINVAL;
960 }
961 code->op = FETCH_OP_IMM;
962 code->immediate = 0;
963 }
964 /* These are fetching from memory */
965 if (++code == end) {
966 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
967 return -EINVAL;
968 }
969 *pcode = code;
970 code->op = FETCH_OP_DEREF;
971 code->offset = offset;
972 break;
973
974 case '+': /* deref memory */
975 case '-':
976 if (arg[1] == 'u') {
977 deref = FETCH_OP_UDEREF;
978 arg[1] = arg[0];
979 arg++;
980 }
981 if (arg[0] == '+')
982 arg++; /* Skip '+', because kstrtol() rejects it. */
983 tmp = strchr(arg, '(');
984 if (!tmp) {
985 trace_probe_log_err(ctx->offset, DEREF_NEED_BRACE);
986 return -EINVAL;
987 }
988 *tmp = '\0';
989 ret = kstrtol(arg, 0, &offset);
990 if (ret) {
991 trace_probe_log_err(ctx->offset, BAD_DEREF_OFFS);
992 break;
993 }
994 ctx->offset += (tmp + 1 - arg) + (arg[0] != '-' ? 1 : 0);
995 arg = tmp + 1;
996 tmp = strrchr(arg, ')');
997 if (!tmp) {
998 trace_probe_log_err(ctx->offset + strlen(arg),
999 DEREF_OPEN_BRACE);
1000 return -EINVAL;
1001 } else {
1002 const struct fetch_type *t2 = find_fetch_type(NULL, ctx->flags);
1003 int cur_offs = ctx->offset;
1004
1005 *tmp = '\0';
1006 ret = parse_probe_arg(arg, t2, &code, end, ctx);
1007 if (ret)
1008 break;
1009 ctx->offset = cur_offs;
1010 if (code->op == FETCH_OP_COMM ||
1011 code->op == FETCH_OP_DATA) {
1012 trace_probe_log_err(ctx->offset, COMM_CANT_DEREF);
1013 return -EINVAL;
1014 }
1015 if (++code == end) {
1016 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1017 return -EINVAL;
1018 }
1019 *pcode = code;
1020
1021 code->op = deref;
1022 code->offset = offset;
1023 /* Reset the last type if used */
1024 ctx->last_type = NULL;
1025 }
1026 break;
1027 case '\\': /* Immediate value */
1028 if (arg[1] == '"') { /* Immediate string */
1029 ret = __parse_imm_string(arg + 2, &tmp, ctx->offset + 2);
1030 if (ret)
1031 break;
1032 code->op = FETCH_OP_DATA;
1033 code->data = tmp;
1034 } else {
1035 ret = str_to_immediate(arg + 1, &code->immediate);
1036 if (ret)
1037 trace_probe_log_err(ctx->offset + 1, BAD_IMM);
1038 else
1039 code->op = FETCH_OP_IMM;
1040 }
1041 break;
1042 default:
1043 if (isalpha(arg[0]) || arg[0] == '_') { /* BTF variable */
1044 if (!tparg_is_function_entry(ctx->flags)) {
1045 trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
1046 return -EINVAL;
1047 }
1048 ret = parse_btf_arg(arg, pcode, end, ctx);
1049 break;
1050 }
1051 }
1052 if (!ret && code->op == FETCH_OP_NOP) {
1053 /* Parsed, but do not find fetch method */
1054 trace_probe_log_err(ctx->offset, BAD_FETCH_ARG);
1055 ret = -EINVAL;
1056 }
1057 return ret;
1058 }
1059
1060 #define BYTES_TO_BITS(nb) ((BITS_PER_LONG * (nb)) / sizeof(long))
1061
1062 /* Bitfield type needs to be parsed into a fetch function */
__parse_bitfield_probe_arg(const char * bf,const struct fetch_type * t,struct fetch_insn ** pcode)1063 static int __parse_bitfield_probe_arg(const char *bf,
1064 const struct fetch_type *t,
1065 struct fetch_insn **pcode)
1066 {
1067 struct fetch_insn *code = *pcode;
1068 unsigned long bw, bo;
1069 char *tail;
1070
1071 if (*bf != 'b')
1072 return 0;
1073
1074 bw = simple_strtoul(bf + 1, &tail, 0); /* Use simple one */
1075
1076 if (bw == 0 || *tail != '@')
1077 return -EINVAL;
1078
1079 bf = tail + 1;
1080 bo = simple_strtoul(bf, &tail, 0);
1081
1082 if (tail == bf || *tail != '/')
1083 return -EINVAL;
1084 code++;
1085 if (code->op != FETCH_OP_NOP)
1086 return -EINVAL;
1087 *pcode = code;
1088
1089 code->op = FETCH_OP_MOD_BF;
1090 code->lshift = BYTES_TO_BITS(t->size) - (bw + bo);
1091 code->rshift = BYTES_TO_BITS(t->size) - bw;
1092 code->basesize = t->size;
1093
1094 return (BYTES_TO_BITS(t->size) < (bw + bo)) ? -EINVAL : 0;
1095 }
1096
1097 /* String length checking wrapper */
traceprobe_parse_probe_arg_body(const char * argv,ssize_t * size,struct probe_arg * parg,struct traceprobe_parse_context * ctx)1098 static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size,
1099 struct probe_arg *parg,
1100 struct traceprobe_parse_context *ctx)
1101 {
1102 struct fetch_insn *code, *scode, *tmp = NULL;
1103 char *t, *t2, *t3;
1104 int ret, len;
1105 char *arg;
1106
1107 arg = kstrdup(argv, GFP_KERNEL);
1108 if (!arg)
1109 return -ENOMEM;
1110
1111 ret = -EINVAL;
1112 len = strlen(arg);
1113 if (len > MAX_ARGSTR_LEN) {
1114 trace_probe_log_err(ctx->offset, ARG_TOO_LONG);
1115 goto out;
1116 } else if (len == 0) {
1117 trace_probe_log_err(ctx->offset, NO_ARG_BODY);
1118 goto out;
1119 }
1120
1121 ret = -ENOMEM;
1122 parg->comm = kstrdup(arg, GFP_KERNEL);
1123 if (!parg->comm)
1124 goto out;
1125
1126 ret = -EINVAL;
1127 t = strchr(arg, ':');
1128 if (t) {
1129 *t = '\0';
1130 t2 = strchr(++t, '[');
1131 if (t2) {
1132 *t2++ = '\0';
1133 t3 = strchr(t2, ']');
1134 if (!t3) {
1135 int offs = t2 + strlen(t2) - arg;
1136
1137 trace_probe_log_err(ctx->offset + offs,
1138 ARRAY_NO_CLOSE);
1139 goto out;
1140 } else if (t3[1] != '\0') {
1141 trace_probe_log_err(ctx->offset + t3 + 1 - arg,
1142 BAD_ARRAY_SUFFIX);
1143 goto out;
1144 }
1145 *t3 = '\0';
1146 if (kstrtouint(t2, 0, &parg->count) || !parg->count) {
1147 trace_probe_log_err(ctx->offset + t2 - arg,
1148 BAD_ARRAY_NUM);
1149 goto out;
1150 }
1151 if (parg->count > MAX_ARRAY_LEN) {
1152 trace_probe_log_err(ctx->offset + t2 - arg,
1153 ARRAY_TOO_BIG);
1154 goto out;
1155 }
1156 }
1157 }
1158
1159 /*
1160 * Since $comm and immediate string can not be dereferenced,
1161 * we can find those by strcmp. But ignore for eprobes.
1162 */
1163 if (!(ctx->flags & TPARG_FL_TEVENT) &&
1164 (strcmp(arg, "$comm") == 0 || strcmp(arg, "$COMM") == 0 ||
1165 strncmp(arg, "\\\"", 2) == 0)) {
1166 /* The type of $comm must be "string", and not an array type. */
1167 if (parg->count || (t && strcmp(t, "string"))) {
1168 trace_probe_log_err(ctx->offset + (t ? (t - arg) : 0),
1169 NEED_STRING_TYPE);
1170 goto out;
1171 }
1172 parg->type = find_fetch_type("string", ctx->flags);
1173 } else
1174 parg->type = find_fetch_type(t, ctx->flags);
1175 if (!parg->type) {
1176 trace_probe_log_err(ctx->offset + (t ? (t - arg) : 0), BAD_TYPE);
1177 goto out;
1178 }
1179
1180 code = tmp = kcalloc(FETCH_INSN_MAX, sizeof(*code), GFP_KERNEL);
1181 if (!code)
1182 goto out;
1183 code[FETCH_INSN_MAX - 1].op = FETCH_OP_END;
1184
1185 ctx->last_type = NULL;
1186 ret = parse_probe_arg(arg, parg->type, &code, &code[FETCH_INSN_MAX - 1],
1187 ctx);
1188 if (ret)
1189 goto fail;
1190
1191 /* Update storing type if BTF is available */
1192 if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) &&
1193 ctx->last_type) {
1194 if (!t) {
1195 parg->type = find_fetch_type_from_btf_type(ctx);
1196 } else if (strstr(t, "string")) {
1197 ret = check_prepare_btf_string_fetch(t, &code, ctx);
1198 if (ret)
1199 goto fail;
1200 }
1201 }
1202 parg->offset = *size;
1203 *size += parg->type->size * (parg->count ?: 1);
1204
1205 if (parg->count) {
1206 len = strlen(parg->type->fmttype) + 6;
1207 parg->fmt = kmalloc(len, GFP_KERNEL);
1208 if (!parg->fmt) {
1209 ret = -ENOMEM;
1210 goto out;
1211 }
1212 snprintf(parg->fmt, len, "%s[%d]", parg->type->fmttype,
1213 parg->count);
1214 }
1215
1216 ret = -EINVAL;
1217 /* Store operation */
1218 if (parg->type->is_string) {
1219 if (!strcmp(parg->type->name, "symstr")) {
1220 if (code->op != FETCH_OP_REG && code->op != FETCH_OP_STACK &&
1221 code->op != FETCH_OP_RETVAL && code->op != FETCH_OP_ARG &&
1222 code->op != FETCH_OP_DEREF && code->op != FETCH_OP_TP_ARG) {
1223 trace_probe_log_err(ctx->offset + (t ? (t - arg) : 0),
1224 BAD_SYMSTRING);
1225 goto fail;
1226 }
1227 } else {
1228 if (code->op != FETCH_OP_DEREF && code->op != FETCH_OP_UDEREF &&
1229 code->op != FETCH_OP_IMM && code->op != FETCH_OP_COMM &&
1230 code->op != FETCH_OP_DATA && code->op != FETCH_OP_TP_ARG) {
1231 trace_probe_log_err(ctx->offset + (t ? (t - arg) : 0),
1232 BAD_STRING);
1233 goto fail;
1234 }
1235 }
1236 if (!strcmp(parg->type->name, "symstr") ||
1237 (code->op == FETCH_OP_IMM || code->op == FETCH_OP_COMM ||
1238 code->op == FETCH_OP_DATA) || code->op == FETCH_OP_TP_ARG ||
1239 parg->count) {
1240 /*
1241 * IMM, DATA and COMM is pointing actual address, those
1242 * must be kept, and if parg->count != 0, this is an
1243 * array of string pointers instead of string address
1244 * itself.
1245 * For the symstr, it doesn't need to dereference, thus
1246 * it just get the value.
1247 */
1248 code++;
1249 if (code->op != FETCH_OP_NOP) {
1250 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1251 goto fail;
1252 }
1253 }
1254 /* If op == DEREF, replace it with STRING */
1255 if (!strcmp(parg->type->name, "ustring") ||
1256 code->op == FETCH_OP_UDEREF)
1257 code->op = FETCH_OP_ST_USTRING;
1258 else if (!strcmp(parg->type->name, "symstr"))
1259 code->op = FETCH_OP_ST_SYMSTR;
1260 else
1261 code->op = FETCH_OP_ST_STRING;
1262 code->size = parg->type->size;
1263 parg->dynamic = true;
1264 } else if (code->op == FETCH_OP_DEREF) {
1265 code->op = FETCH_OP_ST_MEM;
1266 code->size = parg->type->size;
1267 } else if (code->op == FETCH_OP_UDEREF) {
1268 code->op = FETCH_OP_ST_UMEM;
1269 code->size = parg->type->size;
1270 } else {
1271 code++;
1272 if (code->op != FETCH_OP_NOP) {
1273 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1274 goto fail;
1275 }
1276 code->op = FETCH_OP_ST_RAW;
1277 code->size = parg->type->size;
1278 }
1279 scode = code;
1280 /* Modify operation */
1281 if (t != NULL) {
1282 ret = __parse_bitfield_probe_arg(t, parg->type, &code);
1283 if (ret) {
1284 trace_probe_log_err(ctx->offset + t - arg, BAD_BITFIELD);
1285 goto fail;
1286 }
1287 } else if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) &&
1288 ctx->last_type) {
1289 ret = parse_btf_bitfield(&code, ctx);
1290 if (ret)
1291 goto fail;
1292 }
1293 ret = -EINVAL;
1294 /* Loop(Array) operation */
1295 if (parg->count) {
1296 if (scode->op != FETCH_OP_ST_MEM &&
1297 scode->op != FETCH_OP_ST_STRING &&
1298 scode->op != FETCH_OP_ST_USTRING) {
1299 trace_probe_log_err(ctx->offset + (t ? (t - arg) : 0),
1300 BAD_STRING);
1301 goto fail;
1302 }
1303 code++;
1304 if (code->op != FETCH_OP_NOP) {
1305 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1306 goto fail;
1307 }
1308 code->op = FETCH_OP_LP_ARRAY;
1309 code->param = parg->count;
1310 }
1311 code++;
1312 code->op = FETCH_OP_END;
1313
1314 ret = 0;
1315 /* Shrink down the code buffer */
1316 parg->code = kcalloc(code - tmp + 1, sizeof(*code), GFP_KERNEL);
1317 if (!parg->code)
1318 ret = -ENOMEM;
1319 else
1320 memcpy(parg->code, tmp, sizeof(*code) * (code - tmp + 1));
1321
1322 fail:
1323 if (ret) {
1324 for (code = tmp; code < tmp + FETCH_INSN_MAX; code++)
1325 if (code->op == FETCH_NOP_SYMBOL ||
1326 code->op == FETCH_OP_DATA)
1327 kfree(code->data);
1328 }
1329 kfree(tmp);
1330 out:
1331 kfree(arg);
1332
1333 return ret;
1334 }
1335
1336 /* Return 1 if name is reserved or already used by another argument */
traceprobe_conflict_field_name(const char * name,struct probe_arg * args,int narg)1337 static int traceprobe_conflict_field_name(const char *name,
1338 struct probe_arg *args, int narg)
1339 {
1340 int i;
1341
1342 for (i = 0; i < ARRAY_SIZE(reserved_field_names); i++)
1343 if (strcmp(reserved_field_names[i], name) == 0)
1344 return 1;
1345
1346 for (i = 0; i < narg; i++)
1347 if (strcmp(args[i].name, name) == 0)
1348 return 1;
1349
1350 return 0;
1351 }
1352
generate_probe_arg_name(const char * arg,int idx)1353 static char *generate_probe_arg_name(const char *arg, int idx)
1354 {
1355 char *name = NULL;
1356 const char *end;
1357
1358 /*
1359 * If argument name is omitted, try arg as a name (BTF variable)
1360 * or "argN".
1361 */
1362 if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) {
1363 end = strchr(arg, ':');
1364 if (!end)
1365 end = arg + strlen(arg);
1366
1367 name = kmemdup_nul(arg, end - arg, GFP_KERNEL);
1368 if (!name || !is_good_name(name)) {
1369 kfree(name);
1370 name = NULL;
1371 }
1372 }
1373
1374 if (!name)
1375 name = kasprintf(GFP_KERNEL, "arg%d", idx + 1);
1376
1377 return name;
1378 }
1379
traceprobe_parse_probe_arg(struct trace_probe * tp,int i,const char * arg,struct traceprobe_parse_context * ctx)1380 int traceprobe_parse_probe_arg(struct trace_probe *tp, int i, const char *arg,
1381 struct traceprobe_parse_context *ctx)
1382 {
1383 struct probe_arg *parg = &tp->args[i];
1384 const char *body;
1385
1386 /* Increment count for freeing args in error case */
1387 tp->nr_args++;
1388
1389 body = strchr(arg, '=');
1390 if (body) {
1391 if (body - arg > MAX_ARG_NAME_LEN) {
1392 trace_probe_log_err(0, ARG_NAME_TOO_LONG);
1393 return -EINVAL;
1394 } else if (body == arg) {
1395 trace_probe_log_err(0, NO_ARG_NAME);
1396 return -EINVAL;
1397 }
1398 parg->name = kmemdup_nul(arg, body - arg, GFP_KERNEL);
1399 body++;
1400 } else {
1401 parg->name = generate_probe_arg_name(arg, i);
1402 body = arg;
1403 }
1404 if (!parg->name)
1405 return -ENOMEM;
1406
1407 if (!is_good_name(parg->name)) {
1408 trace_probe_log_err(0, BAD_ARG_NAME);
1409 return -EINVAL;
1410 }
1411 if (traceprobe_conflict_field_name(parg->name, tp->args, i)) {
1412 trace_probe_log_err(0, USED_ARG_NAME);
1413 return -EINVAL;
1414 }
1415 ctx->offset = body - arg;
1416 /* Parse fetch argument */
1417 return traceprobe_parse_probe_arg_body(body, &tp->size, parg, ctx);
1418 }
1419
traceprobe_free_probe_arg(struct probe_arg * arg)1420 void traceprobe_free_probe_arg(struct probe_arg *arg)
1421 {
1422 struct fetch_insn *code = arg->code;
1423
1424 while (code && code->op != FETCH_OP_END) {
1425 if (code->op == FETCH_NOP_SYMBOL ||
1426 code->op == FETCH_OP_DATA)
1427 kfree(code->data);
1428 code++;
1429 }
1430 kfree(arg->code);
1431 kfree(arg->name);
1432 kfree(arg->comm);
1433 kfree(arg->fmt);
1434 }
1435
argv_has_var_arg(int argc,const char * argv[],int * args_idx,struct traceprobe_parse_context * ctx)1436 static int argv_has_var_arg(int argc, const char *argv[], int *args_idx,
1437 struct traceprobe_parse_context *ctx)
1438 {
1439 int i, found = 0;
1440
1441 for (i = 0; i < argc; i++)
1442 if (str_has_prefix(argv[i], "$arg")) {
1443 trace_probe_log_set_index(i + 2);
1444
1445 if (!tparg_is_function_entry(ctx->flags)) {
1446 trace_probe_log_err(0, NOFENTRY_ARGS);
1447 return -EINVAL;
1448 }
1449
1450 if (isdigit(argv[i][4])) {
1451 found = 1;
1452 continue;
1453 }
1454
1455 if (argv[i][4] != '*') {
1456 trace_probe_log_err(0, BAD_VAR);
1457 return -EINVAL;
1458 }
1459
1460 if (*args_idx >= 0 && *args_idx < argc) {
1461 trace_probe_log_err(0, DOUBLE_ARGS);
1462 return -EINVAL;
1463 }
1464 found = 1;
1465 *args_idx = i;
1466 }
1467
1468 return found;
1469 }
1470
sprint_nth_btf_arg(int idx,const char * type,char * buf,int bufsize,struct traceprobe_parse_context * ctx)1471 static int sprint_nth_btf_arg(int idx, const char *type,
1472 char *buf, int bufsize,
1473 struct traceprobe_parse_context *ctx)
1474 {
1475 const char *name;
1476 int ret;
1477
1478 if (idx >= ctx->nr_params) {
1479 trace_probe_log_err(0, NO_BTFARG);
1480 return -ENOENT;
1481 }
1482 name = btf_name_by_offset(ctx->btf, ctx->params[idx].name_off);
1483 if (!name) {
1484 trace_probe_log_err(0, NO_BTF_ENTRY);
1485 return -ENOENT;
1486 }
1487 ret = snprintf(buf, bufsize, "%s%s", name, type);
1488 if (ret >= bufsize) {
1489 trace_probe_log_err(0, ARGS_2LONG);
1490 return -E2BIG;
1491 }
1492 return ret;
1493 }
1494
1495 /* Return new_argv which must be freed after use */
traceprobe_expand_meta_args(int argc,const char * argv[],int * new_argc,char * buf,int bufsize,struct traceprobe_parse_context * ctx)1496 const char **traceprobe_expand_meta_args(int argc, const char *argv[],
1497 int *new_argc, char *buf, int bufsize,
1498 struct traceprobe_parse_context *ctx)
1499 {
1500 const struct btf_param *params = NULL;
1501 int i, j, n, used, ret, args_idx = -1;
1502 const char **new_argv = NULL;
1503
1504 ret = argv_has_var_arg(argc, argv, &args_idx, ctx);
1505 if (ret < 0)
1506 return ERR_PTR(ret);
1507
1508 if (!ret) {
1509 *new_argc = argc;
1510 return NULL;
1511 }
1512
1513 ret = query_btf_context(ctx);
1514 if (ret < 0 || ctx->nr_params == 0) {
1515 if (args_idx != -1) {
1516 /* $arg* requires BTF info */
1517 trace_probe_log_err(0, NOSUP_BTFARG);
1518 return (const char **)params;
1519 }
1520 *new_argc = argc;
1521 return NULL;
1522 }
1523
1524 if (args_idx >= 0)
1525 *new_argc = argc + ctx->nr_params - 1;
1526 else
1527 *new_argc = argc;
1528
1529 new_argv = kcalloc(*new_argc, sizeof(char *), GFP_KERNEL);
1530 if (!new_argv)
1531 return ERR_PTR(-ENOMEM);
1532
1533 used = 0;
1534 for (i = 0, j = 0; i < argc; i++) {
1535 trace_probe_log_set_index(i + 2);
1536 if (i == args_idx) {
1537 for (n = 0; n < ctx->nr_params; n++) {
1538 ret = sprint_nth_btf_arg(n, "", buf + used,
1539 bufsize - used, ctx);
1540 if (ret < 0)
1541 goto error;
1542
1543 new_argv[j++] = buf + used;
1544 used += ret + 1;
1545 }
1546 continue;
1547 }
1548
1549 if (str_has_prefix(argv[i], "$arg")) {
1550 char *type = NULL;
1551
1552 n = simple_strtoul(argv[i] + 4, &type, 10);
1553 if (type && !(*type == ':' || *type == '\0')) {
1554 trace_probe_log_err(0, BAD_VAR);
1555 ret = -ENOENT;
1556 goto error;
1557 }
1558 /* Note: $argN starts from $arg1 */
1559 ret = sprint_nth_btf_arg(n - 1, type, buf + used,
1560 bufsize - used, ctx);
1561 if (ret < 0)
1562 goto error;
1563 new_argv[j++] = buf + used;
1564 used += ret + 1;
1565 } else
1566 new_argv[j++] = argv[i];
1567 }
1568
1569 return new_argv;
1570
1571 error:
1572 kfree(new_argv);
1573 return ERR_PTR(ret);
1574 }
1575
traceprobe_finish_parse(struct traceprobe_parse_context * ctx)1576 void traceprobe_finish_parse(struct traceprobe_parse_context *ctx)
1577 {
1578 clear_btf_context(ctx);
1579 }
1580
traceprobe_update_arg(struct probe_arg * arg)1581 int traceprobe_update_arg(struct probe_arg *arg)
1582 {
1583 struct fetch_insn *code = arg->code;
1584 long offset;
1585 char *tmp;
1586 char c;
1587 int ret = 0;
1588
1589 while (code && code->op != FETCH_OP_END) {
1590 if (code->op == FETCH_NOP_SYMBOL) {
1591 if (code[1].op != FETCH_OP_IMM)
1592 return -EINVAL;
1593
1594 tmp = strpbrk(code->data, "+-");
1595 if (tmp)
1596 c = *tmp;
1597 ret = traceprobe_split_symbol_offset(code->data,
1598 &offset);
1599 if (ret)
1600 return ret;
1601
1602 code[1].immediate =
1603 (unsigned long)kallsyms_lookup_name(code->data);
1604 if (tmp)
1605 *tmp = c;
1606 if (!code[1].immediate)
1607 return -ENOENT;
1608 code[1].immediate += offset;
1609 }
1610 code++;
1611 }
1612 return 0;
1613 }
1614
1615 /* When len=0, we just calculate the needed length */
1616 #define LEN_OR_ZERO (len ? len - pos : 0)
__set_print_fmt(struct trace_probe * tp,char * buf,int len,enum probe_print_type ptype)1617 static int __set_print_fmt(struct trace_probe *tp, char *buf, int len,
1618 enum probe_print_type ptype)
1619 {
1620 struct probe_arg *parg;
1621 int i, j;
1622 int pos = 0;
1623 const char *fmt, *arg;
1624
1625 switch (ptype) {
1626 case PROBE_PRINT_NORMAL:
1627 fmt = "(%lx)";
1628 arg = ", REC->" FIELD_STRING_IP;
1629 break;
1630 case PROBE_PRINT_RETURN:
1631 fmt = "(%lx <- %lx)";
1632 arg = ", REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP;
1633 break;
1634 case PROBE_PRINT_EVENT:
1635 fmt = "";
1636 arg = "";
1637 break;
1638 default:
1639 WARN_ON_ONCE(1);
1640 return 0;
1641 }
1642
1643 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt);
1644
1645 for (i = 0; i < tp->nr_args; i++) {
1646 parg = tp->args + i;
1647 pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=", parg->name);
1648 if (parg->count) {
1649 pos += snprintf(buf + pos, LEN_OR_ZERO, "{%s",
1650 parg->type->fmt);
1651 for (j = 1; j < parg->count; j++)
1652 pos += snprintf(buf + pos, LEN_OR_ZERO, ",%s",
1653 parg->type->fmt);
1654 pos += snprintf(buf + pos, LEN_OR_ZERO, "}");
1655 } else
1656 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s",
1657 parg->type->fmt);
1658 }
1659
1660 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", arg);
1661
1662 for (i = 0; i < tp->nr_args; i++) {
1663 parg = tp->args + i;
1664 if (parg->count) {
1665 if (parg->type->is_string)
1666 fmt = ", __get_str(%s[%d])";
1667 else
1668 fmt = ", REC->%s[%d]";
1669 for (j = 0; j < parg->count; j++)
1670 pos += snprintf(buf + pos, LEN_OR_ZERO,
1671 fmt, parg->name, j);
1672 } else {
1673 if (parg->type->is_string)
1674 fmt = ", __get_str(%s)";
1675 else
1676 fmt = ", REC->%s";
1677 pos += snprintf(buf + pos, LEN_OR_ZERO,
1678 fmt, parg->name);
1679 }
1680 }
1681
1682 /* return the length of print_fmt */
1683 return pos;
1684 }
1685 #undef LEN_OR_ZERO
1686
traceprobe_set_print_fmt(struct trace_probe * tp,enum probe_print_type ptype)1687 int traceprobe_set_print_fmt(struct trace_probe *tp, enum probe_print_type ptype)
1688 {
1689 struct trace_event_call *call = trace_probe_event_call(tp);
1690 int len;
1691 char *print_fmt;
1692
1693 /* First: called with 0 length to calculate the needed length */
1694 len = __set_print_fmt(tp, NULL, 0, ptype);
1695 print_fmt = kmalloc(len + 1, GFP_KERNEL);
1696 if (!print_fmt)
1697 return -ENOMEM;
1698
1699 /* Second: actually write the @print_fmt */
1700 __set_print_fmt(tp, print_fmt, len + 1, ptype);
1701 call->print_fmt = print_fmt;
1702
1703 return 0;
1704 }
1705
traceprobe_define_arg_fields(struct trace_event_call * event_call,size_t offset,struct trace_probe * tp)1706 int traceprobe_define_arg_fields(struct trace_event_call *event_call,
1707 size_t offset, struct trace_probe *tp)
1708 {
1709 int ret, i;
1710
1711 /* Set argument names as fields */
1712 for (i = 0; i < tp->nr_args; i++) {
1713 struct probe_arg *parg = &tp->args[i];
1714 const char *fmt = parg->type->fmttype;
1715 int size = parg->type->size;
1716
1717 if (parg->fmt)
1718 fmt = parg->fmt;
1719 if (parg->count)
1720 size *= parg->count;
1721 ret = trace_define_field(event_call, fmt, parg->name,
1722 offset + parg->offset, size,
1723 parg->type->is_signed,
1724 FILTER_OTHER);
1725 if (ret)
1726 return ret;
1727 }
1728 return 0;
1729 }
1730
trace_probe_event_free(struct trace_probe_event * tpe)1731 static void trace_probe_event_free(struct trace_probe_event *tpe)
1732 {
1733 kfree(tpe->class.system);
1734 kfree(tpe->call.name);
1735 kfree(tpe->call.print_fmt);
1736 kfree(tpe);
1737 }
1738
trace_probe_append(struct trace_probe * tp,struct trace_probe * to)1739 int trace_probe_append(struct trace_probe *tp, struct trace_probe *to)
1740 {
1741 if (trace_probe_has_sibling(tp))
1742 return -EBUSY;
1743
1744 list_del_init(&tp->list);
1745 trace_probe_event_free(tp->event);
1746
1747 tp->event = to->event;
1748 list_add_tail(&tp->list, trace_probe_probe_list(to));
1749
1750 return 0;
1751 }
1752
trace_probe_unlink(struct trace_probe * tp)1753 void trace_probe_unlink(struct trace_probe *tp)
1754 {
1755 list_del_init(&tp->list);
1756 if (list_empty(trace_probe_probe_list(tp)))
1757 trace_probe_event_free(tp->event);
1758 tp->event = NULL;
1759 }
1760
trace_probe_cleanup(struct trace_probe * tp)1761 void trace_probe_cleanup(struct trace_probe *tp)
1762 {
1763 int i;
1764
1765 for (i = 0; i < tp->nr_args; i++)
1766 traceprobe_free_probe_arg(&tp->args[i]);
1767
1768 if (tp->event)
1769 trace_probe_unlink(tp);
1770 }
1771
trace_probe_init(struct trace_probe * tp,const char * event,const char * group,bool alloc_filter)1772 int trace_probe_init(struct trace_probe *tp, const char *event,
1773 const char *group, bool alloc_filter)
1774 {
1775 struct trace_event_call *call;
1776 size_t size = sizeof(struct trace_probe_event);
1777 int ret = 0;
1778
1779 if (!event || !group)
1780 return -EINVAL;
1781
1782 if (alloc_filter)
1783 size += sizeof(struct trace_uprobe_filter);
1784
1785 tp->event = kzalloc(size, GFP_KERNEL);
1786 if (!tp->event)
1787 return -ENOMEM;
1788
1789 INIT_LIST_HEAD(&tp->event->files);
1790 INIT_LIST_HEAD(&tp->event->class.fields);
1791 INIT_LIST_HEAD(&tp->event->probes);
1792 INIT_LIST_HEAD(&tp->list);
1793 list_add(&tp->list, &tp->event->probes);
1794
1795 call = trace_probe_event_call(tp);
1796 call->class = &tp->event->class;
1797 call->name = kstrdup(event, GFP_KERNEL);
1798 if (!call->name) {
1799 ret = -ENOMEM;
1800 goto error;
1801 }
1802
1803 tp->event->class.system = kstrdup(group, GFP_KERNEL);
1804 if (!tp->event->class.system) {
1805 ret = -ENOMEM;
1806 goto error;
1807 }
1808
1809 return 0;
1810
1811 error:
1812 trace_probe_cleanup(tp);
1813 return ret;
1814 }
1815
1816 static struct trace_event_call *
find_trace_event_call(const char * system,const char * event_name)1817 find_trace_event_call(const char *system, const char *event_name)
1818 {
1819 struct trace_event_call *tp_event;
1820 const char *name;
1821
1822 list_for_each_entry(tp_event, &ftrace_events, list) {
1823 if (!tp_event->class->system ||
1824 strcmp(system, tp_event->class->system))
1825 continue;
1826 name = trace_event_name(tp_event);
1827 if (!name || strcmp(event_name, name))
1828 continue;
1829 return tp_event;
1830 }
1831
1832 return NULL;
1833 }
1834
trace_probe_register_event_call(struct trace_probe * tp)1835 int trace_probe_register_event_call(struct trace_probe *tp)
1836 {
1837 struct trace_event_call *call = trace_probe_event_call(tp);
1838 int ret;
1839
1840 lockdep_assert_held(&event_mutex);
1841
1842 if (find_trace_event_call(trace_probe_group_name(tp),
1843 trace_probe_name(tp)))
1844 return -EEXIST;
1845
1846 ret = register_trace_event(&call->event);
1847 if (!ret)
1848 return -ENODEV;
1849
1850 ret = trace_add_event_call(call);
1851 if (ret)
1852 unregister_trace_event(&call->event);
1853
1854 return ret;
1855 }
1856
trace_probe_add_file(struct trace_probe * tp,struct trace_event_file * file)1857 int trace_probe_add_file(struct trace_probe *tp, struct trace_event_file *file)
1858 {
1859 struct event_file_link *link;
1860
1861 link = kmalloc(sizeof(*link), GFP_KERNEL);
1862 if (!link)
1863 return -ENOMEM;
1864
1865 link->file = file;
1866 INIT_LIST_HEAD(&link->list);
1867 list_add_tail_rcu(&link->list, &tp->event->files);
1868 trace_probe_set_flag(tp, TP_FLAG_TRACE);
1869 return 0;
1870 }
1871
trace_probe_get_file_link(struct trace_probe * tp,struct trace_event_file * file)1872 struct event_file_link *trace_probe_get_file_link(struct trace_probe *tp,
1873 struct trace_event_file *file)
1874 {
1875 struct event_file_link *link;
1876
1877 trace_probe_for_each_link(link, tp) {
1878 if (link->file == file)
1879 return link;
1880 }
1881
1882 return NULL;
1883 }
1884
trace_probe_remove_file(struct trace_probe * tp,struct trace_event_file * file)1885 int trace_probe_remove_file(struct trace_probe *tp,
1886 struct trace_event_file *file)
1887 {
1888 struct event_file_link *link;
1889
1890 link = trace_probe_get_file_link(tp, file);
1891 if (!link)
1892 return -ENOENT;
1893
1894 list_del_rcu(&link->list);
1895 kvfree_rcu_mightsleep(link);
1896
1897 if (list_empty(&tp->event->files))
1898 trace_probe_clear_flag(tp, TP_FLAG_TRACE);
1899
1900 return 0;
1901 }
1902
1903 /*
1904 * Return the smallest index of different type argument (start from 1).
1905 * If all argument types and name are same, return 0.
1906 */
trace_probe_compare_arg_type(struct trace_probe * a,struct trace_probe * b)1907 int trace_probe_compare_arg_type(struct trace_probe *a, struct trace_probe *b)
1908 {
1909 int i;
1910
1911 /* In case of more arguments */
1912 if (a->nr_args < b->nr_args)
1913 return a->nr_args + 1;
1914 if (a->nr_args > b->nr_args)
1915 return b->nr_args + 1;
1916
1917 for (i = 0; i < a->nr_args; i++) {
1918 if ((b->nr_args <= i) ||
1919 ((a->args[i].type != b->args[i].type) ||
1920 (a->args[i].count != b->args[i].count) ||
1921 strcmp(a->args[i].name, b->args[i].name)))
1922 return i + 1;
1923 }
1924
1925 return 0;
1926 }
1927
trace_probe_match_command_args(struct trace_probe * tp,int argc,const char ** argv)1928 bool trace_probe_match_command_args(struct trace_probe *tp,
1929 int argc, const char **argv)
1930 {
1931 char buf[MAX_ARGSTR_LEN + 1];
1932 int i;
1933
1934 if (tp->nr_args < argc)
1935 return false;
1936
1937 for (i = 0; i < argc; i++) {
1938 snprintf(buf, sizeof(buf), "%s=%s",
1939 tp->args[i].name, tp->args[i].comm);
1940 if (strcmp(buf, argv[i]))
1941 return false;
1942 }
1943 return true;
1944 }
1945
trace_probe_create(const char * raw_command,int (* createfn)(int,const char **))1946 int trace_probe_create(const char *raw_command, int (*createfn)(int, const char **))
1947 {
1948 int argc = 0, ret = 0;
1949 char **argv;
1950
1951 argv = argv_split(GFP_KERNEL, raw_command, &argc);
1952 if (!argv)
1953 return -ENOMEM;
1954
1955 if (argc)
1956 ret = createfn(argc, (const char **)argv);
1957
1958 argv_free(argv);
1959
1960 return ret;
1961 }
1962
trace_probe_print_args(struct trace_seq * s,struct probe_arg * args,int nr_args,u8 * data,void * field)1963 int trace_probe_print_args(struct trace_seq *s, struct probe_arg *args, int nr_args,
1964 u8 *data, void *field)
1965 {
1966 void *p;
1967 int i, j;
1968
1969 for (i = 0; i < nr_args; i++) {
1970 struct probe_arg *a = args + i;
1971
1972 trace_seq_printf(s, " %s=", a->name);
1973 if (likely(!a->count)) {
1974 if (!a->type->print(s, data + a->offset, field))
1975 return -ENOMEM;
1976 continue;
1977 }
1978 trace_seq_putc(s, '{');
1979 p = data + a->offset;
1980 for (j = 0; j < a->count; j++) {
1981 if (!a->type->print(s, p, field))
1982 return -ENOMEM;
1983 trace_seq_putc(s, j == a->count - 1 ? '}' : ',');
1984 p += a->type->size;
1985 }
1986 }
1987 return 0;
1988 }
1989