1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * trace_events_hist - trace event hist triggers 4 * 5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com> 6 */ 7 8 #include <linux/module.h> 9 #include <linux/kallsyms.h> 10 #include <linux/mutex.h> 11 #include <linux/slab.h> 12 #include <linux/stacktrace.h> 13 #include <linux/rculist.h> 14 #include <linux/tracefs.h> 15 16 #include "tracing_map.h" 17 #include "trace.h" 18 19 #define SYNTH_SYSTEM "synthetic" 20 #define SYNTH_FIELDS_MAX 16 21 22 #define STR_VAR_LEN_MAX 32 /* must be multiple of sizeof(u64) */ 23 24 struct hist_field; 25 26 typedef u64 (*hist_field_fn_t) (struct hist_field *field, 27 struct tracing_map_elt *elt, 28 struct ring_buffer_event *rbe, 29 void *event); 30 31 #define HIST_FIELD_OPERANDS_MAX 2 32 #define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX) 33 #define HIST_ACTIONS_MAX 8 34 35 enum field_op_id { 36 FIELD_OP_NONE, 37 FIELD_OP_PLUS, 38 FIELD_OP_MINUS, 39 FIELD_OP_UNARY_MINUS, 40 }; 41 42 struct hist_var { 43 char *name; 44 struct hist_trigger_data *hist_data; 45 unsigned int idx; 46 }; 47 48 struct hist_field { 49 struct ftrace_event_field *field; 50 unsigned long flags; 51 hist_field_fn_t fn; 52 unsigned int size; 53 unsigned int offset; 54 unsigned int is_signed; 55 const char *type; 56 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX]; 57 struct hist_trigger_data *hist_data; 58 struct hist_var var; 59 enum field_op_id operator; 60 char *system; 61 char *event_name; 62 char *name; 63 unsigned int var_idx; 64 unsigned int var_ref_idx; 65 bool read_once; 66 }; 67 68 static u64 hist_field_none(struct hist_field *field, 69 struct tracing_map_elt *elt, 70 struct ring_buffer_event *rbe, 71 void *event) 72 { 73 return 0; 74 } 75 76 static u64 hist_field_counter(struct hist_field *field, 77 struct tracing_map_elt *elt, 78 struct ring_buffer_event *rbe, 79 void *event) 80 { 81 return 1; 82 } 83 84 static u64 hist_field_string(struct hist_field *hist_field, 85 struct tracing_map_elt *elt, 86 struct ring_buffer_event *rbe, 87 void *event) 88 { 89 char *addr = (char *)(event + hist_field->field->offset); 90 91 return (u64)(unsigned long)addr; 92 } 93 94 static u64 hist_field_dynstring(struct hist_field *hist_field, 95 struct tracing_map_elt *elt, 96 struct ring_buffer_event *rbe, 97 void *event) 98 { 99 u32 str_item = *(u32 *)(event + hist_field->field->offset); 100 int str_loc = str_item & 0xffff; 101 char *addr = (char *)(event + str_loc); 102 103 return (u64)(unsigned long)addr; 104 } 105 106 static u64 hist_field_pstring(struct hist_field *hist_field, 107 struct tracing_map_elt *elt, 108 struct ring_buffer_event *rbe, 109 void *event) 110 { 111 char **addr = (char **)(event + hist_field->field->offset); 112 113 return (u64)(unsigned long)*addr; 114 } 115 116 static u64 hist_field_log2(struct hist_field *hist_field, 117 struct tracing_map_elt *elt, 118 struct ring_buffer_event *rbe, 119 void *event) 120 { 121 struct hist_field *operand = hist_field->operands[0]; 122 123 u64 val = operand->fn(operand, elt, rbe, event); 124 125 return (u64) ilog2(roundup_pow_of_two(val)); 126 } 127 128 static u64 hist_field_plus(struct hist_field *hist_field, 129 struct tracing_map_elt *elt, 130 struct ring_buffer_event *rbe, 131 void *event) 132 { 133 struct hist_field *operand1 = hist_field->operands[0]; 134 struct hist_field *operand2 = hist_field->operands[1]; 135 136 u64 val1 = operand1->fn(operand1, elt, rbe, event); 137 u64 val2 = operand2->fn(operand2, elt, rbe, event); 138 139 return val1 + val2; 140 } 141 142 static u64 hist_field_minus(struct hist_field *hist_field, 143 struct tracing_map_elt *elt, 144 struct ring_buffer_event *rbe, 145 void *event) 146 { 147 struct hist_field *operand1 = hist_field->operands[0]; 148 struct hist_field *operand2 = hist_field->operands[1]; 149 150 u64 val1 = operand1->fn(operand1, elt, rbe, event); 151 u64 val2 = operand2->fn(operand2, elt, rbe, event); 152 153 return val1 - val2; 154 } 155 156 static u64 hist_field_unary_minus(struct hist_field *hist_field, 157 struct tracing_map_elt *elt, 158 struct ring_buffer_event *rbe, 159 void *event) 160 { 161 struct hist_field *operand = hist_field->operands[0]; 162 163 s64 sval = (s64)operand->fn(operand, elt, rbe, event); 164 u64 val = (u64)-sval; 165 166 return val; 167 } 168 169 #define DEFINE_HIST_FIELD_FN(type) \ 170 static u64 hist_field_##type(struct hist_field *hist_field, \ 171 struct tracing_map_elt *elt, \ 172 struct ring_buffer_event *rbe, \ 173 void *event) \ 174 { \ 175 type *addr = (type *)(event + hist_field->field->offset); \ 176 \ 177 return (u64)(unsigned long)*addr; \ 178 } 179 180 DEFINE_HIST_FIELD_FN(s64); 181 DEFINE_HIST_FIELD_FN(u64); 182 DEFINE_HIST_FIELD_FN(s32); 183 DEFINE_HIST_FIELD_FN(u32); 184 DEFINE_HIST_FIELD_FN(s16); 185 DEFINE_HIST_FIELD_FN(u16); 186 DEFINE_HIST_FIELD_FN(s8); 187 DEFINE_HIST_FIELD_FN(u8); 188 189 #define for_each_hist_field(i, hist_data) \ 190 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++) 191 192 #define for_each_hist_val_field(i, hist_data) \ 193 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++) 194 195 #define for_each_hist_key_field(i, hist_data) \ 196 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++) 197 198 #define HIST_STACKTRACE_DEPTH 16 199 #define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long)) 200 #define HIST_STACKTRACE_SKIP 5 201 202 #define HITCOUNT_IDX 0 203 #define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE) 204 205 enum hist_field_flags { 206 HIST_FIELD_FL_HITCOUNT = 1 << 0, 207 HIST_FIELD_FL_KEY = 1 << 1, 208 HIST_FIELD_FL_STRING = 1 << 2, 209 HIST_FIELD_FL_HEX = 1 << 3, 210 HIST_FIELD_FL_SYM = 1 << 4, 211 HIST_FIELD_FL_SYM_OFFSET = 1 << 5, 212 HIST_FIELD_FL_EXECNAME = 1 << 6, 213 HIST_FIELD_FL_SYSCALL = 1 << 7, 214 HIST_FIELD_FL_STACKTRACE = 1 << 8, 215 HIST_FIELD_FL_LOG2 = 1 << 9, 216 HIST_FIELD_FL_TIMESTAMP = 1 << 10, 217 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11, 218 HIST_FIELD_FL_VAR = 1 << 12, 219 HIST_FIELD_FL_EXPR = 1 << 13, 220 HIST_FIELD_FL_VAR_REF = 1 << 14, 221 HIST_FIELD_FL_CPU = 1 << 15, 222 HIST_FIELD_FL_ALIAS = 1 << 16, 223 }; 224 225 struct var_defs { 226 unsigned int n_vars; 227 char *name[TRACING_MAP_VARS_MAX]; 228 char *expr[TRACING_MAP_VARS_MAX]; 229 }; 230 231 struct hist_trigger_attrs { 232 char *keys_str; 233 char *vals_str; 234 char *sort_key_str; 235 char *name; 236 char *clock; 237 bool pause; 238 bool cont; 239 bool clear; 240 bool ts_in_usecs; 241 unsigned int map_bits; 242 243 char *assignment_str[TRACING_MAP_VARS_MAX]; 244 unsigned int n_assignments; 245 246 char *action_str[HIST_ACTIONS_MAX]; 247 unsigned int n_actions; 248 249 struct var_defs var_defs; 250 }; 251 252 struct field_var { 253 struct hist_field *var; 254 struct hist_field *val; 255 }; 256 257 struct field_var_hist { 258 struct hist_trigger_data *hist_data; 259 char *cmd; 260 }; 261 262 struct hist_trigger_data { 263 struct hist_field *fields[HIST_FIELDS_MAX]; 264 unsigned int n_vals; 265 unsigned int n_keys; 266 unsigned int n_fields; 267 unsigned int n_vars; 268 unsigned int key_size; 269 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX]; 270 unsigned int n_sort_keys; 271 struct trace_event_file *event_file; 272 struct hist_trigger_attrs *attrs; 273 struct tracing_map *map; 274 bool enable_timestamps; 275 bool remove; 276 struct hist_field *var_refs[TRACING_MAP_VARS_MAX]; 277 unsigned int n_var_refs; 278 279 struct action_data *actions[HIST_ACTIONS_MAX]; 280 unsigned int n_actions; 281 282 struct hist_field *synth_var_refs[SYNTH_FIELDS_MAX]; 283 unsigned int n_synth_var_refs; 284 struct field_var *field_vars[SYNTH_FIELDS_MAX]; 285 unsigned int n_field_vars; 286 unsigned int n_field_var_str; 287 struct field_var_hist *field_var_hists[SYNTH_FIELDS_MAX]; 288 unsigned int n_field_var_hists; 289 290 struct field_var *max_vars[SYNTH_FIELDS_MAX]; 291 unsigned int n_max_vars; 292 unsigned int n_max_var_str; 293 }; 294 295 struct synth_field { 296 char *type; 297 char *name; 298 size_t size; 299 bool is_signed; 300 bool is_string; 301 }; 302 303 struct synth_event { 304 struct list_head list; 305 int ref; 306 char *name; 307 struct synth_field **fields; 308 unsigned int n_fields; 309 unsigned int n_u64; 310 struct trace_event_class class; 311 struct trace_event_call call; 312 struct tracepoint *tp; 313 }; 314 315 struct action_data; 316 317 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data, 318 struct tracing_map_elt *elt, void *rec, 319 struct ring_buffer_event *rbe, 320 struct action_data *data, u64 *var_ref_vals); 321 322 struct action_data { 323 action_fn_t fn; 324 unsigned int n_params; 325 char *params[SYNTH_FIELDS_MAX]; 326 327 union { 328 struct { 329 unsigned int var_ref_idx; 330 char *match_event; 331 char *match_event_system; 332 char *synth_event_name; 333 struct synth_event *synth_event; 334 } onmatch; 335 336 struct { 337 char *var_str; 338 char *fn_name; 339 unsigned int max_var_ref_idx; 340 struct hist_field *max_var; 341 struct hist_field *var; 342 } onmax; 343 }; 344 }; 345 346 347 static char last_hist_cmd[MAX_FILTER_STR_VAL]; 348 static char hist_err_str[MAX_FILTER_STR_VAL]; 349 350 static void last_cmd_set(char *str) 351 { 352 if (!str) 353 return; 354 355 strncpy(last_hist_cmd, str, MAX_FILTER_STR_VAL - 1); 356 } 357 358 static void hist_err(char *str, char *var) 359 { 360 int maxlen = MAX_FILTER_STR_VAL - 1; 361 362 if (!str) 363 return; 364 365 if (strlen(hist_err_str)) 366 return; 367 368 if (!var) 369 var = ""; 370 371 if (strlen(hist_err_str) + strlen(str) + strlen(var) > maxlen) 372 return; 373 374 strcat(hist_err_str, str); 375 strcat(hist_err_str, var); 376 } 377 378 static void hist_err_event(char *str, char *system, char *event, char *var) 379 { 380 char err[MAX_FILTER_STR_VAL]; 381 382 if (system && var) 383 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s.%s", system, event, var); 384 else if (system) 385 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s", system, event); 386 else 387 strscpy(err, var, MAX_FILTER_STR_VAL); 388 389 hist_err(str, err); 390 } 391 392 static void hist_err_clear(void) 393 { 394 hist_err_str[0] = '\0'; 395 } 396 397 static bool have_hist_err(void) 398 { 399 if (strlen(hist_err_str)) 400 return true; 401 402 return false; 403 } 404 405 static LIST_HEAD(synth_event_list); 406 static DEFINE_MUTEX(synth_event_mutex); 407 408 struct synth_trace_event { 409 struct trace_entry ent; 410 u64 fields[]; 411 }; 412 413 static int synth_event_define_fields(struct trace_event_call *call) 414 { 415 struct synth_trace_event trace; 416 int offset = offsetof(typeof(trace), fields); 417 struct synth_event *event = call->data; 418 unsigned int i, size, n_u64; 419 char *name, *type; 420 bool is_signed; 421 int ret = 0; 422 423 for (i = 0, n_u64 = 0; i < event->n_fields; i++) { 424 size = event->fields[i]->size; 425 is_signed = event->fields[i]->is_signed; 426 type = event->fields[i]->type; 427 name = event->fields[i]->name; 428 ret = trace_define_field(call, type, name, offset, size, 429 is_signed, FILTER_OTHER); 430 if (ret) 431 break; 432 433 if (event->fields[i]->is_string) { 434 offset += STR_VAR_LEN_MAX; 435 n_u64 += STR_VAR_LEN_MAX / sizeof(u64); 436 } else { 437 offset += sizeof(u64); 438 n_u64++; 439 } 440 } 441 442 event->n_u64 = n_u64; 443 444 return ret; 445 } 446 447 static bool synth_field_signed(char *type) 448 { 449 if (strncmp(type, "u", 1) == 0) 450 return false; 451 452 return true; 453 } 454 455 static int synth_field_is_string(char *type) 456 { 457 if (strstr(type, "char[") != NULL) 458 return true; 459 460 return false; 461 } 462 463 static int synth_field_string_size(char *type) 464 { 465 char buf[4], *end, *start; 466 unsigned int len; 467 int size, err; 468 469 start = strstr(type, "char["); 470 if (start == NULL) 471 return -EINVAL; 472 start += strlen("char["); 473 474 end = strchr(type, ']'); 475 if (!end || end < start) 476 return -EINVAL; 477 478 len = end - start; 479 if (len > 3) 480 return -EINVAL; 481 482 strncpy(buf, start, len); 483 buf[len] = '\0'; 484 485 err = kstrtouint(buf, 0, &size); 486 if (err) 487 return err; 488 489 if (size > STR_VAR_LEN_MAX) 490 return -EINVAL; 491 492 return size; 493 } 494 495 static int synth_field_size(char *type) 496 { 497 int size = 0; 498 499 if (strcmp(type, "s64") == 0) 500 size = sizeof(s64); 501 else if (strcmp(type, "u64") == 0) 502 size = sizeof(u64); 503 else if (strcmp(type, "s32") == 0) 504 size = sizeof(s32); 505 else if (strcmp(type, "u32") == 0) 506 size = sizeof(u32); 507 else if (strcmp(type, "s16") == 0) 508 size = sizeof(s16); 509 else if (strcmp(type, "u16") == 0) 510 size = sizeof(u16); 511 else if (strcmp(type, "s8") == 0) 512 size = sizeof(s8); 513 else if (strcmp(type, "u8") == 0) 514 size = sizeof(u8); 515 else if (strcmp(type, "char") == 0) 516 size = sizeof(char); 517 else if (strcmp(type, "unsigned char") == 0) 518 size = sizeof(unsigned char); 519 else if (strcmp(type, "int") == 0) 520 size = sizeof(int); 521 else if (strcmp(type, "unsigned int") == 0) 522 size = sizeof(unsigned int); 523 else if (strcmp(type, "long") == 0) 524 size = sizeof(long); 525 else if (strcmp(type, "unsigned long") == 0) 526 size = sizeof(unsigned long); 527 else if (strcmp(type, "pid_t") == 0) 528 size = sizeof(pid_t); 529 else if (synth_field_is_string(type)) 530 size = synth_field_string_size(type); 531 532 return size; 533 } 534 535 static const char *synth_field_fmt(char *type) 536 { 537 const char *fmt = "%llu"; 538 539 if (strcmp(type, "s64") == 0) 540 fmt = "%lld"; 541 else if (strcmp(type, "u64") == 0) 542 fmt = "%llu"; 543 else if (strcmp(type, "s32") == 0) 544 fmt = "%d"; 545 else if (strcmp(type, "u32") == 0) 546 fmt = "%u"; 547 else if (strcmp(type, "s16") == 0) 548 fmt = "%d"; 549 else if (strcmp(type, "u16") == 0) 550 fmt = "%u"; 551 else if (strcmp(type, "s8") == 0) 552 fmt = "%d"; 553 else if (strcmp(type, "u8") == 0) 554 fmt = "%u"; 555 else if (strcmp(type, "char") == 0) 556 fmt = "%d"; 557 else if (strcmp(type, "unsigned char") == 0) 558 fmt = "%u"; 559 else if (strcmp(type, "int") == 0) 560 fmt = "%d"; 561 else if (strcmp(type, "unsigned int") == 0) 562 fmt = "%u"; 563 else if (strcmp(type, "long") == 0) 564 fmt = "%ld"; 565 else if (strcmp(type, "unsigned long") == 0) 566 fmt = "%lu"; 567 else if (strcmp(type, "pid_t") == 0) 568 fmt = "%d"; 569 else if (synth_field_is_string(type)) 570 fmt = "%s"; 571 572 return fmt; 573 } 574 575 static enum print_line_t print_synth_event(struct trace_iterator *iter, 576 int flags, 577 struct trace_event *event) 578 { 579 struct trace_array *tr = iter->tr; 580 struct trace_seq *s = &iter->seq; 581 struct synth_trace_event *entry; 582 struct synth_event *se; 583 unsigned int i, n_u64; 584 char print_fmt[32]; 585 const char *fmt; 586 587 entry = (struct synth_trace_event *)iter->ent; 588 se = container_of(event, struct synth_event, call.event); 589 590 trace_seq_printf(s, "%s: ", se->name); 591 592 for (i = 0, n_u64 = 0; i < se->n_fields; i++) { 593 if (trace_seq_has_overflowed(s)) 594 goto end; 595 596 fmt = synth_field_fmt(se->fields[i]->type); 597 598 /* parameter types */ 599 if (tr->trace_flags & TRACE_ITER_VERBOSE) 600 trace_seq_printf(s, "%s ", fmt); 601 602 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt); 603 604 /* parameter values */ 605 if (se->fields[i]->is_string) { 606 trace_seq_printf(s, print_fmt, se->fields[i]->name, 607 (char *)&entry->fields[n_u64], 608 i == se->n_fields - 1 ? "" : " "); 609 n_u64 += STR_VAR_LEN_MAX / sizeof(u64); 610 } else { 611 trace_seq_printf(s, print_fmt, se->fields[i]->name, 612 entry->fields[n_u64], 613 i == se->n_fields - 1 ? "" : " "); 614 n_u64++; 615 } 616 } 617 end: 618 trace_seq_putc(s, '\n'); 619 620 return trace_handle_return(s); 621 } 622 623 static struct trace_event_functions synth_event_funcs = { 624 .trace = print_synth_event 625 }; 626 627 static notrace void trace_event_raw_event_synth(void *__data, 628 u64 *var_ref_vals, 629 unsigned int var_ref_idx) 630 { 631 struct trace_event_file *trace_file = __data; 632 struct synth_trace_event *entry; 633 struct trace_event_buffer fbuffer; 634 struct ring_buffer *buffer; 635 struct synth_event *event; 636 unsigned int i, n_u64; 637 int fields_size = 0; 638 639 event = trace_file->event_call->data; 640 641 if (trace_trigger_soft_disabled(trace_file)) 642 return; 643 644 fields_size = event->n_u64 * sizeof(u64); 645 646 /* 647 * Avoid ring buffer recursion detection, as this event 648 * is being performed within another event. 649 */ 650 buffer = trace_file->tr->trace_buffer.buffer; 651 ring_buffer_nest_start(buffer); 652 653 entry = trace_event_buffer_reserve(&fbuffer, trace_file, 654 sizeof(*entry) + fields_size); 655 if (!entry) 656 goto out; 657 658 for (i = 0, n_u64 = 0; i < event->n_fields; i++) { 659 if (event->fields[i]->is_string) { 660 char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i]; 661 char *str_field = (char *)&entry->fields[n_u64]; 662 663 strscpy(str_field, str_val, STR_VAR_LEN_MAX); 664 n_u64 += STR_VAR_LEN_MAX / sizeof(u64); 665 } else { 666 entry->fields[n_u64] = var_ref_vals[var_ref_idx + i]; 667 n_u64++; 668 } 669 } 670 671 trace_event_buffer_commit(&fbuffer); 672 out: 673 ring_buffer_nest_end(buffer); 674 } 675 676 static void free_synth_event_print_fmt(struct trace_event_call *call) 677 { 678 if (call) { 679 kfree(call->print_fmt); 680 call->print_fmt = NULL; 681 } 682 } 683 684 static int __set_synth_event_print_fmt(struct synth_event *event, 685 char *buf, int len) 686 { 687 const char *fmt; 688 int pos = 0; 689 int i; 690 691 /* When len=0, we just calculate the needed length */ 692 #define LEN_OR_ZERO (len ? len - pos : 0) 693 694 pos += snprintf(buf + pos, LEN_OR_ZERO, "\""); 695 for (i = 0; i < event->n_fields; i++) { 696 fmt = synth_field_fmt(event->fields[i]->type); 697 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s", 698 event->fields[i]->name, fmt, 699 i == event->n_fields - 1 ? "" : ", "); 700 } 701 pos += snprintf(buf + pos, LEN_OR_ZERO, "\""); 702 703 for (i = 0; i < event->n_fields; i++) { 704 pos += snprintf(buf + pos, LEN_OR_ZERO, 705 ", REC->%s", event->fields[i]->name); 706 } 707 708 #undef LEN_OR_ZERO 709 710 /* return the length of print_fmt */ 711 return pos; 712 } 713 714 static int set_synth_event_print_fmt(struct trace_event_call *call) 715 { 716 struct synth_event *event = call->data; 717 char *print_fmt; 718 int len; 719 720 /* First: called with 0 length to calculate the needed length */ 721 len = __set_synth_event_print_fmt(event, NULL, 0); 722 723 print_fmt = kmalloc(len + 1, GFP_KERNEL); 724 if (!print_fmt) 725 return -ENOMEM; 726 727 /* Second: actually write the @print_fmt */ 728 __set_synth_event_print_fmt(event, print_fmt, len + 1); 729 call->print_fmt = print_fmt; 730 731 return 0; 732 } 733 734 static void free_synth_field(struct synth_field *field) 735 { 736 kfree(field->type); 737 kfree(field->name); 738 kfree(field); 739 } 740 741 static struct synth_field *parse_synth_field(char *field_type, 742 char *field_name) 743 { 744 struct synth_field *field; 745 int len, ret = 0; 746 char *array; 747 748 if (field_type[0] == ';') 749 field_type++; 750 751 len = strlen(field_name); 752 if (field_name[len - 1] == ';') 753 field_name[len - 1] = '\0'; 754 755 field = kzalloc(sizeof(*field), GFP_KERNEL); 756 if (!field) 757 return ERR_PTR(-ENOMEM); 758 759 len = strlen(field_type) + 1; 760 array = strchr(field_name, '['); 761 if (array) 762 len += strlen(array); 763 field->type = kzalloc(len, GFP_KERNEL); 764 if (!field->type) { 765 ret = -ENOMEM; 766 goto free; 767 } 768 strcat(field->type, field_type); 769 if (array) { 770 strcat(field->type, array); 771 *array = '\0'; 772 } 773 774 field->size = synth_field_size(field->type); 775 if (!field->size) { 776 ret = -EINVAL; 777 goto free; 778 } 779 780 if (synth_field_is_string(field->type)) 781 field->is_string = true; 782 783 field->is_signed = synth_field_signed(field->type); 784 785 field->name = kstrdup(field_name, GFP_KERNEL); 786 if (!field->name) { 787 ret = -ENOMEM; 788 goto free; 789 } 790 out: 791 return field; 792 free: 793 free_synth_field(field); 794 field = ERR_PTR(ret); 795 goto out; 796 } 797 798 static void free_synth_tracepoint(struct tracepoint *tp) 799 { 800 if (!tp) 801 return; 802 803 kfree(tp->name); 804 kfree(tp); 805 } 806 807 static struct tracepoint *alloc_synth_tracepoint(char *name) 808 { 809 struct tracepoint *tp; 810 811 tp = kzalloc(sizeof(*tp), GFP_KERNEL); 812 if (!tp) 813 return ERR_PTR(-ENOMEM); 814 815 tp->name = kstrdup(name, GFP_KERNEL); 816 if (!tp->name) { 817 kfree(tp); 818 return ERR_PTR(-ENOMEM); 819 } 820 821 return tp; 822 } 823 824 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals, 825 unsigned int var_ref_idx); 826 827 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals, 828 unsigned int var_ref_idx) 829 { 830 struct tracepoint *tp = event->tp; 831 832 if (unlikely(atomic_read(&tp->key.enabled) > 0)) { 833 struct tracepoint_func *probe_func_ptr; 834 synth_probe_func_t probe_func; 835 void *__data; 836 837 if (!(cpu_online(raw_smp_processor_id()))) 838 return; 839 840 probe_func_ptr = rcu_dereference_sched((tp)->funcs); 841 if (probe_func_ptr) { 842 do { 843 probe_func = probe_func_ptr->func; 844 __data = probe_func_ptr->data; 845 probe_func(__data, var_ref_vals, var_ref_idx); 846 } while ((++probe_func_ptr)->func); 847 } 848 } 849 } 850 851 static struct synth_event *find_synth_event(const char *name) 852 { 853 struct synth_event *event; 854 855 list_for_each_entry(event, &synth_event_list, list) { 856 if (strcmp(event->name, name) == 0) 857 return event; 858 } 859 860 return NULL; 861 } 862 863 static int register_synth_event(struct synth_event *event) 864 { 865 struct trace_event_call *call = &event->call; 866 int ret = 0; 867 868 event->call.class = &event->class; 869 event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL); 870 if (!event->class.system) { 871 ret = -ENOMEM; 872 goto out; 873 } 874 875 event->tp = alloc_synth_tracepoint(event->name); 876 if (IS_ERR(event->tp)) { 877 ret = PTR_ERR(event->tp); 878 event->tp = NULL; 879 goto out; 880 } 881 882 INIT_LIST_HEAD(&call->class->fields); 883 call->event.funcs = &synth_event_funcs; 884 call->class->define_fields = synth_event_define_fields; 885 886 ret = register_trace_event(&call->event); 887 if (!ret) { 888 ret = -ENODEV; 889 goto out; 890 } 891 call->flags = TRACE_EVENT_FL_TRACEPOINT; 892 call->class->reg = trace_event_reg; 893 call->class->probe = trace_event_raw_event_synth; 894 call->data = event; 895 call->tp = event->tp; 896 897 ret = trace_add_event_call(call); 898 if (ret) { 899 pr_warn("Failed to register synthetic event: %s\n", 900 trace_event_name(call)); 901 goto err; 902 } 903 904 ret = set_synth_event_print_fmt(call); 905 if (ret < 0) { 906 trace_remove_event_call(call); 907 goto err; 908 } 909 out: 910 return ret; 911 err: 912 unregister_trace_event(&call->event); 913 goto out; 914 } 915 916 static int unregister_synth_event(struct synth_event *event) 917 { 918 struct trace_event_call *call = &event->call; 919 int ret; 920 921 ret = trace_remove_event_call(call); 922 923 return ret; 924 } 925 926 static void free_synth_event(struct synth_event *event) 927 { 928 unsigned int i; 929 930 if (!event) 931 return; 932 933 for (i = 0; i < event->n_fields; i++) 934 free_synth_field(event->fields[i]); 935 936 kfree(event->fields); 937 kfree(event->name); 938 kfree(event->class.system); 939 free_synth_tracepoint(event->tp); 940 free_synth_event_print_fmt(&event->call); 941 kfree(event); 942 } 943 944 static struct synth_event *alloc_synth_event(char *event_name, int n_fields, 945 struct synth_field **fields) 946 { 947 struct synth_event *event; 948 unsigned int i; 949 950 event = kzalloc(sizeof(*event), GFP_KERNEL); 951 if (!event) { 952 event = ERR_PTR(-ENOMEM); 953 goto out; 954 } 955 956 event->name = kstrdup(event_name, GFP_KERNEL); 957 if (!event->name) { 958 kfree(event); 959 event = ERR_PTR(-ENOMEM); 960 goto out; 961 } 962 963 event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL); 964 if (!event->fields) { 965 free_synth_event(event); 966 event = ERR_PTR(-ENOMEM); 967 goto out; 968 } 969 970 for (i = 0; i < n_fields; i++) 971 event->fields[i] = fields[i]; 972 973 event->n_fields = n_fields; 974 out: 975 return event; 976 } 977 978 static void action_trace(struct hist_trigger_data *hist_data, 979 struct tracing_map_elt *elt, void *rec, 980 struct ring_buffer_event *rbe, 981 struct action_data *data, u64 *var_ref_vals) 982 { 983 struct synth_event *event = data->onmatch.synth_event; 984 985 trace_synth(event, var_ref_vals, data->onmatch.var_ref_idx); 986 } 987 988 struct hist_var_data { 989 struct list_head list; 990 struct hist_trigger_data *hist_data; 991 }; 992 993 static void add_or_delete_synth_event(struct synth_event *event, int delete) 994 { 995 if (delete) 996 free_synth_event(event); 997 else { 998 mutex_lock(&synth_event_mutex); 999 if (!find_synth_event(event->name)) 1000 list_add(&event->list, &synth_event_list); 1001 else 1002 free_synth_event(event); 1003 mutex_unlock(&synth_event_mutex); 1004 } 1005 } 1006 1007 static int create_synth_event(int argc, char **argv) 1008 { 1009 struct synth_field *field, *fields[SYNTH_FIELDS_MAX]; 1010 struct synth_event *event = NULL; 1011 bool delete_event = false; 1012 int i, n_fields = 0, ret = 0; 1013 char *name; 1014 1015 mutex_lock(&synth_event_mutex); 1016 1017 /* 1018 * Argument syntax: 1019 * - Add synthetic event: <event_name> field[;field] ... 1020 * - Remove synthetic event: !<event_name> field[;field] ... 1021 * where 'field' = type field_name 1022 */ 1023 if (argc < 1) { 1024 ret = -EINVAL; 1025 goto out; 1026 } 1027 1028 name = argv[0]; 1029 if (name[0] == '!') { 1030 delete_event = true; 1031 name++; 1032 } 1033 1034 event = find_synth_event(name); 1035 if (event) { 1036 if (delete_event) { 1037 if (event->ref) { 1038 event = NULL; 1039 ret = -EBUSY; 1040 goto out; 1041 } 1042 list_del(&event->list); 1043 goto out; 1044 } 1045 event = NULL; 1046 ret = -EEXIST; 1047 goto out; 1048 } else if (delete_event) 1049 goto out; 1050 1051 if (argc < 2) { 1052 ret = -EINVAL; 1053 goto out; 1054 } 1055 1056 for (i = 1; i < argc - 1; i++) { 1057 if (strcmp(argv[i], ";") == 0) 1058 continue; 1059 if (n_fields == SYNTH_FIELDS_MAX) { 1060 ret = -EINVAL; 1061 goto err; 1062 } 1063 1064 field = parse_synth_field(argv[i], argv[i + 1]); 1065 if (IS_ERR(field)) { 1066 ret = PTR_ERR(field); 1067 goto err; 1068 } 1069 fields[n_fields] = field; 1070 i++; n_fields++; 1071 } 1072 1073 if (i < argc) { 1074 ret = -EINVAL; 1075 goto err; 1076 } 1077 1078 event = alloc_synth_event(name, n_fields, fields); 1079 if (IS_ERR(event)) { 1080 ret = PTR_ERR(event); 1081 event = NULL; 1082 goto err; 1083 } 1084 out: 1085 mutex_unlock(&synth_event_mutex); 1086 1087 if (event) { 1088 if (delete_event) { 1089 ret = unregister_synth_event(event); 1090 add_or_delete_synth_event(event, !ret); 1091 } else { 1092 ret = register_synth_event(event); 1093 add_or_delete_synth_event(event, ret); 1094 } 1095 } 1096 1097 return ret; 1098 err: 1099 mutex_unlock(&synth_event_mutex); 1100 1101 for (i = 0; i < n_fields; i++) 1102 free_synth_field(fields[i]); 1103 free_synth_event(event); 1104 1105 return ret; 1106 } 1107 1108 static int release_all_synth_events(void) 1109 { 1110 struct list_head release_events; 1111 struct synth_event *event, *e; 1112 int ret = 0; 1113 1114 INIT_LIST_HEAD(&release_events); 1115 1116 mutex_lock(&synth_event_mutex); 1117 1118 list_for_each_entry(event, &synth_event_list, list) { 1119 if (event->ref) { 1120 mutex_unlock(&synth_event_mutex); 1121 return -EBUSY; 1122 } 1123 } 1124 1125 list_splice_init(&event->list, &release_events); 1126 1127 mutex_unlock(&synth_event_mutex); 1128 1129 list_for_each_entry_safe(event, e, &release_events, list) { 1130 list_del(&event->list); 1131 1132 ret = unregister_synth_event(event); 1133 add_or_delete_synth_event(event, !ret); 1134 } 1135 1136 return ret; 1137 } 1138 1139 1140 static void *synth_events_seq_start(struct seq_file *m, loff_t *pos) 1141 { 1142 mutex_lock(&synth_event_mutex); 1143 1144 return seq_list_start(&synth_event_list, *pos); 1145 } 1146 1147 static void *synth_events_seq_next(struct seq_file *m, void *v, loff_t *pos) 1148 { 1149 return seq_list_next(v, &synth_event_list, pos); 1150 } 1151 1152 static void synth_events_seq_stop(struct seq_file *m, void *v) 1153 { 1154 mutex_unlock(&synth_event_mutex); 1155 } 1156 1157 static int synth_events_seq_show(struct seq_file *m, void *v) 1158 { 1159 struct synth_field *field; 1160 struct synth_event *event = v; 1161 unsigned int i; 1162 1163 seq_printf(m, "%s\t", event->name); 1164 1165 for (i = 0; i < event->n_fields; i++) { 1166 field = event->fields[i]; 1167 1168 /* parameter values */ 1169 seq_printf(m, "%s %s%s", field->type, field->name, 1170 i == event->n_fields - 1 ? "" : "; "); 1171 } 1172 1173 seq_putc(m, '\n'); 1174 1175 return 0; 1176 } 1177 1178 static const struct seq_operations synth_events_seq_op = { 1179 .start = synth_events_seq_start, 1180 .next = synth_events_seq_next, 1181 .stop = synth_events_seq_stop, 1182 .show = synth_events_seq_show 1183 }; 1184 1185 static int synth_events_open(struct inode *inode, struct file *file) 1186 { 1187 int ret; 1188 1189 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 1190 ret = release_all_synth_events(); 1191 if (ret < 0) 1192 return ret; 1193 } 1194 1195 return seq_open(file, &synth_events_seq_op); 1196 } 1197 1198 static ssize_t synth_events_write(struct file *file, 1199 const char __user *buffer, 1200 size_t count, loff_t *ppos) 1201 { 1202 return trace_parse_run_command(file, buffer, count, ppos, 1203 create_synth_event); 1204 } 1205 1206 static const struct file_operations synth_events_fops = { 1207 .open = synth_events_open, 1208 .write = synth_events_write, 1209 .read = seq_read, 1210 .llseek = seq_lseek, 1211 .release = seq_release, 1212 }; 1213 1214 static u64 hist_field_timestamp(struct hist_field *hist_field, 1215 struct tracing_map_elt *elt, 1216 struct ring_buffer_event *rbe, 1217 void *event) 1218 { 1219 struct hist_trigger_data *hist_data = hist_field->hist_data; 1220 struct trace_array *tr = hist_data->event_file->tr; 1221 1222 u64 ts = ring_buffer_event_time_stamp(rbe); 1223 1224 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr)) 1225 ts = ns2usecs(ts); 1226 1227 return ts; 1228 } 1229 1230 static u64 hist_field_cpu(struct hist_field *hist_field, 1231 struct tracing_map_elt *elt, 1232 struct ring_buffer_event *rbe, 1233 void *event) 1234 { 1235 int cpu = smp_processor_id(); 1236 1237 return cpu; 1238 } 1239 1240 static struct hist_field * 1241 check_field_for_var_ref(struct hist_field *hist_field, 1242 struct hist_trigger_data *var_data, 1243 unsigned int var_idx) 1244 { 1245 struct hist_field *found = NULL; 1246 1247 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF) { 1248 if (hist_field->var.idx == var_idx && 1249 hist_field->var.hist_data == var_data) { 1250 found = hist_field; 1251 } 1252 } 1253 1254 return found; 1255 } 1256 1257 static struct hist_field * 1258 check_field_for_var_refs(struct hist_trigger_data *hist_data, 1259 struct hist_field *hist_field, 1260 struct hist_trigger_data *var_data, 1261 unsigned int var_idx, 1262 unsigned int level) 1263 { 1264 struct hist_field *found = NULL; 1265 unsigned int i; 1266 1267 if (level > 3) 1268 return found; 1269 1270 if (!hist_field) 1271 return found; 1272 1273 found = check_field_for_var_ref(hist_field, var_data, var_idx); 1274 if (found) 1275 return found; 1276 1277 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) { 1278 struct hist_field *operand; 1279 1280 operand = hist_field->operands[i]; 1281 found = check_field_for_var_refs(hist_data, operand, var_data, 1282 var_idx, level + 1); 1283 if (found) 1284 return found; 1285 } 1286 1287 return found; 1288 } 1289 1290 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data, 1291 struct hist_trigger_data *var_data, 1292 unsigned int var_idx) 1293 { 1294 struct hist_field *hist_field, *found = NULL; 1295 unsigned int i; 1296 1297 for_each_hist_field(i, hist_data) { 1298 hist_field = hist_data->fields[i]; 1299 found = check_field_for_var_refs(hist_data, hist_field, 1300 var_data, var_idx, 0); 1301 if (found) 1302 return found; 1303 } 1304 1305 for (i = 0; i < hist_data->n_synth_var_refs; i++) { 1306 hist_field = hist_data->synth_var_refs[i]; 1307 found = check_field_for_var_refs(hist_data, hist_field, 1308 var_data, var_idx, 0); 1309 if (found) 1310 return found; 1311 } 1312 1313 return found; 1314 } 1315 1316 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data, 1317 unsigned int var_idx) 1318 { 1319 struct trace_array *tr = hist_data->event_file->tr; 1320 struct hist_field *found = NULL; 1321 struct hist_var_data *var_data; 1322 1323 list_for_each_entry(var_data, &tr->hist_vars, list) { 1324 if (var_data->hist_data == hist_data) 1325 continue; 1326 found = find_var_ref(var_data->hist_data, hist_data, var_idx); 1327 if (found) 1328 break; 1329 } 1330 1331 return found; 1332 } 1333 1334 static bool check_var_refs(struct hist_trigger_data *hist_data) 1335 { 1336 struct hist_field *field; 1337 bool found = false; 1338 int i; 1339 1340 for_each_hist_field(i, hist_data) { 1341 field = hist_data->fields[i]; 1342 if (field && field->flags & HIST_FIELD_FL_VAR) { 1343 if (find_any_var_ref(hist_data, field->var.idx)) { 1344 found = true; 1345 break; 1346 } 1347 } 1348 } 1349 1350 return found; 1351 } 1352 1353 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data) 1354 { 1355 struct trace_array *tr = hist_data->event_file->tr; 1356 struct hist_var_data *var_data, *found = NULL; 1357 1358 list_for_each_entry(var_data, &tr->hist_vars, list) { 1359 if (var_data->hist_data == hist_data) { 1360 found = var_data; 1361 break; 1362 } 1363 } 1364 1365 return found; 1366 } 1367 1368 static bool field_has_hist_vars(struct hist_field *hist_field, 1369 unsigned int level) 1370 { 1371 int i; 1372 1373 if (level > 3) 1374 return false; 1375 1376 if (!hist_field) 1377 return false; 1378 1379 if (hist_field->flags & HIST_FIELD_FL_VAR || 1380 hist_field->flags & HIST_FIELD_FL_VAR_REF) 1381 return true; 1382 1383 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) { 1384 struct hist_field *operand; 1385 1386 operand = hist_field->operands[i]; 1387 if (field_has_hist_vars(operand, level + 1)) 1388 return true; 1389 } 1390 1391 return false; 1392 } 1393 1394 static bool has_hist_vars(struct hist_trigger_data *hist_data) 1395 { 1396 struct hist_field *hist_field; 1397 int i; 1398 1399 for_each_hist_field(i, hist_data) { 1400 hist_field = hist_data->fields[i]; 1401 if (field_has_hist_vars(hist_field, 0)) 1402 return true; 1403 } 1404 1405 return false; 1406 } 1407 1408 static int save_hist_vars(struct hist_trigger_data *hist_data) 1409 { 1410 struct trace_array *tr = hist_data->event_file->tr; 1411 struct hist_var_data *var_data; 1412 1413 var_data = find_hist_vars(hist_data); 1414 if (var_data) 1415 return 0; 1416 1417 if (trace_array_get(tr) < 0) 1418 return -ENODEV; 1419 1420 var_data = kzalloc(sizeof(*var_data), GFP_KERNEL); 1421 if (!var_data) { 1422 trace_array_put(tr); 1423 return -ENOMEM; 1424 } 1425 1426 var_data->hist_data = hist_data; 1427 list_add(&var_data->list, &tr->hist_vars); 1428 1429 return 0; 1430 } 1431 1432 static void remove_hist_vars(struct hist_trigger_data *hist_data) 1433 { 1434 struct trace_array *tr = hist_data->event_file->tr; 1435 struct hist_var_data *var_data; 1436 1437 var_data = find_hist_vars(hist_data); 1438 if (!var_data) 1439 return; 1440 1441 if (WARN_ON(check_var_refs(hist_data))) 1442 return; 1443 1444 list_del(&var_data->list); 1445 1446 kfree(var_data); 1447 1448 trace_array_put(tr); 1449 } 1450 1451 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data, 1452 const char *var_name) 1453 { 1454 struct hist_field *hist_field, *found = NULL; 1455 int i; 1456 1457 for_each_hist_field(i, hist_data) { 1458 hist_field = hist_data->fields[i]; 1459 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR && 1460 strcmp(hist_field->var.name, var_name) == 0) { 1461 found = hist_field; 1462 break; 1463 } 1464 } 1465 1466 return found; 1467 } 1468 1469 static struct hist_field *find_var(struct hist_trigger_data *hist_data, 1470 struct trace_event_file *file, 1471 const char *var_name) 1472 { 1473 struct hist_trigger_data *test_data; 1474 struct event_trigger_data *test; 1475 struct hist_field *hist_field; 1476 1477 hist_field = find_var_field(hist_data, var_name); 1478 if (hist_field) 1479 return hist_field; 1480 1481 list_for_each_entry_rcu(test, &file->triggers, list) { 1482 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1483 test_data = test->private_data; 1484 hist_field = find_var_field(test_data, var_name); 1485 if (hist_field) 1486 return hist_field; 1487 } 1488 } 1489 1490 return NULL; 1491 } 1492 1493 static struct trace_event_file *find_var_file(struct trace_array *tr, 1494 char *system, 1495 char *event_name, 1496 char *var_name) 1497 { 1498 struct hist_trigger_data *var_hist_data; 1499 struct hist_var_data *var_data; 1500 struct trace_event_file *file, *found = NULL; 1501 1502 if (system) 1503 return find_event_file(tr, system, event_name); 1504 1505 list_for_each_entry(var_data, &tr->hist_vars, list) { 1506 var_hist_data = var_data->hist_data; 1507 file = var_hist_data->event_file; 1508 if (file == found) 1509 continue; 1510 1511 if (find_var_field(var_hist_data, var_name)) { 1512 if (found) { 1513 hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name); 1514 return NULL; 1515 } 1516 1517 found = file; 1518 } 1519 } 1520 1521 return found; 1522 } 1523 1524 static struct hist_field *find_file_var(struct trace_event_file *file, 1525 const char *var_name) 1526 { 1527 struct hist_trigger_data *test_data; 1528 struct event_trigger_data *test; 1529 struct hist_field *hist_field; 1530 1531 list_for_each_entry_rcu(test, &file->triggers, list) { 1532 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1533 test_data = test->private_data; 1534 hist_field = find_var_field(test_data, var_name); 1535 if (hist_field) 1536 return hist_field; 1537 } 1538 } 1539 1540 return NULL; 1541 } 1542 1543 static struct hist_field * 1544 find_match_var(struct hist_trigger_data *hist_data, char *var_name) 1545 { 1546 struct trace_array *tr = hist_data->event_file->tr; 1547 struct hist_field *hist_field, *found = NULL; 1548 struct trace_event_file *file; 1549 unsigned int i; 1550 1551 for (i = 0; i < hist_data->n_actions; i++) { 1552 struct action_data *data = hist_data->actions[i]; 1553 1554 if (data->fn == action_trace) { 1555 char *system = data->onmatch.match_event_system; 1556 char *event_name = data->onmatch.match_event; 1557 1558 file = find_var_file(tr, system, event_name, var_name); 1559 if (!file) 1560 continue; 1561 hist_field = find_file_var(file, var_name); 1562 if (hist_field) { 1563 if (found) { 1564 hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name); 1565 return ERR_PTR(-EINVAL); 1566 } 1567 1568 found = hist_field; 1569 } 1570 } 1571 } 1572 return found; 1573 } 1574 1575 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data, 1576 char *system, 1577 char *event_name, 1578 char *var_name) 1579 { 1580 struct trace_array *tr = hist_data->event_file->tr; 1581 struct hist_field *hist_field = NULL; 1582 struct trace_event_file *file; 1583 1584 if (!system || !event_name) { 1585 hist_field = find_match_var(hist_data, var_name); 1586 if (IS_ERR(hist_field)) 1587 return NULL; 1588 if (hist_field) 1589 return hist_field; 1590 } 1591 1592 file = find_var_file(tr, system, event_name, var_name); 1593 if (!file) 1594 return NULL; 1595 1596 hist_field = find_file_var(file, var_name); 1597 1598 return hist_field; 1599 } 1600 1601 struct hist_elt_data { 1602 char *comm; 1603 u64 *var_ref_vals; 1604 char *field_var_str[SYNTH_FIELDS_MAX]; 1605 }; 1606 1607 static u64 hist_field_var_ref(struct hist_field *hist_field, 1608 struct tracing_map_elt *elt, 1609 struct ring_buffer_event *rbe, 1610 void *event) 1611 { 1612 struct hist_elt_data *elt_data; 1613 u64 var_val = 0; 1614 1615 elt_data = elt->private_data; 1616 var_val = elt_data->var_ref_vals[hist_field->var_ref_idx]; 1617 1618 return var_val; 1619 } 1620 1621 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key, 1622 u64 *var_ref_vals, bool self) 1623 { 1624 struct hist_trigger_data *var_data; 1625 struct tracing_map_elt *var_elt; 1626 struct hist_field *hist_field; 1627 unsigned int i, var_idx; 1628 bool resolved = true; 1629 u64 var_val = 0; 1630 1631 for (i = 0; i < hist_data->n_var_refs; i++) { 1632 hist_field = hist_data->var_refs[i]; 1633 var_idx = hist_field->var.idx; 1634 var_data = hist_field->var.hist_data; 1635 1636 if (var_data == NULL) { 1637 resolved = false; 1638 break; 1639 } 1640 1641 if ((self && var_data != hist_data) || 1642 (!self && var_data == hist_data)) 1643 continue; 1644 1645 var_elt = tracing_map_lookup(var_data->map, key); 1646 if (!var_elt) { 1647 resolved = false; 1648 break; 1649 } 1650 1651 if (!tracing_map_var_set(var_elt, var_idx)) { 1652 resolved = false; 1653 break; 1654 } 1655 1656 if (self || !hist_field->read_once) 1657 var_val = tracing_map_read_var(var_elt, var_idx); 1658 else 1659 var_val = tracing_map_read_var_once(var_elt, var_idx); 1660 1661 var_ref_vals[i] = var_val; 1662 } 1663 1664 return resolved; 1665 } 1666 1667 static const char *hist_field_name(struct hist_field *field, 1668 unsigned int level) 1669 { 1670 const char *field_name = ""; 1671 1672 if (level > 1) 1673 return field_name; 1674 1675 if (field->field) 1676 field_name = field->field->name; 1677 else if (field->flags & HIST_FIELD_FL_LOG2 || 1678 field->flags & HIST_FIELD_FL_ALIAS) 1679 field_name = hist_field_name(field->operands[0], ++level); 1680 else if (field->flags & HIST_FIELD_FL_CPU) 1681 field_name = "cpu"; 1682 else if (field->flags & HIST_FIELD_FL_EXPR || 1683 field->flags & HIST_FIELD_FL_VAR_REF) { 1684 if (field->system) { 1685 static char full_name[MAX_FILTER_STR_VAL]; 1686 1687 strcat(full_name, field->system); 1688 strcat(full_name, "."); 1689 strcat(full_name, field->event_name); 1690 strcat(full_name, "."); 1691 strcat(full_name, field->name); 1692 field_name = full_name; 1693 } else 1694 field_name = field->name; 1695 } else if (field->flags & HIST_FIELD_FL_TIMESTAMP) 1696 field_name = "common_timestamp"; 1697 1698 if (field_name == NULL) 1699 field_name = ""; 1700 1701 return field_name; 1702 } 1703 1704 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed) 1705 { 1706 hist_field_fn_t fn = NULL; 1707 1708 switch (field_size) { 1709 case 8: 1710 if (field_is_signed) 1711 fn = hist_field_s64; 1712 else 1713 fn = hist_field_u64; 1714 break; 1715 case 4: 1716 if (field_is_signed) 1717 fn = hist_field_s32; 1718 else 1719 fn = hist_field_u32; 1720 break; 1721 case 2: 1722 if (field_is_signed) 1723 fn = hist_field_s16; 1724 else 1725 fn = hist_field_u16; 1726 break; 1727 case 1: 1728 if (field_is_signed) 1729 fn = hist_field_s8; 1730 else 1731 fn = hist_field_u8; 1732 break; 1733 } 1734 1735 return fn; 1736 } 1737 1738 static int parse_map_size(char *str) 1739 { 1740 unsigned long size, map_bits; 1741 int ret; 1742 1743 strsep(&str, "="); 1744 if (!str) { 1745 ret = -EINVAL; 1746 goto out; 1747 } 1748 1749 ret = kstrtoul(str, 0, &size); 1750 if (ret) 1751 goto out; 1752 1753 map_bits = ilog2(roundup_pow_of_two(size)); 1754 if (map_bits < TRACING_MAP_BITS_MIN || 1755 map_bits > TRACING_MAP_BITS_MAX) 1756 ret = -EINVAL; 1757 else 1758 ret = map_bits; 1759 out: 1760 return ret; 1761 } 1762 1763 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs) 1764 { 1765 unsigned int i; 1766 1767 if (!attrs) 1768 return; 1769 1770 for (i = 0; i < attrs->n_assignments; i++) 1771 kfree(attrs->assignment_str[i]); 1772 1773 for (i = 0; i < attrs->n_actions; i++) 1774 kfree(attrs->action_str[i]); 1775 1776 kfree(attrs->name); 1777 kfree(attrs->sort_key_str); 1778 kfree(attrs->keys_str); 1779 kfree(attrs->vals_str); 1780 kfree(attrs->clock); 1781 kfree(attrs); 1782 } 1783 1784 static int parse_action(char *str, struct hist_trigger_attrs *attrs) 1785 { 1786 int ret = -EINVAL; 1787 1788 if (attrs->n_actions >= HIST_ACTIONS_MAX) 1789 return ret; 1790 1791 if ((strncmp(str, "onmatch(", strlen("onmatch(")) == 0) || 1792 (strncmp(str, "onmax(", strlen("onmax(")) == 0)) { 1793 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL); 1794 if (!attrs->action_str[attrs->n_actions]) { 1795 ret = -ENOMEM; 1796 return ret; 1797 } 1798 attrs->n_actions++; 1799 ret = 0; 1800 } 1801 1802 return ret; 1803 } 1804 1805 static int parse_assignment(char *str, struct hist_trigger_attrs *attrs) 1806 { 1807 int ret = 0; 1808 1809 if ((strncmp(str, "key=", strlen("key=")) == 0) || 1810 (strncmp(str, "keys=", strlen("keys=")) == 0)) { 1811 attrs->keys_str = kstrdup(str, GFP_KERNEL); 1812 if (!attrs->keys_str) { 1813 ret = -ENOMEM; 1814 goto out; 1815 } 1816 } else if ((strncmp(str, "val=", strlen("val=")) == 0) || 1817 (strncmp(str, "vals=", strlen("vals=")) == 0) || 1818 (strncmp(str, "values=", strlen("values=")) == 0)) { 1819 attrs->vals_str = kstrdup(str, GFP_KERNEL); 1820 if (!attrs->vals_str) { 1821 ret = -ENOMEM; 1822 goto out; 1823 } 1824 } else if (strncmp(str, "sort=", strlen("sort=")) == 0) { 1825 attrs->sort_key_str = kstrdup(str, GFP_KERNEL); 1826 if (!attrs->sort_key_str) { 1827 ret = -ENOMEM; 1828 goto out; 1829 } 1830 } else if (strncmp(str, "name=", strlen("name=")) == 0) { 1831 attrs->name = kstrdup(str, GFP_KERNEL); 1832 if (!attrs->name) { 1833 ret = -ENOMEM; 1834 goto out; 1835 } 1836 } else if (strncmp(str, "clock=", strlen("clock=")) == 0) { 1837 strsep(&str, "="); 1838 if (!str) { 1839 ret = -EINVAL; 1840 goto out; 1841 } 1842 1843 str = strstrip(str); 1844 attrs->clock = kstrdup(str, GFP_KERNEL); 1845 if (!attrs->clock) { 1846 ret = -ENOMEM; 1847 goto out; 1848 } 1849 } else if (strncmp(str, "size=", strlen("size=")) == 0) { 1850 int map_bits = parse_map_size(str); 1851 1852 if (map_bits < 0) { 1853 ret = map_bits; 1854 goto out; 1855 } 1856 attrs->map_bits = map_bits; 1857 } else { 1858 char *assignment; 1859 1860 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) { 1861 hist_err("Too many variables defined: ", str); 1862 ret = -EINVAL; 1863 goto out; 1864 } 1865 1866 assignment = kstrdup(str, GFP_KERNEL); 1867 if (!assignment) { 1868 ret = -ENOMEM; 1869 goto out; 1870 } 1871 1872 attrs->assignment_str[attrs->n_assignments++] = assignment; 1873 } 1874 out: 1875 return ret; 1876 } 1877 1878 static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str) 1879 { 1880 struct hist_trigger_attrs *attrs; 1881 int ret = 0; 1882 1883 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL); 1884 if (!attrs) 1885 return ERR_PTR(-ENOMEM); 1886 1887 while (trigger_str) { 1888 char *str = strsep(&trigger_str, ":"); 1889 1890 if (strchr(str, '=')) { 1891 ret = parse_assignment(str, attrs); 1892 if (ret) 1893 goto free; 1894 } else if (strcmp(str, "pause") == 0) 1895 attrs->pause = true; 1896 else if ((strcmp(str, "cont") == 0) || 1897 (strcmp(str, "continue") == 0)) 1898 attrs->cont = true; 1899 else if (strcmp(str, "clear") == 0) 1900 attrs->clear = true; 1901 else { 1902 ret = parse_action(str, attrs); 1903 if (ret) 1904 goto free; 1905 } 1906 } 1907 1908 if (!attrs->keys_str) { 1909 ret = -EINVAL; 1910 goto free; 1911 } 1912 1913 if (!attrs->clock) { 1914 attrs->clock = kstrdup("global", GFP_KERNEL); 1915 if (!attrs->clock) { 1916 ret = -ENOMEM; 1917 goto free; 1918 } 1919 } 1920 1921 return attrs; 1922 free: 1923 destroy_hist_trigger_attrs(attrs); 1924 1925 return ERR_PTR(ret); 1926 } 1927 1928 static inline void save_comm(char *comm, struct task_struct *task) 1929 { 1930 if (!task->pid) { 1931 strcpy(comm, "<idle>"); 1932 return; 1933 } 1934 1935 if (WARN_ON_ONCE(task->pid < 0)) { 1936 strcpy(comm, "<XXX>"); 1937 return; 1938 } 1939 1940 memcpy(comm, task->comm, TASK_COMM_LEN); 1941 } 1942 1943 static void hist_elt_data_free(struct hist_elt_data *elt_data) 1944 { 1945 unsigned int i; 1946 1947 for (i = 0; i < SYNTH_FIELDS_MAX; i++) 1948 kfree(elt_data->field_var_str[i]); 1949 1950 kfree(elt_data->comm); 1951 kfree(elt_data); 1952 } 1953 1954 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt) 1955 { 1956 struct hist_elt_data *elt_data = elt->private_data; 1957 1958 hist_elt_data_free(elt_data); 1959 } 1960 1961 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt) 1962 { 1963 struct hist_trigger_data *hist_data = elt->map->private_data; 1964 unsigned int size = TASK_COMM_LEN; 1965 struct hist_elt_data *elt_data; 1966 struct hist_field *key_field; 1967 unsigned int i, n_str; 1968 1969 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL); 1970 if (!elt_data) 1971 return -ENOMEM; 1972 1973 for_each_hist_key_field(i, hist_data) { 1974 key_field = hist_data->fields[i]; 1975 1976 if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 1977 elt_data->comm = kzalloc(size, GFP_KERNEL); 1978 if (!elt_data->comm) { 1979 kfree(elt_data); 1980 return -ENOMEM; 1981 } 1982 break; 1983 } 1984 } 1985 1986 n_str = hist_data->n_field_var_str + hist_data->n_max_var_str; 1987 1988 size = STR_VAR_LEN_MAX; 1989 1990 for (i = 0; i < n_str; i++) { 1991 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL); 1992 if (!elt_data->field_var_str[i]) { 1993 hist_elt_data_free(elt_data); 1994 return -ENOMEM; 1995 } 1996 } 1997 1998 elt->private_data = elt_data; 1999 2000 return 0; 2001 } 2002 2003 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt) 2004 { 2005 struct hist_elt_data *elt_data = elt->private_data; 2006 2007 if (elt_data->comm) 2008 save_comm(elt_data->comm, current); 2009 } 2010 2011 static const struct tracing_map_ops hist_trigger_elt_data_ops = { 2012 .elt_alloc = hist_trigger_elt_data_alloc, 2013 .elt_free = hist_trigger_elt_data_free, 2014 .elt_init = hist_trigger_elt_data_init, 2015 }; 2016 2017 static const char *get_hist_field_flags(struct hist_field *hist_field) 2018 { 2019 const char *flags_str = NULL; 2020 2021 if (hist_field->flags & HIST_FIELD_FL_HEX) 2022 flags_str = "hex"; 2023 else if (hist_field->flags & HIST_FIELD_FL_SYM) 2024 flags_str = "sym"; 2025 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET) 2026 flags_str = "sym-offset"; 2027 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME) 2028 flags_str = "execname"; 2029 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL) 2030 flags_str = "syscall"; 2031 else if (hist_field->flags & HIST_FIELD_FL_LOG2) 2032 flags_str = "log2"; 2033 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS) 2034 flags_str = "usecs"; 2035 2036 return flags_str; 2037 } 2038 2039 static void expr_field_str(struct hist_field *field, char *expr) 2040 { 2041 if (field->flags & HIST_FIELD_FL_VAR_REF) 2042 strcat(expr, "$"); 2043 2044 strcat(expr, hist_field_name(field, 0)); 2045 2046 if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) { 2047 const char *flags_str = get_hist_field_flags(field); 2048 2049 if (flags_str) { 2050 strcat(expr, "."); 2051 strcat(expr, flags_str); 2052 } 2053 } 2054 } 2055 2056 static char *expr_str(struct hist_field *field, unsigned int level) 2057 { 2058 char *expr; 2059 2060 if (level > 1) 2061 return NULL; 2062 2063 expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2064 if (!expr) 2065 return NULL; 2066 2067 if (!field->operands[0]) { 2068 expr_field_str(field, expr); 2069 return expr; 2070 } 2071 2072 if (field->operator == FIELD_OP_UNARY_MINUS) { 2073 char *subexpr; 2074 2075 strcat(expr, "-("); 2076 subexpr = expr_str(field->operands[0], ++level); 2077 if (!subexpr) { 2078 kfree(expr); 2079 return NULL; 2080 } 2081 strcat(expr, subexpr); 2082 strcat(expr, ")"); 2083 2084 kfree(subexpr); 2085 2086 return expr; 2087 } 2088 2089 expr_field_str(field->operands[0], expr); 2090 2091 switch (field->operator) { 2092 case FIELD_OP_MINUS: 2093 strcat(expr, "-"); 2094 break; 2095 case FIELD_OP_PLUS: 2096 strcat(expr, "+"); 2097 break; 2098 default: 2099 kfree(expr); 2100 return NULL; 2101 } 2102 2103 expr_field_str(field->operands[1], expr); 2104 2105 return expr; 2106 } 2107 2108 static int contains_operator(char *str) 2109 { 2110 enum field_op_id field_op = FIELD_OP_NONE; 2111 char *op; 2112 2113 op = strpbrk(str, "+-"); 2114 if (!op) 2115 return FIELD_OP_NONE; 2116 2117 switch (*op) { 2118 case '-': 2119 if (*str == '-') 2120 field_op = FIELD_OP_UNARY_MINUS; 2121 else 2122 field_op = FIELD_OP_MINUS; 2123 break; 2124 case '+': 2125 field_op = FIELD_OP_PLUS; 2126 break; 2127 default: 2128 break; 2129 } 2130 2131 return field_op; 2132 } 2133 2134 static void destroy_hist_field(struct hist_field *hist_field, 2135 unsigned int level) 2136 { 2137 unsigned int i; 2138 2139 if (level > 3) 2140 return; 2141 2142 if (!hist_field) 2143 return; 2144 2145 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) 2146 destroy_hist_field(hist_field->operands[i], level + 1); 2147 2148 kfree(hist_field->var.name); 2149 kfree(hist_field->name); 2150 kfree(hist_field->type); 2151 2152 kfree(hist_field); 2153 } 2154 2155 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, 2156 struct ftrace_event_field *field, 2157 unsigned long flags, 2158 char *var_name) 2159 { 2160 struct hist_field *hist_field; 2161 2162 if (field && is_function_field(field)) 2163 return NULL; 2164 2165 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL); 2166 if (!hist_field) 2167 return NULL; 2168 2169 hist_field->hist_data = hist_data; 2170 2171 if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS) 2172 goto out; /* caller will populate */ 2173 2174 if (flags & HIST_FIELD_FL_VAR_REF) { 2175 hist_field->fn = hist_field_var_ref; 2176 goto out; 2177 } 2178 2179 if (flags & HIST_FIELD_FL_HITCOUNT) { 2180 hist_field->fn = hist_field_counter; 2181 hist_field->size = sizeof(u64); 2182 hist_field->type = kstrdup("u64", GFP_KERNEL); 2183 if (!hist_field->type) 2184 goto free; 2185 goto out; 2186 } 2187 2188 if (flags & HIST_FIELD_FL_STACKTRACE) { 2189 hist_field->fn = hist_field_none; 2190 goto out; 2191 } 2192 2193 if (flags & HIST_FIELD_FL_LOG2) { 2194 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2; 2195 hist_field->fn = hist_field_log2; 2196 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL); 2197 hist_field->size = hist_field->operands[0]->size; 2198 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL); 2199 if (!hist_field->type) 2200 goto free; 2201 goto out; 2202 } 2203 2204 if (flags & HIST_FIELD_FL_TIMESTAMP) { 2205 hist_field->fn = hist_field_timestamp; 2206 hist_field->size = sizeof(u64); 2207 hist_field->type = kstrdup("u64", GFP_KERNEL); 2208 if (!hist_field->type) 2209 goto free; 2210 goto out; 2211 } 2212 2213 if (flags & HIST_FIELD_FL_CPU) { 2214 hist_field->fn = hist_field_cpu; 2215 hist_field->size = sizeof(int); 2216 hist_field->type = kstrdup("unsigned int", GFP_KERNEL); 2217 if (!hist_field->type) 2218 goto free; 2219 goto out; 2220 } 2221 2222 if (WARN_ON_ONCE(!field)) 2223 goto out; 2224 2225 if (is_string_field(field)) { 2226 flags |= HIST_FIELD_FL_STRING; 2227 2228 hist_field->size = MAX_FILTER_STR_VAL; 2229 hist_field->type = kstrdup(field->type, GFP_KERNEL); 2230 if (!hist_field->type) 2231 goto free; 2232 2233 if (field->filter_type == FILTER_STATIC_STRING) 2234 hist_field->fn = hist_field_string; 2235 else if (field->filter_type == FILTER_DYN_STRING) 2236 hist_field->fn = hist_field_dynstring; 2237 else 2238 hist_field->fn = hist_field_pstring; 2239 } else { 2240 hist_field->size = field->size; 2241 hist_field->is_signed = field->is_signed; 2242 hist_field->type = kstrdup(field->type, GFP_KERNEL); 2243 if (!hist_field->type) 2244 goto free; 2245 2246 hist_field->fn = select_value_fn(field->size, 2247 field->is_signed); 2248 if (!hist_field->fn) { 2249 destroy_hist_field(hist_field, 0); 2250 return NULL; 2251 } 2252 } 2253 out: 2254 hist_field->field = field; 2255 hist_field->flags = flags; 2256 2257 if (var_name) { 2258 hist_field->var.name = kstrdup(var_name, GFP_KERNEL); 2259 if (!hist_field->var.name) 2260 goto free; 2261 } 2262 2263 return hist_field; 2264 free: 2265 destroy_hist_field(hist_field, 0); 2266 return NULL; 2267 } 2268 2269 static void destroy_hist_fields(struct hist_trigger_data *hist_data) 2270 { 2271 unsigned int i; 2272 2273 for (i = 0; i < HIST_FIELDS_MAX; i++) { 2274 if (hist_data->fields[i]) { 2275 destroy_hist_field(hist_data->fields[i], 0); 2276 hist_data->fields[i] = NULL; 2277 } 2278 } 2279 } 2280 2281 static int init_var_ref(struct hist_field *ref_field, 2282 struct hist_field *var_field, 2283 char *system, char *event_name) 2284 { 2285 int err = 0; 2286 2287 ref_field->var.idx = var_field->var.idx; 2288 ref_field->var.hist_data = var_field->hist_data; 2289 ref_field->size = var_field->size; 2290 ref_field->is_signed = var_field->is_signed; 2291 ref_field->flags |= var_field->flags & 2292 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2293 2294 if (system) { 2295 ref_field->system = kstrdup(system, GFP_KERNEL); 2296 if (!ref_field->system) 2297 return -ENOMEM; 2298 } 2299 2300 if (event_name) { 2301 ref_field->event_name = kstrdup(event_name, GFP_KERNEL); 2302 if (!ref_field->event_name) { 2303 err = -ENOMEM; 2304 goto free; 2305 } 2306 } 2307 2308 if (var_field->var.name) { 2309 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL); 2310 if (!ref_field->name) { 2311 err = -ENOMEM; 2312 goto free; 2313 } 2314 } else if (var_field->name) { 2315 ref_field->name = kstrdup(var_field->name, GFP_KERNEL); 2316 if (!ref_field->name) { 2317 err = -ENOMEM; 2318 goto free; 2319 } 2320 } 2321 2322 ref_field->type = kstrdup(var_field->type, GFP_KERNEL); 2323 if (!ref_field->type) { 2324 err = -ENOMEM; 2325 goto free; 2326 } 2327 out: 2328 return err; 2329 free: 2330 kfree(ref_field->system); 2331 kfree(ref_field->event_name); 2332 kfree(ref_field->name); 2333 2334 goto out; 2335 } 2336 2337 static struct hist_field *create_var_ref(struct hist_field *var_field, 2338 char *system, char *event_name) 2339 { 2340 unsigned long flags = HIST_FIELD_FL_VAR_REF; 2341 struct hist_field *ref_field; 2342 2343 ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL); 2344 if (ref_field) { 2345 if (init_var_ref(ref_field, var_field, system, event_name)) { 2346 destroy_hist_field(ref_field, 0); 2347 return NULL; 2348 } 2349 } 2350 2351 return ref_field; 2352 } 2353 2354 static bool is_var_ref(char *var_name) 2355 { 2356 if (!var_name || strlen(var_name) < 2 || var_name[0] != '$') 2357 return false; 2358 2359 return true; 2360 } 2361 2362 static char *field_name_from_var(struct hist_trigger_data *hist_data, 2363 char *var_name) 2364 { 2365 char *name, *field; 2366 unsigned int i; 2367 2368 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 2369 name = hist_data->attrs->var_defs.name[i]; 2370 2371 if (strcmp(var_name, name) == 0) { 2372 field = hist_data->attrs->var_defs.expr[i]; 2373 if (contains_operator(field) || is_var_ref(field)) 2374 continue; 2375 return field; 2376 } 2377 } 2378 2379 return NULL; 2380 } 2381 2382 static char *local_field_var_ref(struct hist_trigger_data *hist_data, 2383 char *system, char *event_name, 2384 char *var_name) 2385 { 2386 struct trace_event_call *call; 2387 2388 if (system && event_name) { 2389 call = hist_data->event_file->event_call; 2390 2391 if (strcmp(system, call->class->system) != 0) 2392 return NULL; 2393 2394 if (strcmp(event_name, trace_event_name(call)) != 0) 2395 return NULL; 2396 } 2397 2398 if (!!system != !!event_name) 2399 return NULL; 2400 2401 if (!is_var_ref(var_name)) 2402 return NULL; 2403 2404 var_name++; 2405 2406 return field_name_from_var(hist_data, var_name); 2407 } 2408 2409 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data, 2410 char *system, char *event_name, 2411 char *var_name) 2412 { 2413 struct hist_field *var_field = NULL, *ref_field = NULL; 2414 2415 if (!is_var_ref(var_name)) 2416 return NULL; 2417 2418 var_name++; 2419 2420 var_field = find_event_var(hist_data, system, event_name, var_name); 2421 if (var_field) 2422 ref_field = create_var_ref(var_field, system, event_name); 2423 2424 if (!ref_field) 2425 hist_err_event("Couldn't find variable: $", 2426 system, event_name, var_name); 2427 2428 return ref_field; 2429 } 2430 2431 static struct ftrace_event_field * 2432 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, 2433 char *field_str, unsigned long *flags) 2434 { 2435 struct ftrace_event_field *field = NULL; 2436 char *field_name, *modifier, *str; 2437 2438 modifier = str = kstrdup(field_str, GFP_KERNEL); 2439 if (!modifier) 2440 return ERR_PTR(-ENOMEM); 2441 2442 field_name = strsep(&modifier, "."); 2443 if (modifier) { 2444 if (strcmp(modifier, "hex") == 0) 2445 *flags |= HIST_FIELD_FL_HEX; 2446 else if (strcmp(modifier, "sym") == 0) 2447 *flags |= HIST_FIELD_FL_SYM; 2448 else if (strcmp(modifier, "sym-offset") == 0) 2449 *flags |= HIST_FIELD_FL_SYM_OFFSET; 2450 else if ((strcmp(modifier, "execname") == 0) && 2451 (strcmp(field_name, "common_pid") == 0)) 2452 *flags |= HIST_FIELD_FL_EXECNAME; 2453 else if (strcmp(modifier, "syscall") == 0) 2454 *flags |= HIST_FIELD_FL_SYSCALL; 2455 else if (strcmp(modifier, "log2") == 0) 2456 *flags |= HIST_FIELD_FL_LOG2; 2457 else if (strcmp(modifier, "usecs") == 0) 2458 *flags |= HIST_FIELD_FL_TIMESTAMP_USECS; 2459 else { 2460 hist_err("Invalid field modifier: ", modifier); 2461 field = ERR_PTR(-EINVAL); 2462 goto out; 2463 } 2464 } 2465 2466 if (strcmp(field_name, "common_timestamp") == 0) { 2467 *flags |= HIST_FIELD_FL_TIMESTAMP; 2468 hist_data->enable_timestamps = true; 2469 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS) 2470 hist_data->attrs->ts_in_usecs = true; 2471 } else if (strcmp(field_name, "cpu") == 0) 2472 *flags |= HIST_FIELD_FL_CPU; 2473 else { 2474 field = trace_find_event_field(file->event_call, field_name); 2475 if (!field || !field->size) { 2476 hist_err("Couldn't find field: ", field_name); 2477 field = ERR_PTR(-EINVAL); 2478 goto out; 2479 } 2480 } 2481 out: 2482 kfree(str); 2483 2484 return field; 2485 } 2486 2487 static struct hist_field *create_alias(struct hist_trigger_data *hist_data, 2488 struct hist_field *var_ref, 2489 char *var_name) 2490 { 2491 struct hist_field *alias = NULL; 2492 unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR; 2493 2494 alias = create_hist_field(hist_data, NULL, flags, var_name); 2495 if (!alias) 2496 return NULL; 2497 2498 alias->fn = var_ref->fn; 2499 alias->operands[0] = var_ref; 2500 2501 if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) { 2502 destroy_hist_field(alias, 0); 2503 return NULL; 2504 } 2505 2506 return alias; 2507 } 2508 2509 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data, 2510 struct trace_event_file *file, char *str, 2511 unsigned long *flags, char *var_name) 2512 { 2513 char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str; 2514 struct ftrace_event_field *field = NULL; 2515 struct hist_field *hist_field = NULL; 2516 int ret = 0; 2517 2518 s = strchr(str, '.'); 2519 if (s) { 2520 s = strchr(++s, '.'); 2521 if (s) { 2522 ref_system = strsep(&str, "."); 2523 if (!str) { 2524 ret = -EINVAL; 2525 goto out; 2526 } 2527 ref_event = strsep(&str, "."); 2528 if (!str) { 2529 ret = -EINVAL; 2530 goto out; 2531 } 2532 ref_var = str; 2533 } 2534 } 2535 2536 s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var); 2537 if (!s) { 2538 hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var); 2539 if (hist_field) { 2540 hist_data->var_refs[hist_data->n_var_refs] = hist_field; 2541 hist_field->var_ref_idx = hist_data->n_var_refs++; 2542 if (var_name) { 2543 hist_field = create_alias(hist_data, hist_field, var_name); 2544 if (!hist_field) { 2545 ret = -ENOMEM; 2546 goto out; 2547 } 2548 } 2549 return hist_field; 2550 } 2551 } else 2552 str = s; 2553 2554 field = parse_field(hist_data, file, str, flags); 2555 if (IS_ERR(field)) { 2556 ret = PTR_ERR(field); 2557 goto out; 2558 } 2559 2560 hist_field = create_hist_field(hist_data, field, *flags, var_name); 2561 if (!hist_field) { 2562 ret = -ENOMEM; 2563 goto out; 2564 } 2565 2566 return hist_field; 2567 out: 2568 return ERR_PTR(ret); 2569 } 2570 2571 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2572 struct trace_event_file *file, 2573 char *str, unsigned long flags, 2574 char *var_name, unsigned int level); 2575 2576 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data, 2577 struct trace_event_file *file, 2578 char *str, unsigned long flags, 2579 char *var_name, unsigned int level) 2580 { 2581 struct hist_field *operand1, *expr = NULL; 2582 unsigned long operand_flags; 2583 int ret = 0; 2584 char *s; 2585 2586 /* we support only -(xxx) i.e. explicit parens required */ 2587 2588 if (level > 3) { 2589 hist_err("Too many subexpressions (3 max): ", str); 2590 ret = -EINVAL; 2591 goto free; 2592 } 2593 2594 str++; /* skip leading '-' */ 2595 2596 s = strchr(str, '('); 2597 if (s) 2598 str++; 2599 else { 2600 ret = -EINVAL; 2601 goto free; 2602 } 2603 2604 s = strrchr(str, ')'); 2605 if (s) 2606 *s = '\0'; 2607 else { 2608 ret = -EINVAL; /* no closing ')' */ 2609 goto free; 2610 } 2611 2612 flags |= HIST_FIELD_FL_EXPR; 2613 expr = create_hist_field(hist_data, NULL, flags, var_name); 2614 if (!expr) { 2615 ret = -ENOMEM; 2616 goto free; 2617 } 2618 2619 operand_flags = 0; 2620 operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level); 2621 if (IS_ERR(operand1)) { 2622 ret = PTR_ERR(operand1); 2623 goto free; 2624 } 2625 2626 expr->flags |= operand1->flags & 2627 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2628 expr->fn = hist_field_unary_minus; 2629 expr->operands[0] = operand1; 2630 expr->operator = FIELD_OP_UNARY_MINUS; 2631 expr->name = expr_str(expr, 0); 2632 expr->type = kstrdup(operand1->type, GFP_KERNEL); 2633 if (!expr->type) { 2634 ret = -ENOMEM; 2635 goto free; 2636 } 2637 2638 return expr; 2639 free: 2640 destroy_hist_field(expr, 0); 2641 return ERR_PTR(ret); 2642 } 2643 2644 static int check_expr_operands(struct hist_field *operand1, 2645 struct hist_field *operand2) 2646 { 2647 unsigned long operand1_flags = operand1->flags; 2648 unsigned long operand2_flags = operand2->flags; 2649 2650 if ((operand1_flags & HIST_FIELD_FL_VAR_REF) || 2651 (operand1_flags & HIST_FIELD_FL_ALIAS)) { 2652 struct hist_field *var; 2653 2654 var = find_var_field(operand1->var.hist_data, operand1->name); 2655 if (!var) 2656 return -EINVAL; 2657 operand1_flags = var->flags; 2658 } 2659 2660 if ((operand2_flags & HIST_FIELD_FL_VAR_REF) || 2661 (operand2_flags & HIST_FIELD_FL_ALIAS)) { 2662 struct hist_field *var; 2663 2664 var = find_var_field(operand2->var.hist_data, operand2->name); 2665 if (!var) 2666 return -EINVAL; 2667 operand2_flags = var->flags; 2668 } 2669 2670 if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) != 2671 (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) { 2672 hist_err("Timestamp units in expression don't match", NULL); 2673 return -EINVAL; 2674 } 2675 2676 return 0; 2677 } 2678 2679 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2680 struct trace_event_file *file, 2681 char *str, unsigned long flags, 2682 char *var_name, unsigned int level) 2683 { 2684 struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL; 2685 unsigned long operand_flags; 2686 int field_op, ret = -EINVAL; 2687 char *sep, *operand1_str; 2688 2689 if (level > 3) { 2690 hist_err("Too many subexpressions (3 max): ", str); 2691 return ERR_PTR(-EINVAL); 2692 } 2693 2694 field_op = contains_operator(str); 2695 2696 if (field_op == FIELD_OP_NONE) 2697 return parse_atom(hist_data, file, str, &flags, var_name); 2698 2699 if (field_op == FIELD_OP_UNARY_MINUS) 2700 return parse_unary(hist_data, file, str, flags, var_name, ++level); 2701 2702 switch (field_op) { 2703 case FIELD_OP_MINUS: 2704 sep = "-"; 2705 break; 2706 case FIELD_OP_PLUS: 2707 sep = "+"; 2708 break; 2709 default: 2710 goto free; 2711 } 2712 2713 operand1_str = strsep(&str, sep); 2714 if (!operand1_str || !str) 2715 goto free; 2716 2717 operand_flags = 0; 2718 operand1 = parse_atom(hist_data, file, operand1_str, 2719 &operand_flags, NULL); 2720 if (IS_ERR(operand1)) { 2721 ret = PTR_ERR(operand1); 2722 operand1 = NULL; 2723 goto free; 2724 } 2725 2726 /* rest of string could be another expression e.g. b+c in a+b+c */ 2727 operand_flags = 0; 2728 operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level); 2729 if (IS_ERR(operand2)) { 2730 ret = PTR_ERR(operand2); 2731 operand2 = NULL; 2732 goto free; 2733 } 2734 2735 ret = check_expr_operands(operand1, operand2); 2736 if (ret) 2737 goto free; 2738 2739 flags |= HIST_FIELD_FL_EXPR; 2740 2741 flags |= operand1->flags & 2742 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2743 2744 expr = create_hist_field(hist_data, NULL, flags, var_name); 2745 if (!expr) { 2746 ret = -ENOMEM; 2747 goto free; 2748 } 2749 2750 operand1->read_once = true; 2751 operand2->read_once = true; 2752 2753 expr->operands[0] = operand1; 2754 expr->operands[1] = operand2; 2755 expr->operator = field_op; 2756 expr->name = expr_str(expr, 0); 2757 expr->type = kstrdup(operand1->type, GFP_KERNEL); 2758 if (!expr->type) { 2759 ret = -ENOMEM; 2760 goto free; 2761 } 2762 2763 switch (field_op) { 2764 case FIELD_OP_MINUS: 2765 expr->fn = hist_field_minus; 2766 break; 2767 case FIELD_OP_PLUS: 2768 expr->fn = hist_field_plus; 2769 break; 2770 default: 2771 ret = -EINVAL; 2772 goto free; 2773 } 2774 2775 return expr; 2776 free: 2777 destroy_hist_field(operand1, 0); 2778 destroy_hist_field(operand2, 0); 2779 destroy_hist_field(expr, 0); 2780 2781 return ERR_PTR(ret); 2782 } 2783 2784 static char *find_trigger_filter(struct hist_trigger_data *hist_data, 2785 struct trace_event_file *file) 2786 { 2787 struct event_trigger_data *test; 2788 2789 list_for_each_entry_rcu(test, &file->triggers, list) { 2790 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2791 if (test->private_data == hist_data) 2792 return test->filter_str; 2793 } 2794 } 2795 2796 return NULL; 2797 } 2798 2799 static struct event_command trigger_hist_cmd; 2800 static int event_hist_trigger_func(struct event_command *cmd_ops, 2801 struct trace_event_file *file, 2802 char *glob, char *cmd, char *param); 2803 2804 static bool compatible_keys(struct hist_trigger_data *target_hist_data, 2805 struct hist_trigger_data *hist_data, 2806 unsigned int n_keys) 2807 { 2808 struct hist_field *target_hist_field, *hist_field; 2809 unsigned int n, i, j; 2810 2811 if (hist_data->n_fields - hist_data->n_vals != n_keys) 2812 return false; 2813 2814 i = hist_data->n_vals; 2815 j = target_hist_data->n_vals; 2816 2817 for (n = 0; n < n_keys; n++) { 2818 hist_field = hist_data->fields[i + n]; 2819 target_hist_field = target_hist_data->fields[j + n]; 2820 2821 if (strcmp(hist_field->type, target_hist_field->type) != 0) 2822 return false; 2823 if (hist_field->size != target_hist_field->size) 2824 return false; 2825 if (hist_field->is_signed != target_hist_field->is_signed) 2826 return false; 2827 } 2828 2829 return true; 2830 } 2831 2832 static struct hist_trigger_data * 2833 find_compatible_hist(struct hist_trigger_data *target_hist_data, 2834 struct trace_event_file *file) 2835 { 2836 struct hist_trigger_data *hist_data; 2837 struct event_trigger_data *test; 2838 unsigned int n_keys; 2839 2840 n_keys = target_hist_data->n_fields - target_hist_data->n_vals; 2841 2842 list_for_each_entry_rcu(test, &file->triggers, list) { 2843 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2844 hist_data = test->private_data; 2845 2846 if (compatible_keys(target_hist_data, hist_data, n_keys)) 2847 return hist_data; 2848 } 2849 } 2850 2851 return NULL; 2852 } 2853 2854 static struct trace_event_file *event_file(struct trace_array *tr, 2855 char *system, char *event_name) 2856 { 2857 struct trace_event_file *file; 2858 2859 file = __find_event_file(tr, system, event_name); 2860 if (!file) 2861 return ERR_PTR(-EINVAL); 2862 2863 return file; 2864 } 2865 2866 static struct hist_field * 2867 find_synthetic_field_var(struct hist_trigger_data *target_hist_data, 2868 char *system, char *event_name, char *field_name) 2869 { 2870 struct hist_field *event_var; 2871 char *synthetic_name; 2872 2873 synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2874 if (!synthetic_name) 2875 return ERR_PTR(-ENOMEM); 2876 2877 strcpy(synthetic_name, "synthetic_"); 2878 strcat(synthetic_name, field_name); 2879 2880 event_var = find_event_var(target_hist_data, system, event_name, synthetic_name); 2881 2882 kfree(synthetic_name); 2883 2884 return event_var; 2885 } 2886 2887 /** 2888 * create_field_var_hist - Automatically create a histogram and var for a field 2889 * @target_hist_data: The target hist trigger 2890 * @subsys_name: Optional subsystem name 2891 * @event_name: Optional event name 2892 * @field_name: The name of the field (and the resulting variable) 2893 * 2894 * Hist trigger actions fetch data from variables, not directly from 2895 * events. However, for convenience, users are allowed to directly 2896 * specify an event field in an action, which will be automatically 2897 * converted into a variable on their behalf. 2898 2899 * If a user specifies a field on an event that isn't the event the 2900 * histogram currently being defined (the target event histogram), the 2901 * only way that can be accomplished is if a new hist trigger is 2902 * created and the field variable defined on that. 2903 * 2904 * This function creates a new histogram compatible with the target 2905 * event (meaning a histogram with the same key as the target 2906 * histogram), and creates a variable for the specified field, but 2907 * with 'synthetic_' prepended to the variable name in order to avoid 2908 * collision with normal field variables. 2909 * 2910 * Return: The variable created for the field. 2911 */ 2912 static struct hist_field * 2913 create_field_var_hist(struct hist_trigger_data *target_hist_data, 2914 char *subsys_name, char *event_name, char *field_name) 2915 { 2916 struct trace_array *tr = target_hist_data->event_file->tr; 2917 struct hist_field *event_var = ERR_PTR(-EINVAL); 2918 struct hist_trigger_data *hist_data; 2919 unsigned int i, n, first = true; 2920 struct field_var_hist *var_hist; 2921 struct trace_event_file *file; 2922 struct hist_field *key_field; 2923 char *saved_filter; 2924 char *cmd; 2925 int ret; 2926 2927 if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) { 2928 hist_err_event("onmatch: Too many field variables defined: ", 2929 subsys_name, event_name, field_name); 2930 return ERR_PTR(-EINVAL); 2931 } 2932 2933 file = event_file(tr, subsys_name, event_name); 2934 2935 if (IS_ERR(file)) { 2936 hist_err_event("onmatch: Event file not found: ", 2937 subsys_name, event_name, field_name); 2938 ret = PTR_ERR(file); 2939 return ERR_PTR(ret); 2940 } 2941 2942 /* 2943 * Look for a histogram compatible with target. We'll use the 2944 * found histogram specification to create a new matching 2945 * histogram with our variable on it. target_hist_data is not 2946 * yet a registered histogram so we can't use that. 2947 */ 2948 hist_data = find_compatible_hist(target_hist_data, file); 2949 if (!hist_data) { 2950 hist_err_event("onmatch: Matching event histogram not found: ", 2951 subsys_name, event_name, field_name); 2952 return ERR_PTR(-EINVAL); 2953 } 2954 2955 /* See if a synthetic field variable has already been created */ 2956 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 2957 event_name, field_name); 2958 if (!IS_ERR_OR_NULL(event_var)) 2959 return event_var; 2960 2961 var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL); 2962 if (!var_hist) 2963 return ERR_PTR(-ENOMEM); 2964 2965 cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2966 if (!cmd) { 2967 kfree(var_hist); 2968 return ERR_PTR(-ENOMEM); 2969 } 2970 2971 /* Use the same keys as the compatible histogram */ 2972 strcat(cmd, "keys="); 2973 2974 for_each_hist_key_field(i, hist_data) { 2975 key_field = hist_data->fields[i]; 2976 if (!first) 2977 strcat(cmd, ","); 2978 strcat(cmd, key_field->field->name); 2979 first = false; 2980 } 2981 2982 /* Create the synthetic field variable specification */ 2983 strcat(cmd, ":synthetic_"); 2984 strcat(cmd, field_name); 2985 strcat(cmd, "="); 2986 strcat(cmd, field_name); 2987 2988 /* Use the same filter as the compatible histogram */ 2989 saved_filter = find_trigger_filter(hist_data, file); 2990 if (saved_filter) { 2991 strcat(cmd, " if "); 2992 strcat(cmd, saved_filter); 2993 } 2994 2995 var_hist->cmd = kstrdup(cmd, GFP_KERNEL); 2996 if (!var_hist->cmd) { 2997 kfree(cmd); 2998 kfree(var_hist); 2999 return ERR_PTR(-ENOMEM); 3000 } 3001 3002 /* Save the compatible histogram information */ 3003 var_hist->hist_data = hist_data; 3004 3005 /* Create the new histogram with our variable */ 3006 ret = event_hist_trigger_func(&trigger_hist_cmd, file, 3007 "", "hist", cmd); 3008 if (ret) { 3009 kfree(cmd); 3010 kfree(var_hist->cmd); 3011 kfree(var_hist); 3012 hist_err_event("onmatch: Couldn't create histogram for field: ", 3013 subsys_name, event_name, field_name); 3014 return ERR_PTR(ret); 3015 } 3016 3017 kfree(cmd); 3018 3019 /* If we can't find the variable, something went wrong */ 3020 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 3021 event_name, field_name); 3022 if (IS_ERR_OR_NULL(event_var)) { 3023 kfree(var_hist->cmd); 3024 kfree(var_hist); 3025 hist_err_event("onmatch: Couldn't find synthetic variable: ", 3026 subsys_name, event_name, field_name); 3027 return ERR_PTR(-EINVAL); 3028 } 3029 3030 n = target_hist_data->n_field_var_hists; 3031 target_hist_data->field_var_hists[n] = var_hist; 3032 target_hist_data->n_field_var_hists++; 3033 3034 return event_var; 3035 } 3036 3037 static struct hist_field * 3038 find_target_event_var(struct hist_trigger_data *hist_data, 3039 char *subsys_name, char *event_name, char *var_name) 3040 { 3041 struct trace_event_file *file = hist_data->event_file; 3042 struct hist_field *hist_field = NULL; 3043 3044 if (subsys_name) { 3045 struct trace_event_call *call; 3046 3047 if (!event_name) 3048 return NULL; 3049 3050 call = file->event_call; 3051 3052 if (strcmp(subsys_name, call->class->system) != 0) 3053 return NULL; 3054 3055 if (strcmp(event_name, trace_event_name(call)) != 0) 3056 return NULL; 3057 } 3058 3059 hist_field = find_var_field(hist_data, var_name); 3060 3061 return hist_field; 3062 } 3063 3064 static inline void __update_field_vars(struct tracing_map_elt *elt, 3065 struct ring_buffer_event *rbe, 3066 void *rec, 3067 struct field_var **field_vars, 3068 unsigned int n_field_vars, 3069 unsigned int field_var_str_start) 3070 { 3071 struct hist_elt_data *elt_data = elt->private_data; 3072 unsigned int i, j, var_idx; 3073 u64 var_val; 3074 3075 for (i = 0, j = field_var_str_start; i < n_field_vars; i++) { 3076 struct field_var *field_var = field_vars[i]; 3077 struct hist_field *var = field_var->var; 3078 struct hist_field *val = field_var->val; 3079 3080 var_val = val->fn(val, elt, rbe, rec); 3081 var_idx = var->var.idx; 3082 3083 if (val->flags & HIST_FIELD_FL_STRING) { 3084 char *str = elt_data->field_var_str[j++]; 3085 char *val_str = (char *)(uintptr_t)var_val; 3086 3087 strscpy(str, val_str, STR_VAR_LEN_MAX); 3088 var_val = (u64)(uintptr_t)str; 3089 } 3090 tracing_map_set_var(elt, var_idx, var_val); 3091 } 3092 } 3093 3094 static void update_field_vars(struct hist_trigger_data *hist_data, 3095 struct tracing_map_elt *elt, 3096 struct ring_buffer_event *rbe, 3097 void *rec) 3098 { 3099 __update_field_vars(elt, rbe, rec, hist_data->field_vars, 3100 hist_data->n_field_vars, 0); 3101 } 3102 3103 static void update_max_vars(struct hist_trigger_data *hist_data, 3104 struct tracing_map_elt *elt, 3105 struct ring_buffer_event *rbe, 3106 void *rec) 3107 { 3108 __update_field_vars(elt, rbe, rec, hist_data->max_vars, 3109 hist_data->n_max_vars, hist_data->n_field_var_str); 3110 } 3111 3112 static struct hist_field *create_var(struct hist_trigger_data *hist_data, 3113 struct trace_event_file *file, 3114 char *name, int size, const char *type) 3115 { 3116 struct hist_field *var; 3117 int idx; 3118 3119 if (find_var(hist_data, file, name) && !hist_data->remove) { 3120 var = ERR_PTR(-EINVAL); 3121 goto out; 3122 } 3123 3124 var = kzalloc(sizeof(struct hist_field), GFP_KERNEL); 3125 if (!var) { 3126 var = ERR_PTR(-ENOMEM); 3127 goto out; 3128 } 3129 3130 idx = tracing_map_add_var(hist_data->map); 3131 if (idx < 0) { 3132 kfree(var); 3133 var = ERR_PTR(-EINVAL); 3134 goto out; 3135 } 3136 3137 var->flags = HIST_FIELD_FL_VAR; 3138 var->var.idx = idx; 3139 var->var.hist_data = var->hist_data = hist_data; 3140 var->size = size; 3141 var->var.name = kstrdup(name, GFP_KERNEL); 3142 var->type = kstrdup(type, GFP_KERNEL); 3143 if (!var->var.name || !var->type) { 3144 kfree(var->var.name); 3145 kfree(var->type); 3146 kfree(var); 3147 var = ERR_PTR(-ENOMEM); 3148 } 3149 out: 3150 return var; 3151 } 3152 3153 static struct field_var *create_field_var(struct hist_trigger_data *hist_data, 3154 struct trace_event_file *file, 3155 char *field_name) 3156 { 3157 struct hist_field *val = NULL, *var = NULL; 3158 unsigned long flags = HIST_FIELD_FL_VAR; 3159 struct field_var *field_var; 3160 int ret = 0; 3161 3162 if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) { 3163 hist_err("Too many field variables defined: ", field_name); 3164 ret = -EINVAL; 3165 goto err; 3166 } 3167 3168 val = parse_atom(hist_data, file, field_name, &flags, NULL); 3169 if (IS_ERR(val)) { 3170 hist_err("Couldn't parse field variable: ", field_name); 3171 ret = PTR_ERR(val); 3172 goto err; 3173 } 3174 3175 var = create_var(hist_data, file, field_name, val->size, val->type); 3176 if (IS_ERR(var)) { 3177 hist_err("Couldn't create or find variable: ", field_name); 3178 kfree(val); 3179 ret = PTR_ERR(var); 3180 goto err; 3181 } 3182 3183 field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL); 3184 if (!field_var) { 3185 kfree(val); 3186 kfree(var); 3187 ret = -ENOMEM; 3188 goto err; 3189 } 3190 3191 field_var->var = var; 3192 field_var->val = val; 3193 out: 3194 return field_var; 3195 err: 3196 field_var = ERR_PTR(ret); 3197 goto out; 3198 } 3199 3200 /** 3201 * create_target_field_var - Automatically create a variable for a field 3202 * @target_hist_data: The target hist trigger 3203 * @subsys_name: Optional subsystem name 3204 * @event_name: Optional event name 3205 * @var_name: The name of the field (and the resulting variable) 3206 * 3207 * Hist trigger actions fetch data from variables, not directly from 3208 * events. However, for convenience, users are allowed to directly 3209 * specify an event field in an action, which will be automatically 3210 * converted into a variable on their behalf. 3211 3212 * This function creates a field variable with the name var_name on 3213 * the hist trigger currently being defined on the target event. If 3214 * subsys_name and event_name are specified, this function simply 3215 * verifies that they do in fact match the target event subsystem and 3216 * event name. 3217 * 3218 * Return: The variable created for the field. 3219 */ 3220 static struct field_var * 3221 create_target_field_var(struct hist_trigger_data *target_hist_data, 3222 char *subsys_name, char *event_name, char *var_name) 3223 { 3224 struct trace_event_file *file = target_hist_data->event_file; 3225 3226 if (subsys_name) { 3227 struct trace_event_call *call; 3228 3229 if (!event_name) 3230 return NULL; 3231 3232 call = file->event_call; 3233 3234 if (strcmp(subsys_name, call->class->system) != 0) 3235 return NULL; 3236 3237 if (strcmp(event_name, trace_event_name(call)) != 0) 3238 return NULL; 3239 } 3240 3241 return create_field_var(target_hist_data, file, var_name); 3242 } 3243 3244 static void onmax_print(struct seq_file *m, 3245 struct hist_trigger_data *hist_data, 3246 struct tracing_map_elt *elt, 3247 struct action_data *data) 3248 { 3249 unsigned int i, save_var_idx, max_idx = data->onmax.max_var->var.idx; 3250 3251 seq_printf(m, "\n\tmax: %10llu", tracing_map_read_var(elt, max_idx)); 3252 3253 for (i = 0; i < hist_data->n_max_vars; i++) { 3254 struct hist_field *save_val = hist_data->max_vars[i]->val; 3255 struct hist_field *save_var = hist_data->max_vars[i]->var; 3256 u64 val; 3257 3258 save_var_idx = save_var->var.idx; 3259 3260 val = tracing_map_read_var(elt, save_var_idx); 3261 3262 if (save_val->flags & HIST_FIELD_FL_STRING) { 3263 seq_printf(m, " %s: %-32s", save_var->var.name, 3264 (char *)(uintptr_t)(val)); 3265 } else 3266 seq_printf(m, " %s: %10llu", save_var->var.name, val); 3267 } 3268 } 3269 3270 static void onmax_save(struct hist_trigger_data *hist_data, 3271 struct tracing_map_elt *elt, void *rec, 3272 struct ring_buffer_event *rbe, 3273 struct action_data *data, u64 *var_ref_vals) 3274 { 3275 unsigned int max_idx = data->onmax.max_var->var.idx; 3276 unsigned int max_var_ref_idx = data->onmax.max_var_ref_idx; 3277 3278 u64 var_val, max_val; 3279 3280 var_val = var_ref_vals[max_var_ref_idx]; 3281 max_val = tracing_map_read_var(elt, max_idx); 3282 3283 if (var_val <= max_val) 3284 return; 3285 3286 tracing_map_set_var(elt, max_idx, var_val); 3287 3288 update_max_vars(hist_data, elt, rbe, rec); 3289 } 3290 3291 static void onmax_destroy(struct action_data *data) 3292 { 3293 unsigned int i; 3294 3295 destroy_hist_field(data->onmax.max_var, 0); 3296 destroy_hist_field(data->onmax.var, 0); 3297 3298 kfree(data->onmax.var_str); 3299 kfree(data->onmax.fn_name); 3300 3301 for (i = 0; i < data->n_params; i++) 3302 kfree(data->params[i]); 3303 3304 kfree(data); 3305 } 3306 3307 static int onmax_create(struct hist_trigger_data *hist_data, 3308 struct action_data *data) 3309 { 3310 struct trace_event_file *file = hist_data->event_file; 3311 struct hist_field *var_field, *ref_field, *max_var; 3312 unsigned int var_ref_idx = hist_data->n_var_refs; 3313 struct field_var *field_var; 3314 char *onmax_var_str, *param; 3315 unsigned long flags; 3316 unsigned int i; 3317 int ret = 0; 3318 3319 onmax_var_str = data->onmax.var_str; 3320 if (onmax_var_str[0] != '$') { 3321 hist_err("onmax: For onmax(x), x must be a variable: ", onmax_var_str); 3322 return -EINVAL; 3323 } 3324 onmax_var_str++; 3325 3326 var_field = find_target_event_var(hist_data, NULL, NULL, onmax_var_str); 3327 if (!var_field) { 3328 hist_err("onmax: Couldn't find onmax variable: ", onmax_var_str); 3329 return -EINVAL; 3330 } 3331 3332 flags = HIST_FIELD_FL_VAR_REF; 3333 ref_field = create_hist_field(hist_data, NULL, flags, NULL); 3334 if (!ref_field) 3335 return -ENOMEM; 3336 3337 if (init_var_ref(ref_field, var_field, NULL, NULL)) { 3338 destroy_hist_field(ref_field, 0); 3339 ret = -ENOMEM; 3340 goto out; 3341 } 3342 hist_data->var_refs[hist_data->n_var_refs] = ref_field; 3343 ref_field->var_ref_idx = hist_data->n_var_refs++; 3344 data->onmax.var = ref_field; 3345 3346 data->fn = onmax_save; 3347 data->onmax.max_var_ref_idx = var_ref_idx; 3348 max_var = create_var(hist_data, file, "max", sizeof(u64), "u64"); 3349 if (IS_ERR(max_var)) { 3350 hist_err("onmax: Couldn't create onmax variable: ", "max"); 3351 ret = PTR_ERR(max_var); 3352 goto out; 3353 } 3354 data->onmax.max_var = max_var; 3355 3356 for (i = 0; i < data->n_params; i++) { 3357 param = kstrdup(data->params[i], GFP_KERNEL); 3358 if (!param) { 3359 ret = -ENOMEM; 3360 goto out; 3361 } 3362 3363 field_var = create_target_field_var(hist_data, NULL, NULL, param); 3364 if (IS_ERR(field_var)) { 3365 hist_err("onmax: Couldn't create field variable: ", param); 3366 ret = PTR_ERR(field_var); 3367 kfree(param); 3368 goto out; 3369 } 3370 3371 hist_data->max_vars[hist_data->n_max_vars++] = field_var; 3372 if (field_var->val->flags & HIST_FIELD_FL_STRING) 3373 hist_data->n_max_var_str++; 3374 3375 kfree(param); 3376 } 3377 out: 3378 return ret; 3379 } 3380 3381 static int parse_action_params(char *params, struct action_data *data) 3382 { 3383 char *param, *saved_param; 3384 int ret = 0; 3385 3386 while (params) { 3387 if (data->n_params >= SYNTH_FIELDS_MAX) 3388 goto out; 3389 3390 param = strsep(¶ms, ","); 3391 if (!param) { 3392 ret = -EINVAL; 3393 goto out; 3394 } 3395 3396 param = strstrip(param); 3397 if (strlen(param) < 2) { 3398 hist_err("Invalid action param: ", param); 3399 ret = -EINVAL; 3400 goto out; 3401 } 3402 3403 saved_param = kstrdup(param, GFP_KERNEL); 3404 if (!saved_param) { 3405 ret = -ENOMEM; 3406 goto out; 3407 } 3408 3409 data->params[data->n_params++] = saved_param; 3410 } 3411 out: 3412 return ret; 3413 } 3414 3415 static struct action_data *onmax_parse(char *str) 3416 { 3417 char *onmax_fn_name, *onmax_var_str; 3418 struct action_data *data; 3419 int ret = -EINVAL; 3420 3421 data = kzalloc(sizeof(*data), GFP_KERNEL); 3422 if (!data) 3423 return ERR_PTR(-ENOMEM); 3424 3425 onmax_var_str = strsep(&str, ")"); 3426 if (!onmax_var_str || !str) { 3427 ret = -EINVAL; 3428 goto free; 3429 } 3430 3431 data->onmax.var_str = kstrdup(onmax_var_str, GFP_KERNEL); 3432 if (!data->onmax.var_str) { 3433 ret = -ENOMEM; 3434 goto free; 3435 } 3436 3437 strsep(&str, "."); 3438 if (!str) 3439 goto free; 3440 3441 onmax_fn_name = strsep(&str, "("); 3442 if (!onmax_fn_name || !str) 3443 goto free; 3444 3445 if (strncmp(onmax_fn_name, "save", strlen("save")) == 0) { 3446 char *params = strsep(&str, ")"); 3447 3448 if (!params) { 3449 ret = -EINVAL; 3450 goto free; 3451 } 3452 3453 ret = parse_action_params(params, data); 3454 if (ret) 3455 goto free; 3456 } else 3457 goto free; 3458 3459 data->onmax.fn_name = kstrdup(onmax_fn_name, GFP_KERNEL); 3460 if (!data->onmax.fn_name) { 3461 ret = -ENOMEM; 3462 goto free; 3463 } 3464 out: 3465 return data; 3466 free: 3467 onmax_destroy(data); 3468 data = ERR_PTR(ret); 3469 goto out; 3470 } 3471 3472 static void onmatch_destroy(struct action_data *data) 3473 { 3474 unsigned int i; 3475 3476 mutex_lock(&synth_event_mutex); 3477 3478 kfree(data->onmatch.match_event); 3479 kfree(data->onmatch.match_event_system); 3480 kfree(data->onmatch.synth_event_name); 3481 3482 for (i = 0; i < data->n_params; i++) 3483 kfree(data->params[i]); 3484 3485 if (data->onmatch.synth_event) 3486 data->onmatch.synth_event->ref--; 3487 3488 kfree(data); 3489 3490 mutex_unlock(&synth_event_mutex); 3491 } 3492 3493 static void destroy_field_var(struct field_var *field_var) 3494 { 3495 if (!field_var) 3496 return; 3497 3498 destroy_hist_field(field_var->var, 0); 3499 destroy_hist_field(field_var->val, 0); 3500 3501 kfree(field_var); 3502 } 3503 3504 static void destroy_field_vars(struct hist_trigger_data *hist_data) 3505 { 3506 unsigned int i; 3507 3508 for (i = 0; i < hist_data->n_field_vars; i++) 3509 destroy_field_var(hist_data->field_vars[i]); 3510 } 3511 3512 static void save_field_var(struct hist_trigger_data *hist_data, 3513 struct field_var *field_var) 3514 { 3515 hist_data->field_vars[hist_data->n_field_vars++] = field_var; 3516 3517 if (field_var->val->flags & HIST_FIELD_FL_STRING) 3518 hist_data->n_field_var_str++; 3519 } 3520 3521 3522 static void destroy_synth_var_refs(struct hist_trigger_data *hist_data) 3523 { 3524 unsigned int i; 3525 3526 for (i = 0; i < hist_data->n_synth_var_refs; i++) 3527 destroy_hist_field(hist_data->synth_var_refs[i], 0); 3528 } 3529 3530 static void save_synth_var_ref(struct hist_trigger_data *hist_data, 3531 struct hist_field *var_ref) 3532 { 3533 hist_data->synth_var_refs[hist_data->n_synth_var_refs++] = var_ref; 3534 3535 hist_data->var_refs[hist_data->n_var_refs] = var_ref; 3536 var_ref->var_ref_idx = hist_data->n_var_refs++; 3537 } 3538 3539 static int check_synth_field(struct synth_event *event, 3540 struct hist_field *hist_field, 3541 unsigned int field_pos) 3542 { 3543 struct synth_field *field; 3544 3545 if (field_pos >= event->n_fields) 3546 return -EINVAL; 3547 3548 field = event->fields[field_pos]; 3549 3550 if (strcmp(field->type, hist_field->type) != 0) 3551 return -EINVAL; 3552 3553 return 0; 3554 } 3555 3556 static struct hist_field * 3557 onmatch_find_var(struct hist_trigger_data *hist_data, struct action_data *data, 3558 char *system, char *event, char *var) 3559 { 3560 struct hist_field *hist_field; 3561 3562 var++; /* skip '$' */ 3563 3564 hist_field = find_target_event_var(hist_data, system, event, var); 3565 if (!hist_field) { 3566 if (!system) { 3567 system = data->onmatch.match_event_system; 3568 event = data->onmatch.match_event; 3569 } 3570 3571 hist_field = find_event_var(hist_data, system, event, var); 3572 } 3573 3574 if (!hist_field) 3575 hist_err_event("onmatch: Couldn't find onmatch param: $", system, event, var); 3576 3577 return hist_field; 3578 } 3579 3580 static struct hist_field * 3581 onmatch_create_field_var(struct hist_trigger_data *hist_data, 3582 struct action_data *data, char *system, 3583 char *event, char *var) 3584 { 3585 struct hist_field *hist_field = NULL; 3586 struct field_var *field_var; 3587 3588 /* 3589 * First try to create a field var on the target event (the 3590 * currently being defined). This will create a variable for 3591 * unqualified fields on the target event, or if qualified, 3592 * target fields that have qualified names matching the target. 3593 */ 3594 field_var = create_target_field_var(hist_data, system, event, var); 3595 3596 if (field_var && !IS_ERR(field_var)) { 3597 save_field_var(hist_data, field_var); 3598 hist_field = field_var->var; 3599 } else { 3600 field_var = NULL; 3601 /* 3602 * If no explicit system.event is specfied, default to 3603 * looking for fields on the onmatch(system.event.xxx) 3604 * event. 3605 */ 3606 if (!system) { 3607 system = data->onmatch.match_event_system; 3608 event = data->onmatch.match_event; 3609 } 3610 3611 /* 3612 * At this point, we're looking at a field on another 3613 * event. Because we can't modify a hist trigger on 3614 * another event to add a variable for a field, we need 3615 * to create a new trigger on that event and create the 3616 * variable at the same time. 3617 */ 3618 hist_field = create_field_var_hist(hist_data, system, event, var); 3619 if (IS_ERR(hist_field)) 3620 goto free; 3621 } 3622 out: 3623 return hist_field; 3624 free: 3625 destroy_field_var(field_var); 3626 hist_field = NULL; 3627 goto out; 3628 } 3629 3630 static int onmatch_create(struct hist_trigger_data *hist_data, 3631 struct trace_event_file *file, 3632 struct action_data *data) 3633 { 3634 char *event_name, *param, *system = NULL; 3635 struct hist_field *hist_field, *var_ref; 3636 unsigned int i, var_ref_idx; 3637 unsigned int field_pos = 0; 3638 struct synth_event *event; 3639 int ret = 0; 3640 3641 mutex_lock(&synth_event_mutex); 3642 event = find_synth_event(data->onmatch.synth_event_name); 3643 if (!event) { 3644 hist_err("onmatch: Couldn't find synthetic event: ", data->onmatch.synth_event_name); 3645 mutex_unlock(&synth_event_mutex); 3646 return -EINVAL; 3647 } 3648 event->ref++; 3649 mutex_unlock(&synth_event_mutex); 3650 3651 var_ref_idx = hist_data->n_var_refs; 3652 3653 for (i = 0; i < data->n_params; i++) { 3654 char *p; 3655 3656 p = param = kstrdup(data->params[i], GFP_KERNEL); 3657 if (!param) { 3658 ret = -ENOMEM; 3659 goto err; 3660 } 3661 3662 system = strsep(¶m, "."); 3663 if (!param) { 3664 param = (char *)system; 3665 system = event_name = NULL; 3666 } else { 3667 event_name = strsep(¶m, "."); 3668 if (!param) { 3669 kfree(p); 3670 ret = -EINVAL; 3671 goto err; 3672 } 3673 } 3674 3675 if (param[0] == '$') 3676 hist_field = onmatch_find_var(hist_data, data, system, 3677 event_name, param); 3678 else 3679 hist_field = onmatch_create_field_var(hist_data, data, 3680 system, 3681 event_name, 3682 param); 3683 3684 if (!hist_field) { 3685 kfree(p); 3686 ret = -EINVAL; 3687 goto err; 3688 } 3689 3690 if (check_synth_field(event, hist_field, field_pos) == 0) { 3691 var_ref = create_var_ref(hist_field, system, event_name); 3692 if (!var_ref) { 3693 kfree(p); 3694 ret = -ENOMEM; 3695 goto err; 3696 } 3697 3698 save_synth_var_ref(hist_data, var_ref); 3699 field_pos++; 3700 kfree(p); 3701 continue; 3702 } 3703 3704 hist_err_event("onmatch: Param type doesn't match synthetic event field type: ", 3705 system, event_name, param); 3706 kfree(p); 3707 ret = -EINVAL; 3708 goto err; 3709 } 3710 3711 if (field_pos != event->n_fields) { 3712 hist_err("onmatch: Param count doesn't match synthetic event field count: ", event->name); 3713 ret = -EINVAL; 3714 goto err; 3715 } 3716 3717 data->fn = action_trace; 3718 data->onmatch.synth_event = event; 3719 data->onmatch.var_ref_idx = var_ref_idx; 3720 out: 3721 return ret; 3722 err: 3723 mutex_lock(&synth_event_mutex); 3724 event->ref--; 3725 mutex_unlock(&synth_event_mutex); 3726 3727 goto out; 3728 } 3729 3730 static struct action_data *onmatch_parse(struct trace_array *tr, char *str) 3731 { 3732 char *match_event, *match_event_system; 3733 char *synth_event_name, *params; 3734 struct action_data *data; 3735 int ret = -EINVAL; 3736 3737 data = kzalloc(sizeof(*data), GFP_KERNEL); 3738 if (!data) 3739 return ERR_PTR(-ENOMEM); 3740 3741 match_event = strsep(&str, ")"); 3742 if (!match_event || !str) { 3743 hist_err("onmatch: Missing closing paren: ", match_event); 3744 goto free; 3745 } 3746 3747 match_event_system = strsep(&match_event, "."); 3748 if (!match_event) { 3749 hist_err("onmatch: Missing subsystem for match event: ", match_event_system); 3750 goto free; 3751 } 3752 3753 if (IS_ERR(event_file(tr, match_event_system, match_event))) { 3754 hist_err_event("onmatch: Invalid subsystem or event name: ", 3755 match_event_system, match_event, NULL); 3756 goto free; 3757 } 3758 3759 data->onmatch.match_event = kstrdup(match_event, GFP_KERNEL); 3760 if (!data->onmatch.match_event) { 3761 ret = -ENOMEM; 3762 goto free; 3763 } 3764 3765 data->onmatch.match_event_system = kstrdup(match_event_system, GFP_KERNEL); 3766 if (!data->onmatch.match_event_system) { 3767 ret = -ENOMEM; 3768 goto free; 3769 } 3770 3771 strsep(&str, "."); 3772 if (!str) { 3773 hist_err("onmatch: Missing . after onmatch(): ", str); 3774 goto free; 3775 } 3776 3777 synth_event_name = strsep(&str, "("); 3778 if (!synth_event_name || !str) { 3779 hist_err("onmatch: Missing opening paramlist paren: ", synth_event_name); 3780 goto free; 3781 } 3782 3783 data->onmatch.synth_event_name = kstrdup(synth_event_name, GFP_KERNEL); 3784 if (!data->onmatch.synth_event_name) { 3785 ret = -ENOMEM; 3786 goto free; 3787 } 3788 3789 params = strsep(&str, ")"); 3790 if (!params || !str || (str && strlen(str))) { 3791 hist_err("onmatch: Missing closing paramlist paren: ", params); 3792 goto free; 3793 } 3794 3795 ret = parse_action_params(params, data); 3796 if (ret) 3797 goto free; 3798 out: 3799 return data; 3800 free: 3801 onmatch_destroy(data); 3802 data = ERR_PTR(ret); 3803 goto out; 3804 } 3805 3806 static int create_hitcount_val(struct hist_trigger_data *hist_data) 3807 { 3808 hist_data->fields[HITCOUNT_IDX] = 3809 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL); 3810 if (!hist_data->fields[HITCOUNT_IDX]) 3811 return -ENOMEM; 3812 3813 hist_data->n_vals++; 3814 hist_data->n_fields++; 3815 3816 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX)) 3817 return -EINVAL; 3818 3819 return 0; 3820 } 3821 3822 static int __create_val_field(struct hist_trigger_data *hist_data, 3823 unsigned int val_idx, 3824 struct trace_event_file *file, 3825 char *var_name, char *field_str, 3826 unsigned long flags) 3827 { 3828 struct hist_field *hist_field; 3829 int ret = 0; 3830 3831 hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0); 3832 if (IS_ERR(hist_field)) { 3833 ret = PTR_ERR(hist_field); 3834 goto out; 3835 } 3836 3837 hist_data->fields[val_idx] = hist_field; 3838 3839 ++hist_data->n_vals; 3840 ++hist_data->n_fields; 3841 3842 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 3843 ret = -EINVAL; 3844 out: 3845 return ret; 3846 } 3847 3848 static int create_val_field(struct hist_trigger_data *hist_data, 3849 unsigned int val_idx, 3850 struct trace_event_file *file, 3851 char *field_str) 3852 { 3853 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX)) 3854 return -EINVAL; 3855 3856 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0); 3857 } 3858 3859 static int create_var_field(struct hist_trigger_data *hist_data, 3860 unsigned int val_idx, 3861 struct trace_event_file *file, 3862 char *var_name, char *expr_str) 3863 { 3864 unsigned long flags = 0; 3865 3866 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 3867 return -EINVAL; 3868 3869 if (find_var(hist_data, file, var_name) && !hist_data->remove) { 3870 hist_err("Variable already defined: ", var_name); 3871 return -EINVAL; 3872 } 3873 3874 flags |= HIST_FIELD_FL_VAR; 3875 hist_data->n_vars++; 3876 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX)) 3877 return -EINVAL; 3878 3879 return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags); 3880 } 3881 3882 static int create_val_fields(struct hist_trigger_data *hist_data, 3883 struct trace_event_file *file) 3884 { 3885 char *fields_str, *field_str; 3886 unsigned int i, j = 1; 3887 int ret; 3888 3889 ret = create_hitcount_val(hist_data); 3890 if (ret) 3891 goto out; 3892 3893 fields_str = hist_data->attrs->vals_str; 3894 if (!fields_str) 3895 goto out; 3896 3897 strsep(&fields_str, "="); 3898 if (!fields_str) 3899 goto out; 3900 3901 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX && 3902 j < TRACING_MAP_VALS_MAX; i++) { 3903 field_str = strsep(&fields_str, ","); 3904 if (!field_str) 3905 break; 3906 3907 if (strcmp(field_str, "hitcount") == 0) 3908 continue; 3909 3910 ret = create_val_field(hist_data, j++, file, field_str); 3911 if (ret) 3912 goto out; 3913 } 3914 3915 if (fields_str && (strcmp(fields_str, "hitcount") != 0)) 3916 ret = -EINVAL; 3917 out: 3918 return ret; 3919 } 3920 3921 static int create_key_field(struct hist_trigger_data *hist_data, 3922 unsigned int key_idx, 3923 unsigned int key_offset, 3924 struct trace_event_file *file, 3925 char *field_str) 3926 { 3927 struct hist_field *hist_field = NULL; 3928 3929 unsigned long flags = 0; 3930 unsigned int key_size; 3931 int ret = 0; 3932 3933 if (WARN_ON(key_idx >= HIST_FIELDS_MAX)) 3934 return -EINVAL; 3935 3936 flags |= HIST_FIELD_FL_KEY; 3937 3938 if (strcmp(field_str, "stacktrace") == 0) { 3939 flags |= HIST_FIELD_FL_STACKTRACE; 3940 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH; 3941 hist_field = create_hist_field(hist_data, NULL, flags, NULL); 3942 } else { 3943 hist_field = parse_expr(hist_data, file, field_str, flags, 3944 NULL, 0); 3945 if (IS_ERR(hist_field)) { 3946 ret = PTR_ERR(hist_field); 3947 goto out; 3948 } 3949 3950 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) { 3951 hist_err("Using variable references as keys not supported: ", field_str); 3952 destroy_hist_field(hist_field, 0); 3953 ret = -EINVAL; 3954 goto out; 3955 } 3956 3957 key_size = hist_field->size; 3958 } 3959 3960 hist_data->fields[key_idx] = hist_field; 3961 3962 key_size = ALIGN(key_size, sizeof(u64)); 3963 hist_data->fields[key_idx]->size = key_size; 3964 hist_data->fields[key_idx]->offset = key_offset; 3965 3966 hist_data->key_size += key_size; 3967 3968 if (hist_data->key_size > HIST_KEY_SIZE_MAX) { 3969 ret = -EINVAL; 3970 goto out; 3971 } 3972 3973 hist_data->n_keys++; 3974 hist_data->n_fields++; 3975 3976 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX)) 3977 return -EINVAL; 3978 3979 ret = key_size; 3980 out: 3981 return ret; 3982 } 3983 3984 static int create_key_fields(struct hist_trigger_data *hist_data, 3985 struct trace_event_file *file) 3986 { 3987 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals; 3988 char *fields_str, *field_str; 3989 int ret = -EINVAL; 3990 3991 fields_str = hist_data->attrs->keys_str; 3992 if (!fields_str) 3993 goto out; 3994 3995 strsep(&fields_str, "="); 3996 if (!fields_str) 3997 goto out; 3998 3999 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) { 4000 field_str = strsep(&fields_str, ","); 4001 if (!field_str) 4002 break; 4003 ret = create_key_field(hist_data, i, key_offset, 4004 file, field_str); 4005 if (ret < 0) 4006 goto out; 4007 key_offset += ret; 4008 } 4009 if (fields_str) { 4010 ret = -EINVAL; 4011 goto out; 4012 } 4013 ret = 0; 4014 out: 4015 return ret; 4016 } 4017 4018 static int create_var_fields(struct hist_trigger_data *hist_data, 4019 struct trace_event_file *file) 4020 { 4021 unsigned int i, j = hist_data->n_vals; 4022 int ret = 0; 4023 4024 unsigned int n_vars = hist_data->attrs->var_defs.n_vars; 4025 4026 for (i = 0; i < n_vars; i++) { 4027 char *var_name = hist_data->attrs->var_defs.name[i]; 4028 char *expr = hist_data->attrs->var_defs.expr[i]; 4029 4030 ret = create_var_field(hist_data, j++, file, var_name, expr); 4031 if (ret) 4032 goto out; 4033 } 4034 out: 4035 return ret; 4036 } 4037 4038 static void free_var_defs(struct hist_trigger_data *hist_data) 4039 { 4040 unsigned int i; 4041 4042 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 4043 kfree(hist_data->attrs->var_defs.name[i]); 4044 kfree(hist_data->attrs->var_defs.expr[i]); 4045 } 4046 4047 hist_data->attrs->var_defs.n_vars = 0; 4048 } 4049 4050 static int parse_var_defs(struct hist_trigger_data *hist_data) 4051 { 4052 char *s, *str, *var_name, *field_str; 4053 unsigned int i, j, n_vars = 0; 4054 int ret = 0; 4055 4056 for (i = 0; i < hist_data->attrs->n_assignments; i++) { 4057 str = hist_data->attrs->assignment_str[i]; 4058 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) { 4059 field_str = strsep(&str, ","); 4060 if (!field_str) 4061 break; 4062 4063 var_name = strsep(&field_str, "="); 4064 if (!var_name || !field_str) { 4065 hist_err("Malformed assignment: ", var_name); 4066 ret = -EINVAL; 4067 goto free; 4068 } 4069 4070 if (n_vars == TRACING_MAP_VARS_MAX) { 4071 hist_err("Too many variables defined: ", var_name); 4072 ret = -EINVAL; 4073 goto free; 4074 } 4075 4076 s = kstrdup(var_name, GFP_KERNEL); 4077 if (!s) { 4078 ret = -ENOMEM; 4079 goto free; 4080 } 4081 hist_data->attrs->var_defs.name[n_vars] = s; 4082 4083 s = kstrdup(field_str, GFP_KERNEL); 4084 if (!s) { 4085 kfree(hist_data->attrs->var_defs.name[n_vars]); 4086 ret = -ENOMEM; 4087 goto free; 4088 } 4089 hist_data->attrs->var_defs.expr[n_vars++] = s; 4090 4091 hist_data->attrs->var_defs.n_vars = n_vars; 4092 } 4093 } 4094 4095 return ret; 4096 free: 4097 free_var_defs(hist_data); 4098 4099 return ret; 4100 } 4101 4102 static int create_hist_fields(struct hist_trigger_data *hist_data, 4103 struct trace_event_file *file) 4104 { 4105 int ret; 4106 4107 ret = parse_var_defs(hist_data); 4108 if (ret) 4109 goto out; 4110 4111 ret = create_val_fields(hist_data, file); 4112 if (ret) 4113 goto out; 4114 4115 ret = create_var_fields(hist_data, file); 4116 if (ret) 4117 goto out; 4118 4119 ret = create_key_fields(hist_data, file); 4120 if (ret) 4121 goto out; 4122 out: 4123 free_var_defs(hist_data); 4124 4125 return ret; 4126 } 4127 4128 static int is_descending(const char *str) 4129 { 4130 if (!str) 4131 return 0; 4132 4133 if (strcmp(str, "descending") == 0) 4134 return 1; 4135 4136 if (strcmp(str, "ascending") == 0) 4137 return 0; 4138 4139 return -EINVAL; 4140 } 4141 4142 static int create_sort_keys(struct hist_trigger_data *hist_data) 4143 { 4144 char *fields_str = hist_data->attrs->sort_key_str; 4145 struct tracing_map_sort_key *sort_key; 4146 int descending, ret = 0; 4147 unsigned int i, j, k; 4148 4149 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */ 4150 4151 if (!fields_str) 4152 goto out; 4153 4154 strsep(&fields_str, "="); 4155 if (!fields_str) { 4156 ret = -EINVAL; 4157 goto out; 4158 } 4159 4160 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) { 4161 struct hist_field *hist_field; 4162 char *field_str, *field_name; 4163 const char *test_name; 4164 4165 sort_key = &hist_data->sort_keys[i]; 4166 4167 field_str = strsep(&fields_str, ","); 4168 if (!field_str) { 4169 if (i == 0) 4170 ret = -EINVAL; 4171 break; 4172 } 4173 4174 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) { 4175 ret = -EINVAL; 4176 break; 4177 } 4178 4179 field_name = strsep(&field_str, "."); 4180 if (!field_name) { 4181 ret = -EINVAL; 4182 break; 4183 } 4184 4185 if (strcmp(field_name, "hitcount") == 0) { 4186 descending = is_descending(field_str); 4187 if (descending < 0) { 4188 ret = descending; 4189 break; 4190 } 4191 sort_key->descending = descending; 4192 continue; 4193 } 4194 4195 for (j = 1, k = 1; j < hist_data->n_fields; j++) { 4196 unsigned int idx; 4197 4198 hist_field = hist_data->fields[j]; 4199 if (hist_field->flags & HIST_FIELD_FL_VAR) 4200 continue; 4201 4202 idx = k++; 4203 4204 test_name = hist_field_name(hist_field, 0); 4205 4206 if (strcmp(field_name, test_name) == 0) { 4207 sort_key->field_idx = idx; 4208 descending = is_descending(field_str); 4209 if (descending < 0) { 4210 ret = descending; 4211 goto out; 4212 } 4213 sort_key->descending = descending; 4214 break; 4215 } 4216 } 4217 if (j == hist_data->n_fields) { 4218 ret = -EINVAL; 4219 break; 4220 } 4221 } 4222 4223 hist_data->n_sort_keys = i; 4224 out: 4225 return ret; 4226 } 4227 4228 static void destroy_actions(struct hist_trigger_data *hist_data) 4229 { 4230 unsigned int i; 4231 4232 for (i = 0; i < hist_data->n_actions; i++) { 4233 struct action_data *data = hist_data->actions[i]; 4234 4235 if (data->fn == action_trace) 4236 onmatch_destroy(data); 4237 else if (data->fn == onmax_save) 4238 onmax_destroy(data); 4239 else 4240 kfree(data); 4241 } 4242 } 4243 4244 static int parse_actions(struct hist_trigger_data *hist_data) 4245 { 4246 struct trace_array *tr = hist_data->event_file->tr; 4247 struct action_data *data; 4248 unsigned int i; 4249 int ret = 0; 4250 char *str; 4251 4252 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4253 str = hist_data->attrs->action_str[i]; 4254 4255 if (strncmp(str, "onmatch(", strlen("onmatch(")) == 0) { 4256 char *action_str = str + strlen("onmatch("); 4257 4258 data = onmatch_parse(tr, action_str); 4259 if (IS_ERR(data)) { 4260 ret = PTR_ERR(data); 4261 break; 4262 } 4263 data->fn = action_trace; 4264 } else if (strncmp(str, "onmax(", strlen("onmax(")) == 0) { 4265 char *action_str = str + strlen("onmax("); 4266 4267 data = onmax_parse(action_str); 4268 if (IS_ERR(data)) { 4269 ret = PTR_ERR(data); 4270 break; 4271 } 4272 data->fn = onmax_save; 4273 } else { 4274 ret = -EINVAL; 4275 break; 4276 } 4277 4278 hist_data->actions[hist_data->n_actions++] = data; 4279 } 4280 4281 return ret; 4282 } 4283 4284 static int create_actions(struct hist_trigger_data *hist_data, 4285 struct trace_event_file *file) 4286 { 4287 struct action_data *data; 4288 unsigned int i; 4289 int ret = 0; 4290 4291 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4292 data = hist_data->actions[i]; 4293 4294 if (data->fn == action_trace) { 4295 ret = onmatch_create(hist_data, file, data); 4296 if (ret) 4297 return ret; 4298 } else if (data->fn == onmax_save) { 4299 ret = onmax_create(hist_data, data); 4300 if (ret) 4301 return ret; 4302 } 4303 } 4304 4305 return ret; 4306 } 4307 4308 static void print_actions(struct seq_file *m, 4309 struct hist_trigger_data *hist_data, 4310 struct tracing_map_elt *elt) 4311 { 4312 unsigned int i; 4313 4314 for (i = 0; i < hist_data->n_actions; i++) { 4315 struct action_data *data = hist_data->actions[i]; 4316 4317 if (data->fn == onmax_save) 4318 onmax_print(m, hist_data, elt, data); 4319 } 4320 } 4321 4322 static void print_onmax_spec(struct seq_file *m, 4323 struct hist_trigger_data *hist_data, 4324 struct action_data *data) 4325 { 4326 unsigned int i; 4327 4328 seq_puts(m, ":onmax("); 4329 seq_printf(m, "%s", data->onmax.var_str); 4330 seq_printf(m, ").%s(", data->onmax.fn_name); 4331 4332 for (i = 0; i < hist_data->n_max_vars; i++) { 4333 seq_printf(m, "%s", hist_data->max_vars[i]->var->var.name); 4334 if (i < hist_data->n_max_vars - 1) 4335 seq_puts(m, ","); 4336 } 4337 seq_puts(m, ")"); 4338 } 4339 4340 static void print_onmatch_spec(struct seq_file *m, 4341 struct hist_trigger_data *hist_data, 4342 struct action_data *data) 4343 { 4344 unsigned int i; 4345 4346 seq_printf(m, ":onmatch(%s.%s).", data->onmatch.match_event_system, 4347 data->onmatch.match_event); 4348 4349 seq_printf(m, "%s(", data->onmatch.synth_event->name); 4350 4351 for (i = 0; i < data->n_params; i++) { 4352 if (i) 4353 seq_puts(m, ","); 4354 seq_printf(m, "%s", data->params[i]); 4355 } 4356 4357 seq_puts(m, ")"); 4358 } 4359 4360 static bool actions_match(struct hist_trigger_data *hist_data, 4361 struct hist_trigger_data *hist_data_test) 4362 { 4363 unsigned int i, j; 4364 4365 if (hist_data->n_actions != hist_data_test->n_actions) 4366 return false; 4367 4368 for (i = 0; i < hist_data->n_actions; i++) { 4369 struct action_data *data = hist_data->actions[i]; 4370 struct action_data *data_test = hist_data_test->actions[i]; 4371 4372 if (data->fn != data_test->fn) 4373 return false; 4374 4375 if (data->n_params != data_test->n_params) 4376 return false; 4377 4378 for (j = 0; j < data->n_params; j++) { 4379 if (strcmp(data->params[j], data_test->params[j]) != 0) 4380 return false; 4381 } 4382 4383 if (data->fn == action_trace) { 4384 if (strcmp(data->onmatch.synth_event_name, 4385 data_test->onmatch.synth_event_name) != 0) 4386 return false; 4387 if (strcmp(data->onmatch.match_event_system, 4388 data_test->onmatch.match_event_system) != 0) 4389 return false; 4390 if (strcmp(data->onmatch.match_event, 4391 data_test->onmatch.match_event) != 0) 4392 return false; 4393 } else if (data->fn == onmax_save) { 4394 if (strcmp(data->onmax.var_str, 4395 data_test->onmax.var_str) != 0) 4396 return false; 4397 if (strcmp(data->onmax.fn_name, 4398 data_test->onmax.fn_name) != 0) 4399 return false; 4400 } 4401 } 4402 4403 return true; 4404 } 4405 4406 4407 static void print_actions_spec(struct seq_file *m, 4408 struct hist_trigger_data *hist_data) 4409 { 4410 unsigned int i; 4411 4412 for (i = 0; i < hist_data->n_actions; i++) { 4413 struct action_data *data = hist_data->actions[i]; 4414 4415 if (data->fn == action_trace) 4416 print_onmatch_spec(m, hist_data, data); 4417 else if (data->fn == onmax_save) 4418 print_onmax_spec(m, hist_data, data); 4419 } 4420 } 4421 4422 static void destroy_field_var_hists(struct hist_trigger_data *hist_data) 4423 { 4424 unsigned int i; 4425 4426 for (i = 0; i < hist_data->n_field_var_hists; i++) { 4427 kfree(hist_data->field_var_hists[i]->cmd); 4428 kfree(hist_data->field_var_hists[i]); 4429 } 4430 } 4431 4432 static void destroy_hist_data(struct hist_trigger_data *hist_data) 4433 { 4434 if (!hist_data) 4435 return; 4436 4437 destroy_hist_trigger_attrs(hist_data->attrs); 4438 destroy_hist_fields(hist_data); 4439 tracing_map_destroy(hist_data->map); 4440 4441 destroy_actions(hist_data); 4442 destroy_field_vars(hist_data); 4443 destroy_field_var_hists(hist_data); 4444 destroy_synth_var_refs(hist_data); 4445 4446 kfree(hist_data); 4447 } 4448 4449 static int create_tracing_map_fields(struct hist_trigger_data *hist_data) 4450 { 4451 struct tracing_map *map = hist_data->map; 4452 struct ftrace_event_field *field; 4453 struct hist_field *hist_field; 4454 int i, idx = 0; 4455 4456 for_each_hist_field(i, hist_data) { 4457 hist_field = hist_data->fields[i]; 4458 if (hist_field->flags & HIST_FIELD_FL_KEY) { 4459 tracing_map_cmp_fn_t cmp_fn; 4460 4461 field = hist_field->field; 4462 4463 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE) 4464 cmp_fn = tracing_map_cmp_none; 4465 else if (!field) 4466 cmp_fn = tracing_map_cmp_num(hist_field->size, 4467 hist_field->is_signed); 4468 else if (is_string_field(field)) 4469 cmp_fn = tracing_map_cmp_string; 4470 else 4471 cmp_fn = tracing_map_cmp_num(field->size, 4472 field->is_signed); 4473 idx = tracing_map_add_key_field(map, 4474 hist_field->offset, 4475 cmp_fn); 4476 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR)) 4477 idx = tracing_map_add_sum_field(map); 4478 4479 if (idx < 0) 4480 return idx; 4481 4482 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4483 idx = tracing_map_add_var(map); 4484 if (idx < 0) 4485 return idx; 4486 hist_field->var.idx = idx; 4487 hist_field->var.hist_data = hist_data; 4488 } 4489 } 4490 4491 return 0; 4492 } 4493 4494 static struct hist_trigger_data * 4495 create_hist_data(unsigned int map_bits, 4496 struct hist_trigger_attrs *attrs, 4497 struct trace_event_file *file, 4498 bool remove) 4499 { 4500 const struct tracing_map_ops *map_ops = NULL; 4501 struct hist_trigger_data *hist_data; 4502 int ret = 0; 4503 4504 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL); 4505 if (!hist_data) 4506 return ERR_PTR(-ENOMEM); 4507 4508 hist_data->attrs = attrs; 4509 hist_data->remove = remove; 4510 hist_data->event_file = file; 4511 4512 ret = parse_actions(hist_data); 4513 if (ret) 4514 goto free; 4515 4516 ret = create_hist_fields(hist_data, file); 4517 if (ret) 4518 goto free; 4519 4520 ret = create_sort_keys(hist_data); 4521 if (ret) 4522 goto free; 4523 4524 map_ops = &hist_trigger_elt_data_ops; 4525 4526 hist_data->map = tracing_map_create(map_bits, hist_data->key_size, 4527 map_ops, hist_data); 4528 if (IS_ERR(hist_data->map)) { 4529 ret = PTR_ERR(hist_data->map); 4530 hist_data->map = NULL; 4531 goto free; 4532 } 4533 4534 ret = create_tracing_map_fields(hist_data); 4535 if (ret) 4536 goto free; 4537 out: 4538 return hist_data; 4539 free: 4540 hist_data->attrs = NULL; 4541 4542 destroy_hist_data(hist_data); 4543 4544 hist_data = ERR_PTR(ret); 4545 4546 goto out; 4547 } 4548 4549 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, 4550 struct tracing_map_elt *elt, void *rec, 4551 struct ring_buffer_event *rbe, 4552 u64 *var_ref_vals) 4553 { 4554 struct hist_elt_data *elt_data; 4555 struct hist_field *hist_field; 4556 unsigned int i, var_idx; 4557 u64 hist_val; 4558 4559 elt_data = elt->private_data; 4560 elt_data->var_ref_vals = var_ref_vals; 4561 4562 for_each_hist_val_field(i, hist_data) { 4563 hist_field = hist_data->fields[i]; 4564 hist_val = hist_field->fn(hist_field, elt, rbe, rec); 4565 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4566 var_idx = hist_field->var.idx; 4567 tracing_map_set_var(elt, var_idx, hist_val); 4568 continue; 4569 } 4570 tracing_map_update_sum(elt, i, hist_val); 4571 } 4572 4573 for_each_hist_key_field(i, hist_data) { 4574 hist_field = hist_data->fields[i]; 4575 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4576 hist_val = hist_field->fn(hist_field, elt, rbe, rec); 4577 var_idx = hist_field->var.idx; 4578 tracing_map_set_var(elt, var_idx, hist_val); 4579 } 4580 } 4581 4582 update_field_vars(hist_data, elt, rbe, rec); 4583 } 4584 4585 static inline void add_to_key(char *compound_key, void *key, 4586 struct hist_field *key_field, void *rec) 4587 { 4588 size_t size = key_field->size; 4589 4590 if (key_field->flags & HIST_FIELD_FL_STRING) { 4591 struct ftrace_event_field *field; 4592 4593 field = key_field->field; 4594 if (field->filter_type == FILTER_DYN_STRING) 4595 size = *(u32 *)(rec + field->offset) >> 16; 4596 else if (field->filter_type == FILTER_PTR_STRING) 4597 size = strlen(key); 4598 else if (field->filter_type == FILTER_STATIC_STRING) 4599 size = field->size; 4600 4601 /* ensure NULL-termination */ 4602 if (size > key_field->size - 1) 4603 size = key_field->size - 1; 4604 } 4605 4606 memcpy(compound_key + key_field->offset, key, size); 4607 } 4608 4609 static void 4610 hist_trigger_actions(struct hist_trigger_data *hist_data, 4611 struct tracing_map_elt *elt, void *rec, 4612 struct ring_buffer_event *rbe, u64 *var_ref_vals) 4613 { 4614 struct action_data *data; 4615 unsigned int i; 4616 4617 for (i = 0; i < hist_data->n_actions; i++) { 4618 data = hist_data->actions[i]; 4619 data->fn(hist_data, elt, rec, rbe, data, var_ref_vals); 4620 } 4621 } 4622 4623 static void event_hist_trigger(struct event_trigger_data *data, void *rec, 4624 struct ring_buffer_event *rbe) 4625 { 4626 struct hist_trigger_data *hist_data = data->private_data; 4627 bool use_compound_key = (hist_data->n_keys > 1); 4628 unsigned long entries[HIST_STACKTRACE_DEPTH]; 4629 u64 var_ref_vals[TRACING_MAP_VARS_MAX]; 4630 char compound_key[HIST_KEY_SIZE_MAX]; 4631 struct tracing_map_elt *elt = NULL; 4632 struct stack_trace stacktrace; 4633 struct hist_field *key_field; 4634 u64 field_contents; 4635 void *key = NULL; 4636 unsigned int i; 4637 4638 memset(compound_key, 0, hist_data->key_size); 4639 4640 for_each_hist_key_field(i, hist_data) { 4641 key_field = hist_data->fields[i]; 4642 4643 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 4644 stacktrace.max_entries = HIST_STACKTRACE_DEPTH; 4645 stacktrace.entries = entries; 4646 stacktrace.nr_entries = 0; 4647 stacktrace.skip = HIST_STACKTRACE_SKIP; 4648 4649 memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE); 4650 save_stack_trace(&stacktrace); 4651 4652 key = entries; 4653 } else { 4654 field_contents = key_field->fn(key_field, elt, rbe, rec); 4655 if (key_field->flags & HIST_FIELD_FL_STRING) { 4656 key = (void *)(unsigned long)field_contents; 4657 use_compound_key = true; 4658 } else 4659 key = (void *)&field_contents; 4660 } 4661 4662 if (use_compound_key) 4663 add_to_key(compound_key, key, key_field, rec); 4664 } 4665 4666 if (use_compound_key) 4667 key = compound_key; 4668 4669 if (hist_data->n_var_refs && 4670 !resolve_var_refs(hist_data, key, var_ref_vals, false)) 4671 return; 4672 4673 elt = tracing_map_insert(hist_data->map, key); 4674 if (!elt) 4675 return; 4676 4677 hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals); 4678 4679 if (resolve_var_refs(hist_data, key, var_ref_vals, true)) 4680 hist_trigger_actions(hist_data, elt, rec, rbe, var_ref_vals); 4681 } 4682 4683 static void hist_trigger_stacktrace_print(struct seq_file *m, 4684 unsigned long *stacktrace_entries, 4685 unsigned int max_entries) 4686 { 4687 char str[KSYM_SYMBOL_LEN]; 4688 unsigned int spaces = 8; 4689 unsigned int i; 4690 4691 for (i = 0; i < max_entries; i++) { 4692 if (stacktrace_entries[i] == ULONG_MAX) 4693 return; 4694 4695 seq_printf(m, "%*c", 1 + spaces, ' '); 4696 sprint_symbol(str, stacktrace_entries[i]); 4697 seq_printf(m, "%s\n", str); 4698 } 4699 } 4700 4701 static void 4702 hist_trigger_entry_print(struct seq_file *m, 4703 struct hist_trigger_data *hist_data, void *key, 4704 struct tracing_map_elt *elt) 4705 { 4706 struct hist_field *key_field; 4707 char str[KSYM_SYMBOL_LEN]; 4708 bool multiline = false; 4709 const char *field_name; 4710 unsigned int i; 4711 u64 uval; 4712 4713 seq_puts(m, "{ "); 4714 4715 for_each_hist_key_field(i, hist_data) { 4716 key_field = hist_data->fields[i]; 4717 4718 if (i > hist_data->n_vals) 4719 seq_puts(m, ", "); 4720 4721 field_name = hist_field_name(key_field, 0); 4722 4723 if (key_field->flags & HIST_FIELD_FL_HEX) { 4724 uval = *(u64 *)(key + key_field->offset); 4725 seq_printf(m, "%s: %llx", field_name, uval); 4726 } else if (key_field->flags & HIST_FIELD_FL_SYM) { 4727 uval = *(u64 *)(key + key_field->offset); 4728 sprint_symbol_no_offset(str, uval); 4729 seq_printf(m, "%s: [%llx] %-45s", field_name, 4730 uval, str); 4731 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) { 4732 uval = *(u64 *)(key + key_field->offset); 4733 sprint_symbol(str, uval); 4734 seq_printf(m, "%s: [%llx] %-55s", field_name, 4735 uval, str); 4736 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 4737 struct hist_elt_data *elt_data = elt->private_data; 4738 char *comm; 4739 4740 if (WARN_ON_ONCE(!elt_data)) 4741 return; 4742 4743 comm = elt_data->comm; 4744 4745 uval = *(u64 *)(key + key_field->offset); 4746 seq_printf(m, "%s: %-16s[%10llu]", field_name, 4747 comm, uval); 4748 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) { 4749 const char *syscall_name; 4750 4751 uval = *(u64 *)(key + key_field->offset); 4752 syscall_name = get_syscall_name(uval); 4753 if (!syscall_name) 4754 syscall_name = "unknown_syscall"; 4755 4756 seq_printf(m, "%s: %-30s[%3llu]", field_name, 4757 syscall_name, uval); 4758 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 4759 seq_puts(m, "stacktrace:\n"); 4760 hist_trigger_stacktrace_print(m, 4761 key + key_field->offset, 4762 HIST_STACKTRACE_DEPTH); 4763 multiline = true; 4764 } else if (key_field->flags & HIST_FIELD_FL_LOG2) { 4765 seq_printf(m, "%s: ~ 2^%-2llu", field_name, 4766 *(u64 *)(key + key_field->offset)); 4767 } else if (key_field->flags & HIST_FIELD_FL_STRING) { 4768 seq_printf(m, "%s: %-50s", field_name, 4769 (char *)(key + key_field->offset)); 4770 } else { 4771 uval = *(u64 *)(key + key_field->offset); 4772 seq_printf(m, "%s: %10llu", field_name, uval); 4773 } 4774 } 4775 4776 if (!multiline) 4777 seq_puts(m, " "); 4778 4779 seq_puts(m, "}"); 4780 4781 seq_printf(m, " hitcount: %10llu", 4782 tracing_map_read_sum(elt, HITCOUNT_IDX)); 4783 4784 for (i = 1; i < hist_data->n_vals; i++) { 4785 field_name = hist_field_name(hist_data->fields[i], 0); 4786 4787 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR || 4788 hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR) 4789 continue; 4790 4791 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) { 4792 seq_printf(m, " %s: %10llx", field_name, 4793 tracing_map_read_sum(elt, i)); 4794 } else { 4795 seq_printf(m, " %s: %10llu", field_name, 4796 tracing_map_read_sum(elt, i)); 4797 } 4798 } 4799 4800 print_actions(m, hist_data, elt); 4801 4802 seq_puts(m, "\n"); 4803 } 4804 4805 static int print_entries(struct seq_file *m, 4806 struct hist_trigger_data *hist_data) 4807 { 4808 struct tracing_map_sort_entry **sort_entries = NULL; 4809 struct tracing_map *map = hist_data->map; 4810 int i, n_entries; 4811 4812 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys, 4813 hist_data->n_sort_keys, 4814 &sort_entries); 4815 if (n_entries < 0) 4816 return n_entries; 4817 4818 for (i = 0; i < n_entries; i++) 4819 hist_trigger_entry_print(m, hist_data, 4820 sort_entries[i]->key, 4821 sort_entries[i]->elt); 4822 4823 tracing_map_destroy_sort_entries(sort_entries, n_entries); 4824 4825 return n_entries; 4826 } 4827 4828 static void hist_trigger_show(struct seq_file *m, 4829 struct event_trigger_data *data, int n) 4830 { 4831 struct hist_trigger_data *hist_data; 4832 int n_entries; 4833 4834 if (n > 0) 4835 seq_puts(m, "\n\n"); 4836 4837 seq_puts(m, "# event histogram\n#\n# trigger info: "); 4838 data->ops->print(m, data->ops, data); 4839 seq_puts(m, "#\n\n"); 4840 4841 hist_data = data->private_data; 4842 n_entries = print_entries(m, hist_data); 4843 if (n_entries < 0) 4844 n_entries = 0; 4845 4846 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n", 4847 (u64)atomic64_read(&hist_data->map->hits), 4848 n_entries, (u64)atomic64_read(&hist_data->map->drops)); 4849 } 4850 4851 static int hist_show(struct seq_file *m, void *v) 4852 { 4853 struct event_trigger_data *data; 4854 struct trace_event_file *event_file; 4855 int n = 0, ret = 0; 4856 4857 mutex_lock(&event_mutex); 4858 4859 event_file = event_file_data(m->private); 4860 if (unlikely(!event_file)) { 4861 ret = -ENODEV; 4862 goto out_unlock; 4863 } 4864 4865 list_for_each_entry_rcu(data, &event_file->triggers, list) { 4866 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 4867 hist_trigger_show(m, data, n++); 4868 } 4869 4870 if (have_hist_err()) { 4871 seq_printf(m, "\nERROR: %s\n", hist_err_str); 4872 seq_printf(m, " Last command: %s\n", last_hist_cmd); 4873 } 4874 4875 out_unlock: 4876 mutex_unlock(&event_mutex); 4877 4878 return ret; 4879 } 4880 4881 static int event_hist_open(struct inode *inode, struct file *file) 4882 { 4883 return single_open(file, hist_show, file); 4884 } 4885 4886 const struct file_operations event_hist_fops = { 4887 .open = event_hist_open, 4888 .read = seq_read, 4889 .llseek = seq_lseek, 4890 .release = single_release, 4891 }; 4892 4893 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field) 4894 { 4895 const char *field_name = hist_field_name(hist_field, 0); 4896 4897 if (hist_field->var.name) 4898 seq_printf(m, "%s=", hist_field->var.name); 4899 4900 if (hist_field->flags & HIST_FIELD_FL_CPU) 4901 seq_puts(m, "cpu"); 4902 else if (field_name) { 4903 if (hist_field->flags & HIST_FIELD_FL_VAR_REF || 4904 hist_field->flags & HIST_FIELD_FL_ALIAS) 4905 seq_putc(m, '$'); 4906 seq_printf(m, "%s", field_name); 4907 } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP) 4908 seq_puts(m, "common_timestamp"); 4909 4910 if (hist_field->flags) { 4911 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) && 4912 !(hist_field->flags & HIST_FIELD_FL_EXPR)) { 4913 const char *flags = get_hist_field_flags(hist_field); 4914 4915 if (flags) 4916 seq_printf(m, ".%s", flags); 4917 } 4918 } 4919 } 4920 4921 static int event_hist_trigger_print(struct seq_file *m, 4922 struct event_trigger_ops *ops, 4923 struct event_trigger_data *data) 4924 { 4925 struct hist_trigger_data *hist_data = data->private_data; 4926 struct hist_field *field; 4927 bool have_var = false; 4928 unsigned int i; 4929 4930 seq_puts(m, "hist:"); 4931 4932 if (data->name) 4933 seq_printf(m, "%s:", data->name); 4934 4935 seq_puts(m, "keys="); 4936 4937 for_each_hist_key_field(i, hist_data) { 4938 field = hist_data->fields[i]; 4939 4940 if (i > hist_data->n_vals) 4941 seq_puts(m, ","); 4942 4943 if (field->flags & HIST_FIELD_FL_STACKTRACE) 4944 seq_puts(m, "stacktrace"); 4945 else 4946 hist_field_print(m, field); 4947 } 4948 4949 seq_puts(m, ":vals="); 4950 4951 for_each_hist_val_field(i, hist_data) { 4952 field = hist_data->fields[i]; 4953 if (field->flags & HIST_FIELD_FL_VAR) { 4954 have_var = true; 4955 continue; 4956 } 4957 4958 if (i == HITCOUNT_IDX) 4959 seq_puts(m, "hitcount"); 4960 else { 4961 seq_puts(m, ","); 4962 hist_field_print(m, field); 4963 } 4964 } 4965 4966 if (have_var) { 4967 unsigned int n = 0; 4968 4969 seq_puts(m, ":"); 4970 4971 for_each_hist_val_field(i, hist_data) { 4972 field = hist_data->fields[i]; 4973 4974 if (field->flags & HIST_FIELD_FL_VAR) { 4975 if (n++) 4976 seq_puts(m, ","); 4977 hist_field_print(m, field); 4978 } 4979 } 4980 } 4981 4982 seq_puts(m, ":sort="); 4983 4984 for (i = 0; i < hist_data->n_sort_keys; i++) { 4985 struct tracing_map_sort_key *sort_key; 4986 unsigned int idx, first_key_idx; 4987 4988 /* skip VAR vals */ 4989 first_key_idx = hist_data->n_vals - hist_data->n_vars; 4990 4991 sort_key = &hist_data->sort_keys[i]; 4992 idx = sort_key->field_idx; 4993 4994 if (WARN_ON(idx >= HIST_FIELDS_MAX)) 4995 return -EINVAL; 4996 4997 if (i > 0) 4998 seq_puts(m, ","); 4999 5000 if (idx == HITCOUNT_IDX) 5001 seq_puts(m, "hitcount"); 5002 else { 5003 if (idx >= first_key_idx) 5004 idx += hist_data->n_vars; 5005 hist_field_print(m, hist_data->fields[idx]); 5006 } 5007 5008 if (sort_key->descending) 5009 seq_puts(m, ".descending"); 5010 } 5011 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits)); 5012 if (hist_data->enable_timestamps) 5013 seq_printf(m, ":clock=%s", hist_data->attrs->clock); 5014 5015 print_actions_spec(m, hist_data); 5016 5017 if (data->filter_str) 5018 seq_printf(m, " if %s", data->filter_str); 5019 5020 if (data->paused) 5021 seq_puts(m, " [paused]"); 5022 else 5023 seq_puts(m, " [active]"); 5024 5025 seq_putc(m, '\n'); 5026 5027 return 0; 5028 } 5029 5030 static int event_hist_trigger_init(struct event_trigger_ops *ops, 5031 struct event_trigger_data *data) 5032 { 5033 struct hist_trigger_data *hist_data = data->private_data; 5034 5035 if (!data->ref && hist_data->attrs->name) 5036 save_named_trigger(hist_data->attrs->name, data); 5037 5038 data->ref++; 5039 5040 return 0; 5041 } 5042 5043 static void unregister_field_var_hists(struct hist_trigger_data *hist_data) 5044 { 5045 struct trace_event_file *file; 5046 unsigned int i; 5047 char *cmd; 5048 int ret; 5049 5050 for (i = 0; i < hist_data->n_field_var_hists; i++) { 5051 file = hist_data->field_var_hists[i]->hist_data->event_file; 5052 cmd = hist_data->field_var_hists[i]->cmd; 5053 ret = event_hist_trigger_func(&trigger_hist_cmd, file, 5054 "!hist", "hist", cmd); 5055 } 5056 } 5057 5058 static void event_hist_trigger_free(struct event_trigger_ops *ops, 5059 struct event_trigger_data *data) 5060 { 5061 struct hist_trigger_data *hist_data = data->private_data; 5062 5063 if (WARN_ON_ONCE(data->ref <= 0)) 5064 return; 5065 5066 data->ref--; 5067 if (!data->ref) { 5068 if (data->name) 5069 del_named_trigger(data); 5070 5071 trigger_data_free(data); 5072 5073 remove_hist_vars(hist_data); 5074 5075 unregister_field_var_hists(hist_data); 5076 5077 destroy_hist_data(hist_data); 5078 } 5079 } 5080 5081 static struct event_trigger_ops event_hist_trigger_ops = { 5082 .func = event_hist_trigger, 5083 .print = event_hist_trigger_print, 5084 .init = event_hist_trigger_init, 5085 .free = event_hist_trigger_free, 5086 }; 5087 5088 static int event_hist_trigger_named_init(struct event_trigger_ops *ops, 5089 struct event_trigger_data *data) 5090 { 5091 data->ref++; 5092 5093 save_named_trigger(data->named_data->name, data); 5094 5095 event_hist_trigger_init(ops, data->named_data); 5096 5097 return 0; 5098 } 5099 5100 static void event_hist_trigger_named_free(struct event_trigger_ops *ops, 5101 struct event_trigger_data *data) 5102 { 5103 if (WARN_ON_ONCE(data->ref <= 0)) 5104 return; 5105 5106 event_hist_trigger_free(ops, data->named_data); 5107 5108 data->ref--; 5109 if (!data->ref) { 5110 del_named_trigger(data); 5111 trigger_data_free(data); 5112 } 5113 } 5114 5115 static struct event_trigger_ops event_hist_trigger_named_ops = { 5116 .func = event_hist_trigger, 5117 .print = event_hist_trigger_print, 5118 .init = event_hist_trigger_named_init, 5119 .free = event_hist_trigger_named_free, 5120 }; 5121 5122 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd, 5123 char *param) 5124 { 5125 return &event_hist_trigger_ops; 5126 } 5127 5128 static void hist_clear(struct event_trigger_data *data) 5129 { 5130 struct hist_trigger_data *hist_data = data->private_data; 5131 5132 if (data->name) 5133 pause_named_trigger(data); 5134 5135 tracepoint_synchronize_unregister(); 5136 5137 tracing_map_clear(hist_data->map); 5138 5139 if (data->name) 5140 unpause_named_trigger(data); 5141 } 5142 5143 static bool compatible_field(struct ftrace_event_field *field, 5144 struct ftrace_event_field *test_field) 5145 { 5146 if (field == test_field) 5147 return true; 5148 if (field == NULL || test_field == NULL) 5149 return false; 5150 if (strcmp(field->name, test_field->name) != 0) 5151 return false; 5152 if (strcmp(field->type, test_field->type) != 0) 5153 return false; 5154 if (field->size != test_field->size) 5155 return false; 5156 if (field->is_signed != test_field->is_signed) 5157 return false; 5158 5159 return true; 5160 } 5161 5162 static bool hist_trigger_match(struct event_trigger_data *data, 5163 struct event_trigger_data *data_test, 5164 struct event_trigger_data *named_data, 5165 bool ignore_filter) 5166 { 5167 struct tracing_map_sort_key *sort_key, *sort_key_test; 5168 struct hist_trigger_data *hist_data, *hist_data_test; 5169 struct hist_field *key_field, *key_field_test; 5170 unsigned int i; 5171 5172 if (named_data && (named_data != data_test) && 5173 (named_data != data_test->named_data)) 5174 return false; 5175 5176 if (!named_data && is_named_trigger(data_test)) 5177 return false; 5178 5179 hist_data = data->private_data; 5180 hist_data_test = data_test->private_data; 5181 5182 if (hist_data->n_vals != hist_data_test->n_vals || 5183 hist_data->n_fields != hist_data_test->n_fields || 5184 hist_data->n_sort_keys != hist_data_test->n_sort_keys) 5185 return false; 5186 5187 if (!ignore_filter) { 5188 if ((data->filter_str && !data_test->filter_str) || 5189 (!data->filter_str && data_test->filter_str)) 5190 return false; 5191 } 5192 5193 for_each_hist_field(i, hist_data) { 5194 key_field = hist_data->fields[i]; 5195 key_field_test = hist_data_test->fields[i]; 5196 5197 if (key_field->flags != key_field_test->flags) 5198 return false; 5199 if (!compatible_field(key_field->field, key_field_test->field)) 5200 return false; 5201 if (key_field->offset != key_field_test->offset) 5202 return false; 5203 if (key_field->size != key_field_test->size) 5204 return false; 5205 if (key_field->is_signed != key_field_test->is_signed) 5206 return false; 5207 if (!!key_field->var.name != !!key_field_test->var.name) 5208 return false; 5209 if (key_field->var.name && 5210 strcmp(key_field->var.name, key_field_test->var.name) != 0) 5211 return false; 5212 } 5213 5214 for (i = 0; i < hist_data->n_sort_keys; i++) { 5215 sort_key = &hist_data->sort_keys[i]; 5216 sort_key_test = &hist_data_test->sort_keys[i]; 5217 5218 if (sort_key->field_idx != sort_key_test->field_idx || 5219 sort_key->descending != sort_key_test->descending) 5220 return false; 5221 } 5222 5223 if (!ignore_filter && data->filter_str && 5224 (strcmp(data->filter_str, data_test->filter_str) != 0)) 5225 return false; 5226 5227 if (!actions_match(hist_data, hist_data_test)) 5228 return false; 5229 5230 return true; 5231 } 5232 5233 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops, 5234 struct event_trigger_data *data, 5235 struct trace_event_file *file) 5236 { 5237 struct hist_trigger_data *hist_data = data->private_data; 5238 struct event_trigger_data *test, *named_data = NULL; 5239 int ret = 0; 5240 5241 if (hist_data->attrs->name) { 5242 named_data = find_named_trigger(hist_data->attrs->name); 5243 if (named_data) { 5244 if (!hist_trigger_match(data, named_data, named_data, 5245 true)) { 5246 hist_err("Named hist trigger doesn't match existing named trigger (includes variables): ", hist_data->attrs->name); 5247 ret = -EINVAL; 5248 goto out; 5249 } 5250 } 5251 } 5252 5253 if (hist_data->attrs->name && !named_data) 5254 goto new; 5255 5256 list_for_each_entry_rcu(test, &file->triggers, list) { 5257 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5258 if (!hist_trigger_match(data, test, named_data, false)) 5259 continue; 5260 if (hist_data->attrs->pause) 5261 test->paused = true; 5262 else if (hist_data->attrs->cont) 5263 test->paused = false; 5264 else if (hist_data->attrs->clear) 5265 hist_clear(test); 5266 else { 5267 hist_err("Hist trigger already exists", NULL); 5268 ret = -EEXIST; 5269 } 5270 goto out; 5271 } 5272 } 5273 new: 5274 if (hist_data->attrs->cont || hist_data->attrs->clear) { 5275 hist_err("Can't clear or continue a nonexistent hist trigger", NULL); 5276 ret = -ENOENT; 5277 goto out; 5278 } 5279 5280 if (hist_data->attrs->pause) 5281 data->paused = true; 5282 5283 if (named_data) { 5284 data->private_data = named_data->private_data; 5285 set_named_trigger_data(data, named_data); 5286 data->ops = &event_hist_trigger_named_ops; 5287 } 5288 5289 if (data->ops->init) { 5290 ret = data->ops->init(data->ops, data); 5291 if (ret < 0) 5292 goto out; 5293 } 5294 5295 if (hist_data->enable_timestamps) { 5296 char *clock = hist_data->attrs->clock; 5297 5298 ret = tracing_set_clock(file->tr, hist_data->attrs->clock); 5299 if (ret) { 5300 hist_err("Couldn't set trace_clock: ", clock); 5301 goto out; 5302 } 5303 5304 tracing_set_time_stamp_abs(file->tr, true); 5305 } 5306 5307 if (named_data) 5308 destroy_hist_data(hist_data); 5309 5310 ret++; 5311 out: 5312 return ret; 5313 } 5314 5315 static int hist_trigger_enable(struct event_trigger_data *data, 5316 struct trace_event_file *file) 5317 { 5318 int ret = 0; 5319 5320 list_add_tail_rcu(&data->list, &file->triggers); 5321 5322 update_cond_flag(file); 5323 5324 if (trace_event_trigger_enable_disable(file, 1) < 0) { 5325 list_del_rcu(&data->list); 5326 update_cond_flag(file); 5327 ret--; 5328 } 5329 5330 return ret; 5331 } 5332 5333 static bool have_hist_trigger_match(struct event_trigger_data *data, 5334 struct trace_event_file *file) 5335 { 5336 struct hist_trigger_data *hist_data = data->private_data; 5337 struct event_trigger_data *test, *named_data = NULL; 5338 bool match = false; 5339 5340 if (hist_data->attrs->name) 5341 named_data = find_named_trigger(hist_data->attrs->name); 5342 5343 list_for_each_entry_rcu(test, &file->triggers, list) { 5344 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5345 if (hist_trigger_match(data, test, named_data, false)) { 5346 match = true; 5347 break; 5348 } 5349 } 5350 } 5351 5352 return match; 5353 } 5354 5355 static bool hist_trigger_check_refs(struct event_trigger_data *data, 5356 struct trace_event_file *file) 5357 { 5358 struct hist_trigger_data *hist_data = data->private_data; 5359 struct event_trigger_data *test, *named_data = NULL; 5360 5361 if (hist_data->attrs->name) 5362 named_data = find_named_trigger(hist_data->attrs->name); 5363 5364 list_for_each_entry_rcu(test, &file->triggers, list) { 5365 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5366 if (!hist_trigger_match(data, test, named_data, false)) 5367 continue; 5368 hist_data = test->private_data; 5369 if (check_var_refs(hist_data)) 5370 return true; 5371 break; 5372 } 5373 } 5374 5375 return false; 5376 } 5377 5378 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops, 5379 struct event_trigger_data *data, 5380 struct trace_event_file *file) 5381 { 5382 struct hist_trigger_data *hist_data = data->private_data; 5383 struct event_trigger_data *test, *named_data = NULL; 5384 bool unregistered = false; 5385 5386 if (hist_data->attrs->name) 5387 named_data = find_named_trigger(hist_data->attrs->name); 5388 5389 list_for_each_entry_rcu(test, &file->triggers, list) { 5390 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5391 if (!hist_trigger_match(data, test, named_data, false)) 5392 continue; 5393 unregistered = true; 5394 list_del_rcu(&test->list); 5395 trace_event_trigger_enable_disable(file, 0); 5396 update_cond_flag(file); 5397 break; 5398 } 5399 } 5400 5401 if (unregistered && test->ops->free) 5402 test->ops->free(test->ops, test); 5403 5404 if (hist_data->enable_timestamps) { 5405 if (!hist_data->remove || unregistered) 5406 tracing_set_time_stamp_abs(file->tr, false); 5407 } 5408 } 5409 5410 static bool hist_file_check_refs(struct trace_event_file *file) 5411 { 5412 struct hist_trigger_data *hist_data; 5413 struct event_trigger_data *test; 5414 5415 list_for_each_entry_rcu(test, &file->triggers, list) { 5416 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5417 hist_data = test->private_data; 5418 if (check_var_refs(hist_data)) 5419 return true; 5420 } 5421 } 5422 5423 return false; 5424 } 5425 5426 static void hist_unreg_all(struct trace_event_file *file) 5427 { 5428 struct event_trigger_data *test, *n; 5429 struct hist_trigger_data *hist_data; 5430 struct synth_event *se; 5431 const char *se_name; 5432 5433 if (hist_file_check_refs(file)) 5434 return; 5435 5436 list_for_each_entry_safe(test, n, &file->triggers, list) { 5437 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5438 hist_data = test->private_data; 5439 list_del_rcu(&test->list); 5440 trace_event_trigger_enable_disable(file, 0); 5441 5442 mutex_lock(&synth_event_mutex); 5443 se_name = trace_event_name(file->event_call); 5444 se = find_synth_event(se_name); 5445 if (se) 5446 se->ref--; 5447 mutex_unlock(&synth_event_mutex); 5448 5449 update_cond_flag(file); 5450 if (hist_data->enable_timestamps) 5451 tracing_set_time_stamp_abs(file->tr, false); 5452 if (test->ops->free) 5453 test->ops->free(test->ops, test); 5454 } 5455 } 5456 } 5457 5458 static int event_hist_trigger_func(struct event_command *cmd_ops, 5459 struct trace_event_file *file, 5460 char *glob, char *cmd, char *param) 5461 { 5462 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT; 5463 struct event_trigger_data *trigger_data; 5464 struct hist_trigger_attrs *attrs; 5465 struct event_trigger_ops *trigger_ops; 5466 struct hist_trigger_data *hist_data; 5467 struct synth_event *se; 5468 const char *se_name; 5469 bool remove = false; 5470 char *trigger, *p; 5471 int ret = 0; 5472 5473 if (glob && strlen(glob)) { 5474 last_cmd_set(param); 5475 hist_err_clear(); 5476 } 5477 5478 if (!param) 5479 return -EINVAL; 5480 5481 if (glob[0] == '!') 5482 remove = true; 5483 5484 /* 5485 * separate the trigger from the filter (k:v [if filter]) 5486 * allowing for whitespace in the trigger 5487 */ 5488 p = trigger = param; 5489 do { 5490 p = strstr(p, "if"); 5491 if (!p) 5492 break; 5493 if (p == param) 5494 return -EINVAL; 5495 if (*(p - 1) != ' ' && *(p - 1) != '\t') { 5496 p++; 5497 continue; 5498 } 5499 if (p >= param + strlen(param) - strlen("if") - 1) 5500 return -EINVAL; 5501 if (*(p + strlen("if")) != ' ' && *(p + strlen("if")) != '\t') { 5502 p++; 5503 continue; 5504 } 5505 break; 5506 } while (p); 5507 5508 if (!p) 5509 param = NULL; 5510 else { 5511 *(p - 1) = '\0'; 5512 param = strstrip(p); 5513 trigger = strstrip(trigger); 5514 } 5515 5516 attrs = parse_hist_trigger_attrs(trigger); 5517 if (IS_ERR(attrs)) 5518 return PTR_ERR(attrs); 5519 5520 if (attrs->map_bits) 5521 hist_trigger_bits = attrs->map_bits; 5522 5523 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove); 5524 if (IS_ERR(hist_data)) { 5525 destroy_hist_trigger_attrs(attrs); 5526 return PTR_ERR(hist_data); 5527 } 5528 5529 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger); 5530 5531 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL); 5532 if (!trigger_data) { 5533 ret = -ENOMEM; 5534 goto out_free; 5535 } 5536 5537 trigger_data->count = -1; 5538 trigger_data->ops = trigger_ops; 5539 trigger_data->cmd_ops = cmd_ops; 5540 5541 INIT_LIST_HEAD(&trigger_data->list); 5542 RCU_INIT_POINTER(trigger_data->filter, NULL); 5543 5544 trigger_data->private_data = hist_data; 5545 5546 /* if param is non-empty, it's supposed to be a filter */ 5547 if (param && cmd_ops->set_filter) { 5548 ret = cmd_ops->set_filter(param, trigger_data, file); 5549 if (ret < 0) 5550 goto out_free; 5551 } 5552 5553 if (remove) { 5554 if (!have_hist_trigger_match(trigger_data, file)) 5555 goto out_free; 5556 5557 if (hist_trigger_check_refs(trigger_data, file)) { 5558 ret = -EBUSY; 5559 goto out_free; 5560 } 5561 5562 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file); 5563 5564 mutex_lock(&synth_event_mutex); 5565 se_name = trace_event_name(file->event_call); 5566 se = find_synth_event(se_name); 5567 if (se) 5568 se->ref--; 5569 mutex_unlock(&synth_event_mutex); 5570 5571 ret = 0; 5572 goto out_free; 5573 } 5574 5575 ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file); 5576 /* 5577 * The above returns on success the # of triggers registered, 5578 * but if it didn't register any it returns zero. Consider no 5579 * triggers registered a failure too. 5580 */ 5581 if (!ret) { 5582 if (!(attrs->pause || attrs->cont || attrs->clear)) 5583 ret = -ENOENT; 5584 goto out_free; 5585 } else if (ret < 0) 5586 goto out_free; 5587 5588 if (get_named_trigger_data(trigger_data)) 5589 goto enable; 5590 5591 if (has_hist_vars(hist_data)) 5592 save_hist_vars(hist_data); 5593 5594 ret = create_actions(hist_data, file); 5595 if (ret) 5596 goto out_unreg; 5597 5598 ret = tracing_map_init(hist_data->map); 5599 if (ret) 5600 goto out_unreg; 5601 enable: 5602 ret = hist_trigger_enable(trigger_data, file); 5603 if (ret) 5604 goto out_unreg; 5605 5606 mutex_lock(&synth_event_mutex); 5607 se_name = trace_event_name(file->event_call); 5608 se = find_synth_event(se_name); 5609 if (se) 5610 se->ref++; 5611 mutex_unlock(&synth_event_mutex); 5612 5613 /* Just return zero, not the number of registered triggers */ 5614 ret = 0; 5615 out: 5616 if (ret == 0) 5617 hist_err_clear(); 5618 5619 return ret; 5620 out_unreg: 5621 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file); 5622 out_free: 5623 if (cmd_ops->set_filter) 5624 cmd_ops->set_filter(NULL, trigger_data, NULL); 5625 5626 remove_hist_vars(hist_data); 5627 5628 kfree(trigger_data); 5629 5630 destroy_hist_data(hist_data); 5631 goto out; 5632 } 5633 5634 static struct event_command trigger_hist_cmd = { 5635 .name = "hist", 5636 .trigger_type = ETT_EVENT_HIST, 5637 .flags = EVENT_CMD_FL_NEEDS_REC, 5638 .func = event_hist_trigger_func, 5639 .reg = hist_register_trigger, 5640 .unreg = hist_unregister_trigger, 5641 .unreg_all = hist_unreg_all, 5642 .get_trigger_ops = event_hist_get_trigger_ops, 5643 .set_filter = set_trigger_filter, 5644 }; 5645 5646 __init int register_trigger_hist_cmd(void) 5647 { 5648 int ret; 5649 5650 ret = register_event_command(&trigger_hist_cmd); 5651 WARN_ON(ret < 0); 5652 5653 return ret; 5654 } 5655 5656 static void 5657 hist_enable_trigger(struct event_trigger_data *data, void *rec, 5658 struct ring_buffer_event *event) 5659 { 5660 struct enable_trigger_data *enable_data = data->private_data; 5661 struct event_trigger_data *test; 5662 5663 list_for_each_entry_rcu(test, &enable_data->file->triggers, list) { 5664 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5665 if (enable_data->enable) 5666 test->paused = false; 5667 else 5668 test->paused = true; 5669 } 5670 } 5671 } 5672 5673 static void 5674 hist_enable_count_trigger(struct event_trigger_data *data, void *rec, 5675 struct ring_buffer_event *event) 5676 { 5677 if (!data->count) 5678 return; 5679 5680 if (data->count != -1) 5681 (data->count)--; 5682 5683 hist_enable_trigger(data, rec, event); 5684 } 5685 5686 static struct event_trigger_ops hist_enable_trigger_ops = { 5687 .func = hist_enable_trigger, 5688 .print = event_enable_trigger_print, 5689 .init = event_trigger_init, 5690 .free = event_enable_trigger_free, 5691 }; 5692 5693 static struct event_trigger_ops hist_enable_count_trigger_ops = { 5694 .func = hist_enable_count_trigger, 5695 .print = event_enable_trigger_print, 5696 .init = event_trigger_init, 5697 .free = event_enable_trigger_free, 5698 }; 5699 5700 static struct event_trigger_ops hist_disable_trigger_ops = { 5701 .func = hist_enable_trigger, 5702 .print = event_enable_trigger_print, 5703 .init = event_trigger_init, 5704 .free = event_enable_trigger_free, 5705 }; 5706 5707 static struct event_trigger_ops hist_disable_count_trigger_ops = { 5708 .func = hist_enable_count_trigger, 5709 .print = event_enable_trigger_print, 5710 .init = event_trigger_init, 5711 .free = event_enable_trigger_free, 5712 }; 5713 5714 static struct event_trigger_ops * 5715 hist_enable_get_trigger_ops(char *cmd, char *param) 5716 { 5717 struct event_trigger_ops *ops; 5718 bool enable; 5719 5720 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0); 5721 5722 if (enable) 5723 ops = param ? &hist_enable_count_trigger_ops : 5724 &hist_enable_trigger_ops; 5725 else 5726 ops = param ? &hist_disable_count_trigger_ops : 5727 &hist_disable_trigger_ops; 5728 5729 return ops; 5730 } 5731 5732 static void hist_enable_unreg_all(struct trace_event_file *file) 5733 { 5734 struct event_trigger_data *test, *n; 5735 5736 list_for_each_entry_safe(test, n, &file->triggers, list) { 5737 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) { 5738 list_del_rcu(&test->list); 5739 update_cond_flag(file); 5740 trace_event_trigger_enable_disable(file, 0); 5741 if (test->ops->free) 5742 test->ops->free(test->ops, test); 5743 } 5744 } 5745 } 5746 5747 static struct event_command trigger_hist_enable_cmd = { 5748 .name = ENABLE_HIST_STR, 5749 .trigger_type = ETT_HIST_ENABLE, 5750 .func = event_enable_trigger_func, 5751 .reg = event_enable_register_trigger, 5752 .unreg = event_enable_unregister_trigger, 5753 .unreg_all = hist_enable_unreg_all, 5754 .get_trigger_ops = hist_enable_get_trigger_ops, 5755 .set_filter = set_trigger_filter, 5756 }; 5757 5758 static struct event_command trigger_hist_disable_cmd = { 5759 .name = DISABLE_HIST_STR, 5760 .trigger_type = ETT_HIST_ENABLE, 5761 .func = event_enable_trigger_func, 5762 .reg = event_enable_register_trigger, 5763 .unreg = event_enable_unregister_trigger, 5764 .unreg_all = hist_enable_unreg_all, 5765 .get_trigger_ops = hist_enable_get_trigger_ops, 5766 .set_filter = set_trigger_filter, 5767 }; 5768 5769 static __init void unregister_trigger_hist_enable_disable_cmds(void) 5770 { 5771 unregister_event_command(&trigger_hist_enable_cmd); 5772 unregister_event_command(&trigger_hist_disable_cmd); 5773 } 5774 5775 __init int register_trigger_hist_enable_disable_cmds(void) 5776 { 5777 int ret; 5778 5779 ret = register_event_command(&trigger_hist_enable_cmd); 5780 if (WARN_ON(ret < 0)) 5781 return ret; 5782 ret = register_event_command(&trigger_hist_disable_cmd); 5783 if (WARN_ON(ret < 0)) 5784 unregister_trigger_hist_enable_disable_cmds(); 5785 5786 return ret; 5787 } 5788 5789 static __init int trace_events_hist_init(void) 5790 { 5791 struct dentry *entry = NULL; 5792 struct dentry *d_tracer; 5793 int err = 0; 5794 5795 d_tracer = tracing_init_dentry(); 5796 if (IS_ERR(d_tracer)) { 5797 err = PTR_ERR(d_tracer); 5798 goto err; 5799 } 5800 5801 entry = tracefs_create_file("synthetic_events", 0644, d_tracer, 5802 NULL, &synth_events_fops); 5803 if (!entry) { 5804 err = -ENODEV; 5805 goto err; 5806 } 5807 5808 return err; 5809 err: 5810 pr_warn("Could not create tracefs 'synthetic_events' entry\n"); 5811 5812 return err; 5813 } 5814 5815 fs_initcall(trace_events_hist_init); 5816