1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * trace_events_hist - trace event hist triggers 4 * 5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com> 6 */ 7 8 #include <linux/module.h> 9 #include <linux/kallsyms.h> 10 #include <linux/mutex.h> 11 #include <linux/slab.h> 12 #include <linux/stacktrace.h> 13 #include <linux/rculist.h> 14 #include <linux/tracefs.h> 15 16 #include "tracing_map.h" 17 #include "trace.h" 18 19 #define SYNTH_SYSTEM "synthetic" 20 #define SYNTH_FIELDS_MAX 16 21 22 #define STR_VAR_LEN_MAX 32 /* must be multiple of sizeof(u64) */ 23 24 struct hist_field; 25 26 typedef u64 (*hist_field_fn_t) (struct hist_field *field, 27 struct tracing_map_elt *elt, 28 struct ring_buffer_event *rbe, 29 void *event); 30 31 #define HIST_FIELD_OPERANDS_MAX 2 32 #define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX) 33 #define HIST_ACTIONS_MAX 8 34 35 enum field_op_id { 36 FIELD_OP_NONE, 37 FIELD_OP_PLUS, 38 FIELD_OP_MINUS, 39 FIELD_OP_UNARY_MINUS, 40 }; 41 42 struct hist_var { 43 char *name; 44 struct hist_trigger_data *hist_data; 45 unsigned int idx; 46 }; 47 48 struct hist_field { 49 struct ftrace_event_field *field; 50 unsigned long flags; 51 hist_field_fn_t fn; 52 unsigned int size; 53 unsigned int offset; 54 unsigned int is_signed; 55 const char *type; 56 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX]; 57 struct hist_trigger_data *hist_data; 58 struct hist_var var; 59 enum field_op_id operator; 60 char *system; 61 char *event_name; 62 char *name; 63 unsigned int var_idx; 64 unsigned int var_ref_idx; 65 bool read_once; 66 }; 67 68 static u64 hist_field_none(struct hist_field *field, 69 struct tracing_map_elt *elt, 70 struct ring_buffer_event *rbe, 71 void *event) 72 { 73 return 0; 74 } 75 76 static u64 hist_field_counter(struct hist_field *field, 77 struct tracing_map_elt *elt, 78 struct ring_buffer_event *rbe, 79 void *event) 80 { 81 return 1; 82 } 83 84 static u64 hist_field_string(struct hist_field *hist_field, 85 struct tracing_map_elt *elt, 86 struct ring_buffer_event *rbe, 87 void *event) 88 { 89 char *addr = (char *)(event + hist_field->field->offset); 90 91 return (u64)(unsigned long)addr; 92 } 93 94 static u64 hist_field_dynstring(struct hist_field *hist_field, 95 struct tracing_map_elt *elt, 96 struct ring_buffer_event *rbe, 97 void *event) 98 { 99 u32 str_item = *(u32 *)(event + hist_field->field->offset); 100 int str_loc = str_item & 0xffff; 101 char *addr = (char *)(event + str_loc); 102 103 return (u64)(unsigned long)addr; 104 } 105 106 static u64 hist_field_pstring(struct hist_field *hist_field, 107 struct tracing_map_elt *elt, 108 struct ring_buffer_event *rbe, 109 void *event) 110 { 111 char **addr = (char **)(event + hist_field->field->offset); 112 113 return (u64)(unsigned long)*addr; 114 } 115 116 static u64 hist_field_log2(struct hist_field *hist_field, 117 struct tracing_map_elt *elt, 118 struct ring_buffer_event *rbe, 119 void *event) 120 { 121 struct hist_field *operand = hist_field->operands[0]; 122 123 u64 val = operand->fn(operand, elt, rbe, event); 124 125 return (u64) ilog2(roundup_pow_of_two(val)); 126 } 127 128 static u64 hist_field_plus(struct hist_field *hist_field, 129 struct tracing_map_elt *elt, 130 struct ring_buffer_event *rbe, 131 void *event) 132 { 133 struct hist_field *operand1 = hist_field->operands[0]; 134 struct hist_field *operand2 = hist_field->operands[1]; 135 136 u64 val1 = operand1->fn(operand1, elt, rbe, event); 137 u64 val2 = operand2->fn(operand2, elt, rbe, event); 138 139 return val1 + val2; 140 } 141 142 static u64 hist_field_minus(struct hist_field *hist_field, 143 struct tracing_map_elt *elt, 144 struct ring_buffer_event *rbe, 145 void *event) 146 { 147 struct hist_field *operand1 = hist_field->operands[0]; 148 struct hist_field *operand2 = hist_field->operands[1]; 149 150 u64 val1 = operand1->fn(operand1, elt, rbe, event); 151 u64 val2 = operand2->fn(operand2, elt, rbe, event); 152 153 return val1 - val2; 154 } 155 156 static u64 hist_field_unary_minus(struct hist_field *hist_field, 157 struct tracing_map_elt *elt, 158 struct ring_buffer_event *rbe, 159 void *event) 160 { 161 struct hist_field *operand = hist_field->operands[0]; 162 163 s64 sval = (s64)operand->fn(operand, elt, rbe, event); 164 u64 val = (u64)-sval; 165 166 return val; 167 } 168 169 #define DEFINE_HIST_FIELD_FN(type) \ 170 static u64 hist_field_##type(struct hist_field *hist_field, \ 171 struct tracing_map_elt *elt, \ 172 struct ring_buffer_event *rbe, \ 173 void *event) \ 174 { \ 175 type *addr = (type *)(event + hist_field->field->offset); \ 176 \ 177 return (u64)(unsigned long)*addr; \ 178 } 179 180 DEFINE_HIST_FIELD_FN(s64); 181 DEFINE_HIST_FIELD_FN(u64); 182 DEFINE_HIST_FIELD_FN(s32); 183 DEFINE_HIST_FIELD_FN(u32); 184 DEFINE_HIST_FIELD_FN(s16); 185 DEFINE_HIST_FIELD_FN(u16); 186 DEFINE_HIST_FIELD_FN(s8); 187 DEFINE_HIST_FIELD_FN(u8); 188 189 #define for_each_hist_field(i, hist_data) \ 190 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++) 191 192 #define for_each_hist_val_field(i, hist_data) \ 193 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++) 194 195 #define for_each_hist_key_field(i, hist_data) \ 196 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++) 197 198 #define HIST_STACKTRACE_DEPTH 16 199 #define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long)) 200 #define HIST_STACKTRACE_SKIP 5 201 202 #define HITCOUNT_IDX 0 203 #define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE) 204 205 enum hist_field_flags { 206 HIST_FIELD_FL_HITCOUNT = 1 << 0, 207 HIST_FIELD_FL_KEY = 1 << 1, 208 HIST_FIELD_FL_STRING = 1 << 2, 209 HIST_FIELD_FL_HEX = 1 << 3, 210 HIST_FIELD_FL_SYM = 1 << 4, 211 HIST_FIELD_FL_SYM_OFFSET = 1 << 5, 212 HIST_FIELD_FL_EXECNAME = 1 << 6, 213 HIST_FIELD_FL_SYSCALL = 1 << 7, 214 HIST_FIELD_FL_STACKTRACE = 1 << 8, 215 HIST_FIELD_FL_LOG2 = 1 << 9, 216 HIST_FIELD_FL_TIMESTAMP = 1 << 10, 217 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11, 218 HIST_FIELD_FL_VAR = 1 << 12, 219 HIST_FIELD_FL_EXPR = 1 << 13, 220 HIST_FIELD_FL_VAR_REF = 1 << 14, 221 HIST_FIELD_FL_CPU = 1 << 15, 222 HIST_FIELD_FL_ALIAS = 1 << 16, 223 }; 224 225 struct var_defs { 226 unsigned int n_vars; 227 char *name[TRACING_MAP_VARS_MAX]; 228 char *expr[TRACING_MAP_VARS_MAX]; 229 }; 230 231 struct hist_trigger_attrs { 232 char *keys_str; 233 char *vals_str; 234 char *sort_key_str; 235 char *name; 236 char *clock; 237 bool pause; 238 bool cont; 239 bool clear; 240 bool ts_in_usecs; 241 unsigned int map_bits; 242 243 char *assignment_str[TRACING_MAP_VARS_MAX]; 244 unsigned int n_assignments; 245 246 char *action_str[HIST_ACTIONS_MAX]; 247 unsigned int n_actions; 248 249 struct var_defs var_defs; 250 }; 251 252 struct field_var { 253 struct hist_field *var; 254 struct hist_field *val; 255 }; 256 257 struct field_var_hist { 258 struct hist_trigger_data *hist_data; 259 char *cmd; 260 }; 261 262 struct hist_trigger_data { 263 struct hist_field *fields[HIST_FIELDS_MAX]; 264 unsigned int n_vals; 265 unsigned int n_keys; 266 unsigned int n_fields; 267 unsigned int n_vars; 268 unsigned int key_size; 269 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX]; 270 unsigned int n_sort_keys; 271 struct trace_event_file *event_file; 272 struct hist_trigger_attrs *attrs; 273 struct tracing_map *map; 274 bool enable_timestamps; 275 bool remove; 276 struct hist_field *var_refs[TRACING_MAP_VARS_MAX]; 277 unsigned int n_var_refs; 278 279 struct action_data *actions[HIST_ACTIONS_MAX]; 280 unsigned int n_actions; 281 282 struct hist_field *synth_var_refs[SYNTH_FIELDS_MAX]; 283 unsigned int n_synth_var_refs; 284 struct field_var *field_vars[SYNTH_FIELDS_MAX]; 285 unsigned int n_field_vars; 286 unsigned int n_field_var_str; 287 struct field_var_hist *field_var_hists[SYNTH_FIELDS_MAX]; 288 unsigned int n_field_var_hists; 289 290 struct field_var *max_vars[SYNTH_FIELDS_MAX]; 291 unsigned int n_max_vars; 292 unsigned int n_max_var_str; 293 }; 294 295 struct synth_field { 296 char *type; 297 char *name; 298 size_t size; 299 bool is_signed; 300 bool is_string; 301 }; 302 303 struct synth_event { 304 struct list_head list; 305 int ref; 306 char *name; 307 struct synth_field **fields; 308 unsigned int n_fields; 309 unsigned int n_u64; 310 struct trace_event_class class; 311 struct trace_event_call call; 312 struct tracepoint *tp; 313 }; 314 315 struct action_data; 316 317 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data, 318 struct tracing_map_elt *elt, void *rec, 319 struct ring_buffer_event *rbe, 320 struct action_data *data, u64 *var_ref_vals); 321 322 struct action_data { 323 action_fn_t fn; 324 unsigned int n_params; 325 char *params[SYNTH_FIELDS_MAX]; 326 327 union { 328 struct { 329 unsigned int var_ref_idx; 330 char *match_event; 331 char *match_event_system; 332 char *synth_event_name; 333 struct synth_event *synth_event; 334 } onmatch; 335 336 struct { 337 char *var_str; 338 char *fn_name; 339 unsigned int max_var_ref_idx; 340 struct hist_field *max_var; 341 struct hist_field *var; 342 } onmax; 343 }; 344 }; 345 346 347 static char last_hist_cmd[MAX_FILTER_STR_VAL]; 348 static char hist_err_str[MAX_FILTER_STR_VAL]; 349 350 static void last_cmd_set(char *str) 351 { 352 if (!str) 353 return; 354 355 strncpy(last_hist_cmd, str, MAX_FILTER_STR_VAL - 1); 356 } 357 358 static void hist_err(char *str, char *var) 359 { 360 int maxlen = MAX_FILTER_STR_VAL - 1; 361 362 if (!str) 363 return; 364 365 if (strlen(hist_err_str)) 366 return; 367 368 if (!var) 369 var = ""; 370 371 if (strlen(hist_err_str) + strlen(str) + strlen(var) > maxlen) 372 return; 373 374 strcat(hist_err_str, str); 375 strcat(hist_err_str, var); 376 } 377 378 static void hist_err_event(char *str, char *system, char *event, char *var) 379 { 380 char err[MAX_FILTER_STR_VAL]; 381 382 if (system && var) 383 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s.%s", system, event, var); 384 else if (system) 385 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s", system, event); 386 else 387 strscpy(err, var, MAX_FILTER_STR_VAL); 388 389 hist_err(str, err); 390 } 391 392 static void hist_err_clear(void) 393 { 394 hist_err_str[0] = '\0'; 395 } 396 397 static bool have_hist_err(void) 398 { 399 if (strlen(hist_err_str)) 400 return true; 401 402 return false; 403 } 404 405 static LIST_HEAD(synth_event_list); 406 static DEFINE_MUTEX(synth_event_mutex); 407 408 struct synth_trace_event { 409 struct trace_entry ent; 410 u64 fields[]; 411 }; 412 413 static int synth_event_define_fields(struct trace_event_call *call) 414 { 415 struct synth_trace_event trace; 416 int offset = offsetof(typeof(trace), fields); 417 struct synth_event *event = call->data; 418 unsigned int i, size, n_u64; 419 char *name, *type; 420 bool is_signed; 421 int ret = 0; 422 423 for (i = 0, n_u64 = 0; i < event->n_fields; i++) { 424 size = event->fields[i]->size; 425 is_signed = event->fields[i]->is_signed; 426 type = event->fields[i]->type; 427 name = event->fields[i]->name; 428 ret = trace_define_field(call, type, name, offset, size, 429 is_signed, FILTER_OTHER); 430 if (ret) 431 break; 432 433 if (event->fields[i]->is_string) { 434 offset += STR_VAR_LEN_MAX; 435 n_u64 += STR_VAR_LEN_MAX / sizeof(u64); 436 } else { 437 offset += sizeof(u64); 438 n_u64++; 439 } 440 } 441 442 event->n_u64 = n_u64; 443 444 return ret; 445 } 446 447 static bool synth_field_signed(char *type) 448 { 449 if (strncmp(type, "u", 1) == 0) 450 return false; 451 452 return true; 453 } 454 455 static int synth_field_is_string(char *type) 456 { 457 if (strstr(type, "char[") != NULL) 458 return true; 459 460 return false; 461 } 462 463 static int synth_field_string_size(char *type) 464 { 465 char buf[4], *end, *start; 466 unsigned int len; 467 int size, err; 468 469 start = strstr(type, "char["); 470 if (start == NULL) 471 return -EINVAL; 472 start += strlen("char["); 473 474 end = strchr(type, ']'); 475 if (!end || end < start) 476 return -EINVAL; 477 478 len = end - start; 479 if (len > 3) 480 return -EINVAL; 481 482 strncpy(buf, start, len); 483 buf[len] = '\0'; 484 485 err = kstrtouint(buf, 0, &size); 486 if (err) 487 return err; 488 489 if (size > STR_VAR_LEN_MAX) 490 return -EINVAL; 491 492 return size; 493 } 494 495 static int synth_field_size(char *type) 496 { 497 int size = 0; 498 499 if (strcmp(type, "s64") == 0) 500 size = sizeof(s64); 501 else if (strcmp(type, "u64") == 0) 502 size = sizeof(u64); 503 else if (strcmp(type, "s32") == 0) 504 size = sizeof(s32); 505 else if (strcmp(type, "u32") == 0) 506 size = sizeof(u32); 507 else if (strcmp(type, "s16") == 0) 508 size = sizeof(s16); 509 else if (strcmp(type, "u16") == 0) 510 size = sizeof(u16); 511 else if (strcmp(type, "s8") == 0) 512 size = sizeof(s8); 513 else if (strcmp(type, "u8") == 0) 514 size = sizeof(u8); 515 else if (strcmp(type, "char") == 0) 516 size = sizeof(char); 517 else if (strcmp(type, "unsigned char") == 0) 518 size = sizeof(unsigned char); 519 else if (strcmp(type, "int") == 0) 520 size = sizeof(int); 521 else if (strcmp(type, "unsigned int") == 0) 522 size = sizeof(unsigned int); 523 else if (strcmp(type, "long") == 0) 524 size = sizeof(long); 525 else if (strcmp(type, "unsigned long") == 0) 526 size = sizeof(unsigned long); 527 else if (strcmp(type, "pid_t") == 0) 528 size = sizeof(pid_t); 529 else if (synth_field_is_string(type)) 530 size = synth_field_string_size(type); 531 532 return size; 533 } 534 535 static const char *synth_field_fmt(char *type) 536 { 537 const char *fmt = "%llu"; 538 539 if (strcmp(type, "s64") == 0) 540 fmt = "%lld"; 541 else if (strcmp(type, "u64") == 0) 542 fmt = "%llu"; 543 else if (strcmp(type, "s32") == 0) 544 fmt = "%d"; 545 else if (strcmp(type, "u32") == 0) 546 fmt = "%u"; 547 else if (strcmp(type, "s16") == 0) 548 fmt = "%d"; 549 else if (strcmp(type, "u16") == 0) 550 fmt = "%u"; 551 else if (strcmp(type, "s8") == 0) 552 fmt = "%d"; 553 else if (strcmp(type, "u8") == 0) 554 fmt = "%u"; 555 else if (strcmp(type, "char") == 0) 556 fmt = "%d"; 557 else if (strcmp(type, "unsigned char") == 0) 558 fmt = "%u"; 559 else if (strcmp(type, "int") == 0) 560 fmt = "%d"; 561 else if (strcmp(type, "unsigned int") == 0) 562 fmt = "%u"; 563 else if (strcmp(type, "long") == 0) 564 fmt = "%ld"; 565 else if (strcmp(type, "unsigned long") == 0) 566 fmt = "%lu"; 567 else if (strcmp(type, "pid_t") == 0) 568 fmt = "%d"; 569 else if (synth_field_is_string(type)) 570 fmt = "%s"; 571 572 return fmt; 573 } 574 575 static enum print_line_t print_synth_event(struct trace_iterator *iter, 576 int flags, 577 struct trace_event *event) 578 { 579 struct trace_array *tr = iter->tr; 580 struct trace_seq *s = &iter->seq; 581 struct synth_trace_event *entry; 582 struct synth_event *se; 583 unsigned int i, n_u64; 584 char print_fmt[32]; 585 const char *fmt; 586 587 entry = (struct synth_trace_event *)iter->ent; 588 se = container_of(event, struct synth_event, call.event); 589 590 trace_seq_printf(s, "%s: ", se->name); 591 592 for (i = 0, n_u64 = 0; i < se->n_fields; i++) { 593 if (trace_seq_has_overflowed(s)) 594 goto end; 595 596 fmt = synth_field_fmt(se->fields[i]->type); 597 598 /* parameter types */ 599 if (tr->trace_flags & TRACE_ITER_VERBOSE) 600 trace_seq_printf(s, "%s ", fmt); 601 602 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt); 603 604 /* parameter values */ 605 if (se->fields[i]->is_string) { 606 trace_seq_printf(s, print_fmt, se->fields[i]->name, 607 (char *)&entry->fields[n_u64], 608 i == se->n_fields - 1 ? "" : " "); 609 n_u64 += STR_VAR_LEN_MAX / sizeof(u64); 610 } else { 611 trace_seq_printf(s, print_fmt, se->fields[i]->name, 612 entry->fields[n_u64], 613 i == se->n_fields - 1 ? "" : " "); 614 n_u64++; 615 } 616 } 617 end: 618 trace_seq_putc(s, '\n'); 619 620 return trace_handle_return(s); 621 } 622 623 static struct trace_event_functions synth_event_funcs = { 624 .trace = print_synth_event 625 }; 626 627 static notrace void trace_event_raw_event_synth(void *__data, 628 u64 *var_ref_vals, 629 unsigned int var_ref_idx) 630 { 631 struct trace_event_file *trace_file = __data; 632 struct synth_trace_event *entry; 633 struct trace_event_buffer fbuffer; 634 struct ring_buffer *buffer; 635 struct synth_event *event; 636 unsigned int i, n_u64; 637 int fields_size = 0; 638 639 event = trace_file->event_call->data; 640 641 if (trace_trigger_soft_disabled(trace_file)) 642 return; 643 644 fields_size = event->n_u64 * sizeof(u64); 645 646 /* 647 * Avoid ring buffer recursion detection, as this event 648 * is being performed within another event. 649 */ 650 buffer = trace_file->tr->trace_buffer.buffer; 651 ring_buffer_nest_start(buffer); 652 653 entry = trace_event_buffer_reserve(&fbuffer, trace_file, 654 sizeof(*entry) + fields_size); 655 if (!entry) 656 goto out; 657 658 for (i = 0, n_u64 = 0; i < event->n_fields; i++) { 659 if (event->fields[i]->is_string) { 660 char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i]; 661 char *str_field = (char *)&entry->fields[n_u64]; 662 663 strscpy(str_field, str_val, STR_VAR_LEN_MAX); 664 n_u64 += STR_VAR_LEN_MAX / sizeof(u64); 665 } else { 666 entry->fields[n_u64] = var_ref_vals[var_ref_idx + i]; 667 n_u64++; 668 } 669 } 670 671 trace_event_buffer_commit(&fbuffer); 672 out: 673 ring_buffer_nest_end(buffer); 674 } 675 676 static void free_synth_event_print_fmt(struct trace_event_call *call) 677 { 678 if (call) { 679 kfree(call->print_fmt); 680 call->print_fmt = NULL; 681 } 682 } 683 684 static int __set_synth_event_print_fmt(struct synth_event *event, 685 char *buf, int len) 686 { 687 const char *fmt; 688 int pos = 0; 689 int i; 690 691 /* When len=0, we just calculate the needed length */ 692 #define LEN_OR_ZERO (len ? len - pos : 0) 693 694 pos += snprintf(buf + pos, LEN_OR_ZERO, "\""); 695 for (i = 0; i < event->n_fields; i++) { 696 fmt = synth_field_fmt(event->fields[i]->type); 697 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s", 698 event->fields[i]->name, fmt, 699 i == event->n_fields - 1 ? "" : ", "); 700 } 701 pos += snprintf(buf + pos, LEN_OR_ZERO, "\""); 702 703 for (i = 0; i < event->n_fields; i++) { 704 pos += snprintf(buf + pos, LEN_OR_ZERO, 705 ", REC->%s", event->fields[i]->name); 706 } 707 708 #undef LEN_OR_ZERO 709 710 /* return the length of print_fmt */ 711 return pos; 712 } 713 714 static int set_synth_event_print_fmt(struct trace_event_call *call) 715 { 716 struct synth_event *event = call->data; 717 char *print_fmt; 718 int len; 719 720 /* First: called with 0 length to calculate the needed length */ 721 len = __set_synth_event_print_fmt(event, NULL, 0); 722 723 print_fmt = kmalloc(len + 1, GFP_KERNEL); 724 if (!print_fmt) 725 return -ENOMEM; 726 727 /* Second: actually write the @print_fmt */ 728 __set_synth_event_print_fmt(event, print_fmt, len + 1); 729 call->print_fmt = print_fmt; 730 731 return 0; 732 } 733 734 static void free_synth_field(struct synth_field *field) 735 { 736 kfree(field->type); 737 kfree(field->name); 738 kfree(field); 739 } 740 741 static struct synth_field *parse_synth_field(int argc, char **argv, 742 int *consumed) 743 { 744 struct synth_field *field; 745 const char *prefix = NULL; 746 char *field_type = argv[0], *field_name; 747 int len, ret = 0; 748 char *array; 749 750 if (field_type[0] == ';') 751 field_type++; 752 753 if (!strcmp(field_type, "unsigned")) { 754 if (argc < 3) 755 return ERR_PTR(-EINVAL); 756 prefix = "unsigned "; 757 field_type = argv[1]; 758 field_name = argv[2]; 759 *consumed = 3; 760 } else { 761 field_name = argv[1]; 762 *consumed = 2; 763 } 764 765 len = strlen(field_name); 766 if (field_name[len - 1] == ';') 767 field_name[len - 1] = '\0'; 768 769 field = kzalloc(sizeof(*field), GFP_KERNEL); 770 if (!field) 771 return ERR_PTR(-ENOMEM); 772 773 len = strlen(field_type) + 1; 774 array = strchr(field_name, '['); 775 if (array) 776 len += strlen(array); 777 if (prefix) 778 len += strlen(prefix); 779 field->type = kzalloc(len, GFP_KERNEL); 780 if (!field->type) { 781 ret = -ENOMEM; 782 goto free; 783 } 784 if (prefix) 785 strcat(field->type, prefix); 786 strcat(field->type, field_type); 787 if (array) { 788 strcat(field->type, array); 789 *array = '\0'; 790 } 791 792 field->size = synth_field_size(field->type); 793 if (!field->size) { 794 ret = -EINVAL; 795 goto free; 796 } 797 798 if (synth_field_is_string(field->type)) 799 field->is_string = true; 800 801 field->is_signed = synth_field_signed(field->type); 802 803 field->name = kstrdup(field_name, GFP_KERNEL); 804 if (!field->name) { 805 ret = -ENOMEM; 806 goto free; 807 } 808 out: 809 return field; 810 free: 811 free_synth_field(field); 812 field = ERR_PTR(ret); 813 goto out; 814 } 815 816 static void free_synth_tracepoint(struct tracepoint *tp) 817 { 818 if (!tp) 819 return; 820 821 kfree(tp->name); 822 kfree(tp); 823 } 824 825 static struct tracepoint *alloc_synth_tracepoint(char *name) 826 { 827 struct tracepoint *tp; 828 829 tp = kzalloc(sizeof(*tp), GFP_KERNEL); 830 if (!tp) 831 return ERR_PTR(-ENOMEM); 832 833 tp->name = kstrdup(name, GFP_KERNEL); 834 if (!tp->name) { 835 kfree(tp); 836 return ERR_PTR(-ENOMEM); 837 } 838 839 return tp; 840 } 841 842 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals, 843 unsigned int var_ref_idx); 844 845 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals, 846 unsigned int var_ref_idx) 847 { 848 struct tracepoint *tp = event->tp; 849 850 if (unlikely(atomic_read(&tp->key.enabled) > 0)) { 851 struct tracepoint_func *probe_func_ptr; 852 synth_probe_func_t probe_func; 853 void *__data; 854 855 if (!(cpu_online(raw_smp_processor_id()))) 856 return; 857 858 probe_func_ptr = rcu_dereference_sched((tp)->funcs); 859 if (probe_func_ptr) { 860 do { 861 probe_func = probe_func_ptr->func; 862 __data = probe_func_ptr->data; 863 probe_func(__data, var_ref_vals, var_ref_idx); 864 } while ((++probe_func_ptr)->func); 865 } 866 } 867 } 868 869 static struct synth_event *find_synth_event(const char *name) 870 { 871 struct synth_event *event; 872 873 list_for_each_entry(event, &synth_event_list, list) { 874 if (strcmp(event->name, name) == 0) 875 return event; 876 } 877 878 return NULL; 879 } 880 881 static int register_synth_event(struct synth_event *event) 882 { 883 struct trace_event_call *call = &event->call; 884 int ret = 0; 885 886 event->call.class = &event->class; 887 event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL); 888 if (!event->class.system) { 889 ret = -ENOMEM; 890 goto out; 891 } 892 893 event->tp = alloc_synth_tracepoint(event->name); 894 if (IS_ERR(event->tp)) { 895 ret = PTR_ERR(event->tp); 896 event->tp = NULL; 897 goto out; 898 } 899 900 INIT_LIST_HEAD(&call->class->fields); 901 call->event.funcs = &synth_event_funcs; 902 call->class->define_fields = synth_event_define_fields; 903 904 ret = register_trace_event(&call->event); 905 if (!ret) { 906 ret = -ENODEV; 907 goto out; 908 } 909 call->flags = TRACE_EVENT_FL_TRACEPOINT; 910 call->class->reg = trace_event_reg; 911 call->class->probe = trace_event_raw_event_synth; 912 call->data = event; 913 call->tp = event->tp; 914 915 ret = trace_add_event_call(call); 916 if (ret) { 917 pr_warn("Failed to register synthetic event: %s\n", 918 trace_event_name(call)); 919 goto err; 920 } 921 922 ret = set_synth_event_print_fmt(call); 923 if (ret < 0) { 924 trace_remove_event_call(call); 925 goto err; 926 } 927 out: 928 return ret; 929 err: 930 unregister_trace_event(&call->event); 931 goto out; 932 } 933 934 static int unregister_synth_event(struct synth_event *event) 935 { 936 struct trace_event_call *call = &event->call; 937 int ret; 938 939 ret = trace_remove_event_call(call); 940 941 return ret; 942 } 943 944 static void free_synth_event(struct synth_event *event) 945 { 946 unsigned int i; 947 948 if (!event) 949 return; 950 951 for (i = 0; i < event->n_fields; i++) 952 free_synth_field(event->fields[i]); 953 954 kfree(event->fields); 955 kfree(event->name); 956 kfree(event->class.system); 957 free_synth_tracepoint(event->tp); 958 free_synth_event_print_fmt(&event->call); 959 kfree(event); 960 } 961 962 static struct synth_event *alloc_synth_event(char *event_name, int n_fields, 963 struct synth_field **fields) 964 { 965 struct synth_event *event; 966 unsigned int i; 967 968 event = kzalloc(sizeof(*event), GFP_KERNEL); 969 if (!event) { 970 event = ERR_PTR(-ENOMEM); 971 goto out; 972 } 973 974 event->name = kstrdup(event_name, GFP_KERNEL); 975 if (!event->name) { 976 kfree(event); 977 event = ERR_PTR(-ENOMEM); 978 goto out; 979 } 980 981 event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL); 982 if (!event->fields) { 983 free_synth_event(event); 984 event = ERR_PTR(-ENOMEM); 985 goto out; 986 } 987 988 for (i = 0; i < n_fields; i++) 989 event->fields[i] = fields[i]; 990 991 event->n_fields = n_fields; 992 out: 993 return event; 994 } 995 996 static void action_trace(struct hist_trigger_data *hist_data, 997 struct tracing_map_elt *elt, void *rec, 998 struct ring_buffer_event *rbe, 999 struct action_data *data, u64 *var_ref_vals) 1000 { 1001 struct synth_event *event = data->onmatch.synth_event; 1002 1003 trace_synth(event, var_ref_vals, data->onmatch.var_ref_idx); 1004 } 1005 1006 struct hist_var_data { 1007 struct list_head list; 1008 struct hist_trigger_data *hist_data; 1009 }; 1010 1011 static void add_or_delete_synth_event(struct synth_event *event, int delete) 1012 { 1013 if (delete) 1014 free_synth_event(event); 1015 else { 1016 mutex_lock(&synth_event_mutex); 1017 if (!find_synth_event(event->name)) 1018 list_add(&event->list, &synth_event_list); 1019 else 1020 free_synth_event(event); 1021 mutex_unlock(&synth_event_mutex); 1022 } 1023 } 1024 1025 static int create_synth_event(int argc, char **argv) 1026 { 1027 struct synth_field *field, *fields[SYNTH_FIELDS_MAX]; 1028 struct synth_event *event = NULL; 1029 bool delete_event = false; 1030 int i, consumed = 0, n_fields = 0, ret = 0; 1031 char *name; 1032 1033 mutex_lock(&synth_event_mutex); 1034 1035 /* 1036 * Argument syntax: 1037 * - Add synthetic event: <event_name> field[;field] ... 1038 * - Remove synthetic event: !<event_name> field[;field] ... 1039 * where 'field' = type field_name 1040 */ 1041 if (argc < 1) { 1042 ret = -EINVAL; 1043 goto out; 1044 } 1045 1046 name = argv[0]; 1047 if (name[0] == '!') { 1048 delete_event = true; 1049 name++; 1050 } 1051 1052 event = find_synth_event(name); 1053 if (event) { 1054 if (delete_event) { 1055 if (event->ref) { 1056 event = NULL; 1057 ret = -EBUSY; 1058 goto out; 1059 } 1060 list_del(&event->list); 1061 goto out; 1062 } 1063 event = NULL; 1064 ret = -EEXIST; 1065 goto out; 1066 } else if (delete_event) { 1067 ret = -ENOENT; 1068 goto out; 1069 } 1070 1071 if (argc < 2) { 1072 ret = -EINVAL; 1073 goto out; 1074 } 1075 1076 for (i = 1; i < argc - 1; i++) { 1077 if (strcmp(argv[i], ";") == 0) 1078 continue; 1079 if (n_fields == SYNTH_FIELDS_MAX) { 1080 ret = -EINVAL; 1081 goto err; 1082 } 1083 1084 field = parse_synth_field(argc - i, &argv[i], &consumed); 1085 if (IS_ERR(field)) { 1086 ret = PTR_ERR(field); 1087 goto err; 1088 } 1089 fields[n_fields++] = field; 1090 i += consumed - 1; 1091 } 1092 1093 if (i < argc && strcmp(argv[i], ";") != 0) { 1094 ret = -EINVAL; 1095 goto err; 1096 } 1097 1098 event = alloc_synth_event(name, n_fields, fields); 1099 if (IS_ERR(event)) { 1100 ret = PTR_ERR(event); 1101 event = NULL; 1102 goto err; 1103 } 1104 out: 1105 mutex_unlock(&synth_event_mutex); 1106 1107 if (event) { 1108 if (delete_event) { 1109 ret = unregister_synth_event(event); 1110 add_or_delete_synth_event(event, !ret); 1111 } else { 1112 ret = register_synth_event(event); 1113 add_or_delete_synth_event(event, ret); 1114 } 1115 } 1116 1117 return ret; 1118 err: 1119 mutex_unlock(&synth_event_mutex); 1120 1121 for (i = 0; i < n_fields; i++) 1122 free_synth_field(fields[i]); 1123 free_synth_event(event); 1124 1125 return ret; 1126 } 1127 1128 static int release_all_synth_events(void) 1129 { 1130 struct list_head release_events; 1131 struct synth_event *event, *e; 1132 int ret = 0; 1133 1134 INIT_LIST_HEAD(&release_events); 1135 1136 mutex_lock(&synth_event_mutex); 1137 1138 list_for_each_entry(event, &synth_event_list, list) { 1139 if (event->ref) { 1140 mutex_unlock(&synth_event_mutex); 1141 return -EBUSY; 1142 } 1143 } 1144 1145 list_splice_init(&event->list, &release_events); 1146 1147 mutex_unlock(&synth_event_mutex); 1148 1149 list_for_each_entry_safe(event, e, &release_events, list) { 1150 list_del(&event->list); 1151 1152 ret = unregister_synth_event(event); 1153 add_or_delete_synth_event(event, !ret); 1154 } 1155 1156 return ret; 1157 } 1158 1159 1160 static void *synth_events_seq_start(struct seq_file *m, loff_t *pos) 1161 { 1162 mutex_lock(&synth_event_mutex); 1163 1164 return seq_list_start(&synth_event_list, *pos); 1165 } 1166 1167 static void *synth_events_seq_next(struct seq_file *m, void *v, loff_t *pos) 1168 { 1169 return seq_list_next(v, &synth_event_list, pos); 1170 } 1171 1172 static void synth_events_seq_stop(struct seq_file *m, void *v) 1173 { 1174 mutex_unlock(&synth_event_mutex); 1175 } 1176 1177 static int synth_events_seq_show(struct seq_file *m, void *v) 1178 { 1179 struct synth_field *field; 1180 struct synth_event *event = v; 1181 unsigned int i; 1182 1183 seq_printf(m, "%s\t", event->name); 1184 1185 for (i = 0; i < event->n_fields; i++) { 1186 field = event->fields[i]; 1187 1188 /* parameter values */ 1189 seq_printf(m, "%s %s%s", field->type, field->name, 1190 i == event->n_fields - 1 ? "" : "; "); 1191 } 1192 1193 seq_putc(m, '\n'); 1194 1195 return 0; 1196 } 1197 1198 static const struct seq_operations synth_events_seq_op = { 1199 .start = synth_events_seq_start, 1200 .next = synth_events_seq_next, 1201 .stop = synth_events_seq_stop, 1202 .show = synth_events_seq_show 1203 }; 1204 1205 static int synth_events_open(struct inode *inode, struct file *file) 1206 { 1207 int ret; 1208 1209 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 1210 ret = release_all_synth_events(); 1211 if (ret < 0) 1212 return ret; 1213 } 1214 1215 return seq_open(file, &synth_events_seq_op); 1216 } 1217 1218 static ssize_t synth_events_write(struct file *file, 1219 const char __user *buffer, 1220 size_t count, loff_t *ppos) 1221 { 1222 return trace_parse_run_command(file, buffer, count, ppos, 1223 create_synth_event); 1224 } 1225 1226 static const struct file_operations synth_events_fops = { 1227 .open = synth_events_open, 1228 .write = synth_events_write, 1229 .read = seq_read, 1230 .llseek = seq_lseek, 1231 .release = seq_release, 1232 }; 1233 1234 static u64 hist_field_timestamp(struct hist_field *hist_field, 1235 struct tracing_map_elt *elt, 1236 struct ring_buffer_event *rbe, 1237 void *event) 1238 { 1239 struct hist_trigger_data *hist_data = hist_field->hist_data; 1240 struct trace_array *tr = hist_data->event_file->tr; 1241 1242 u64 ts = ring_buffer_event_time_stamp(rbe); 1243 1244 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr)) 1245 ts = ns2usecs(ts); 1246 1247 return ts; 1248 } 1249 1250 static u64 hist_field_cpu(struct hist_field *hist_field, 1251 struct tracing_map_elt *elt, 1252 struct ring_buffer_event *rbe, 1253 void *event) 1254 { 1255 int cpu = smp_processor_id(); 1256 1257 return cpu; 1258 } 1259 1260 static struct hist_field * 1261 check_field_for_var_ref(struct hist_field *hist_field, 1262 struct hist_trigger_data *var_data, 1263 unsigned int var_idx) 1264 { 1265 struct hist_field *found = NULL; 1266 1267 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF) { 1268 if (hist_field->var.idx == var_idx && 1269 hist_field->var.hist_data == var_data) { 1270 found = hist_field; 1271 } 1272 } 1273 1274 return found; 1275 } 1276 1277 static struct hist_field * 1278 check_field_for_var_refs(struct hist_trigger_data *hist_data, 1279 struct hist_field *hist_field, 1280 struct hist_trigger_data *var_data, 1281 unsigned int var_idx, 1282 unsigned int level) 1283 { 1284 struct hist_field *found = NULL; 1285 unsigned int i; 1286 1287 if (level > 3) 1288 return found; 1289 1290 if (!hist_field) 1291 return found; 1292 1293 found = check_field_for_var_ref(hist_field, var_data, var_idx); 1294 if (found) 1295 return found; 1296 1297 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) { 1298 struct hist_field *operand; 1299 1300 operand = hist_field->operands[i]; 1301 found = check_field_for_var_refs(hist_data, operand, var_data, 1302 var_idx, level + 1); 1303 if (found) 1304 return found; 1305 } 1306 1307 return found; 1308 } 1309 1310 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data, 1311 struct hist_trigger_data *var_data, 1312 unsigned int var_idx) 1313 { 1314 struct hist_field *hist_field, *found = NULL; 1315 unsigned int i; 1316 1317 for_each_hist_field(i, hist_data) { 1318 hist_field = hist_data->fields[i]; 1319 found = check_field_for_var_refs(hist_data, hist_field, 1320 var_data, var_idx, 0); 1321 if (found) 1322 return found; 1323 } 1324 1325 for (i = 0; i < hist_data->n_synth_var_refs; i++) { 1326 hist_field = hist_data->synth_var_refs[i]; 1327 found = check_field_for_var_refs(hist_data, hist_field, 1328 var_data, var_idx, 0); 1329 if (found) 1330 return found; 1331 } 1332 1333 return found; 1334 } 1335 1336 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data, 1337 unsigned int var_idx) 1338 { 1339 struct trace_array *tr = hist_data->event_file->tr; 1340 struct hist_field *found = NULL; 1341 struct hist_var_data *var_data; 1342 1343 list_for_each_entry(var_data, &tr->hist_vars, list) { 1344 if (var_data->hist_data == hist_data) 1345 continue; 1346 found = find_var_ref(var_data->hist_data, hist_data, var_idx); 1347 if (found) 1348 break; 1349 } 1350 1351 return found; 1352 } 1353 1354 static bool check_var_refs(struct hist_trigger_data *hist_data) 1355 { 1356 struct hist_field *field; 1357 bool found = false; 1358 int i; 1359 1360 for_each_hist_field(i, hist_data) { 1361 field = hist_data->fields[i]; 1362 if (field && field->flags & HIST_FIELD_FL_VAR) { 1363 if (find_any_var_ref(hist_data, field->var.idx)) { 1364 found = true; 1365 break; 1366 } 1367 } 1368 } 1369 1370 return found; 1371 } 1372 1373 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data) 1374 { 1375 struct trace_array *tr = hist_data->event_file->tr; 1376 struct hist_var_data *var_data, *found = NULL; 1377 1378 list_for_each_entry(var_data, &tr->hist_vars, list) { 1379 if (var_data->hist_data == hist_data) { 1380 found = var_data; 1381 break; 1382 } 1383 } 1384 1385 return found; 1386 } 1387 1388 static bool field_has_hist_vars(struct hist_field *hist_field, 1389 unsigned int level) 1390 { 1391 int i; 1392 1393 if (level > 3) 1394 return false; 1395 1396 if (!hist_field) 1397 return false; 1398 1399 if (hist_field->flags & HIST_FIELD_FL_VAR || 1400 hist_field->flags & HIST_FIELD_FL_VAR_REF) 1401 return true; 1402 1403 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) { 1404 struct hist_field *operand; 1405 1406 operand = hist_field->operands[i]; 1407 if (field_has_hist_vars(operand, level + 1)) 1408 return true; 1409 } 1410 1411 return false; 1412 } 1413 1414 static bool has_hist_vars(struct hist_trigger_data *hist_data) 1415 { 1416 struct hist_field *hist_field; 1417 int i; 1418 1419 for_each_hist_field(i, hist_data) { 1420 hist_field = hist_data->fields[i]; 1421 if (field_has_hist_vars(hist_field, 0)) 1422 return true; 1423 } 1424 1425 return false; 1426 } 1427 1428 static int save_hist_vars(struct hist_trigger_data *hist_data) 1429 { 1430 struct trace_array *tr = hist_data->event_file->tr; 1431 struct hist_var_data *var_data; 1432 1433 var_data = find_hist_vars(hist_data); 1434 if (var_data) 1435 return 0; 1436 1437 if (trace_array_get(tr) < 0) 1438 return -ENODEV; 1439 1440 var_data = kzalloc(sizeof(*var_data), GFP_KERNEL); 1441 if (!var_data) { 1442 trace_array_put(tr); 1443 return -ENOMEM; 1444 } 1445 1446 var_data->hist_data = hist_data; 1447 list_add(&var_data->list, &tr->hist_vars); 1448 1449 return 0; 1450 } 1451 1452 static void remove_hist_vars(struct hist_trigger_data *hist_data) 1453 { 1454 struct trace_array *tr = hist_data->event_file->tr; 1455 struct hist_var_data *var_data; 1456 1457 var_data = find_hist_vars(hist_data); 1458 if (!var_data) 1459 return; 1460 1461 if (WARN_ON(check_var_refs(hist_data))) 1462 return; 1463 1464 list_del(&var_data->list); 1465 1466 kfree(var_data); 1467 1468 trace_array_put(tr); 1469 } 1470 1471 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data, 1472 const char *var_name) 1473 { 1474 struct hist_field *hist_field, *found = NULL; 1475 int i; 1476 1477 for_each_hist_field(i, hist_data) { 1478 hist_field = hist_data->fields[i]; 1479 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR && 1480 strcmp(hist_field->var.name, var_name) == 0) { 1481 found = hist_field; 1482 break; 1483 } 1484 } 1485 1486 return found; 1487 } 1488 1489 static struct hist_field *find_var(struct hist_trigger_data *hist_data, 1490 struct trace_event_file *file, 1491 const char *var_name) 1492 { 1493 struct hist_trigger_data *test_data; 1494 struct event_trigger_data *test; 1495 struct hist_field *hist_field; 1496 1497 hist_field = find_var_field(hist_data, var_name); 1498 if (hist_field) 1499 return hist_field; 1500 1501 list_for_each_entry_rcu(test, &file->triggers, list) { 1502 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1503 test_data = test->private_data; 1504 hist_field = find_var_field(test_data, var_name); 1505 if (hist_field) 1506 return hist_field; 1507 } 1508 } 1509 1510 return NULL; 1511 } 1512 1513 static struct trace_event_file *find_var_file(struct trace_array *tr, 1514 char *system, 1515 char *event_name, 1516 char *var_name) 1517 { 1518 struct hist_trigger_data *var_hist_data; 1519 struct hist_var_data *var_data; 1520 struct trace_event_file *file, *found = NULL; 1521 1522 if (system) 1523 return find_event_file(tr, system, event_name); 1524 1525 list_for_each_entry(var_data, &tr->hist_vars, list) { 1526 var_hist_data = var_data->hist_data; 1527 file = var_hist_data->event_file; 1528 if (file == found) 1529 continue; 1530 1531 if (find_var_field(var_hist_data, var_name)) { 1532 if (found) { 1533 hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name); 1534 return NULL; 1535 } 1536 1537 found = file; 1538 } 1539 } 1540 1541 return found; 1542 } 1543 1544 static struct hist_field *find_file_var(struct trace_event_file *file, 1545 const char *var_name) 1546 { 1547 struct hist_trigger_data *test_data; 1548 struct event_trigger_data *test; 1549 struct hist_field *hist_field; 1550 1551 list_for_each_entry_rcu(test, &file->triggers, list) { 1552 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1553 test_data = test->private_data; 1554 hist_field = find_var_field(test_data, var_name); 1555 if (hist_field) 1556 return hist_field; 1557 } 1558 } 1559 1560 return NULL; 1561 } 1562 1563 static struct hist_field * 1564 find_match_var(struct hist_trigger_data *hist_data, char *var_name) 1565 { 1566 struct trace_array *tr = hist_data->event_file->tr; 1567 struct hist_field *hist_field, *found = NULL; 1568 struct trace_event_file *file; 1569 unsigned int i; 1570 1571 for (i = 0; i < hist_data->n_actions; i++) { 1572 struct action_data *data = hist_data->actions[i]; 1573 1574 if (data->fn == action_trace) { 1575 char *system = data->onmatch.match_event_system; 1576 char *event_name = data->onmatch.match_event; 1577 1578 file = find_var_file(tr, system, event_name, var_name); 1579 if (!file) 1580 continue; 1581 hist_field = find_file_var(file, var_name); 1582 if (hist_field) { 1583 if (found) { 1584 hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name); 1585 return ERR_PTR(-EINVAL); 1586 } 1587 1588 found = hist_field; 1589 } 1590 } 1591 } 1592 return found; 1593 } 1594 1595 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data, 1596 char *system, 1597 char *event_name, 1598 char *var_name) 1599 { 1600 struct trace_array *tr = hist_data->event_file->tr; 1601 struct hist_field *hist_field = NULL; 1602 struct trace_event_file *file; 1603 1604 if (!system || !event_name) { 1605 hist_field = find_match_var(hist_data, var_name); 1606 if (IS_ERR(hist_field)) 1607 return NULL; 1608 if (hist_field) 1609 return hist_field; 1610 } 1611 1612 file = find_var_file(tr, system, event_name, var_name); 1613 if (!file) 1614 return NULL; 1615 1616 hist_field = find_file_var(file, var_name); 1617 1618 return hist_field; 1619 } 1620 1621 struct hist_elt_data { 1622 char *comm; 1623 u64 *var_ref_vals; 1624 char *field_var_str[SYNTH_FIELDS_MAX]; 1625 }; 1626 1627 static u64 hist_field_var_ref(struct hist_field *hist_field, 1628 struct tracing_map_elt *elt, 1629 struct ring_buffer_event *rbe, 1630 void *event) 1631 { 1632 struct hist_elt_data *elt_data; 1633 u64 var_val = 0; 1634 1635 elt_data = elt->private_data; 1636 var_val = elt_data->var_ref_vals[hist_field->var_ref_idx]; 1637 1638 return var_val; 1639 } 1640 1641 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key, 1642 u64 *var_ref_vals, bool self) 1643 { 1644 struct hist_trigger_data *var_data; 1645 struct tracing_map_elt *var_elt; 1646 struct hist_field *hist_field; 1647 unsigned int i, var_idx; 1648 bool resolved = true; 1649 u64 var_val = 0; 1650 1651 for (i = 0; i < hist_data->n_var_refs; i++) { 1652 hist_field = hist_data->var_refs[i]; 1653 var_idx = hist_field->var.idx; 1654 var_data = hist_field->var.hist_data; 1655 1656 if (var_data == NULL) { 1657 resolved = false; 1658 break; 1659 } 1660 1661 if ((self && var_data != hist_data) || 1662 (!self && var_data == hist_data)) 1663 continue; 1664 1665 var_elt = tracing_map_lookup(var_data->map, key); 1666 if (!var_elt) { 1667 resolved = false; 1668 break; 1669 } 1670 1671 if (!tracing_map_var_set(var_elt, var_idx)) { 1672 resolved = false; 1673 break; 1674 } 1675 1676 if (self || !hist_field->read_once) 1677 var_val = tracing_map_read_var(var_elt, var_idx); 1678 else 1679 var_val = tracing_map_read_var_once(var_elt, var_idx); 1680 1681 var_ref_vals[i] = var_val; 1682 } 1683 1684 return resolved; 1685 } 1686 1687 static const char *hist_field_name(struct hist_field *field, 1688 unsigned int level) 1689 { 1690 const char *field_name = ""; 1691 1692 if (level > 1) 1693 return field_name; 1694 1695 if (field->field) 1696 field_name = field->field->name; 1697 else if (field->flags & HIST_FIELD_FL_LOG2 || 1698 field->flags & HIST_FIELD_FL_ALIAS) 1699 field_name = hist_field_name(field->operands[0], ++level); 1700 else if (field->flags & HIST_FIELD_FL_CPU) 1701 field_name = "cpu"; 1702 else if (field->flags & HIST_FIELD_FL_EXPR || 1703 field->flags & HIST_FIELD_FL_VAR_REF) { 1704 if (field->system) { 1705 static char full_name[MAX_FILTER_STR_VAL]; 1706 1707 strcat(full_name, field->system); 1708 strcat(full_name, "."); 1709 strcat(full_name, field->event_name); 1710 strcat(full_name, "."); 1711 strcat(full_name, field->name); 1712 field_name = full_name; 1713 } else 1714 field_name = field->name; 1715 } else if (field->flags & HIST_FIELD_FL_TIMESTAMP) 1716 field_name = "common_timestamp"; 1717 1718 if (field_name == NULL) 1719 field_name = ""; 1720 1721 return field_name; 1722 } 1723 1724 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed) 1725 { 1726 hist_field_fn_t fn = NULL; 1727 1728 switch (field_size) { 1729 case 8: 1730 if (field_is_signed) 1731 fn = hist_field_s64; 1732 else 1733 fn = hist_field_u64; 1734 break; 1735 case 4: 1736 if (field_is_signed) 1737 fn = hist_field_s32; 1738 else 1739 fn = hist_field_u32; 1740 break; 1741 case 2: 1742 if (field_is_signed) 1743 fn = hist_field_s16; 1744 else 1745 fn = hist_field_u16; 1746 break; 1747 case 1: 1748 if (field_is_signed) 1749 fn = hist_field_s8; 1750 else 1751 fn = hist_field_u8; 1752 break; 1753 } 1754 1755 return fn; 1756 } 1757 1758 static int parse_map_size(char *str) 1759 { 1760 unsigned long size, map_bits; 1761 int ret; 1762 1763 strsep(&str, "="); 1764 if (!str) { 1765 ret = -EINVAL; 1766 goto out; 1767 } 1768 1769 ret = kstrtoul(str, 0, &size); 1770 if (ret) 1771 goto out; 1772 1773 map_bits = ilog2(roundup_pow_of_two(size)); 1774 if (map_bits < TRACING_MAP_BITS_MIN || 1775 map_bits > TRACING_MAP_BITS_MAX) 1776 ret = -EINVAL; 1777 else 1778 ret = map_bits; 1779 out: 1780 return ret; 1781 } 1782 1783 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs) 1784 { 1785 unsigned int i; 1786 1787 if (!attrs) 1788 return; 1789 1790 for (i = 0; i < attrs->n_assignments; i++) 1791 kfree(attrs->assignment_str[i]); 1792 1793 for (i = 0; i < attrs->n_actions; i++) 1794 kfree(attrs->action_str[i]); 1795 1796 kfree(attrs->name); 1797 kfree(attrs->sort_key_str); 1798 kfree(attrs->keys_str); 1799 kfree(attrs->vals_str); 1800 kfree(attrs->clock); 1801 kfree(attrs); 1802 } 1803 1804 static int parse_action(char *str, struct hist_trigger_attrs *attrs) 1805 { 1806 int ret = -EINVAL; 1807 1808 if (attrs->n_actions >= HIST_ACTIONS_MAX) 1809 return ret; 1810 1811 if ((strncmp(str, "onmatch(", strlen("onmatch(")) == 0) || 1812 (strncmp(str, "onmax(", strlen("onmax(")) == 0)) { 1813 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL); 1814 if (!attrs->action_str[attrs->n_actions]) { 1815 ret = -ENOMEM; 1816 return ret; 1817 } 1818 attrs->n_actions++; 1819 ret = 0; 1820 } 1821 1822 return ret; 1823 } 1824 1825 static int parse_assignment(char *str, struct hist_trigger_attrs *attrs) 1826 { 1827 int ret = 0; 1828 1829 if ((strncmp(str, "key=", strlen("key=")) == 0) || 1830 (strncmp(str, "keys=", strlen("keys=")) == 0)) { 1831 attrs->keys_str = kstrdup(str, GFP_KERNEL); 1832 if (!attrs->keys_str) { 1833 ret = -ENOMEM; 1834 goto out; 1835 } 1836 } else if ((strncmp(str, "val=", strlen("val=")) == 0) || 1837 (strncmp(str, "vals=", strlen("vals=")) == 0) || 1838 (strncmp(str, "values=", strlen("values=")) == 0)) { 1839 attrs->vals_str = kstrdup(str, GFP_KERNEL); 1840 if (!attrs->vals_str) { 1841 ret = -ENOMEM; 1842 goto out; 1843 } 1844 } else if (strncmp(str, "sort=", strlen("sort=")) == 0) { 1845 attrs->sort_key_str = kstrdup(str, GFP_KERNEL); 1846 if (!attrs->sort_key_str) { 1847 ret = -ENOMEM; 1848 goto out; 1849 } 1850 } else if (strncmp(str, "name=", strlen("name=")) == 0) { 1851 attrs->name = kstrdup(str, GFP_KERNEL); 1852 if (!attrs->name) { 1853 ret = -ENOMEM; 1854 goto out; 1855 } 1856 } else if (strncmp(str, "clock=", strlen("clock=")) == 0) { 1857 strsep(&str, "="); 1858 if (!str) { 1859 ret = -EINVAL; 1860 goto out; 1861 } 1862 1863 str = strstrip(str); 1864 attrs->clock = kstrdup(str, GFP_KERNEL); 1865 if (!attrs->clock) { 1866 ret = -ENOMEM; 1867 goto out; 1868 } 1869 } else if (strncmp(str, "size=", strlen("size=")) == 0) { 1870 int map_bits = parse_map_size(str); 1871 1872 if (map_bits < 0) { 1873 ret = map_bits; 1874 goto out; 1875 } 1876 attrs->map_bits = map_bits; 1877 } else { 1878 char *assignment; 1879 1880 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) { 1881 hist_err("Too many variables defined: ", str); 1882 ret = -EINVAL; 1883 goto out; 1884 } 1885 1886 assignment = kstrdup(str, GFP_KERNEL); 1887 if (!assignment) { 1888 ret = -ENOMEM; 1889 goto out; 1890 } 1891 1892 attrs->assignment_str[attrs->n_assignments++] = assignment; 1893 } 1894 out: 1895 return ret; 1896 } 1897 1898 static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str) 1899 { 1900 struct hist_trigger_attrs *attrs; 1901 int ret = 0; 1902 1903 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL); 1904 if (!attrs) 1905 return ERR_PTR(-ENOMEM); 1906 1907 while (trigger_str) { 1908 char *str = strsep(&trigger_str, ":"); 1909 1910 if (strchr(str, '=')) { 1911 ret = parse_assignment(str, attrs); 1912 if (ret) 1913 goto free; 1914 } else if (strcmp(str, "pause") == 0) 1915 attrs->pause = true; 1916 else if ((strcmp(str, "cont") == 0) || 1917 (strcmp(str, "continue") == 0)) 1918 attrs->cont = true; 1919 else if (strcmp(str, "clear") == 0) 1920 attrs->clear = true; 1921 else { 1922 ret = parse_action(str, attrs); 1923 if (ret) 1924 goto free; 1925 } 1926 } 1927 1928 if (!attrs->keys_str) { 1929 ret = -EINVAL; 1930 goto free; 1931 } 1932 1933 if (!attrs->clock) { 1934 attrs->clock = kstrdup("global", GFP_KERNEL); 1935 if (!attrs->clock) { 1936 ret = -ENOMEM; 1937 goto free; 1938 } 1939 } 1940 1941 return attrs; 1942 free: 1943 destroy_hist_trigger_attrs(attrs); 1944 1945 return ERR_PTR(ret); 1946 } 1947 1948 static inline void save_comm(char *comm, struct task_struct *task) 1949 { 1950 if (!task->pid) { 1951 strcpy(comm, "<idle>"); 1952 return; 1953 } 1954 1955 if (WARN_ON_ONCE(task->pid < 0)) { 1956 strcpy(comm, "<XXX>"); 1957 return; 1958 } 1959 1960 memcpy(comm, task->comm, TASK_COMM_LEN); 1961 } 1962 1963 static void hist_elt_data_free(struct hist_elt_data *elt_data) 1964 { 1965 unsigned int i; 1966 1967 for (i = 0; i < SYNTH_FIELDS_MAX; i++) 1968 kfree(elt_data->field_var_str[i]); 1969 1970 kfree(elt_data->comm); 1971 kfree(elt_data); 1972 } 1973 1974 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt) 1975 { 1976 struct hist_elt_data *elt_data = elt->private_data; 1977 1978 hist_elt_data_free(elt_data); 1979 } 1980 1981 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt) 1982 { 1983 struct hist_trigger_data *hist_data = elt->map->private_data; 1984 unsigned int size = TASK_COMM_LEN; 1985 struct hist_elt_data *elt_data; 1986 struct hist_field *key_field; 1987 unsigned int i, n_str; 1988 1989 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL); 1990 if (!elt_data) 1991 return -ENOMEM; 1992 1993 for_each_hist_key_field(i, hist_data) { 1994 key_field = hist_data->fields[i]; 1995 1996 if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 1997 elt_data->comm = kzalloc(size, GFP_KERNEL); 1998 if (!elt_data->comm) { 1999 kfree(elt_data); 2000 return -ENOMEM; 2001 } 2002 break; 2003 } 2004 } 2005 2006 n_str = hist_data->n_field_var_str + hist_data->n_max_var_str; 2007 2008 size = STR_VAR_LEN_MAX; 2009 2010 for (i = 0; i < n_str; i++) { 2011 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL); 2012 if (!elt_data->field_var_str[i]) { 2013 hist_elt_data_free(elt_data); 2014 return -ENOMEM; 2015 } 2016 } 2017 2018 elt->private_data = elt_data; 2019 2020 return 0; 2021 } 2022 2023 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt) 2024 { 2025 struct hist_elt_data *elt_data = elt->private_data; 2026 2027 if (elt_data->comm) 2028 save_comm(elt_data->comm, current); 2029 } 2030 2031 static const struct tracing_map_ops hist_trigger_elt_data_ops = { 2032 .elt_alloc = hist_trigger_elt_data_alloc, 2033 .elt_free = hist_trigger_elt_data_free, 2034 .elt_init = hist_trigger_elt_data_init, 2035 }; 2036 2037 static const char *get_hist_field_flags(struct hist_field *hist_field) 2038 { 2039 const char *flags_str = NULL; 2040 2041 if (hist_field->flags & HIST_FIELD_FL_HEX) 2042 flags_str = "hex"; 2043 else if (hist_field->flags & HIST_FIELD_FL_SYM) 2044 flags_str = "sym"; 2045 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET) 2046 flags_str = "sym-offset"; 2047 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME) 2048 flags_str = "execname"; 2049 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL) 2050 flags_str = "syscall"; 2051 else if (hist_field->flags & HIST_FIELD_FL_LOG2) 2052 flags_str = "log2"; 2053 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS) 2054 flags_str = "usecs"; 2055 2056 return flags_str; 2057 } 2058 2059 static void expr_field_str(struct hist_field *field, char *expr) 2060 { 2061 if (field->flags & HIST_FIELD_FL_VAR_REF) 2062 strcat(expr, "$"); 2063 2064 strcat(expr, hist_field_name(field, 0)); 2065 2066 if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) { 2067 const char *flags_str = get_hist_field_flags(field); 2068 2069 if (flags_str) { 2070 strcat(expr, "."); 2071 strcat(expr, flags_str); 2072 } 2073 } 2074 } 2075 2076 static char *expr_str(struct hist_field *field, unsigned int level) 2077 { 2078 char *expr; 2079 2080 if (level > 1) 2081 return NULL; 2082 2083 expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2084 if (!expr) 2085 return NULL; 2086 2087 if (!field->operands[0]) { 2088 expr_field_str(field, expr); 2089 return expr; 2090 } 2091 2092 if (field->operator == FIELD_OP_UNARY_MINUS) { 2093 char *subexpr; 2094 2095 strcat(expr, "-("); 2096 subexpr = expr_str(field->operands[0], ++level); 2097 if (!subexpr) { 2098 kfree(expr); 2099 return NULL; 2100 } 2101 strcat(expr, subexpr); 2102 strcat(expr, ")"); 2103 2104 kfree(subexpr); 2105 2106 return expr; 2107 } 2108 2109 expr_field_str(field->operands[0], expr); 2110 2111 switch (field->operator) { 2112 case FIELD_OP_MINUS: 2113 strcat(expr, "-"); 2114 break; 2115 case FIELD_OP_PLUS: 2116 strcat(expr, "+"); 2117 break; 2118 default: 2119 kfree(expr); 2120 return NULL; 2121 } 2122 2123 expr_field_str(field->operands[1], expr); 2124 2125 return expr; 2126 } 2127 2128 static int contains_operator(char *str) 2129 { 2130 enum field_op_id field_op = FIELD_OP_NONE; 2131 char *op; 2132 2133 op = strpbrk(str, "+-"); 2134 if (!op) 2135 return FIELD_OP_NONE; 2136 2137 switch (*op) { 2138 case '-': 2139 if (*str == '-') 2140 field_op = FIELD_OP_UNARY_MINUS; 2141 else 2142 field_op = FIELD_OP_MINUS; 2143 break; 2144 case '+': 2145 field_op = FIELD_OP_PLUS; 2146 break; 2147 default: 2148 break; 2149 } 2150 2151 return field_op; 2152 } 2153 2154 static void destroy_hist_field(struct hist_field *hist_field, 2155 unsigned int level) 2156 { 2157 unsigned int i; 2158 2159 if (level > 3) 2160 return; 2161 2162 if (!hist_field) 2163 return; 2164 2165 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) 2166 destroy_hist_field(hist_field->operands[i], level + 1); 2167 2168 kfree(hist_field->var.name); 2169 kfree(hist_field->name); 2170 kfree(hist_field->type); 2171 2172 kfree(hist_field); 2173 } 2174 2175 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, 2176 struct ftrace_event_field *field, 2177 unsigned long flags, 2178 char *var_name) 2179 { 2180 struct hist_field *hist_field; 2181 2182 if (field && is_function_field(field)) 2183 return NULL; 2184 2185 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL); 2186 if (!hist_field) 2187 return NULL; 2188 2189 hist_field->hist_data = hist_data; 2190 2191 if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS) 2192 goto out; /* caller will populate */ 2193 2194 if (flags & HIST_FIELD_FL_VAR_REF) { 2195 hist_field->fn = hist_field_var_ref; 2196 goto out; 2197 } 2198 2199 if (flags & HIST_FIELD_FL_HITCOUNT) { 2200 hist_field->fn = hist_field_counter; 2201 hist_field->size = sizeof(u64); 2202 hist_field->type = kstrdup("u64", GFP_KERNEL); 2203 if (!hist_field->type) 2204 goto free; 2205 goto out; 2206 } 2207 2208 if (flags & HIST_FIELD_FL_STACKTRACE) { 2209 hist_field->fn = hist_field_none; 2210 goto out; 2211 } 2212 2213 if (flags & HIST_FIELD_FL_LOG2) { 2214 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2; 2215 hist_field->fn = hist_field_log2; 2216 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL); 2217 hist_field->size = hist_field->operands[0]->size; 2218 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL); 2219 if (!hist_field->type) 2220 goto free; 2221 goto out; 2222 } 2223 2224 if (flags & HIST_FIELD_FL_TIMESTAMP) { 2225 hist_field->fn = hist_field_timestamp; 2226 hist_field->size = sizeof(u64); 2227 hist_field->type = kstrdup("u64", GFP_KERNEL); 2228 if (!hist_field->type) 2229 goto free; 2230 goto out; 2231 } 2232 2233 if (flags & HIST_FIELD_FL_CPU) { 2234 hist_field->fn = hist_field_cpu; 2235 hist_field->size = sizeof(int); 2236 hist_field->type = kstrdup("unsigned int", GFP_KERNEL); 2237 if (!hist_field->type) 2238 goto free; 2239 goto out; 2240 } 2241 2242 if (WARN_ON_ONCE(!field)) 2243 goto out; 2244 2245 if (is_string_field(field)) { 2246 flags |= HIST_FIELD_FL_STRING; 2247 2248 hist_field->size = MAX_FILTER_STR_VAL; 2249 hist_field->type = kstrdup(field->type, GFP_KERNEL); 2250 if (!hist_field->type) 2251 goto free; 2252 2253 if (field->filter_type == FILTER_STATIC_STRING) 2254 hist_field->fn = hist_field_string; 2255 else if (field->filter_type == FILTER_DYN_STRING) 2256 hist_field->fn = hist_field_dynstring; 2257 else 2258 hist_field->fn = hist_field_pstring; 2259 } else { 2260 hist_field->size = field->size; 2261 hist_field->is_signed = field->is_signed; 2262 hist_field->type = kstrdup(field->type, GFP_KERNEL); 2263 if (!hist_field->type) 2264 goto free; 2265 2266 hist_field->fn = select_value_fn(field->size, 2267 field->is_signed); 2268 if (!hist_field->fn) { 2269 destroy_hist_field(hist_field, 0); 2270 return NULL; 2271 } 2272 } 2273 out: 2274 hist_field->field = field; 2275 hist_field->flags = flags; 2276 2277 if (var_name) { 2278 hist_field->var.name = kstrdup(var_name, GFP_KERNEL); 2279 if (!hist_field->var.name) 2280 goto free; 2281 } 2282 2283 return hist_field; 2284 free: 2285 destroy_hist_field(hist_field, 0); 2286 return NULL; 2287 } 2288 2289 static void destroy_hist_fields(struct hist_trigger_data *hist_data) 2290 { 2291 unsigned int i; 2292 2293 for (i = 0; i < HIST_FIELDS_MAX; i++) { 2294 if (hist_data->fields[i]) { 2295 destroy_hist_field(hist_data->fields[i], 0); 2296 hist_data->fields[i] = NULL; 2297 } 2298 } 2299 } 2300 2301 static int init_var_ref(struct hist_field *ref_field, 2302 struct hist_field *var_field, 2303 char *system, char *event_name) 2304 { 2305 int err = 0; 2306 2307 ref_field->var.idx = var_field->var.idx; 2308 ref_field->var.hist_data = var_field->hist_data; 2309 ref_field->size = var_field->size; 2310 ref_field->is_signed = var_field->is_signed; 2311 ref_field->flags |= var_field->flags & 2312 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2313 2314 if (system) { 2315 ref_field->system = kstrdup(system, GFP_KERNEL); 2316 if (!ref_field->system) 2317 return -ENOMEM; 2318 } 2319 2320 if (event_name) { 2321 ref_field->event_name = kstrdup(event_name, GFP_KERNEL); 2322 if (!ref_field->event_name) { 2323 err = -ENOMEM; 2324 goto free; 2325 } 2326 } 2327 2328 if (var_field->var.name) { 2329 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL); 2330 if (!ref_field->name) { 2331 err = -ENOMEM; 2332 goto free; 2333 } 2334 } else if (var_field->name) { 2335 ref_field->name = kstrdup(var_field->name, GFP_KERNEL); 2336 if (!ref_field->name) { 2337 err = -ENOMEM; 2338 goto free; 2339 } 2340 } 2341 2342 ref_field->type = kstrdup(var_field->type, GFP_KERNEL); 2343 if (!ref_field->type) { 2344 err = -ENOMEM; 2345 goto free; 2346 } 2347 out: 2348 return err; 2349 free: 2350 kfree(ref_field->system); 2351 kfree(ref_field->event_name); 2352 kfree(ref_field->name); 2353 2354 goto out; 2355 } 2356 2357 static struct hist_field *create_var_ref(struct hist_field *var_field, 2358 char *system, char *event_name) 2359 { 2360 unsigned long flags = HIST_FIELD_FL_VAR_REF; 2361 struct hist_field *ref_field; 2362 2363 ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL); 2364 if (ref_field) { 2365 if (init_var_ref(ref_field, var_field, system, event_name)) { 2366 destroy_hist_field(ref_field, 0); 2367 return NULL; 2368 } 2369 } 2370 2371 return ref_field; 2372 } 2373 2374 static bool is_var_ref(char *var_name) 2375 { 2376 if (!var_name || strlen(var_name) < 2 || var_name[0] != '$') 2377 return false; 2378 2379 return true; 2380 } 2381 2382 static char *field_name_from_var(struct hist_trigger_data *hist_data, 2383 char *var_name) 2384 { 2385 char *name, *field; 2386 unsigned int i; 2387 2388 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 2389 name = hist_data->attrs->var_defs.name[i]; 2390 2391 if (strcmp(var_name, name) == 0) { 2392 field = hist_data->attrs->var_defs.expr[i]; 2393 if (contains_operator(field) || is_var_ref(field)) 2394 continue; 2395 return field; 2396 } 2397 } 2398 2399 return NULL; 2400 } 2401 2402 static char *local_field_var_ref(struct hist_trigger_data *hist_data, 2403 char *system, char *event_name, 2404 char *var_name) 2405 { 2406 struct trace_event_call *call; 2407 2408 if (system && event_name) { 2409 call = hist_data->event_file->event_call; 2410 2411 if (strcmp(system, call->class->system) != 0) 2412 return NULL; 2413 2414 if (strcmp(event_name, trace_event_name(call)) != 0) 2415 return NULL; 2416 } 2417 2418 if (!!system != !!event_name) 2419 return NULL; 2420 2421 if (!is_var_ref(var_name)) 2422 return NULL; 2423 2424 var_name++; 2425 2426 return field_name_from_var(hist_data, var_name); 2427 } 2428 2429 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data, 2430 char *system, char *event_name, 2431 char *var_name) 2432 { 2433 struct hist_field *var_field = NULL, *ref_field = NULL; 2434 2435 if (!is_var_ref(var_name)) 2436 return NULL; 2437 2438 var_name++; 2439 2440 var_field = find_event_var(hist_data, system, event_name, var_name); 2441 if (var_field) 2442 ref_field = create_var_ref(var_field, system, event_name); 2443 2444 if (!ref_field) 2445 hist_err_event("Couldn't find variable: $", 2446 system, event_name, var_name); 2447 2448 return ref_field; 2449 } 2450 2451 static struct ftrace_event_field * 2452 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, 2453 char *field_str, unsigned long *flags) 2454 { 2455 struct ftrace_event_field *field = NULL; 2456 char *field_name, *modifier, *str; 2457 2458 modifier = str = kstrdup(field_str, GFP_KERNEL); 2459 if (!modifier) 2460 return ERR_PTR(-ENOMEM); 2461 2462 field_name = strsep(&modifier, "."); 2463 if (modifier) { 2464 if (strcmp(modifier, "hex") == 0) 2465 *flags |= HIST_FIELD_FL_HEX; 2466 else if (strcmp(modifier, "sym") == 0) 2467 *flags |= HIST_FIELD_FL_SYM; 2468 else if (strcmp(modifier, "sym-offset") == 0) 2469 *flags |= HIST_FIELD_FL_SYM_OFFSET; 2470 else if ((strcmp(modifier, "execname") == 0) && 2471 (strcmp(field_name, "common_pid") == 0)) 2472 *flags |= HIST_FIELD_FL_EXECNAME; 2473 else if (strcmp(modifier, "syscall") == 0) 2474 *flags |= HIST_FIELD_FL_SYSCALL; 2475 else if (strcmp(modifier, "log2") == 0) 2476 *flags |= HIST_FIELD_FL_LOG2; 2477 else if (strcmp(modifier, "usecs") == 0) 2478 *flags |= HIST_FIELD_FL_TIMESTAMP_USECS; 2479 else { 2480 hist_err("Invalid field modifier: ", modifier); 2481 field = ERR_PTR(-EINVAL); 2482 goto out; 2483 } 2484 } 2485 2486 if (strcmp(field_name, "common_timestamp") == 0) { 2487 *flags |= HIST_FIELD_FL_TIMESTAMP; 2488 hist_data->enable_timestamps = true; 2489 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS) 2490 hist_data->attrs->ts_in_usecs = true; 2491 } else if (strcmp(field_name, "cpu") == 0) 2492 *flags |= HIST_FIELD_FL_CPU; 2493 else { 2494 field = trace_find_event_field(file->event_call, field_name); 2495 if (!field || !field->size) { 2496 hist_err("Couldn't find field: ", field_name); 2497 field = ERR_PTR(-EINVAL); 2498 goto out; 2499 } 2500 } 2501 out: 2502 kfree(str); 2503 2504 return field; 2505 } 2506 2507 static struct hist_field *create_alias(struct hist_trigger_data *hist_data, 2508 struct hist_field *var_ref, 2509 char *var_name) 2510 { 2511 struct hist_field *alias = NULL; 2512 unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR; 2513 2514 alias = create_hist_field(hist_data, NULL, flags, var_name); 2515 if (!alias) 2516 return NULL; 2517 2518 alias->fn = var_ref->fn; 2519 alias->operands[0] = var_ref; 2520 2521 if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) { 2522 destroy_hist_field(alias, 0); 2523 return NULL; 2524 } 2525 2526 return alias; 2527 } 2528 2529 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data, 2530 struct trace_event_file *file, char *str, 2531 unsigned long *flags, char *var_name) 2532 { 2533 char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str; 2534 struct ftrace_event_field *field = NULL; 2535 struct hist_field *hist_field = NULL; 2536 int ret = 0; 2537 2538 s = strchr(str, '.'); 2539 if (s) { 2540 s = strchr(++s, '.'); 2541 if (s) { 2542 ref_system = strsep(&str, "."); 2543 if (!str) { 2544 ret = -EINVAL; 2545 goto out; 2546 } 2547 ref_event = strsep(&str, "."); 2548 if (!str) { 2549 ret = -EINVAL; 2550 goto out; 2551 } 2552 ref_var = str; 2553 } 2554 } 2555 2556 s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var); 2557 if (!s) { 2558 hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var); 2559 if (hist_field) { 2560 hist_data->var_refs[hist_data->n_var_refs] = hist_field; 2561 hist_field->var_ref_idx = hist_data->n_var_refs++; 2562 if (var_name) { 2563 hist_field = create_alias(hist_data, hist_field, var_name); 2564 if (!hist_field) { 2565 ret = -ENOMEM; 2566 goto out; 2567 } 2568 } 2569 return hist_field; 2570 } 2571 } else 2572 str = s; 2573 2574 field = parse_field(hist_data, file, str, flags); 2575 if (IS_ERR(field)) { 2576 ret = PTR_ERR(field); 2577 goto out; 2578 } 2579 2580 hist_field = create_hist_field(hist_data, field, *flags, var_name); 2581 if (!hist_field) { 2582 ret = -ENOMEM; 2583 goto out; 2584 } 2585 2586 return hist_field; 2587 out: 2588 return ERR_PTR(ret); 2589 } 2590 2591 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2592 struct trace_event_file *file, 2593 char *str, unsigned long flags, 2594 char *var_name, unsigned int level); 2595 2596 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data, 2597 struct trace_event_file *file, 2598 char *str, unsigned long flags, 2599 char *var_name, unsigned int level) 2600 { 2601 struct hist_field *operand1, *expr = NULL; 2602 unsigned long operand_flags; 2603 int ret = 0; 2604 char *s; 2605 2606 /* we support only -(xxx) i.e. explicit parens required */ 2607 2608 if (level > 3) { 2609 hist_err("Too many subexpressions (3 max): ", str); 2610 ret = -EINVAL; 2611 goto free; 2612 } 2613 2614 str++; /* skip leading '-' */ 2615 2616 s = strchr(str, '('); 2617 if (s) 2618 str++; 2619 else { 2620 ret = -EINVAL; 2621 goto free; 2622 } 2623 2624 s = strrchr(str, ')'); 2625 if (s) 2626 *s = '\0'; 2627 else { 2628 ret = -EINVAL; /* no closing ')' */ 2629 goto free; 2630 } 2631 2632 flags |= HIST_FIELD_FL_EXPR; 2633 expr = create_hist_field(hist_data, NULL, flags, var_name); 2634 if (!expr) { 2635 ret = -ENOMEM; 2636 goto free; 2637 } 2638 2639 operand_flags = 0; 2640 operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level); 2641 if (IS_ERR(operand1)) { 2642 ret = PTR_ERR(operand1); 2643 goto free; 2644 } 2645 2646 expr->flags |= operand1->flags & 2647 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2648 expr->fn = hist_field_unary_minus; 2649 expr->operands[0] = operand1; 2650 expr->operator = FIELD_OP_UNARY_MINUS; 2651 expr->name = expr_str(expr, 0); 2652 expr->type = kstrdup(operand1->type, GFP_KERNEL); 2653 if (!expr->type) { 2654 ret = -ENOMEM; 2655 goto free; 2656 } 2657 2658 return expr; 2659 free: 2660 destroy_hist_field(expr, 0); 2661 return ERR_PTR(ret); 2662 } 2663 2664 static int check_expr_operands(struct hist_field *operand1, 2665 struct hist_field *operand2) 2666 { 2667 unsigned long operand1_flags = operand1->flags; 2668 unsigned long operand2_flags = operand2->flags; 2669 2670 if ((operand1_flags & HIST_FIELD_FL_VAR_REF) || 2671 (operand1_flags & HIST_FIELD_FL_ALIAS)) { 2672 struct hist_field *var; 2673 2674 var = find_var_field(operand1->var.hist_data, operand1->name); 2675 if (!var) 2676 return -EINVAL; 2677 operand1_flags = var->flags; 2678 } 2679 2680 if ((operand2_flags & HIST_FIELD_FL_VAR_REF) || 2681 (operand2_flags & HIST_FIELD_FL_ALIAS)) { 2682 struct hist_field *var; 2683 2684 var = find_var_field(operand2->var.hist_data, operand2->name); 2685 if (!var) 2686 return -EINVAL; 2687 operand2_flags = var->flags; 2688 } 2689 2690 if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) != 2691 (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) { 2692 hist_err("Timestamp units in expression don't match", NULL); 2693 return -EINVAL; 2694 } 2695 2696 return 0; 2697 } 2698 2699 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2700 struct trace_event_file *file, 2701 char *str, unsigned long flags, 2702 char *var_name, unsigned int level) 2703 { 2704 struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL; 2705 unsigned long operand_flags; 2706 int field_op, ret = -EINVAL; 2707 char *sep, *operand1_str; 2708 2709 if (level > 3) { 2710 hist_err("Too many subexpressions (3 max): ", str); 2711 return ERR_PTR(-EINVAL); 2712 } 2713 2714 field_op = contains_operator(str); 2715 2716 if (field_op == FIELD_OP_NONE) 2717 return parse_atom(hist_data, file, str, &flags, var_name); 2718 2719 if (field_op == FIELD_OP_UNARY_MINUS) 2720 return parse_unary(hist_data, file, str, flags, var_name, ++level); 2721 2722 switch (field_op) { 2723 case FIELD_OP_MINUS: 2724 sep = "-"; 2725 break; 2726 case FIELD_OP_PLUS: 2727 sep = "+"; 2728 break; 2729 default: 2730 goto free; 2731 } 2732 2733 operand1_str = strsep(&str, sep); 2734 if (!operand1_str || !str) 2735 goto free; 2736 2737 operand_flags = 0; 2738 operand1 = parse_atom(hist_data, file, operand1_str, 2739 &operand_flags, NULL); 2740 if (IS_ERR(operand1)) { 2741 ret = PTR_ERR(operand1); 2742 operand1 = NULL; 2743 goto free; 2744 } 2745 2746 /* rest of string could be another expression e.g. b+c in a+b+c */ 2747 operand_flags = 0; 2748 operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level); 2749 if (IS_ERR(operand2)) { 2750 ret = PTR_ERR(operand2); 2751 operand2 = NULL; 2752 goto free; 2753 } 2754 2755 ret = check_expr_operands(operand1, operand2); 2756 if (ret) 2757 goto free; 2758 2759 flags |= HIST_FIELD_FL_EXPR; 2760 2761 flags |= operand1->flags & 2762 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2763 2764 expr = create_hist_field(hist_data, NULL, flags, var_name); 2765 if (!expr) { 2766 ret = -ENOMEM; 2767 goto free; 2768 } 2769 2770 operand1->read_once = true; 2771 operand2->read_once = true; 2772 2773 expr->operands[0] = operand1; 2774 expr->operands[1] = operand2; 2775 expr->operator = field_op; 2776 expr->name = expr_str(expr, 0); 2777 expr->type = kstrdup(operand1->type, GFP_KERNEL); 2778 if (!expr->type) { 2779 ret = -ENOMEM; 2780 goto free; 2781 } 2782 2783 switch (field_op) { 2784 case FIELD_OP_MINUS: 2785 expr->fn = hist_field_minus; 2786 break; 2787 case FIELD_OP_PLUS: 2788 expr->fn = hist_field_plus; 2789 break; 2790 default: 2791 ret = -EINVAL; 2792 goto free; 2793 } 2794 2795 return expr; 2796 free: 2797 destroy_hist_field(operand1, 0); 2798 destroy_hist_field(operand2, 0); 2799 destroy_hist_field(expr, 0); 2800 2801 return ERR_PTR(ret); 2802 } 2803 2804 static char *find_trigger_filter(struct hist_trigger_data *hist_data, 2805 struct trace_event_file *file) 2806 { 2807 struct event_trigger_data *test; 2808 2809 list_for_each_entry_rcu(test, &file->triggers, list) { 2810 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2811 if (test->private_data == hist_data) 2812 return test->filter_str; 2813 } 2814 } 2815 2816 return NULL; 2817 } 2818 2819 static struct event_command trigger_hist_cmd; 2820 static int event_hist_trigger_func(struct event_command *cmd_ops, 2821 struct trace_event_file *file, 2822 char *glob, char *cmd, char *param); 2823 2824 static bool compatible_keys(struct hist_trigger_data *target_hist_data, 2825 struct hist_trigger_data *hist_data, 2826 unsigned int n_keys) 2827 { 2828 struct hist_field *target_hist_field, *hist_field; 2829 unsigned int n, i, j; 2830 2831 if (hist_data->n_fields - hist_data->n_vals != n_keys) 2832 return false; 2833 2834 i = hist_data->n_vals; 2835 j = target_hist_data->n_vals; 2836 2837 for (n = 0; n < n_keys; n++) { 2838 hist_field = hist_data->fields[i + n]; 2839 target_hist_field = target_hist_data->fields[j + n]; 2840 2841 if (strcmp(hist_field->type, target_hist_field->type) != 0) 2842 return false; 2843 if (hist_field->size != target_hist_field->size) 2844 return false; 2845 if (hist_field->is_signed != target_hist_field->is_signed) 2846 return false; 2847 } 2848 2849 return true; 2850 } 2851 2852 static struct hist_trigger_data * 2853 find_compatible_hist(struct hist_trigger_data *target_hist_data, 2854 struct trace_event_file *file) 2855 { 2856 struct hist_trigger_data *hist_data; 2857 struct event_trigger_data *test; 2858 unsigned int n_keys; 2859 2860 n_keys = target_hist_data->n_fields - target_hist_data->n_vals; 2861 2862 list_for_each_entry_rcu(test, &file->triggers, list) { 2863 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2864 hist_data = test->private_data; 2865 2866 if (compatible_keys(target_hist_data, hist_data, n_keys)) 2867 return hist_data; 2868 } 2869 } 2870 2871 return NULL; 2872 } 2873 2874 static struct trace_event_file *event_file(struct trace_array *tr, 2875 char *system, char *event_name) 2876 { 2877 struct trace_event_file *file; 2878 2879 file = __find_event_file(tr, system, event_name); 2880 if (!file) 2881 return ERR_PTR(-EINVAL); 2882 2883 return file; 2884 } 2885 2886 static struct hist_field * 2887 find_synthetic_field_var(struct hist_trigger_data *target_hist_data, 2888 char *system, char *event_name, char *field_name) 2889 { 2890 struct hist_field *event_var; 2891 char *synthetic_name; 2892 2893 synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2894 if (!synthetic_name) 2895 return ERR_PTR(-ENOMEM); 2896 2897 strcpy(synthetic_name, "synthetic_"); 2898 strcat(synthetic_name, field_name); 2899 2900 event_var = find_event_var(target_hist_data, system, event_name, synthetic_name); 2901 2902 kfree(synthetic_name); 2903 2904 return event_var; 2905 } 2906 2907 /** 2908 * create_field_var_hist - Automatically create a histogram and var for a field 2909 * @target_hist_data: The target hist trigger 2910 * @subsys_name: Optional subsystem name 2911 * @event_name: Optional event name 2912 * @field_name: The name of the field (and the resulting variable) 2913 * 2914 * Hist trigger actions fetch data from variables, not directly from 2915 * events. However, for convenience, users are allowed to directly 2916 * specify an event field in an action, which will be automatically 2917 * converted into a variable on their behalf. 2918 2919 * If a user specifies a field on an event that isn't the event the 2920 * histogram currently being defined (the target event histogram), the 2921 * only way that can be accomplished is if a new hist trigger is 2922 * created and the field variable defined on that. 2923 * 2924 * This function creates a new histogram compatible with the target 2925 * event (meaning a histogram with the same key as the target 2926 * histogram), and creates a variable for the specified field, but 2927 * with 'synthetic_' prepended to the variable name in order to avoid 2928 * collision with normal field variables. 2929 * 2930 * Return: The variable created for the field. 2931 */ 2932 static struct hist_field * 2933 create_field_var_hist(struct hist_trigger_data *target_hist_data, 2934 char *subsys_name, char *event_name, char *field_name) 2935 { 2936 struct trace_array *tr = target_hist_data->event_file->tr; 2937 struct hist_field *event_var = ERR_PTR(-EINVAL); 2938 struct hist_trigger_data *hist_data; 2939 unsigned int i, n, first = true; 2940 struct field_var_hist *var_hist; 2941 struct trace_event_file *file; 2942 struct hist_field *key_field; 2943 char *saved_filter; 2944 char *cmd; 2945 int ret; 2946 2947 if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) { 2948 hist_err_event("onmatch: Too many field variables defined: ", 2949 subsys_name, event_name, field_name); 2950 return ERR_PTR(-EINVAL); 2951 } 2952 2953 file = event_file(tr, subsys_name, event_name); 2954 2955 if (IS_ERR(file)) { 2956 hist_err_event("onmatch: Event file not found: ", 2957 subsys_name, event_name, field_name); 2958 ret = PTR_ERR(file); 2959 return ERR_PTR(ret); 2960 } 2961 2962 /* 2963 * Look for a histogram compatible with target. We'll use the 2964 * found histogram specification to create a new matching 2965 * histogram with our variable on it. target_hist_data is not 2966 * yet a registered histogram so we can't use that. 2967 */ 2968 hist_data = find_compatible_hist(target_hist_data, file); 2969 if (!hist_data) { 2970 hist_err_event("onmatch: Matching event histogram not found: ", 2971 subsys_name, event_name, field_name); 2972 return ERR_PTR(-EINVAL); 2973 } 2974 2975 /* See if a synthetic field variable has already been created */ 2976 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 2977 event_name, field_name); 2978 if (!IS_ERR_OR_NULL(event_var)) 2979 return event_var; 2980 2981 var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL); 2982 if (!var_hist) 2983 return ERR_PTR(-ENOMEM); 2984 2985 cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2986 if (!cmd) { 2987 kfree(var_hist); 2988 return ERR_PTR(-ENOMEM); 2989 } 2990 2991 /* Use the same keys as the compatible histogram */ 2992 strcat(cmd, "keys="); 2993 2994 for_each_hist_key_field(i, hist_data) { 2995 key_field = hist_data->fields[i]; 2996 if (!first) 2997 strcat(cmd, ","); 2998 strcat(cmd, key_field->field->name); 2999 first = false; 3000 } 3001 3002 /* Create the synthetic field variable specification */ 3003 strcat(cmd, ":synthetic_"); 3004 strcat(cmd, field_name); 3005 strcat(cmd, "="); 3006 strcat(cmd, field_name); 3007 3008 /* Use the same filter as the compatible histogram */ 3009 saved_filter = find_trigger_filter(hist_data, file); 3010 if (saved_filter) { 3011 strcat(cmd, " if "); 3012 strcat(cmd, saved_filter); 3013 } 3014 3015 var_hist->cmd = kstrdup(cmd, GFP_KERNEL); 3016 if (!var_hist->cmd) { 3017 kfree(cmd); 3018 kfree(var_hist); 3019 return ERR_PTR(-ENOMEM); 3020 } 3021 3022 /* Save the compatible histogram information */ 3023 var_hist->hist_data = hist_data; 3024 3025 /* Create the new histogram with our variable */ 3026 ret = event_hist_trigger_func(&trigger_hist_cmd, file, 3027 "", "hist", cmd); 3028 if (ret) { 3029 kfree(cmd); 3030 kfree(var_hist->cmd); 3031 kfree(var_hist); 3032 hist_err_event("onmatch: Couldn't create histogram for field: ", 3033 subsys_name, event_name, field_name); 3034 return ERR_PTR(ret); 3035 } 3036 3037 kfree(cmd); 3038 3039 /* If we can't find the variable, something went wrong */ 3040 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 3041 event_name, field_name); 3042 if (IS_ERR_OR_NULL(event_var)) { 3043 kfree(var_hist->cmd); 3044 kfree(var_hist); 3045 hist_err_event("onmatch: Couldn't find synthetic variable: ", 3046 subsys_name, event_name, field_name); 3047 return ERR_PTR(-EINVAL); 3048 } 3049 3050 n = target_hist_data->n_field_var_hists; 3051 target_hist_data->field_var_hists[n] = var_hist; 3052 target_hist_data->n_field_var_hists++; 3053 3054 return event_var; 3055 } 3056 3057 static struct hist_field * 3058 find_target_event_var(struct hist_trigger_data *hist_data, 3059 char *subsys_name, char *event_name, char *var_name) 3060 { 3061 struct trace_event_file *file = hist_data->event_file; 3062 struct hist_field *hist_field = NULL; 3063 3064 if (subsys_name) { 3065 struct trace_event_call *call; 3066 3067 if (!event_name) 3068 return NULL; 3069 3070 call = file->event_call; 3071 3072 if (strcmp(subsys_name, call->class->system) != 0) 3073 return NULL; 3074 3075 if (strcmp(event_name, trace_event_name(call)) != 0) 3076 return NULL; 3077 } 3078 3079 hist_field = find_var_field(hist_data, var_name); 3080 3081 return hist_field; 3082 } 3083 3084 static inline void __update_field_vars(struct tracing_map_elt *elt, 3085 struct ring_buffer_event *rbe, 3086 void *rec, 3087 struct field_var **field_vars, 3088 unsigned int n_field_vars, 3089 unsigned int field_var_str_start) 3090 { 3091 struct hist_elt_data *elt_data = elt->private_data; 3092 unsigned int i, j, var_idx; 3093 u64 var_val; 3094 3095 for (i = 0, j = field_var_str_start; i < n_field_vars; i++) { 3096 struct field_var *field_var = field_vars[i]; 3097 struct hist_field *var = field_var->var; 3098 struct hist_field *val = field_var->val; 3099 3100 var_val = val->fn(val, elt, rbe, rec); 3101 var_idx = var->var.idx; 3102 3103 if (val->flags & HIST_FIELD_FL_STRING) { 3104 char *str = elt_data->field_var_str[j++]; 3105 char *val_str = (char *)(uintptr_t)var_val; 3106 3107 strscpy(str, val_str, STR_VAR_LEN_MAX); 3108 var_val = (u64)(uintptr_t)str; 3109 } 3110 tracing_map_set_var(elt, var_idx, var_val); 3111 } 3112 } 3113 3114 static void update_field_vars(struct hist_trigger_data *hist_data, 3115 struct tracing_map_elt *elt, 3116 struct ring_buffer_event *rbe, 3117 void *rec) 3118 { 3119 __update_field_vars(elt, rbe, rec, hist_data->field_vars, 3120 hist_data->n_field_vars, 0); 3121 } 3122 3123 static void update_max_vars(struct hist_trigger_data *hist_data, 3124 struct tracing_map_elt *elt, 3125 struct ring_buffer_event *rbe, 3126 void *rec) 3127 { 3128 __update_field_vars(elt, rbe, rec, hist_data->max_vars, 3129 hist_data->n_max_vars, hist_data->n_field_var_str); 3130 } 3131 3132 static struct hist_field *create_var(struct hist_trigger_data *hist_data, 3133 struct trace_event_file *file, 3134 char *name, int size, const char *type) 3135 { 3136 struct hist_field *var; 3137 int idx; 3138 3139 if (find_var(hist_data, file, name) && !hist_data->remove) { 3140 var = ERR_PTR(-EINVAL); 3141 goto out; 3142 } 3143 3144 var = kzalloc(sizeof(struct hist_field), GFP_KERNEL); 3145 if (!var) { 3146 var = ERR_PTR(-ENOMEM); 3147 goto out; 3148 } 3149 3150 idx = tracing_map_add_var(hist_data->map); 3151 if (idx < 0) { 3152 kfree(var); 3153 var = ERR_PTR(-EINVAL); 3154 goto out; 3155 } 3156 3157 var->flags = HIST_FIELD_FL_VAR; 3158 var->var.idx = idx; 3159 var->var.hist_data = var->hist_data = hist_data; 3160 var->size = size; 3161 var->var.name = kstrdup(name, GFP_KERNEL); 3162 var->type = kstrdup(type, GFP_KERNEL); 3163 if (!var->var.name || !var->type) { 3164 kfree(var->var.name); 3165 kfree(var->type); 3166 kfree(var); 3167 var = ERR_PTR(-ENOMEM); 3168 } 3169 out: 3170 return var; 3171 } 3172 3173 static struct field_var *create_field_var(struct hist_trigger_data *hist_data, 3174 struct trace_event_file *file, 3175 char *field_name) 3176 { 3177 struct hist_field *val = NULL, *var = NULL; 3178 unsigned long flags = HIST_FIELD_FL_VAR; 3179 struct field_var *field_var; 3180 int ret = 0; 3181 3182 if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) { 3183 hist_err("Too many field variables defined: ", field_name); 3184 ret = -EINVAL; 3185 goto err; 3186 } 3187 3188 val = parse_atom(hist_data, file, field_name, &flags, NULL); 3189 if (IS_ERR(val)) { 3190 hist_err("Couldn't parse field variable: ", field_name); 3191 ret = PTR_ERR(val); 3192 goto err; 3193 } 3194 3195 var = create_var(hist_data, file, field_name, val->size, val->type); 3196 if (IS_ERR(var)) { 3197 hist_err("Couldn't create or find variable: ", field_name); 3198 kfree(val); 3199 ret = PTR_ERR(var); 3200 goto err; 3201 } 3202 3203 field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL); 3204 if (!field_var) { 3205 kfree(val); 3206 kfree(var); 3207 ret = -ENOMEM; 3208 goto err; 3209 } 3210 3211 field_var->var = var; 3212 field_var->val = val; 3213 out: 3214 return field_var; 3215 err: 3216 field_var = ERR_PTR(ret); 3217 goto out; 3218 } 3219 3220 /** 3221 * create_target_field_var - Automatically create a variable for a field 3222 * @target_hist_data: The target hist trigger 3223 * @subsys_name: Optional subsystem name 3224 * @event_name: Optional event name 3225 * @var_name: The name of the field (and the resulting variable) 3226 * 3227 * Hist trigger actions fetch data from variables, not directly from 3228 * events. However, for convenience, users are allowed to directly 3229 * specify an event field in an action, which will be automatically 3230 * converted into a variable on their behalf. 3231 3232 * This function creates a field variable with the name var_name on 3233 * the hist trigger currently being defined on the target event. If 3234 * subsys_name and event_name are specified, this function simply 3235 * verifies that they do in fact match the target event subsystem and 3236 * event name. 3237 * 3238 * Return: The variable created for the field. 3239 */ 3240 static struct field_var * 3241 create_target_field_var(struct hist_trigger_data *target_hist_data, 3242 char *subsys_name, char *event_name, char *var_name) 3243 { 3244 struct trace_event_file *file = target_hist_data->event_file; 3245 3246 if (subsys_name) { 3247 struct trace_event_call *call; 3248 3249 if (!event_name) 3250 return NULL; 3251 3252 call = file->event_call; 3253 3254 if (strcmp(subsys_name, call->class->system) != 0) 3255 return NULL; 3256 3257 if (strcmp(event_name, trace_event_name(call)) != 0) 3258 return NULL; 3259 } 3260 3261 return create_field_var(target_hist_data, file, var_name); 3262 } 3263 3264 static void onmax_print(struct seq_file *m, 3265 struct hist_trigger_data *hist_data, 3266 struct tracing_map_elt *elt, 3267 struct action_data *data) 3268 { 3269 unsigned int i, save_var_idx, max_idx = data->onmax.max_var->var.idx; 3270 3271 seq_printf(m, "\n\tmax: %10llu", tracing_map_read_var(elt, max_idx)); 3272 3273 for (i = 0; i < hist_data->n_max_vars; i++) { 3274 struct hist_field *save_val = hist_data->max_vars[i]->val; 3275 struct hist_field *save_var = hist_data->max_vars[i]->var; 3276 u64 val; 3277 3278 save_var_idx = save_var->var.idx; 3279 3280 val = tracing_map_read_var(elt, save_var_idx); 3281 3282 if (save_val->flags & HIST_FIELD_FL_STRING) { 3283 seq_printf(m, " %s: %-32s", save_var->var.name, 3284 (char *)(uintptr_t)(val)); 3285 } else 3286 seq_printf(m, " %s: %10llu", save_var->var.name, val); 3287 } 3288 } 3289 3290 static void onmax_save(struct hist_trigger_data *hist_data, 3291 struct tracing_map_elt *elt, void *rec, 3292 struct ring_buffer_event *rbe, 3293 struct action_data *data, u64 *var_ref_vals) 3294 { 3295 unsigned int max_idx = data->onmax.max_var->var.idx; 3296 unsigned int max_var_ref_idx = data->onmax.max_var_ref_idx; 3297 3298 u64 var_val, max_val; 3299 3300 var_val = var_ref_vals[max_var_ref_idx]; 3301 max_val = tracing_map_read_var(elt, max_idx); 3302 3303 if (var_val <= max_val) 3304 return; 3305 3306 tracing_map_set_var(elt, max_idx, var_val); 3307 3308 update_max_vars(hist_data, elt, rbe, rec); 3309 } 3310 3311 static void onmax_destroy(struct action_data *data) 3312 { 3313 unsigned int i; 3314 3315 destroy_hist_field(data->onmax.max_var, 0); 3316 destroy_hist_field(data->onmax.var, 0); 3317 3318 kfree(data->onmax.var_str); 3319 kfree(data->onmax.fn_name); 3320 3321 for (i = 0; i < data->n_params; i++) 3322 kfree(data->params[i]); 3323 3324 kfree(data); 3325 } 3326 3327 static int onmax_create(struct hist_trigger_data *hist_data, 3328 struct action_data *data) 3329 { 3330 struct trace_event_file *file = hist_data->event_file; 3331 struct hist_field *var_field, *ref_field, *max_var; 3332 unsigned int var_ref_idx = hist_data->n_var_refs; 3333 struct field_var *field_var; 3334 char *onmax_var_str, *param; 3335 unsigned long flags; 3336 unsigned int i; 3337 int ret = 0; 3338 3339 onmax_var_str = data->onmax.var_str; 3340 if (onmax_var_str[0] != '$') { 3341 hist_err("onmax: For onmax(x), x must be a variable: ", onmax_var_str); 3342 return -EINVAL; 3343 } 3344 onmax_var_str++; 3345 3346 var_field = find_target_event_var(hist_data, NULL, NULL, onmax_var_str); 3347 if (!var_field) { 3348 hist_err("onmax: Couldn't find onmax variable: ", onmax_var_str); 3349 return -EINVAL; 3350 } 3351 3352 flags = HIST_FIELD_FL_VAR_REF; 3353 ref_field = create_hist_field(hist_data, NULL, flags, NULL); 3354 if (!ref_field) 3355 return -ENOMEM; 3356 3357 if (init_var_ref(ref_field, var_field, NULL, NULL)) { 3358 destroy_hist_field(ref_field, 0); 3359 ret = -ENOMEM; 3360 goto out; 3361 } 3362 hist_data->var_refs[hist_data->n_var_refs] = ref_field; 3363 ref_field->var_ref_idx = hist_data->n_var_refs++; 3364 data->onmax.var = ref_field; 3365 3366 data->fn = onmax_save; 3367 data->onmax.max_var_ref_idx = var_ref_idx; 3368 max_var = create_var(hist_data, file, "max", sizeof(u64), "u64"); 3369 if (IS_ERR(max_var)) { 3370 hist_err("onmax: Couldn't create onmax variable: ", "max"); 3371 ret = PTR_ERR(max_var); 3372 goto out; 3373 } 3374 data->onmax.max_var = max_var; 3375 3376 for (i = 0; i < data->n_params; i++) { 3377 param = kstrdup(data->params[i], GFP_KERNEL); 3378 if (!param) { 3379 ret = -ENOMEM; 3380 goto out; 3381 } 3382 3383 field_var = create_target_field_var(hist_data, NULL, NULL, param); 3384 if (IS_ERR(field_var)) { 3385 hist_err("onmax: Couldn't create field variable: ", param); 3386 ret = PTR_ERR(field_var); 3387 kfree(param); 3388 goto out; 3389 } 3390 3391 hist_data->max_vars[hist_data->n_max_vars++] = field_var; 3392 if (field_var->val->flags & HIST_FIELD_FL_STRING) 3393 hist_data->n_max_var_str++; 3394 3395 kfree(param); 3396 } 3397 out: 3398 return ret; 3399 } 3400 3401 static int parse_action_params(char *params, struct action_data *data) 3402 { 3403 char *param, *saved_param; 3404 int ret = 0; 3405 3406 while (params) { 3407 if (data->n_params >= SYNTH_FIELDS_MAX) 3408 goto out; 3409 3410 param = strsep(¶ms, ","); 3411 if (!param) { 3412 ret = -EINVAL; 3413 goto out; 3414 } 3415 3416 param = strstrip(param); 3417 if (strlen(param) < 2) { 3418 hist_err("Invalid action param: ", param); 3419 ret = -EINVAL; 3420 goto out; 3421 } 3422 3423 saved_param = kstrdup(param, GFP_KERNEL); 3424 if (!saved_param) { 3425 ret = -ENOMEM; 3426 goto out; 3427 } 3428 3429 data->params[data->n_params++] = saved_param; 3430 } 3431 out: 3432 return ret; 3433 } 3434 3435 static struct action_data *onmax_parse(char *str) 3436 { 3437 char *onmax_fn_name, *onmax_var_str; 3438 struct action_data *data; 3439 int ret = -EINVAL; 3440 3441 data = kzalloc(sizeof(*data), GFP_KERNEL); 3442 if (!data) 3443 return ERR_PTR(-ENOMEM); 3444 3445 onmax_var_str = strsep(&str, ")"); 3446 if (!onmax_var_str || !str) { 3447 ret = -EINVAL; 3448 goto free; 3449 } 3450 3451 data->onmax.var_str = kstrdup(onmax_var_str, GFP_KERNEL); 3452 if (!data->onmax.var_str) { 3453 ret = -ENOMEM; 3454 goto free; 3455 } 3456 3457 strsep(&str, "."); 3458 if (!str) 3459 goto free; 3460 3461 onmax_fn_name = strsep(&str, "("); 3462 if (!onmax_fn_name || !str) 3463 goto free; 3464 3465 if (strncmp(onmax_fn_name, "save", strlen("save")) == 0) { 3466 char *params = strsep(&str, ")"); 3467 3468 if (!params) { 3469 ret = -EINVAL; 3470 goto free; 3471 } 3472 3473 ret = parse_action_params(params, data); 3474 if (ret) 3475 goto free; 3476 } else 3477 goto free; 3478 3479 data->onmax.fn_name = kstrdup(onmax_fn_name, GFP_KERNEL); 3480 if (!data->onmax.fn_name) { 3481 ret = -ENOMEM; 3482 goto free; 3483 } 3484 out: 3485 return data; 3486 free: 3487 onmax_destroy(data); 3488 data = ERR_PTR(ret); 3489 goto out; 3490 } 3491 3492 static void onmatch_destroy(struct action_data *data) 3493 { 3494 unsigned int i; 3495 3496 mutex_lock(&synth_event_mutex); 3497 3498 kfree(data->onmatch.match_event); 3499 kfree(data->onmatch.match_event_system); 3500 kfree(data->onmatch.synth_event_name); 3501 3502 for (i = 0; i < data->n_params; i++) 3503 kfree(data->params[i]); 3504 3505 if (data->onmatch.synth_event) 3506 data->onmatch.synth_event->ref--; 3507 3508 kfree(data); 3509 3510 mutex_unlock(&synth_event_mutex); 3511 } 3512 3513 static void destroy_field_var(struct field_var *field_var) 3514 { 3515 if (!field_var) 3516 return; 3517 3518 destroy_hist_field(field_var->var, 0); 3519 destroy_hist_field(field_var->val, 0); 3520 3521 kfree(field_var); 3522 } 3523 3524 static void destroy_field_vars(struct hist_trigger_data *hist_data) 3525 { 3526 unsigned int i; 3527 3528 for (i = 0; i < hist_data->n_field_vars; i++) 3529 destroy_field_var(hist_data->field_vars[i]); 3530 } 3531 3532 static void save_field_var(struct hist_trigger_data *hist_data, 3533 struct field_var *field_var) 3534 { 3535 hist_data->field_vars[hist_data->n_field_vars++] = field_var; 3536 3537 if (field_var->val->flags & HIST_FIELD_FL_STRING) 3538 hist_data->n_field_var_str++; 3539 } 3540 3541 3542 static void destroy_synth_var_refs(struct hist_trigger_data *hist_data) 3543 { 3544 unsigned int i; 3545 3546 for (i = 0; i < hist_data->n_synth_var_refs; i++) 3547 destroy_hist_field(hist_data->synth_var_refs[i], 0); 3548 } 3549 3550 static void save_synth_var_ref(struct hist_trigger_data *hist_data, 3551 struct hist_field *var_ref) 3552 { 3553 hist_data->synth_var_refs[hist_data->n_synth_var_refs++] = var_ref; 3554 3555 hist_data->var_refs[hist_data->n_var_refs] = var_ref; 3556 var_ref->var_ref_idx = hist_data->n_var_refs++; 3557 } 3558 3559 static int check_synth_field(struct synth_event *event, 3560 struct hist_field *hist_field, 3561 unsigned int field_pos) 3562 { 3563 struct synth_field *field; 3564 3565 if (field_pos >= event->n_fields) 3566 return -EINVAL; 3567 3568 field = event->fields[field_pos]; 3569 3570 if (strcmp(field->type, hist_field->type) != 0) 3571 return -EINVAL; 3572 3573 return 0; 3574 } 3575 3576 static struct hist_field * 3577 onmatch_find_var(struct hist_trigger_data *hist_data, struct action_data *data, 3578 char *system, char *event, char *var) 3579 { 3580 struct hist_field *hist_field; 3581 3582 var++; /* skip '$' */ 3583 3584 hist_field = find_target_event_var(hist_data, system, event, var); 3585 if (!hist_field) { 3586 if (!system) { 3587 system = data->onmatch.match_event_system; 3588 event = data->onmatch.match_event; 3589 } 3590 3591 hist_field = find_event_var(hist_data, system, event, var); 3592 } 3593 3594 if (!hist_field) 3595 hist_err_event("onmatch: Couldn't find onmatch param: $", system, event, var); 3596 3597 return hist_field; 3598 } 3599 3600 static struct hist_field * 3601 onmatch_create_field_var(struct hist_trigger_data *hist_data, 3602 struct action_data *data, char *system, 3603 char *event, char *var) 3604 { 3605 struct hist_field *hist_field = NULL; 3606 struct field_var *field_var; 3607 3608 /* 3609 * First try to create a field var on the target event (the 3610 * currently being defined). This will create a variable for 3611 * unqualified fields on the target event, or if qualified, 3612 * target fields that have qualified names matching the target. 3613 */ 3614 field_var = create_target_field_var(hist_data, system, event, var); 3615 3616 if (field_var && !IS_ERR(field_var)) { 3617 save_field_var(hist_data, field_var); 3618 hist_field = field_var->var; 3619 } else { 3620 field_var = NULL; 3621 /* 3622 * If no explicit system.event is specfied, default to 3623 * looking for fields on the onmatch(system.event.xxx) 3624 * event. 3625 */ 3626 if (!system) { 3627 system = data->onmatch.match_event_system; 3628 event = data->onmatch.match_event; 3629 } 3630 3631 /* 3632 * At this point, we're looking at a field on another 3633 * event. Because we can't modify a hist trigger on 3634 * another event to add a variable for a field, we need 3635 * to create a new trigger on that event and create the 3636 * variable at the same time. 3637 */ 3638 hist_field = create_field_var_hist(hist_data, system, event, var); 3639 if (IS_ERR(hist_field)) 3640 goto free; 3641 } 3642 out: 3643 return hist_field; 3644 free: 3645 destroy_field_var(field_var); 3646 hist_field = NULL; 3647 goto out; 3648 } 3649 3650 static int onmatch_create(struct hist_trigger_data *hist_data, 3651 struct trace_event_file *file, 3652 struct action_data *data) 3653 { 3654 char *event_name, *param, *system = NULL; 3655 struct hist_field *hist_field, *var_ref; 3656 unsigned int i, var_ref_idx; 3657 unsigned int field_pos = 0; 3658 struct synth_event *event; 3659 int ret = 0; 3660 3661 mutex_lock(&synth_event_mutex); 3662 event = find_synth_event(data->onmatch.synth_event_name); 3663 if (!event) { 3664 hist_err("onmatch: Couldn't find synthetic event: ", data->onmatch.synth_event_name); 3665 mutex_unlock(&synth_event_mutex); 3666 return -EINVAL; 3667 } 3668 event->ref++; 3669 mutex_unlock(&synth_event_mutex); 3670 3671 var_ref_idx = hist_data->n_var_refs; 3672 3673 for (i = 0; i < data->n_params; i++) { 3674 char *p; 3675 3676 p = param = kstrdup(data->params[i], GFP_KERNEL); 3677 if (!param) { 3678 ret = -ENOMEM; 3679 goto err; 3680 } 3681 3682 system = strsep(¶m, "."); 3683 if (!param) { 3684 param = (char *)system; 3685 system = event_name = NULL; 3686 } else { 3687 event_name = strsep(¶m, "."); 3688 if (!param) { 3689 kfree(p); 3690 ret = -EINVAL; 3691 goto err; 3692 } 3693 } 3694 3695 if (param[0] == '$') 3696 hist_field = onmatch_find_var(hist_data, data, system, 3697 event_name, param); 3698 else 3699 hist_field = onmatch_create_field_var(hist_data, data, 3700 system, 3701 event_name, 3702 param); 3703 3704 if (!hist_field) { 3705 kfree(p); 3706 ret = -EINVAL; 3707 goto err; 3708 } 3709 3710 if (check_synth_field(event, hist_field, field_pos) == 0) { 3711 var_ref = create_var_ref(hist_field, system, event_name); 3712 if (!var_ref) { 3713 kfree(p); 3714 ret = -ENOMEM; 3715 goto err; 3716 } 3717 3718 save_synth_var_ref(hist_data, var_ref); 3719 field_pos++; 3720 kfree(p); 3721 continue; 3722 } 3723 3724 hist_err_event("onmatch: Param type doesn't match synthetic event field type: ", 3725 system, event_name, param); 3726 kfree(p); 3727 ret = -EINVAL; 3728 goto err; 3729 } 3730 3731 if (field_pos != event->n_fields) { 3732 hist_err("onmatch: Param count doesn't match synthetic event field count: ", event->name); 3733 ret = -EINVAL; 3734 goto err; 3735 } 3736 3737 data->fn = action_trace; 3738 data->onmatch.synth_event = event; 3739 data->onmatch.var_ref_idx = var_ref_idx; 3740 out: 3741 return ret; 3742 err: 3743 mutex_lock(&synth_event_mutex); 3744 event->ref--; 3745 mutex_unlock(&synth_event_mutex); 3746 3747 goto out; 3748 } 3749 3750 static struct action_data *onmatch_parse(struct trace_array *tr, char *str) 3751 { 3752 char *match_event, *match_event_system; 3753 char *synth_event_name, *params; 3754 struct action_data *data; 3755 int ret = -EINVAL; 3756 3757 data = kzalloc(sizeof(*data), GFP_KERNEL); 3758 if (!data) 3759 return ERR_PTR(-ENOMEM); 3760 3761 match_event = strsep(&str, ")"); 3762 if (!match_event || !str) { 3763 hist_err("onmatch: Missing closing paren: ", match_event); 3764 goto free; 3765 } 3766 3767 match_event_system = strsep(&match_event, "."); 3768 if (!match_event) { 3769 hist_err("onmatch: Missing subsystem for match event: ", match_event_system); 3770 goto free; 3771 } 3772 3773 if (IS_ERR(event_file(tr, match_event_system, match_event))) { 3774 hist_err_event("onmatch: Invalid subsystem or event name: ", 3775 match_event_system, match_event, NULL); 3776 goto free; 3777 } 3778 3779 data->onmatch.match_event = kstrdup(match_event, GFP_KERNEL); 3780 if (!data->onmatch.match_event) { 3781 ret = -ENOMEM; 3782 goto free; 3783 } 3784 3785 data->onmatch.match_event_system = kstrdup(match_event_system, GFP_KERNEL); 3786 if (!data->onmatch.match_event_system) { 3787 ret = -ENOMEM; 3788 goto free; 3789 } 3790 3791 strsep(&str, "."); 3792 if (!str) { 3793 hist_err("onmatch: Missing . after onmatch(): ", str); 3794 goto free; 3795 } 3796 3797 synth_event_name = strsep(&str, "("); 3798 if (!synth_event_name || !str) { 3799 hist_err("onmatch: Missing opening paramlist paren: ", synth_event_name); 3800 goto free; 3801 } 3802 3803 data->onmatch.synth_event_name = kstrdup(synth_event_name, GFP_KERNEL); 3804 if (!data->onmatch.synth_event_name) { 3805 ret = -ENOMEM; 3806 goto free; 3807 } 3808 3809 params = strsep(&str, ")"); 3810 if (!params || !str || (str && strlen(str))) { 3811 hist_err("onmatch: Missing closing paramlist paren: ", params); 3812 goto free; 3813 } 3814 3815 ret = parse_action_params(params, data); 3816 if (ret) 3817 goto free; 3818 out: 3819 return data; 3820 free: 3821 onmatch_destroy(data); 3822 data = ERR_PTR(ret); 3823 goto out; 3824 } 3825 3826 static int create_hitcount_val(struct hist_trigger_data *hist_data) 3827 { 3828 hist_data->fields[HITCOUNT_IDX] = 3829 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL); 3830 if (!hist_data->fields[HITCOUNT_IDX]) 3831 return -ENOMEM; 3832 3833 hist_data->n_vals++; 3834 hist_data->n_fields++; 3835 3836 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX)) 3837 return -EINVAL; 3838 3839 return 0; 3840 } 3841 3842 static int __create_val_field(struct hist_trigger_data *hist_data, 3843 unsigned int val_idx, 3844 struct trace_event_file *file, 3845 char *var_name, char *field_str, 3846 unsigned long flags) 3847 { 3848 struct hist_field *hist_field; 3849 int ret = 0; 3850 3851 hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0); 3852 if (IS_ERR(hist_field)) { 3853 ret = PTR_ERR(hist_field); 3854 goto out; 3855 } 3856 3857 hist_data->fields[val_idx] = hist_field; 3858 3859 ++hist_data->n_vals; 3860 ++hist_data->n_fields; 3861 3862 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 3863 ret = -EINVAL; 3864 out: 3865 return ret; 3866 } 3867 3868 static int create_val_field(struct hist_trigger_data *hist_data, 3869 unsigned int val_idx, 3870 struct trace_event_file *file, 3871 char *field_str) 3872 { 3873 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX)) 3874 return -EINVAL; 3875 3876 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0); 3877 } 3878 3879 static int create_var_field(struct hist_trigger_data *hist_data, 3880 unsigned int val_idx, 3881 struct trace_event_file *file, 3882 char *var_name, char *expr_str) 3883 { 3884 unsigned long flags = 0; 3885 3886 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 3887 return -EINVAL; 3888 3889 if (find_var(hist_data, file, var_name) && !hist_data->remove) { 3890 hist_err("Variable already defined: ", var_name); 3891 return -EINVAL; 3892 } 3893 3894 flags |= HIST_FIELD_FL_VAR; 3895 hist_data->n_vars++; 3896 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX)) 3897 return -EINVAL; 3898 3899 return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags); 3900 } 3901 3902 static int create_val_fields(struct hist_trigger_data *hist_data, 3903 struct trace_event_file *file) 3904 { 3905 char *fields_str, *field_str; 3906 unsigned int i, j = 1; 3907 int ret; 3908 3909 ret = create_hitcount_val(hist_data); 3910 if (ret) 3911 goto out; 3912 3913 fields_str = hist_data->attrs->vals_str; 3914 if (!fields_str) 3915 goto out; 3916 3917 strsep(&fields_str, "="); 3918 if (!fields_str) 3919 goto out; 3920 3921 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX && 3922 j < TRACING_MAP_VALS_MAX; i++) { 3923 field_str = strsep(&fields_str, ","); 3924 if (!field_str) 3925 break; 3926 3927 if (strcmp(field_str, "hitcount") == 0) 3928 continue; 3929 3930 ret = create_val_field(hist_data, j++, file, field_str); 3931 if (ret) 3932 goto out; 3933 } 3934 3935 if (fields_str && (strcmp(fields_str, "hitcount") != 0)) 3936 ret = -EINVAL; 3937 out: 3938 return ret; 3939 } 3940 3941 static int create_key_field(struct hist_trigger_data *hist_data, 3942 unsigned int key_idx, 3943 unsigned int key_offset, 3944 struct trace_event_file *file, 3945 char *field_str) 3946 { 3947 struct hist_field *hist_field = NULL; 3948 3949 unsigned long flags = 0; 3950 unsigned int key_size; 3951 int ret = 0; 3952 3953 if (WARN_ON(key_idx >= HIST_FIELDS_MAX)) 3954 return -EINVAL; 3955 3956 flags |= HIST_FIELD_FL_KEY; 3957 3958 if (strcmp(field_str, "stacktrace") == 0) { 3959 flags |= HIST_FIELD_FL_STACKTRACE; 3960 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH; 3961 hist_field = create_hist_field(hist_data, NULL, flags, NULL); 3962 } else { 3963 hist_field = parse_expr(hist_data, file, field_str, flags, 3964 NULL, 0); 3965 if (IS_ERR(hist_field)) { 3966 ret = PTR_ERR(hist_field); 3967 goto out; 3968 } 3969 3970 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) { 3971 hist_err("Using variable references as keys not supported: ", field_str); 3972 destroy_hist_field(hist_field, 0); 3973 ret = -EINVAL; 3974 goto out; 3975 } 3976 3977 key_size = hist_field->size; 3978 } 3979 3980 hist_data->fields[key_idx] = hist_field; 3981 3982 key_size = ALIGN(key_size, sizeof(u64)); 3983 hist_data->fields[key_idx]->size = key_size; 3984 hist_data->fields[key_idx]->offset = key_offset; 3985 3986 hist_data->key_size += key_size; 3987 3988 if (hist_data->key_size > HIST_KEY_SIZE_MAX) { 3989 ret = -EINVAL; 3990 goto out; 3991 } 3992 3993 hist_data->n_keys++; 3994 hist_data->n_fields++; 3995 3996 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX)) 3997 return -EINVAL; 3998 3999 ret = key_size; 4000 out: 4001 return ret; 4002 } 4003 4004 static int create_key_fields(struct hist_trigger_data *hist_data, 4005 struct trace_event_file *file) 4006 { 4007 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals; 4008 char *fields_str, *field_str; 4009 int ret = -EINVAL; 4010 4011 fields_str = hist_data->attrs->keys_str; 4012 if (!fields_str) 4013 goto out; 4014 4015 strsep(&fields_str, "="); 4016 if (!fields_str) 4017 goto out; 4018 4019 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) { 4020 field_str = strsep(&fields_str, ","); 4021 if (!field_str) 4022 break; 4023 ret = create_key_field(hist_data, i, key_offset, 4024 file, field_str); 4025 if (ret < 0) 4026 goto out; 4027 key_offset += ret; 4028 } 4029 if (fields_str) { 4030 ret = -EINVAL; 4031 goto out; 4032 } 4033 ret = 0; 4034 out: 4035 return ret; 4036 } 4037 4038 static int create_var_fields(struct hist_trigger_data *hist_data, 4039 struct trace_event_file *file) 4040 { 4041 unsigned int i, j = hist_data->n_vals; 4042 int ret = 0; 4043 4044 unsigned int n_vars = hist_data->attrs->var_defs.n_vars; 4045 4046 for (i = 0; i < n_vars; i++) { 4047 char *var_name = hist_data->attrs->var_defs.name[i]; 4048 char *expr = hist_data->attrs->var_defs.expr[i]; 4049 4050 ret = create_var_field(hist_data, j++, file, var_name, expr); 4051 if (ret) 4052 goto out; 4053 } 4054 out: 4055 return ret; 4056 } 4057 4058 static void free_var_defs(struct hist_trigger_data *hist_data) 4059 { 4060 unsigned int i; 4061 4062 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 4063 kfree(hist_data->attrs->var_defs.name[i]); 4064 kfree(hist_data->attrs->var_defs.expr[i]); 4065 } 4066 4067 hist_data->attrs->var_defs.n_vars = 0; 4068 } 4069 4070 static int parse_var_defs(struct hist_trigger_data *hist_data) 4071 { 4072 char *s, *str, *var_name, *field_str; 4073 unsigned int i, j, n_vars = 0; 4074 int ret = 0; 4075 4076 for (i = 0; i < hist_data->attrs->n_assignments; i++) { 4077 str = hist_data->attrs->assignment_str[i]; 4078 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) { 4079 field_str = strsep(&str, ","); 4080 if (!field_str) 4081 break; 4082 4083 var_name = strsep(&field_str, "="); 4084 if (!var_name || !field_str) { 4085 hist_err("Malformed assignment: ", var_name); 4086 ret = -EINVAL; 4087 goto free; 4088 } 4089 4090 if (n_vars == TRACING_MAP_VARS_MAX) { 4091 hist_err("Too many variables defined: ", var_name); 4092 ret = -EINVAL; 4093 goto free; 4094 } 4095 4096 s = kstrdup(var_name, GFP_KERNEL); 4097 if (!s) { 4098 ret = -ENOMEM; 4099 goto free; 4100 } 4101 hist_data->attrs->var_defs.name[n_vars] = s; 4102 4103 s = kstrdup(field_str, GFP_KERNEL); 4104 if (!s) { 4105 kfree(hist_data->attrs->var_defs.name[n_vars]); 4106 ret = -ENOMEM; 4107 goto free; 4108 } 4109 hist_data->attrs->var_defs.expr[n_vars++] = s; 4110 4111 hist_data->attrs->var_defs.n_vars = n_vars; 4112 } 4113 } 4114 4115 return ret; 4116 free: 4117 free_var_defs(hist_data); 4118 4119 return ret; 4120 } 4121 4122 static int create_hist_fields(struct hist_trigger_data *hist_data, 4123 struct trace_event_file *file) 4124 { 4125 int ret; 4126 4127 ret = parse_var_defs(hist_data); 4128 if (ret) 4129 goto out; 4130 4131 ret = create_val_fields(hist_data, file); 4132 if (ret) 4133 goto out; 4134 4135 ret = create_var_fields(hist_data, file); 4136 if (ret) 4137 goto out; 4138 4139 ret = create_key_fields(hist_data, file); 4140 if (ret) 4141 goto out; 4142 out: 4143 free_var_defs(hist_data); 4144 4145 return ret; 4146 } 4147 4148 static int is_descending(const char *str) 4149 { 4150 if (!str) 4151 return 0; 4152 4153 if (strcmp(str, "descending") == 0) 4154 return 1; 4155 4156 if (strcmp(str, "ascending") == 0) 4157 return 0; 4158 4159 return -EINVAL; 4160 } 4161 4162 static int create_sort_keys(struct hist_trigger_data *hist_data) 4163 { 4164 char *fields_str = hist_data->attrs->sort_key_str; 4165 struct tracing_map_sort_key *sort_key; 4166 int descending, ret = 0; 4167 unsigned int i, j, k; 4168 4169 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */ 4170 4171 if (!fields_str) 4172 goto out; 4173 4174 strsep(&fields_str, "="); 4175 if (!fields_str) { 4176 ret = -EINVAL; 4177 goto out; 4178 } 4179 4180 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) { 4181 struct hist_field *hist_field; 4182 char *field_str, *field_name; 4183 const char *test_name; 4184 4185 sort_key = &hist_data->sort_keys[i]; 4186 4187 field_str = strsep(&fields_str, ","); 4188 if (!field_str) { 4189 if (i == 0) 4190 ret = -EINVAL; 4191 break; 4192 } 4193 4194 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) { 4195 ret = -EINVAL; 4196 break; 4197 } 4198 4199 field_name = strsep(&field_str, "."); 4200 if (!field_name) { 4201 ret = -EINVAL; 4202 break; 4203 } 4204 4205 if (strcmp(field_name, "hitcount") == 0) { 4206 descending = is_descending(field_str); 4207 if (descending < 0) { 4208 ret = descending; 4209 break; 4210 } 4211 sort_key->descending = descending; 4212 continue; 4213 } 4214 4215 for (j = 1, k = 1; j < hist_data->n_fields; j++) { 4216 unsigned int idx; 4217 4218 hist_field = hist_data->fields[j]; 4219 if (hist_field->flags & HIST_FIELD_FL_VAR) 4220 continue; 4221 4222 idx = k++; 4223 4224 test_name = hist_field_name(hist_field, 0); 4225 4226 if (strcmp(field_name, test_name) == 0) { 4227 sort_key->field_idx = idx; 4228 descending = is_descending(field_str); 4229 if (descending < 0) { 4230 ret = descending; 4231 goto out; 4232 } 4233 sort_key->descending = descending; 4234 break; 4235 } 4236 } 4237 if (j == hist_data->n_fields) { 4238 ret = -EINVAL; 4239 break; 4240 } 4241 } 4242 4243 hist_data->n_sort_keys = i; 4244 out: 4245 return ret; 4246 } 4247 4248 static void destroy_actions(struct hist_trigger_data *hist_data) 4249 { 4250 unsigned int i; 4251 4252 for (i = 0; i < hist_data->n_actions; i++) { 4253 struct action_data *data = hist_data->actions[i]; 4254 4255 if (data->fn == action_trace) 4256 onmatch_destroy(data); 4257 else if (data->fn == onmax_save) 4258 onmax_destroy(data); 4259 else 4260 kfree(data); 4261 } 4262 } 4263 4264 static int parse_actions(struct hist_trigger_data *hist_data) 4265 { 4266 struct trace_array *tr = hist_data->event_file->tr; 4267 struct action_data *data; 4268 unsigned int i; 4269 int ret = 0; 4270 char *str; 4271 4272 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4273 str = hist_data->attrs->action_str[i]; 4274 4275 if (strncmp(str, "onmatch(", strlen("onmatch(")) == 0) { 4276 char *action_str = str + strlen("onmatch("); 4277 4278 data = onmatch_parse(tr, action_str); 4279 if (IS_ERR(data)) { 4280 ret = PTR_ERR(data); 4281 break; 4282 } 4283 data->fn = action_trace; 4284 } else if (strncmp(str, "onmax(", strlen("onmax(")) == 0) { 4285 char *action_str = str + strlen("onmax("); 4286 4287 data = onmax_parse(action_str); 4288 if (IS_ERR(data)) { 4289 ret = PTR_ERR(data); 4290 break; 4291 } 4292 data->fn = onmax_save; 4293 } else { 4294 ret = -EINVAL; 4295 break; 4296 } 4297 4298 hist_data->actions[hist_data->n_actions++] = data; 4299 } 4300 4301 return ret; 4302 } 4303 4304 static int create_actions(struct hist_trigger_data *hist_data, 4305 struct trace_event_file *file) 4306 { 4307 struct action_data *data; 4308 unsigned int i; 4309 int ret = 0; 4310 4311 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4312 data = hist_data->actions[i]; 4313 4314 if (data->fn == action_trace) { 4315 ret = onmatch_create(hist_data, file, data); 4316 if (ret) 4317 return ret; 4318 } else if (data->fn == onmax_save) { 4319 ret = onmax_create(hist_data, data); 4320 if (ret) 4321 return ret; 4322 } 4323 } 4324 4325 return ret; 4326 } 4327 4328 static void print_actions(struct seq_file *m, 4329 struct hist_trigger_data *hist_data, 4330 struct tracing_map_elt *elt) 4331 { 4332 unsigned int i; 4333 4334 for (i = 0; i < hist_data->n_actions; i++) { 4335 struct action_data *data = hist_data->actions[i]; 4336 4337 if (data->fn == onmax_save) 4338 onmax_print(m, hist_data, elt, data); 4339 } 4340 } 4341 4342 static void print_onmax_spec(struct seq_file *m, 4343 struct hist_trigger_data *hist_data, 4344 struct action_data *data) 4345 { 4346 unsigned int i; 4347 4348 seq_puts(m, ":onmax("); 4349 seq_printf(m, "%s", data->onmax.var_str); 4350 seq_printf(m, ").%s(", data->onmax.fn_name); 4351 4352 for (i = 0; i < hist_data->n_max_vars; i++) { 4353 seq_printf(m, "%s", hist_data->max_vars[i]->var->var.name); 4354 if (i < hist_data->n_max_vars - 1) 4355 seq_puts(m, ","); 4356 } 4357 seq_puts(m, ")"); 4358 } 4359 4360 static void print_onmatch_spec(struct seq_file *m, 4361 struct hist_trigger_data *hist_data, 4362 struct action_data *data) 4363 { 4364 unsigned int i; 4365 4366 seq_printf(m, ":onmatch(%s.%s).", data->onmatch.match_event_system, 4367 data->onmatch.match_event); 4368 4369 seq_printf(m, "%s(", data->onmatch.synth_event->name); 4370 4371 for (i = 0; i < data->n_params; i++) { 4372 if (i) 4373 seq_puts(m, ","); 4374 seq_printf(m, "%s", data->params[i]); 4375 } 4376 4377 seq_puts(m, ")"); 4378 } 4379 4380 static bool actions_match(struct hist_trigger_data *hist_data, 4381 struct hist_trigger_data *hist_data_test) 4382 { 4383 unsigned int i, j; 4384 4385 if (hist_data->n_actions != hist_data_test->n_actions) 4386 return false; 4387 4388 for (i = 0; i < hist_data->n_actions; i++) { 4389 struct action_data *data = hist_data->actions[i]; 4390 struct action_data *data_test = hist_data_test->actions[i]; 4391 4392 if (data->fn != data_test->fn) 4393 return false; 4394 4395 if (data->n_params != data_test->n_params) 4396 return false; 4397 4398 for (j = 0; j < data->n_params; j++) { 4399 if (strcmp(data->params[j], data_test->params[j]) != 0) 4400 return false; 4401 } 4402 4403 if (data->fn == action_trace) { 4404 if (strcmp(data->onmatch.synth_event_name, 4405 data_test->onmatch.synth_event_name) != 0) 4406 return false; 4407 if (strcmp(data->onmatch.match_event_system, 4408 data_test->onmatch.match_event_system) != 0) 4409 return false; 4410 if (strcmp(data->onmatch.match_event, 4411 data_test->onmatch.match_event) != 0) 4412 return false; 4413 } else if (data->fn == onmax_save) { 4414 if (strcmp(data->onmax.var_str, 4415 data_test->onmax.var_str) != 0) 4416 return false; 4417 if (strcmp(data->onmax.fn_name, 4418 data_test->onmax.fn_name) != 0) 4419 return false; 4420 } 4421 } 4422 4423 return true; 4424 } 4425 4426 4427 static void print_actions_spec(struct seq_file *m, 4428 struct hist_trigger_data *hist_data) 4429 { 4430 unsigned int i; 4431 4432 for (i = 0; i < hist_data->n_actions; i++) { 4433 struct action_data *data = hist_data->actions[i]; 4434 4435 if (data->fn == action_trace) 4436 print_onmatch_spec(m, hist_data, data); 4437 else if (data->fn == onmax_save) 4438 print_onmax_spec(m, hist_data, data); 4439 } 4440 } 4441 4442 static void destroy_field_var_hists(struct hist_trigger_data *hist_data) 4443 { 4444 unsigned int i; 4445 4446 for (i = 0; i < hist_data->n_field_var_hists; i++) { 4447 kfree(hist_data->field_var_hists[i]->cmd); 4448 kfree(hist_data->field_var_hists[i]); 4449 } 4450 } 4451 4452 static void destroy_hist_data(struct hist_trigger_data *hist_data) 4453 { 4454 if (!hist_data) 4455 return; 4456 4457 destroy_hist_trigger_attrs(hist_data->attrs); 4458 destroy_hist_fields(hist_data); 4459 tracing_map_destroy(hist_data->map); 4460 4461 destroy_actions(hist_data); 4462 destroy_field_vars(hist_data); 4463 destroy_field_var_hists(hist_data); 4464 destroy_synth_var_refs(hist_data); 4465 4466 kfree(hist_data); 4467 } 4468 4469 static int create_tracing_map_fields(struct hist_trigger_data *hist_data) 4470 { 4471 struct tracing_map *map = hist_data->map; 4472 struct ftrace_event_field *field; 4473 struct hist_field *hist_field; 4474 int i, idx = 0; 4475 4476 for_each_hist_field(i, hist_data) { 4477 hist_field = hist_data->fields[i]; 4478 if (hist_field->flags & HIST_FIELD_FL_KEY) { 4479 tracing_map_cmp_fn_t cmp_fn; 4480 4481 field = hist_field->field; 4482 4483 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE) 4484 cmp_fn = tracing_map_cmp_none; 4485 else if (!field) 4486 cmp_fn = tracing_map_cmp_num(hist_field->size, 4487 hist_field->is_signed); 4488 else if (is_string_field(field)) 4489 cmp_fn = tracing_map_cmp_string; 4490 else 4491 cmp_fn = tracing_map_cmp_num(field->size, 4492 field->is_signed); 4493 idx = tracing_map_add_key_field(map, 4494 hist_field->offset, 4495 cmp_fn); 4496 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR)) 4497 idx = tracing_map_add_sum_field(map); 4498 4499 if (idx < 0) 4500 return idx; 4501 4502 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4503 idx = tracing_map_add_var(map); 4504 if (idx < 0) 4505 return idx; 4506 hist_field->var.idx = idx; 4507 hist_field->var.hist_data = hist_data; 4508 } 4509 } 4510 4511 return 0; 4512 } 4513 4514 static struct hist_trigger_data * 4515 create_hist_data(unsigned int map_bits, 4516 struct hist_trigger_attrs *attrs, 4517 struct trace_event_file *file, 4518 bool remove) 4519 { 4520 const struct tracing_map_ops *map_ops = NULL; 4521 struct hist_trigger_data *hist_data; 4522 int ret = 0; 4523 4524 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL); 4525 if (!hist_data) 4526 return ERR_PTR(-ENOMEM); 4527 4528 hist_data->attrs = attrs; 4529 hist_data->remove = remove; 4530 hist_data->event_file = file; 4531 4532 ret = parse_actions(hist_data); 4533 if (ret) 4534 goto free; 4535 4536 ret = create_hist_fields(hist_data, file); 4537 if (ret) 4538 goto free; 4539 4540 ret = create_sort_keys(hist_data); 4541 if (ret) 4542 goto free; 4543 4544 map_ops = &hist_trigger_elt_data_ops; 4545 4546 hist_data->map = tracing_map_create(map_bits, hist_data->key_size, 4547 map_ops, hist_data); 4548 if (IS_ERR(hist_data->map)) { 4549 ret = PTR_ERR(hist_data->map); 4550 hist_data->map = NULL; 4551 goto free; 4552 } 4553 4554 ret = create_tracing_map_fields(hist_data); 4555 if (ret) 4556 goto free; 4557 out: 4558 return hist_data; 4559 free: 4560 hist_data->attrs = NULL; 4561 4562 destroy_hist_data(hist_data); 4563 4564 hist_data = ERR_PTR(ret); 4565 4566 goto out; 4567 } 4568 4569 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, 4570 struct tracing_map_elt *elt, void *rec, 4571 struct ring_buffer_event *rbe, 4572 u64 *var_ref_vals) 4573 { 4574 struct hist_elt_data *elt_data; 4575 struct hist_field *hist_field; 4576 unsigned int i, var_idx; 4577 u64 hist_val; 4578 4579 elt_data = elt->private_data; 4580 elt_data->var_ref_vals = var_ref_vals; 4581 4582 for_each_hist_val_field(i, hist_data) { 4583 hist_field = hist_data->fields[i]; 4584 hist_val = hist_field->fn(hist_field, elt, rbe, rec); 4585 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4586 var_idx = hist_field->var.idx; 4587 tracing_map_set_var(elt, var_idx, hist_val); 4588 continue; 4589 } 4590 tracing_map_update_sum(elt, i, hist_val); 4591 } 4592 4593 for_each_hist_key_field(i, hist_data) { 4594 hist_field = hist_data->fields[i]; 4595 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4596 hist_val = hist_field->fn(hist_field, elt, rbe, rec); 4597 var_idx = hist_field->var.idx; 4598 tracing_map_set_var(elt, var_idx, hist_val); 4599 } 4600 } 4601 4602 update_field_vars(hist_data, elt, rbe, rec); 4603 } 4604 4605 static inline void add_to_key(char *compound_key, void *key, 4606 struct hist_field *key_field, void *rec) 4607 { 4608 size_t size = key_field->size; 4609 4610 if (key_field->flags & HIST_FIELD_FL_STRING) { 4611 struct ftrace_event_field *field; 4612 4613 field = key_field->field; 4614 if (field->filter_type == FILTER_DYN_STRING) 4615 size = *(u32 *)(rec + field->offset) >> 16; 4616 else if (field->filter_type == FILTER_PTR_STRING) 4617 size = strlen(key); 4618 else if (field->filter_type == FILTER_STATIC_STRING) 4619 size = field->size; 4620 4621 /* ensure NULL-termination */ 4622 if (size > key_field->size - 1) 4623 size = key_field->size - 1; 4624 } 4625 4626 memcpy(compound_key + key_field->offset, key, size); 4627 } 4628 4629 static void 4630 hist_trigger_actions(struct hist_trigger_data *hist_data, 4631 struct tracing_map_elt *elt, void *rec, 4632 struct ring_buffer_event *rbe, u64 *var_ref_vals) 4633 { 4634 struct action_data *data; 4635 unsigned int i; 4636 4637 for (i = 0; i < hist_data->n_actions; i++) { 4638 data = hist_data->actions[i]; 4639 data->fn(hist_data, elt, rec, rbe, data, var_ref_vals); 4640 } 4641 } 4642 4643 static void event_hist_trigger(struct event_trigger_data *data, void *rec, 4644 struct ring_buffer_event *rbe) 4645 { 4646 struct hist_trigger_data *hist_data = data->private_data; 4647 bool use_compound_key = (hist_data->n_keys > 1); 4648 unsigned long entries[HIST_STACKTRACE_DEPTH]; 4649 u64 var_ref_vals[TRACING_MAP_VARS_MAX]; 4650 char compound_key[HIST_KEY_SIZE_MAX]; 4651 struct tracing_map_elt *elt = NULL; 4652 struct stack_trace stacktrace; 4653 struct hist_field *key_field; 4654 u64 field_contents; 4655 void *key = NULL; 4656 unsigned int i; 4657 4658 memset(compound_key, 0, hist_data->key_size); 4659 4660 for_each_hist_key_field(i, hist_data) { 4661 key_field = hist_data->fields[i]; 4662 4663 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 4664 stacktrace.max_entries = HIST_STACKTRACE_DEPTH; 4665 stacktrace.entries = entries; 4666 stacktrace.nr_entries = 0; 4667 stacktrace.skip = HIST_STACKTRACE_SKIP; 4668 4669 memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE); 4670 save_stack_trace(&stacktrace); 4671 4672 key = entries; 4673 } else { 4674 field_contents = key_field->fn(key_field, elt, rbe, rec); 4675 if (key_field->flags & HIST_FIELD_FL_STRING) { 4676 key = (void *)(unsigned long)field_contents; 4677 use_compound_key = true; 4678 } else 4679 key = (void *)&field_contents; 4680 } 4681 4682 if (use_compound_key) 4683 add_to_key(compound_key, key, key_field, rec); 4684 } 4685 4686 if (use_compound_key) 4687 key = compound_key; 4688 4689 if (hist_data->n_var_refs && 4690 !resolve_var_refs(hist_data, key, var_ref_vals, false)) 4691 return; 4692 4693 elt = tracing_map_insert(hist_data->map, key); 4694 if (!elt) 4695 return; 4696 4697 hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals); 4698 4699 if (resolve_var_refs(hist_data, key, var_ref_vals, true)) 4700 hist_trigger_actions(hist_data, elt, rec, rbe, var_ref_vals); 4701 } 4702 4703 static void hist_trigger_stacktrace_print(struct seq_file *m, 4704 unsigned long *stacktrace_entries, 4705 unsigned int max_entries) 4706 { 4707 char str[KSYM_SYMBOL_LEN]; 4708 unsigned int spaces = 8; 4709 unsigned int i; 4710 4711 for (i = 0; i < max_entries; i++) { 4712 if (stacktrace_entries[i] == ULONG_MAX) 4713 return; 4714 4715 seq_printf(m, "%*c", 1 + spaces, ' '); 4716 sprint_symbol(str, stacktrace_entries[i]); 4717 seq_printf(m, "%s\n", str); 4718 } 4719 } 4720 4721 static void 4722 hist_trigger_entry_print(struct seq_file *m, 4723 struct hist_trigger_data *hist_data, void *key, 4724 struct tracing_map_elt *elt) 4725 { 4726 struct hist_field *key_field; 4727 char str[KSYM_SYMBOL_LEN]; 4728 bool multiline = false; 4729 const char *field_name; 4730 unsigned int i; 4731 u64 uval; 4732 4733 seq_puts(m, "{ "); 4734 4735 for_each_hist_key_field(i, hist_data) { 4736 key_field = hist_data->fields[i]; 4737 4738 if (i > hist_data->n_vals) 4739 seq_puts(m, ", "); 4740 4741 field_name = hist_field_name(key_field, 0); 4742 4743 if (key_field->flags & HIST_FIELD_FL_HEX) { 4744 uval = *(u64 *)(key + key_field->offset); 4745 seq_printf(m, "%s: %llx", field_name, uval); 4746 } else if (key_field->flags & HIST_FIELD_FL_SYM) { 4747 uval = *(u64 *)(key + key_field->offset); 4748 sprint_symbol_no_offset(str, uval); 4749 seq_printf(m, "%s: [%llx] %-45s", field_name, 4750 uval, str); 4751 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) { 4752 uval = *(u64 *)(key + key_field->offset); 4753 sprint_symbol(str, uval); 4754 seq_printf(m, "%s: [%llx] %-55s", field_name, 4755 uval, str); 4756 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 4757 struct hist_elt_data *elt_data = elt->private_data; 4758 char *comm; 4759 4760 if (WARN_ON_ONCE(!elt_data)) 4761 return; 4762 4763 comm = elt_data->comm; 4764 4765 uval = *(u64 *)(key + key_field->offset); 4766 seq_printf(m, "%s: %-16s[%10llu]", field_name, 4767 comm, uval); 4768 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) { 4769 const char *syscall_name; 4770 4771 uval = *(u64 *)(key + key_field->offset); 4772 syscall_name = get_syscall_name(uval); 4773 if (!syscall_name) 4774 syscall_name = "unknown_syscall"; 4775 4776 seq_printf(m, "%s: %-30s[%3llu]", field_name, 4777 syscall_name, uval); 4778 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 4779 seq_puts(m, "stacktrace:\n"); 4780 hist_trigger_stacktrace_print(m, 4781 key + key_field->offset, 4782 HIST_STACKTRACE_DEPTH); 4783 multiline = true; 4784 } else if (key_field->flags & HIST_FIELD_FL_LOG2) { 4785 seq_printf(m, "%s: ~ 2^%-2llu", field_name, 4786 *(u64 *)(key + key_field->offset)); 4787 } else if (key_field->flags & HIST_FIELD_FL_STRING) { 4788 seq_printf(m, "%s: %-50s", field_name, 4789 (char *)(key + key_field->offset)); 4790 } else { 4791 uval = *(u64 *)(key + key_field->offset); 4792 seq_printf(m, "%s: %10llu", field_name, uval); 4793 } 4794 } 4795 4796 if (!multiline) 4797 seq_puts(m, " "); 4798 4799 seq_puts(m, "}"); 4800 4801 seq_printf(m, " hitcount: %10llu", 4802 tracing_map_read_sum(elt, HITCOUNT_IDX)); 4803 4804 for (i = 1; i < hist_data->n_vals; i++) { 4805 field_name = hist_field_name(hist_data->fields[i], 0); 4806 4807 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR || 4808 hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR) 4809 continue; 4810 4811 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) { 4812 seq_printf(m, " %s: %10llx", field_name, 4813 tracing_map_read_sum(elt, i)); 4814 } else { 4815 seq_printf(m, " %s: %10llu", field_name, 4816 tracing_map_read_sum(elt, i)); 4817 } 4818 } 4819 4820 print_actions(m, hist_data, elt); 4821 4822 seq_puts(m, "\n"); 4823 } 4824 4825 static int print_entries(struct seq_file *m, 4826 struct hist_trigger_data *hist_data) 4827 { 4828 struct tracing_map_sort_entry **sort_entries = NULL; 4829 struct tracing_map *map = hist_data->map; 4830 int i, n_entries; 4831 4832 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys, 4833 hist_data->n_sort_keys, 4834 &sort_entries); 4835 if (n_entries < 0) 4836 return n_entries; 4837 4838 for (i = 0; i < n_entries; i++) 4839 hist_trigger_entry_print(m, hist_data, 4840 sort_entries[i]->key, 4841 sort_entries[i]->elt); 4842 4843 tracing_map_destroy_sort_entries(sort_entries, n_entries); 4844 4845 return n_entries; 4846 } 4847 4848 static void hist_trigger_show(struct seq_file *m, 4849 struct event_trigger_data *data, int n) 4850 { 4851 struct hist_trigger_data *hist_data; 4852 int n_entries; 4853 4854 if (n > 0) 4855 seq_puts(m, "\n\n"); 4856 4857 seq_puts(m, "# event histogram\n#\n# trigger info: "); 4858 data->ops->print(m, data->ops, data); 4859 seq_puts(m, "#\n\n"); 4860 4861 hist_data = data->private_data; 4862 n_entries = print_entries(m, hist_data); 4863 if (n_entries < 0) 4864 n_entries = 0; 4865 4866 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n", 4867 (u64)atomic64_read(&hist_data->map->hits), 4868 n_entries, (u64)atomic64_read(&hist_data->map->drops)); 4869 } 4870 4871 static int hist_show(struct seq_file *m, void *v) 4872 { 4873 struct event_trigger_data *data; 4874 struct trace_event_file *event_file; 4875 int n = 0, ret = 0; 4876 4877 mutex_lock(&event_mutex); 4878 4879 event_file = event_file_data(m->private); 4880 if (unlikely(!event_file)) { 4881 ret = -ENODEV; 4882 goto out_unlock; 4883 } 4884 4885 list_for_each_entry_rcu(data, &event_file->triggers, list) { 4886 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 4887 hist_trigger_show(m, data, n++); 4888 } 4889 4890 if (have_hist_err()) { 4891 seq_printf(m, "\nERROR: %s\n", hist_err_str); 4892 seq_printf(m, " Last command: %s\n", last_hist_cmd); 4893 } 4894 4895 out_unlock: 4896 mutex_unlock(&event_mutex); 4897 4898 return ret; 4899 } 4900 4901 static int event_hist_open(struct inode *inode, struct file *file) 4902 { 4903 return single_open(file, hist_show, file); 4904 } 4905 4906 const struct file_operations event_hist_fops = { 4907 .open = event_hist_open, 4908 .read = seq_read, 4909 .llseek = seq_lseek, 4910 .release = single_release, 4911 }; 4912 4913 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field) 4914 { 4915 const char *field_name = hist_field_name(hist_field, 0); 4916 4917 if (hist_field->var.name) 4918 seq_printf(m, "%s=", hist_field->var.name); 4919 4920 if (hist_field->flags & HIST_FIELD_FL_CPU) 4921 seq_puts(m, "cpu"); 4922 else if (field_name) { 4923 if (hist_field->flags & HIST_FIELD_FL_VAR_REF || 4924 hist_field->flags & HIST_FIELD_FL_ALIAS) 4925 seq_putc(m, '$'); 4926 seq_printf(m, "%s", field_name); 4927 } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP) 4928 seq_puts(m, "common_timestamp"); 4929 4930 if (hist_field->flags) { 4931 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) && 4932 !(hist_field->flags & HIST_FIELD_FL_EXPR)) { 4933 const char *flags = get_hist_field_flags(hist_field); 4934 4935 if (flags) 4936 seq_printf(m, ".%s", flags); 4937 } 4938 } 4939 } 4940 4941 static int event_hist_trigger_print(struct seq_file *m, 4942 struct event_trigger_ops *ops, 4943 struct event_trigger_data *data) 4944 { 4945 struct hist_trigger_data *hist_data = data->private_data; 4946 struct hist_field *field; 4947 bool have_var = false; 4948 unsigned int i; 4949 4950 seq_puts(m, "hist:"); 4951 4952 if (data->name) 4953 seq_printf(m, "%s:", data->name); 4954 4955 seq_puts(m, "keys="); 4956 4957 for_each_hist_key_field(i, hist_data) { 4958 field = hist_data->fields[i]; 4959 4960 if (i > hist_data->n_vals) 4961 seq_puts(m, ","); 4962 4963 if (field->flags & HIST_FIELD_FL_STACKTRACE) 4964 seq_puts(m, "stacktrace"); 4965 else 4966 hist_field_print(m, field); 4967 } 4968 4969 seq_puts(m, ":vals="); 4970 4971 for_each_hist_val_field(i, hist_data) { 4972 field = hist_data->fields[i]; 4973 if (field->flags & HIST_FIELD_FL_VAR) { 4974 have_var = true; 4975 continue; 4976 } 4977 4978 if (i == HITCOUNT_IDX) 4979 seq_puts(m, "hitcount"); 4980 else { 4981 seq_puts(m, ","); 4982 hist_field_print(m, field); 4983 } 4984 } 4985 4986 if (have_var) { 4987 unsigned int n = 0; 4988 4989 seq_puts(m, ":"); 4990 4991 for_each_hist_val_field(i, hist_data) { 4992 field = hist_data->fields[i]; 4993 4994 if (field->flags & HIST_FIELD_FL_VAR) { 4995 if (n++) 4996 seq_puts(m, ","); 4997 hist_field_print(m, field); 4998 } 4999 } 5000 } 5001 5002 seq_puts(m, ":sort="); 5003 5004 for (i = 0; i < hist_data->n_sort_keys; i++) { 5005 struct tracing_map_sort_key *sort_key; 5006 unsigned int idx, first_key_idx; 5007 5008 /* skip VAR vals */ 5009 first_key_idx = hist_data->n_vals - hist_data->n_vars; 5010 5011 sort_key = &hist_data->sort_keys[i]; 5012 idx = sort_key->field_idx; 5013 5014 if (WARN_ON(idx >= HIST_FIELDS_MAX)) 5015 return -EINVAL; 5016 5017 if (i > 0) 5018 seq_puts(m, ","); 5019 5020 if (idx == HITCOUNT_IDX) 5021 seq_puts(m, "hitcount"); 5022 else { 5023 if (idx >= first_key_idx) 5024 idx += hist_data->n_vars; 5025 hist_field_print(m, hist_data->fields[idx]); 5026 } 5027 5028 if (sort_key->descending) 5029 seq_puts(m, ".descending"); 5030 } 5031 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits)); 5032 if (hist_data->enable_timestamps) 5033 seq_printf(m, ":clock=%s", hist_data->attrs->clock); 5034 5035 print_actions_spec(m, hist_data); 5036 5037 if (data->filter_str) 5038 seq_printf(m, " if %s", data->filter_str); 5039 5040 if (data->paused) 5041 seq_puts(m, " [paused]"); 5042 else 5043 seq_puts(m, " [active]"); 5044 5045 seq_putc(m, '\n'); 5046 5047 return 0; 5048 } 5049 5050 static int event_hist_trigger_init(struct event_trigger_ops *ops, 5051 struct event_trigger_data *data) 5052 { 5053 struct hist_trigger_data *hist_data = data->private_data; 5054 5055 if (!data->ref && hist_data->attrs->name) 5056 save_named_trigger(hist_data->attrs->name, data); 5057 5058 data->ref++; 5059 5060 return 0; 5061 } 5062 5063 static void unregister_field_var_hists(struct hist_trigger_data *hist_data) 5064 { 5065 struct trace_event_file *file; 5066 unsigned int i; 5067 char *cmd; 5068 int ret; 5069 5070 for (i = 0; i < hist_data->n_field_var_hists; i++) { 5071 file = hist_data->field_var_hists[i]->hist_data->event_file; 5072 cmd = hist_data->field_var_hists[i]->cmd; 5073 ret = event_hist_trigger_func(&trigger_hist_cmd, file, 5074 "!hist", "hist", cmd); 5075 } 5076 } 5077 5078 static void event_hist_trigger_free(struct event_trigger_ops *ops, 5079 struct event_trigger_data *data) 5080 { 5081 struct hist_trigger_data *hist_data = data->private_data; 5082 5083 if (WARN_ON_ONCE(data->ref <= 0)) 5084 return; 5085 5086 data->ref--; 5087 if (!data->ref) { 5088 if (data->name) 5089 del_named_trigger(data); 5090 5091 trigger_data_free(data); 5092 5093 remove_hist_vars(hist_data); 5094 5095 unregister_field_var_hists(hist_data); 5096 5097 destroy_hist_data(hist_data); 5098 } 5099 } 5100 5101 static struct event_trigger_ops event_hist_trigger_ops = { 5102 .func = event_hist_trigger, 5103 .print = event_hist_trigger_print, 5104 .init = event_hist_trigger_init, 5105 .free = event_hist_trigger_free, 5106 }; 5107 5108 static int event_hist_trigger_named_init(struct event_trigger_ops *ops, 5109 struct event_trigger_data *data) 5110 { 5111 data->ref++; 5112 5113 save_named_trigger(data->named_data->name, data); 5114 5115 event_hist_trigger_init(ops, data->named_data); 5116 5117 return 0; 5118 } 5119 5120 static void event_hist_trigger_named_free(struct event_trigger_ops *ops, 5121 struct event_trigger_data *data) 5122 { 5123 if (WARN_ON_ONCE(data->ref <= 0)) 5124 return; 5125 5126 event_hist_trigger_free(ops, data->named_data); 5127 5128 data->ref--; 5129 if (!data->ref) { 5130 del_named_trigger(data); 5131 trigger_data_free(data); 5132 } 5133 } 5134 5135 static struct event_trigger_ops event_hist_trigger_named_ops = { 5136 .func = event_hist_trigger, 5137 .print = event_hist_trigger_print, 5138 .init = event_hist_trigger_named_init, 5139 .free = event_hist_trigger_named_free, 5140 }; 5141 5142 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd, 5143 char *param) 5144 { 5145 return &event_hist_trigger_ops; 5146 } 5147 5148 static void hist_clear(struct event_trigger_data *data) 5149 { 5150 struct hist_trigger_data *hist_data = data->private_data; 5151 5152 if (data->name) 5153 pause_named_trigger(data); 5154 5155 tracepoint_synchronize_unregister(); 5156 5157 tracing_map_clear(hist_data->map); 5158 5159 if (data->name) 5160 unpause_named_trigger(data); 5161 } 5162 5163 static bool compatible_field(struct ftrace_event_field *field, 5164 struct ftrace_event_field *test_field) 5165 { 5166 if (field == test_field) 5167 return true; 5168 if (field == NULL || test_field == NULL) 5169 return false; 5170 if (strcmp(field->name, test_field->name) != 0) 5171 return false; 5172 if (strcmp(field->type, test_field->type) != 0) 5173 return false; 5174 if (field->size != test_field->size) 5175 return false; 5176 if (field->is_signed != test_field->is_signed) 5177 return false; 5178 5179 return true; 5180 } 5181 5182 static bool hist_trigger_match(struct event_trigger_data *data, 5183 struct event_trigger_data *data_test, 5184 struct event_trigger_data *named_data, 5185 bool ignore_filter) 5186 { 5187 struct tracing_map_sort_key *sort_key, *sort_key_test; 5188 struct hist_trigger_data *hist_data, *hist_data_test; 5189 struct hist_field *key_field, *key_field_test; 5190 unsigned int i; 5191 5192 if (named_data && (named_data != data_test) && 5193 (named_data != data_test->named_data)) 5194 return false; 5195 5196 if (!named_data && is_named_trigger(data_test)) 5197 return false; 5198 5199 hist_data = data->private_data; 5200 hist_data_test = data_test->private_data; 5201 5202 if (hist_data->n_vals != hist_data_test->n_vals || 5203 hist_data->n_fields != hist_data_test->n_fields || 5204 hist_data->n_sort_keys != hist_data_test->n_sort_keys) 5205 return false; 5206 5207 if (!ignore_filter) { 5208 if ((data->filter_str && !data_test->filter_str) || 5209 (!data->filter_str && data_test->filter_str)) 5210 return false; 5211 } 5212 5213 for_each_hist_field(i, hist_data) { 5214 key_field = hist_data->fields[i]; 5215 key_field_test = hist_data_test->fields[i]; 5216 5217 if (key_field->flags != key_field_test->flags) 5218 return false; 5219 if (!compatible_field(key_field->field, key_field_test->field)) 5220 return false; 5221 if (key_field->offset != key_field_test->offset) 5222 return false; 5223 if (key_field->size != key_field_test->size) 5224 return false; 5225 if (key_field->is_signed != key_field_test->is_signed) 5226 return false; 5227 if (!!key_field->var.name != !!key_field_test->var.name) 5228 return false; 5229 if (key_field->var.name && 5230 strcmp(key_field->var.name, key_field_test->var.name) != 0) 5231 return false; 5232 } 5233 5234 for (i = 0; i < hist_data->n_sort_keys; i++) { 5235 sort_key = &hist_data->sort_keys[i]; 5236 sort_key_test = &hist_data_test->sort_keys[i]; 5237 5238 if (sort_key->field_idx != sort_key_test->field_idx || 5239 sort_key->descending != sort_key_test->descending) 5240 return false; 5241 } 5242 5243 if (!ignore_filter && data->filter_str && 5244 (strcmp(data->filter_str, data_test->filter_str) != 0)) 5245 return false; 5246 5247 if (!actions_match(hist_data, hist_data_test)) 5248 return false; 5249 5250 return true; 5251 } 5252 5253 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops, 5254 struct event_trigger_data *data, 5255 struct trace_event_file *file) 5256 { 5257 struct hist_trigger_data *hist_data = data->private_data; 5258 struct event_trigger_data *test, *named_data = NULL; 5259 int ret = 0; 5260 5261 if (hist_data->attrs->name) { 5262 named_data = find_named_trigger(hist_data->attrs->name); 5263 if (named_data) { 5264 if (!hist_trigger_match(data, named_data, named_data, 5265 true)) { 5266 hist_err("Named hist trigger doesn't match existing named trigger (includes variables): ", hist_data->attrs->name); 5267 ret = -EINVAL; 5268 goto out; 5269 } 5270 } 5271 } 5272 5273 if (hist_data->attrs->name && !named_data) 5274 goto new; 5275 5276 list_for_each_entry_rcu(test, &file->triggers, list) { 5277 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5278 if (!hist_trigger_match(data, test, named_data, false)) 5279 continue; 5280 if (hist_data->attrs->pause) 5281 test->paused = true; 5282 else if (hist_data->attrs->cont) 5283 test->paused = false; 5284 else if (hist_data->attrs->clear) 5285 hist_clear(test); 5286 else { 5287 hist_err("Hist trigger already exists", NULL); 5288 ret = -EEXIST; 5289 } 5290 goto out; 5291 } 5292 } 5293 new: 5294 if (hist_data->attrs->cont || hist_data->attrs->clear) { 5295 hist_err("Can't clear or continue a nonexistent hist trigger", NULL); 5296 ret = -ENOENT; 5297 goto out; 5298 } 5299 5300 if (hist_data->attrs->pause) 5301 data->paused = true; 5302 5303 if (named_data) { 5304 data->private_data = named_data->private_data; 5305 set_named_trigger_data(data, named_data); 5306 data->ops = &event_hist_trigger_named_ops; 5307 } 5308 5309 if (data->ops->init) { 5310 ret = data->ops->init(data->ops, data); 5311 if (ret < 0) 5312 goto out; 5313 } 5314 5315 if (hist_data->enable_timestamps) { 5316 char *clock = hist_data->attrs->clock; 5317 5318 ret = tracing_set_clock(file->tr, hist_data->attrs->clock); 5319 if (ret) { 5320 hist_err("Couldn't set trace_clock: ", clock); 5321 goto out; 5322 } 5323 5324 tracing_set_time_stamp_abs(file->tr, true); 5325 } 5326 5327 if (named_data) 5328 destroy_hist_data(hist_data); 5329 5330 ret++; 5331 out: 5332 return ret; 5333 } 5334 5335 static int hist_trigger_enable(struct event_trigger_data *data, 5336 struct trace_event_file *file) 5337 { 5338 int ret = 0; 5339 5340 list_add_tail_rcu(&data->list, &file->triggers); 5341 5342 update_cond_flag(file); 5343 5344 if (trace_event_trigger_enable_disable(file, 1) < 0) { 5345 list_del_rcu(&data->list); 5346 update_cond_flag(file); 5347 ret--; 5348 } 5349 5350 return ret; 5351 } 5352 5353 static bool have_hist_trigger_match(struct event_trigger_data *data, 5354 struct trace_event_file *file) 5355 { 5356 struct hist_trigger_data *hist_data = data->private_data; 5357 struct event_trigger_data *test, *named_data = NULL; 5358 bool match = false; 5359 5360 if (hist_data->attrs->name) 5361 named_data = find_named_trigger(hist_data->attrs->name); 5362 5363 list_for_each_entry_rcu(test, &file->triggers, list) { 5364 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5365 if (hist_trigger_match(data, test, named_data, false)) { 5366 match = true; 5367 break; 5368 } 5369 } 5370 } 5371 5372 return match; 5373 } 5374 5375 static bool hist_trigger_check_refs(struct event_trigger_data *data, 5376 struct trace_event_file *file) 5377 { 5378 struct hist_trigger_data *hist_data = data->private_data; 5379 struct event_trigger_data *test, *named_data = NULL; 5380 5381 if (hist_data->attrs->name) 5382 named_data = find_named_trigger(hist_data->attrs->name); 5383 5384 list_for_each_entry_rcu(test, &file->triggers, list) { 5385 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5386 if (!hist_trigger_match(data, test, named_data, false)) 5387 continue; 5388 hist_data = test->private_data; 5389 if (check_var_refs(hist_data)) 5390 return true; 5391 break; 5392 } 5393 } 5394 5395 return false; 5396 } 5397 5398 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops, 5399 struct event_trigger_data *data, 5400 struct trace_event_file *file) 5401 { 5402 struct hist_trigger_data *hist_data = data->private_data; 5403 struct event_trigger_data *test, *named_data = NULL; 5404 bool unregistered = false; 5405 5406 if (hist_data->attrs->name) 5407 named_data = find_named_trigger(hist_data->attrs->name); 5408 5409 list_for_each_entry_rcu(test, &file->triggers, list) { 5410 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5411 if (!hist_trigger_match(data, test, named_data, false)) 5412 continue; 5413 unregistered = true; 5414 list_del_rcu(&test->list); 5415 trace_event_trigger_enable_disable(file, 0); 5416 update_cond_flag(file); 5417 break; 5418 } 5419 } 5420 5421 if (unregistered && test->ops->free) 5422 test->ops->free(test->ops, test); 5423 5424 if (hist_data->enable_timestamps) { 5425 if (!hist_data->remove || unregistered) 5426 tracing_set_time_stamp_abs(file->tr, false); 5427 } 5428 } 5429 5430 static bool hist_file_check_refs(struct trace_event_file *file) 5431 { 5432 struct hist_trigger_data *hist_data; 5433 struct event_trigger_data *test; 5434 5435 list_for_each_entry_rcu(test, &file->triggers, list) { 5436 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5437 hist_data = test->private_data; 5438 if (check_var_refs(hist_data)) 5439 return true; 5440 } 5441 } 5442 5443 return false; 5444 } 5445 5446 static void hist_unreg_all(struct trace_event_file *file) 5447 { 5448 struct event_trigger_data *test, *n; 5449 struct hist_trigger_data *hist_data; 5450 struct synth_event *se; 5451 const char *se_name; 5452 5453 if (hist_file_check_refs(file)) 5454 return; 5455 5456 list_for_each_entry_safe(test, n, &file->triggers, list) { 5457 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5458 hist_data = test->private_data; 5459 list_del_rcu(&test->list); 5460 trace_event_trigger_enable_disable(file, 0); 5461 5462 mutex_lock(&synth_event_mutex); 5463 se_name = trace_event_name(file->event_call); 5464 se = find_synth_event(se_name); 5465 if (se) 5466 se->ref--; 5467 mutex_unlock(&synth_event_mutex); 5468 5469 update_cond_flag(file); 5470 if (hist_data->enable_timestamps) 5471 tracing_set_time_stamp_abs(file->tr, false); 5472 if (test->ops->free) 5473 test->ops->free(test->ops, test); 5474 } 5475 } 5476 } 5477 5478 static int event_hist_trigger_func(struct event_command *cmd_ops, 5479 struct trace_event_file *file, 5480 char *glob, char *cmd, char *param) 5481 { 5482 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT; 5483 struct event_trigger_data *trigger_data; 5484 struct hist_trigger_attrs *attrs; 5485 struct event_trigger_ops *trigger_ops; 5486 struct hist_trigger_data *hist_data; 5487 struct synth_event *se; 5488 const char *se_name; 5489 bool remove = false; 5490 char *trigger, *p; 5491 int ret = 0; 5492 5493 if (glob && strlen(glob)) { 5494 last_cmd_set(param); 5495 hist_err_clear(); 5496 } 5497 5498 if (!param) 5499 return -EINVAL; 5500 5501 if (glob[0] == '!') 5502 remove = true; 5503 5504 /* 5505 * separate the trigger from the filter (k:v [if filter]) 5506 * allowing for whitespace in the trigger 5507 */ 5508 p = trigger = param; 5509 do { 5510 p = strstr(p, "if"); 5511 if (!p) 5512 break; 5513 if (p == param) 5514 return -EINVAL; 5515 if (*(p - 1) != ' ' && *(p - 1) != '\t') { 5516 p++; 5517 continue; 5518 } 5519 if (p >= param + strlen(param) - strlen("if") - 1) 5520 return -EINVAL; 5521 if (*(p + strlen("if")) != ' ' && *(p + strlen("if")) != '\t') { 5522 p++; 5523 continue; 5524 } 5525 break; 5526 } while (p); 5527 5528 if (!p) 5529 param = NULL; 5530 else { 5531 *(p - 1) = '\0'; 5532 param = strstrip(p); 5533 trigger = strstrip(trigger); 5534 } 5535 5536 attrs = parse_hist_trigger_attrs(trigger); 5537 if (IS_ERR(attrs)) 5538 return PTR_ERR(attrs); 5539 5540 if (attrs->map_bits) 5541 hist_trigger_bits = attrs->map_bits; 5542 5543 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove); 5544 if (IS_ERR(hist_data)) { 5545 destroy_hist_trigger_attrs(attrs); 5546 return PTR_ERR(hist_data); 5547 } 5548 5549 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger); 5550 5551 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL); 5552 if (!trigger_data) { 5553 ret = -ENOMEM; 5554 goto out_free; 5555 } 5556 5557 trigger_data->count = -1; 5558 trigger_data->ops = trigger_ops; 5559 trigger_data->cmd_ops = cmd_ops; 5560 5561 INIT_LIST_HEAD(&trigger_data->list); 5562 RCU_INIT_POINTER(trigger_data->filter, NULL); 5563 5564 trigger_data->private_data = hist_data; 5565 5566 /* if param is non-empty, it's supposed to be a filter */ 5567 if (param && cmd_ops->set_filter) { 5568 ret = cmd_ops->set_filter(param, trigger_data, file); 5569 if (ret < 0) 5570 goto out_free; 5571 } 5572 5573 if (remove) { 5574 if (!have_hist_trigger_match(trigger_data, file)) 5575 goto out_free; 5576 5577 if (hist_trigger_check_refs(trigger_data, file)) { 5578 ret = -EBUSY; 5579 goto out_free; 5580 } 5581 5582 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file); 5583 5584 mutex_lock(&synth_event_mutex); 5585 se_name = trace_event_name(file->event_call); 5586 se = find_synth_event(se_name); 5587 if (se) 5588 se->ref--; 5589 mutex_unlock(&synth_event_mutex); 5590 5591 ret = 0; 5592 goto out_free; 5593 } 5594 5595 ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file); 5596 /* 5597 * The above returns on success the # of triggers registered, 5598 * but if it didn't register any it returns zero. Consider no 5599 * triggers registered a failure too. 5600 */ 5601 if (!ret) { 5602 if (!(attrs->pause || attrs->cont || attrs->clear)) 5603 ret = -ENOENT; 5604 goto out_free; 5605 } else if (ret < 0) 5606 goto out_free; 5607 5608 if (get_named_trigger_data(trigger_data)) 5609 goto enable; 5610 5611 if (has_hist_vars(hist_data)) 5612 save_hist_vars(hist_data); 5613 5614 ret = create_actions(hist_data, file); 5615 if (ret) 5616 goto out_unreg; 5617 5618 ret = tracing_map_init(hist_data->map); 5619 if (ret) 5620 goto out_unreg; 5621 enable: 5622 ret = hist_trigger_enable(trigger_data, file); 5623 if (ret) 5624 goto out_unreg; 5625 5626 mutex_lock(&synth_event_mutex); 5627 se_name = trace_event_name(file->event_call); 5628 se = find_synth_event(se_name); 5629 if (se) 5630 se->ref++; 5631 mutex_unlock(&synth_event_mutex); 5632 5633 /* Just return zero, not the number of registered triggers */ 5634 ret = 0; 5635 out: 5636 if (ret == 0) 5637 hist_err_clear(); 5638 5639 return ret; 5640 out_unreg: 5641 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file); 5642 out_free: 5643 if (cmd_ops->set_filter) 5644 cmd_ops->set_filter(NULL, trigger_data, NULL); 5645 5646 remove_hist_vars(hist_data); 5647 5648 kfree(trigger_data); 5649 5650 destroy_hist_data(hist_data); 5651 goto out; 5652 } 5653 5654 static struct event_command trigger_hist_cmd = { 5655 .name = "hist", 5656 .trigger_type = ETT_EVENT_HIST, 5657 .flags = EVENT_CMD_FL_NEEDS_REC, 5658 .func = event_hist_trigger_func, 5659 .reg = hist_register_trigger, 5660 .unreg = hist_unregister_trigger, 5661 .unreg_all = hist_unreg_all, 5662 .get_trigger_ops = event_hist_get_trigger_ops, 5663 .set_filter = set_trigger_filter, 5664 }; 5665 5666 __init int register_trigger_hist_cmd(void) 5667 { 5668 int ret; 5669 5670 ret = register_event_command(&trigger_hist_cmd); 5671 WARN_ON(ret < 0); 5672 5673 return ret; 5674 } 5675 5676 static void 5677 hist_enable_trigger(struct event_trigger_data *data, void *rec, 5678 struct ring_buffer_event *event) 5679 { 5680 struct enable_trigger_data *enable_data = data->private_data; 5681 struct event_trigger_data *test; 5682 5683 list_for_each_entry_rcu(test, &enable_data->file->triggers, list) { 5684 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5685 if (enable_data->enable) 5686 test->paused = false; 5687 else 5688 test->paused = true; 5689 } 5690 } 5691 } 5692 5693 static void 5694 hist_enable_count_trigger(struct event_trigger_data *data, void *rec, 5695 struct ring_buffer_event *event) 5696 { 5697 if (!data->count) 5698 return; 5699 5700 if (data->count != -1) 5701 (data->count)--; 5702 5703 hist_enable_trigger(data, rec, event); 5704 } 5705 5706 static struct event_trigger_ops hist_enable_trigger_ops = { 5707 .func = hist_enable_trigger, 5708 .print = event_enable_trigger_print, 5709 .init = event_trigger_init, 5710 .free = event_enable_trigger_free, 5711 }; 5712 5713 static struct event_trigger_ops hist_enable_count_trigger_ops = { 5714 .func = hist_enable_count_trigger, 5715 .print = event_enable_trigger_print, 5716 .init = event_trigger_init, 5717 .free = event_enable_trigger_free, 5718 }; 5719 5720 static struct event_trigger_ops hist_disable_trigger_ops = { 5721 .func = hist_enable_trigger, 5722 .print = event_enable_trigger_print, 5723 .init = event_trigger_init, 5724 .free = event_enable_trigger_free, 5725 }; 5726 5727 static struct event_trigger_ops hist_disable_count_trigger_ops = { 5728 .func = hist_enable_count_trigger, 5729 .print = event_enable_trigger_print, 5730 .init = event_trigger_init, 5731 .free = event_enable_trigger_free, 5732 }; 5733 5734 static struct event_trigger_ops * 5735 hist_enable_get_trigger_ops(char *cmd, char *param) 5736 { 5737 struct event_trigger_ops *ops; 5738 bool enable; 5739 5740 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0); 5741 5742 if (enable) 5743 ops = param ? &hist_enable_count_trigger_ops : 5744 &hist_enable_trigger_ops; 5745 else 5746 ops = param ? &hist_disable_count_trigger_ops : 5747 &hist_disable_trigger_ops; 5748 5749 return ops; 5750 } 5751 5752 static void hist_enable_unreg_all(struct trace_event_file *file) 5753 { 5754 struct event_trigger_data *test, *n; 5755 5756 list_for_each_entry_safe(test, n, &file->triggers, list) { 5757 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) { 5758 list_del_rcu(&test->list); 5759 update_cond_flag(file); 5760 trace_event_trigger_enable_disable(file, 0); 5761 if (test->ops->free) 5762 test->ops->free(test->ops, test); 5763 } 5764 } 5765 } 5766 5767 static struct event_command trigger_hist_enable_cmd = { 5768 .name = ENABLE_HIST_STR, 5769 .trigger_type = ETT_HIST_ENABLE, 5770 .func = event_enable_trigger_func, 5771 .reg = event_enable_register_trigger, 5772 .unreg = event_enable_unregister_trigger, 5773 .unreg_all = hist_enable_unreg_all, 5774 .get_trigger_ops = hist_enable_get_trigger_ops, 5775 .set_filter = set_trigger_filter, 5776 }; 5777 5778 static struct event_command trigger_hist_disable_cmd = { 5779 .name = DISABLE_HIST_STR, 5780 .trigger_type = ETT_HIST_ENABLE, 5781 .func = event_enable_trigger_func, 5782 .reg = event_enable_register_trigger, 5783 .unreg = event_enable_unregister_trigger, 5784 .unreg_all = hist_enable_unreg_all, 5785 .get_trigger_ops = hist_enable_get_trigger_ops, 5786 .set_filter = set_trigger_filter, 5787 }; 5788 5789 static __init void unregister_trigger_hist_enable_disable_cmds(void) 5790 { 5791 unregister_event_command(&trigger_hist_enable_cmd); 5792 unregister_event_command(&trigger_hist_disable_cmd); 5793 } 5794 5795 __init int register_trigger_hist_enable_disable_cmds(void) 5796 { 5797 int ret; 5798 5799 ret = register_event_command(&trigger_hist_enable_cmd); 5800 if (WARN_ON(ret < 0)) 5801 return ret; 5802 ret = register_event_command(&trigger_hist_disable_cmd); 5803 if (WARN_ON(ret < 0)) 5804 unregister_trigger_hist_enable_disable_cmds(); 5805 5806 return ret; 5807 } 5808 5809 static __init int trace_events_hist_init(void) 5810 { 5811 struct dentry *entry = NULL; 5812 struct dentry *d_tracer; 5813 int err = 0; 5814 5815 d_tracer = tracing_init_dentry(); 5816 if (IS_ERR(d_tracer)) { 5817 err = PTR_ERR(d_tracer); 5818 goto err; 5819 } 5820 5821 entry = tracefs_create_file("synthetic_events", 0644, d_tracer, 5822 NULL, &synth_events_fops); 5823 if (!entry) { 5824 err = -ENODEV; 5825 goto err; 5826 } 5827 5828 return err; 5829 err: 5830 pr_warn("Could not create tracefs 'synthetic_events' entry\n"); 5831 5832 return err; 5833 } 5834 5835 fs_initcall(trace_events_hist_init); 5836