1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * trace_events_hist - trace event hist triggers 4 * 5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com> 6 */ 7 8 #include <linux/module.h> 9 #include <linux/kallsyms.h> 10 #include <linux/mutex.h> 11 #include <linux/slab.h> 12 #include <linux/stacktrace.h> 13 #include <linux/rculist.h> 14 #include <linux/tracefs.h> 15 16 /* for gfp flag names */ 17 #include <linux/trace_events.h> 18 #include <trace/events/mmflags.h> 19 20 #include "tracing_map.h" 21 #include "trace.h" 22 #include "trace_dynevent.h" 23 24 #define SYNTH_SYSTEM "synthetic" 25 #define SYNTH_FIELDS_MAX 16 26 27 #define STR_VAR_LEN_MAX 32 /* must be multiple of sizeof(u64) */ 28 29 #define ERRORS \ 30 C(NONE, "No error"), \ 31 C(DUPLICATE_VAR, "Variable already defined"), \ 32 C(VAR_NOT_UNIQUE, "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \ 33 C(TOO_MANY_VARS, "Too many variables defined"), \ 34 C(MALFORMED_ASSIGNMENT, "Malformed assignment"), \ 35 C(NAMED_MISMATCH, "Named hist trigger doesn't match existing named trigger (includes variables)"), \ 36 C(TRIGGER_EEXIST, "Hist trigger already exists"), \ 37 C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \ 38 C(SET_CLOCK_FAIL, "Couldn't set trace_clock"), \ 39 C(BAD_FIELD_MODIFIER, "Invalid field modifier"), \ 40 C(TOO_MANY_SUBEXPR, "Too many subexpressions (3 max)"), \ 41 C(TIMESTAMP_MISMATCH, "Timestamp units in expression don't match"), \ 42 C(TOO_MANY_FIELD_VARS, "Too many field variables defined"), \ 43 C(EVENT_FILE_NOT_FOUND, "Event file not found"), \ 44 C(HIST_NOT_FOUND, "Matching event histogram not found"), \ 45 C(HIST_CREATE_FAIL, "Couldn't create histogram for field"), \ 46 C(SYNTH_VAR_NOT_FOUND, "Couldn't find synthetic variable"), \ 47 C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"), \ 48 C(SYNTH_TYPE_MISMATCH, "Param type doesn't match synthetic event field type"), \ 49 C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \ 50 C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"), \ 51 C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"), \ 52 C(ONX_NOT_VAR, "For onmax(x) or onchange(x), x must be a variable"), \ 53 C(ONX_VAR_NOT_FOUND, "Couldn't find onmax or onchange variable"), \ 54 C(ONX_VAR_CREATE_FAIL, "Couldn't create onmax or onchange variable"), \ 55 C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"), \ 56 C(TOO_MANY_PARAMS, "Too many action params"), \ 57 C(PARAM_NOT_FOUND, "Couldn't find param"), \ 58 C(INVALID_PARAM, "Invalid action param"), \ 59 C(ACTION_NOT_FOUND, "No action found"), \ 60 C(NO_SAVE_PARAMS, "No params found for save()"), \ 61 C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \ 62 C(ACTION_MISMATCH, "Handler doesn't support action"), \ 63 C(NO_CLOSING_PAREN, "No closing paren found"), \ 64 C(SUBSYS_NOT_FOUND, "Missing subsystem"), \ 65 C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"), \ 66 C(INVALID_REF_KEY, "Using variable references in keys not supported"), \ 67 C(VAR_NOT_FOUND, "Couldn't find variable"), \ 68 C(FIELD_NOT_FOUND, "Couldn't find field"), 69 70 #undef C 71 #define C(a, b) HIST_ERR_##a 72 73 enum { ERRORS }; 74 75 #undef C 76 #define C(a, b) b 77 78 static const char *err_text[] = { ERRORS }; 79 80 struct hist_field; 81 82 typedef u64 (*hist_field_fn_t) (struct hist_field *field, 83 struct tracing_map_elt *elt, 84 struct ring_buffer_event *rbe, 85 void *event); 86 87 #define HIST_FIELD_OPERANDS_MAX 2 88 #define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX) 89 #define HIST_ACTIONS_MAX 8 90 91 enum field_op_id { 92 FIELD_OP_NONE, 93 FIELD_OP_PLUS, 94 FIELD_OP_MINUS, 95 FIELD_OP_UNARY_MINUS, 96 }; 97 98 /* 99 * A hist_var (histogram variable) contains variable information for 100 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF 101 * flag set. A hist_var has a variable name e.g. ts0, and is 102 * associated with a given histogram trigger, as specified by 103 * hist_data. The hist_var idx is the unique index assigned to the 104 * variable by the hist trigger's tracing_map. The idx is what is 105 * used to set a variable's value and, by a variable reference, to 106 * retrieve it. 107 */ 108 struct hist_var { 109 char *name; 110 struct hist_trigger_data *hist_data; 111 unsigned int idx; 112 }; 113 114 struct hist_field { 115 struct ftrace_event_field *field; 116 unsigned long flags; 117 hist_field_fn_t fn; 118 unsigned int size; 119 unsigned int offset; 120 unsigned int is_signed; 121 const char *type; 122 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX]; 123 struct hist_trigger_data *hist_data; 124 125 /* 126 * Variable fields contain variable-specific info in var. 127 */ 128 struct hist_var var; 129 enum field_op_id operator; 130 char *system; 131 char *event_name; 132 133 /* 134 * The name field is used for EXPR and VAR_REF fields. VAR 135 * fields contain the variable name in var.name. 136 */ 137 char *name; 138 139 /* 140 * When a histogram trigger is hit, if it has any references 141 * to variables, the values of those variables are collected 142 * into a var_ref_vals array by resolve_var_refs(). The 143 * current value of each variable is read from the tracing_map 144 * using the hist field's hist_var.idx and entered into the 145 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx]. 146 */ 147 unsigned int var_ref_idx; 148 bool read_once; 149 }; 150 151 static u64 hist_field_none(struct hist_field *field, 152 struct tracing_map_elt *elt, 153 struct ring_buffer_event *rbe, 154 void *event) 155 { 156 return 0; 157 } 158 159 static u64 hist_field_counter(struct hist_field *field, 160 struct tracing_map_elt *elt, 161 struct ring_buffer_event *rbe, 162 void *event) 163 { 164 return 1; 165 } 166 167 static u64 hist_field_string(struct hist_field *hist_field, 168 struct tracing_map_elt *elt, 169 struct ring_buffer_event *rbe, 170 void *event) 171 { 172 char *addr = (char *)(event + hist_field->field->offset); 173 174 return (u64)(unsigned long)addr; 175 } 176 177 static u64 hist_field_dynstring(struct hist_field *hist_field, 178 struct tracing_map_elt *elt, 179 struct ring_buffer_event *rbe, 180 void *event) 181 { 182 u32 str_item = *(u32 *)(event + hist_field->field->offset); 183 int str_loc = str_item & 0xffff; 184 char *addr = (char *)(event + str_loc); 185 186 return (u64)(unsigned long)addr; 187 } 188 189 static u64 hist_field_pstring(struct hist_field *hist_field, 190 struct tracing_map_elt *elt, 191 struct ring_buffer_event *rbe, 192 void *event) 193 { 194 char **addr = (char **)(event + hist_field->field->offset); 195 196 return (u64)(unsigned long)*addr; 197 } 198 199 static u64 hist_field_log2(struct hist_field *hist_field, 200 struct tracing_map_elt *elt, 201 struct ring_buffer_event *rbe, 202 void *event) 203 { 204 struct hist_field *operand = hist_field->operands[0]; 205 206 u64 val = operand->fn(operand, elt, rbe, event); 207 208 return (u64) ilog2(roundup_pow_of_two(val)); 209 } 210 211 static u64 hist_field_plus(struct hist_field *hist_field, 212 struct tracing_map_elt *elt, 213 struct ring_buffer_event *rbe, 214 void *event) 215 { 216 struct hist_field *operand1 = hist_field->operands[0]; 217 struct hist_field *operand2 = hist_field->operands[1]; 218 219 u64 val1 = operand1->fn(operand1, elt, rbe, event); 220 u64 val2 = operand2->fn(operand2, elt, rbe, event); 221 222 return val1 + val2; 223 } 224 225 static u64 hist_field_minus(struct hist_field *hist_field, 226 struct tracing_map_elt *elt, 227 struct ring_buffer_event *rbe, 228 void *event) 229 { 230 struct hist_field *operand1 = hist_field->operands[0]; 231 struct hist_field *operand2 = hist_field->operands[1]; 232 233 u64 val1 = operand1->fn(operand1, elt, rbe, event); 234 u64 val2 = operand2->fn(operand2, elt, rbe, event); 235 236 return val1 - val2; 237 } 238 239 static u64 hist_field_unary_minus(struct hist_field *hist_field, 240 struct tracing_map_elt *elt, 241 struct ring_buffer_event *rbe, 242 void *event) 243 { 244 struct hist_field *operand = hist_field->operands[0]; 245 246 s64 sval = (s64)operand->fn(operand, elt, rbe, event); 247 u64 val = (u64)-sval; 248 249 return val; 250 } 251 252 #define DEFINE_HIST_FIELD_FN(type) \ 253 static u64 hist_field_##type(struct hist_field *hist_field, \ 254 struct tracing_map_elt *elt, \ 255 struct ring_buffer_event *rbe, \ 256 void *event) \ 257 { \ 258 type *addr = (type *)(event + hist_field->field->offset); \ 259 \ 260 return (u64)(unsigned long)*addr; \ 261 } 262 263 DEFINE_HIST_FIELD_FN(s64); 264 DEFINE_HIST_FIELD_FN(u64); 265 DEFINE_HIST_FIELD_FN(s32); 266 DEFINE_HIST_FIELD_FN(u32); 267 DEFINE_HIST_FIELD_FN(s16); 268 DEFINE_HIST_FIELD_FN(u16); 269 DEFINE_HIST_FIELD_FN(s8); 270 DEFINE_HIST_FIELD_FN(u8); 271 272 #define for_each_hist_field(i, hist_data) \ 273 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++) 274 275 #define for_each_hist_val_field(i, hist_data) \ 276 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++) 277 278 #define for_each_hist_key_field(i, hist_data) \ 279 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++) 280 281 #define HIST_STACKTRACE_DEPTH 16 282 #define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long)) 283 #define HIST_STACKTRACE_SKIP 5 284 285 #define HITCOUNT_IDX 0 286 #define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE) 287 288 enum hist_field_flags { 289 HIST_FIELD_FL_HITCOUNT = 1 << 0, 290 HIST_FIELD_FL_KEY = 1 << 1, 291 HIST_FIELD_FL_STRING = 1 << 2, 292 HIST_FIELD_FL_HEX = 1 << 3, 293 HIST_FIELD_FL_SYM = 1 << 4, 294 HIST_FIELD_FL_SYM_OFFSET = 1 << 5, 295 HIST_FIELD_FL_EXECNAME = 1 << 6, 296 HIST_FIELD_FL_SYSCALL = 1 << 7, 297 HIST_FIELD_FL_STACKTRACE = 1 << 8, 298 HIST_FIELD_FL_LOG2 = 1 << 9, 299 HIST_FIELD_FL_TIMESTAMP = 1 << 10, 300 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11, 301 HIST_FIELD_FL_VAR = 1 << 12, 302 HIST_FIELD_FL_EXPR = 1 << 13, 303 HIST_FIELD_FL_VAR_REF = 1 << 14, 304 HIST_FIELD_FL_CPU = 1 << 15, 305 HIST_FIELD_FL_ALIAS = 1 << 16, 306 }; 307 308 struct var_defs { 309 unsigned int n_vars; 310 char *name[TRACING_MAP_VARS_MAX]; 311 char *expr[TRACING_MAP_VARS_MAX]; 312 }; 313 314 struct hist_trigger_attrs { 315 char *keys_str; 316 char *vals_str; 317 char *sort_key_str; 318 char *name; 319 char *clock; 320 bool pause; 321 bool cont; 322 bool clear; 323 bool ts_in_usecs; 324 unsigned int map_bits; 325 326 char *assignment_str[TRACING_MAP_VARS_MAX]; 327 unsigned int n_assignments; 328 329 char *action_str[HIST_ACTIONS_MAX]; 330 unsigned int n_actions; 331 332 struct var_defs var_defs; 333 }; 334 335 struct field_var { 336 struct hist_field *var; 337 struct hist_field *val; 338 }; 339 340 struct field_var_hist { 341 struct hist_trigger_data *hist_data; 342 char *cmd; 343 }; 344 345 struct hist_trigger_data { 346 struct hist_field *fields[HIST_FIELDS_MAX]; 347 unsigned int n_vals; 348 unsigned int n_keys; 349 unsigned int n_fields; 350 unsigned int n_vars; 351 unsigned int key_size; 352 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX]; 353 unsigned int n_sort_keys; 354 struct trace_event_file *event_file; 355 struct hist_trigger_attrs *attrs; 356 struct tracing_map *map; 357 bool enable_timestamps; 358 bool remove; 359 struct hist_field *var_refs[TRACING_MAP_VARS_MAX]; 360 unsigned int n_var_refs; 361 362 struct action_data *actions[HIST_ACTIONS_MAX]; 363 unsigned int n_actions; 364 365 struct field_var *field_vars[SYNTH_FIELDS_MAX]; 366 unsigned int n_field_vars; 367 unsigned int n_field_var_str; 368 struct field_var_hist *field_var_hists[SYNTH_FIELDS_MAX]; 369 unsigned int n_field_var_hists; 370 371 struct field_var *save_vars[SYNTH_FIELDS_MAX]; 372 unsigned int n_save_vars; 373 unsigned int n_save_var_str; 374 }; 375 376 static int synth_event_create(int argc, const char **argv); 377 static int synth_event_show(struct seq_file *m, struct dyn_event *ev); 378 static int synth_event_release(struct dyn_event *ev); 379 static bool synth_event_is_busy(struct dyn_event *ev); 380 static bool synth_event_match(const char *system, const char *event, 381 int argc, const char **argv, struct dyn_event *ev); 382 383 static struct dyn_event_operations synth_event_ops = { 384 .create = synth_event_create, 385 .show = synth_event_show, 386 .is_busy = synth_event_is_busy, 387 .free = synth_event_release, 388 .match = synth_event_match, 389 }; 390 391 struct synth_field { 392 char *type; 393 char *name; 394 size_t size; 395 bool is_signed; 396 bool is_string; 397 }; 398 399 struct synth_event { 400 struct dyn_event devent; 401 int ref; 402 char *name; 403 struct synth_field **fields; 404 unsigned int n_fields; 405 unsigned int n_u64; 406 struct trace_event_class class; 407 struct trace_event_call call; 408 struct tracepoint *tp; 409 }; 410 411 static bool is_synth_event(struct dyn_event *ev) 412 { 413 return ev->ops == &synth_event_ops; 414 } 415 416 static struct synth_event *to_synth_event(struct dyn_event *ev) 417 { 418 return container_of(ev, struct synth_event, devent); 419 } 420 421 static bool synth_event_is_busy(struct dyn_event *ev) 422 { 423 struct synth_event *event = to_synth_event(ev); 424 425 return event->ref != 0; 426 } 427 428 static bool synth_event_match(const char *system, const char *event, 429 int argc, const char **argv, struct dyn_event *ev) 430 { 431 struct synth_event *sev = to_synth_event(ev); 432 433 return strcmp(sev->name, event) == 0 && 434 (!system || strcmp(system, SYNTH_SYSTEM) == 0); 435 } 436 437 struct action_data; 438 439 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data, 440 struct tracing_map_elt *elt, void *rec, 441 struct ring_buffer_event *rbe, void *key, 442 struct action_data *data, u64 *var_ref_vals); 443 444 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val); 445 446 enum handler_id { 447 HANDLER_ONMATCH = 1, 448 HANDLER_ONMAX, 449 HANDLER_ONCHANGE, 450 }; 451 452 enum action_id { 453 ACTION_SAVE = 1, 454 ACTION_TRACE, 455 ACTION_SNAPSHOT, 456 }; 457 458 struct action_data { 459 enum handler_id handler; 460 enum action_id action; 461 char *action_name; 462 action_fn_t fn; 463 464 unsigned int n_params; 465 char *params[SYNTH_FIELDS_MAX]; 466 467 /* 468 * When a histogram trigger is hit, the values of any 469 * references to variables, including variables being passed 470 * as parameters to synthetic events, are collected into a 471 * var_ref_vals array. This var_ref_idx is the index of the 472 * first param in the array to be passed to the synthetic 473 * event invocation. 474 */ 475 unsigned int var_ref_idx; 476 struct synth_event *synth_event; 477 bool use_trace_keyword; 478 char *synth_event_name; 479 480 union { 481 struct { 482 char *event; 483 char *event_system; 484 } match_data; 485 486 struct { 487 /* 488 * var_str contains the $-unstripped variable 489 * name referenced by var_ref, and used when 490 * printing the action. Because var_ref 491 * creation is deferred to create_actions(), 492 * we need a per-action way to save it until 493 * then, thus var_str. 494 */ 495 char *var_str; 496 497 /* 498 * var_ref refers to the variable being 499 * tracked e.g onmax($var). 500 */ 501 struct hist_field *var_ref; 502 503 /* 504 * track_var contains the 'invisible' tracking 505 * variable created to keep the current 506 * e.g. max value. 507 */ 508 struct hist_field *track_var; 509 510 check_track_val_fn_t check_val; 511 action_fn_t save_data; 512 } track_data; 513 }; 514 }; 515 516 struct track_data { 517 u64 track_val; 518 bool updated; 519 520 unsigned int key_len; 521 void *key; 522 struct tracing_map_elt elt; 523 524 struct action_data *action_data; 525 struct hist_trigger_data *hist_data; 526 }; 527 528 struct hist_elt_data { 529 char *comm; 530 u64 *var_ref_vals; 531 char *field_var_str[SYNTH_FIELDS_MAX]; 532 }; 533 534 struct snapshot_context { 535 struct tracing_map_elt *elt; 536 void *key; 537 }; 538 539 static void track_data_free(struct track_data *track_data) 540 { 541 struct hist_elt_data *elt_data; 542 543 if (!track_data) 544 return; 545 546 kfree(track_data->key); 547 548 elt_data = track_data->elt.private_data; 549 if (elt_data) { 550 kfree(elt_data->comm); 551 kfree(elt_data); 552 } 553 554 kfree(track_data); 555 } 556 557 static struct track_data *track_data_alloc(unsigned int key_len, 558 struct action_data *action_data, 559 struct hist_trigger_data *hist_data) 560 { 561 struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL); 562 struct hist_elt_data *elt_data; 563 564 if (!data) 565 return ERR_PTR(-ENOMEM); 566 567 data->key = kzalloc(key_len, GFP_KERNEL); 568 if (!data->key) { 569 track_data_free(data); 570 return ERR_PTR(-ENOMEM); 571 } 572 573 data->key_len = key_len; 574 data->action_data = action_data; 575 data->hist_data = hist_data; 576 577 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL); 578 if (!elt_data) { 579 track_data_free(data); 580 return ERR_PTR(-ENOMEM); 581 } 582 data->elt.private_data = elt_data; 583 584 elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL); 585 if (!elt_data->comm) { 586 track_data_free(data); 587 return ERR_PTR(-ENOMEM); 588 } 589 590 return data; 591 } 592 593 static char last_cmd[MAX_FILTER_STR_VAL]; 594 static char last_cmd_loc[MAX_FILTER_STR_VAL]; 595 596 static int errpos(char *str) 597 { 598 return err_pos(last_cmd, str); 599 } 600 601 static void last_cmd_set(struct trace_event_file *file, char *str) 602 { 603 const char *system = NULL, *name = NULL; 604 struct trace_event_call *call; 605 606 if (!str) 607 return; 608 609 strncpy(last_cmd, str, MAX_FILTER_STR_VAL - 1); 610 611 if (file) { 612 call = file->event_call; 613 614 system = call->class->system; 615 if (system) { 616 name = trace_event_name(call); 617 if (!name) 618 system = NULL; 619 } 620 } 621 622 if (system) 623 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name); 624 } 625 626 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos) 627 { 628 tracing_log_err(tr, last_cmd_loc, last_cmd, err_text, 629 err_type, err_pos); 630 } 631 632 static void hist_err_clear(void) 633 { 634 last_cmd[0] = '\0'; 635 last_cmd_loc[0] = '\0'; 636 } 637 638 struct synth_trace_event { 639 struct trace_entry ent; 640 u64 fields[]; 641 }; 642 643 static int synth_event_define_fields(struct trace_event_call *call) 644 { 645 struct synth_trace_event trace; 646 int offset = offsetof(typeof(trace), fields); 647 struct synth_event *event = call->data; 648 unsigned int i, size, n_u64; 649 char *name, *type; 650 bool is_signed; 651 int ret = 0; 652 653 for (i = 0, n_u64 = 0; i < event->n_fields; i++) { 654 size = event->fields[i]->size; 655 is_signed = event->fields[i]->is_signed; 656 type = event->fields[i]->type; 657 name = event->fields[i]->name; 658 ret = trace_define_field(call, type, name, offset, size, 659 is_signed, FILTER_OTHER); 660 if (ret) 661 break; 662 663 if (event->fields[i]->is_string) { 664 offset += STR_VAR_LEN_MAX; 665 n_u64 += STR_VAR_LEN_MAX / sizeof(u64); 666 } else { 667 offset += sizeof(u64); 668 n_u64++; 669 } 670 } 671 672 event->n_u64 = n_u64; 673 674 return ret; 675 } 676 677 static bool synth_field_signed(char *type) 678 { 679 if (str_has_prefix(type, "u")) 680 return false; 681 682 return true; 683 } 684 685 static int synth_field_is_string(char *type) 686 { 687 if (strstr(type, "char[") != NULL) 688 return true; 689 690 return false; 691 } 692 693 static int synth_field_string_size(char *type) 694 { 695 char buf[4], *end, *start; 696 unsigned int len; 697 int size, err; 698 699 start = strstr(type, "char["); 700 if (start == NULL) 701 return -EINVAL; 702 start += sizeof("char[") - 1; 703 704 end = strchr(type, ']'); 705 if (!end || end < start) 706 return -EINVAL; 707 708 len = end - start; 709 if (len > 3) 710 return -EINVAL; 711 712 strncpy(buf, start, len); 713 buf[len] = '\0'; 714 715 err = kstrtouint(buf, 0, &size); 716 if (err) 717 return err; 718 719 if (size > STR_VAR_LEN_MAX) 720 return -EINVAL; 721 722 return size; 723 } 724 725 static int synth_field_size(char *type) 726 { 727 int size = 0; 728 729 if (strcmp(type, "s64") == 0) 730 size = sizeof(s64); 731 else if (strcmp(type, "u64") == 0) 732 size = sizeof(u64); 733 else if (strcmp(type, "s32") == 0) 734 size = sizeof(s32); 735 else if (strcmp(type, "u32") == 0) 736 size = sizeof(u32); 737 else if (strcmp(type, "s16") == 0) 738 size = sizeof(s16); 739 else if (strcmp(type, "u16") == 0) 740 size = sizeof(u16); 741 else if (strcmp(type, "s8") == 0) 742 size = sizeof(s8); 743 else if (strcmp(type, "u8") == 0) 744 size = sizeof(u8); 745 else if (strcmp(type, "char") == 0) 746 size = sizeof(char); 747 else if (strcmp(type, "unsigned char") == 0) 748 size = sizeof(unsigned char); 749 else if (strcmp(type, "int") == 0) 750 size = sizeof(int); 751 else if (strcmp(type, "unsigned int") == 0) 752 size = sizeof(unsigned int); 753 else if (strcmp(type, "long") == 0) 754 size = sizeof(long); 755 else if (strcmp(type, "unsigned long") == 0) 756 size = sizeof(unsigned long); 757 else if (strcmp(type, "pid_t") == 0) 758 size = sizeof(pid_t); 759 else if (strcmp(type, "gfp_t") == 0) 760 size = sizeof(gfp_t); 761 else if (synth_field_is_string(type)) 762 size = synth_field_string_size(type); 763 764 return size; 765 } 766 767 static const char *synth_field_fmt(char *type) 768 { 769 const char *fmt = "%llu"; 770 771 if (strcmp(type, "s64") == 0) 772 fmt = "%lld"; 773 else if (strcmp(type, "u64") == 0) 774 fmt = "%llu"; 775 else if (strcmp(type, "s32") == 0) 776 fmt = "%d"; 777 else if (strcmp(type, "u32") == 0) 778 fmt = "%u"; 779 else if (strcmp(type, "s16") == 0) 780 fmt = "%d"; 781 else if (strcmp(type, "u16") == 0) 782 fmt = "%u"; 783 else if (strcmp(type, "s8") == 0) 784 fmt = "%d"; 785 else if (strcmp(type, "u8") == 0) 786 fmt = "%u"; 787 else if (strcmp(type, "char") == 0) 788 fmt = "%d"; 789 else if (strcmp(type, "unsigned char") == 0) 790 fmt = "%u"; 791 else if (strcmp(type, "int") == 0) 792 fmt = "%d"; 793 else if (strcmp(type, "unsigned int") == 0) 794 fmt = "%u"; 795 else if (strcmp(type, "long") == 0) 796 fmt = "%ld"; 797 else if (strcmp(type, "unsigned long") == 0) 798 fmt = "%lu"; 799 else if (strcmp(type, "pid_t") == 0) 800 fmt = "%d"; 801 else if (strcmp(type, "gfp_t") == 0) 802 fmt = "%x"; 803 else if (synth_field_is_string(type)) 804 fmt = "%s"; 805 806 return fmt; 807 } 808 809 static enum print_line_t print_synth_event(struct trace_iterator *iter, 810 int flags, 811 struct trace_event *event) 812 { 813 struct trace_array *tr = iter->tr; 814 struct trace_seq *s = &iter->seq; 815 struct synth_trace_event *entry; 816 struct synth_event *se; 817 unsigned int i, n_u64; 818 char print_fmt[32]; 819 const char *fmt; 820 821 entry = (struct synth_trace_event *)iter->ent; 822 se = container_of(event, struct synth_event, call.event); 823 824 trace_seq_printf(s, "%s: ", se->name); 825 826 for (i = 0, n_u64 = 0; i < se->n_fields; i++) { 827 if (trace_seq_has_overflowed(s)) 828 goto end; 829 830 fmt = synth_field_fmt(se->fields[i]->type); 831 832 /* parameter types */ 833 if (tr->trace_flags & TRACE_ITER_VERBOSE) 834 trace_seq_printf(s, "%s ", fmt); 835 836 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt); 837 838 /* parameter values */ 839 if (se->fields[i]->is_string) { 840 trace_seq_printf(s, print_fmt, se->fields[i]->name, 841 (char *)&entry->fields[n_u64], 842 i == se->n_fields - 1 ? "" : " "); 843 n_u64 += STR_VAR_LEN_MAX / sizeof(u64); 844 } else { 845 struct trace_print_flags __flags[] = { 846 __def_gfpflag_names, {-1, NULL} }; 847 848 trace_seq_printf(s, print_fmt, se->fields[i]->name, 849 entry->fields[n_u64], 850 i == se->n_fields - 1 ? "" : " "); 851 852 if (strcmp(se->fields[i]->type, "gfp_t") == 0) { 853 trace_seq_puts(s, " ("); 854 trace_print_flags_seq(s, "|", 855 entry->fields[n_u64], 856 __flags); 857 trace_seq_putc(s, ')'); 858 } 859 n_u64++; 860 } 861 } 862 end: 863 trace_seq_putc(s, '\n'); 864 865 return trace_handle_return(s); 866 } 867 868 static struct trace_event_functions synth_event_funcs = { 869 .trace = print_synth_event 870 }; 871 872 static notrace void trace_event_raw_event_synth(void *__data, 873 u64 *var_ref_vals, 874 unsigned int var_ref_idx) 875 { 876 struct trace_event_file *trace_file = __data; 877 struct synth_trace_event *entry; 878 struct trace_event_buffer fbuffer; 879 struct ring_buffer *buffer; 880 struct synth_event *event; 881 unsigned int i, n_u64; 882 int fields_size = 0; 883 884 event = trace_file->event_call->data; 885 886 if (trace_trigger_soft_disabled(trace_file)) 887 return; 888 889 fields_size = event->n_u64 * sizeof(u64); 890 891 /* 892 * Avoid ring buffer recursion detection, as this event 893 * is being performed within another event. 894 */ 895 buffer = trace_file->tr->trace_buffer.buffer; 896 ring_buffer_nest_start(buffer); 897 898 entry = trace_event_buffer_reserve(&fbuffer, trace_file, 899 sizeof(*entry) + fields_size); 900 if (!entry) 901 goto out; 902 903 for (i = 0, n_u64 = 0; i < event->n_fields; i++) { 904 if (event->fields[i]->is_string) { 905 char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i]; 906 char *str_field = (char *)&entry->fields[n_u64]; 907 908 strscpy(str_field, str_val, STR_VAR_LEN_MAX); 909 n_u64 += STR_VAR_LEN_MAX / sizeof(u64); 910 } else { 911 entry->fields[n_u64] = var_ref_vals[var_ref_idx + i]; 912 n_u64++; 913 } 914 } 915 916 trace_event_buffer_commit(&fbuffer); 917 out: 918 ring_buffer_nest_end(buffer); 919 } 920 921 static void free_synth_event_print_fmt(struct trace_event_call *call) 922 { 923 if (call) { 924 kfree(call->print_fmt); 925 call->print_fmt = NULL; 926 } 927 } 928 929 static int __set_synth_event_print_fmt(struct synth_event *event, 930 char *buf, int len) 931 { 932 const char *fmt; 933 int pos = 0; 934 int i; 935 936 /* When len=0, we just calculate the needed length */ 937 #define LEN_OR_ZERO (len ? len - pos : 0) 938 939 pos += snprintf(buf + pos, LEN_OR_ZERO, "\""); 940 for (i = 0; i < event->n_fields; i++) { 941 fmt = synth_field_fmt(event->fields[i]->type); 942 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s", 943 event->fields[i]->name, fmt, 944 i == event->n_fields - 1 ? "" : ", "); 945 } 946 pos += snprintf(buf + pos, LEN_OR_ZERO, "\""); 947 948 for (i = 0; i < event->n_fields; i++) { 949 pos += snprintf(buf + pos, LEN_OR_ZERO, 950 ", REC->%s", event->fields[i]->name); 951 } 952 953 #undef LEN_OR_ZERO 954 955 /* return the length of print_fmt */ 956 return pos; 957 } 958 959 static int set_synth_event_print_fmt(struct trace_event_call *call) 960 { 961 struct synth_event *event = call->data; 962 char *print_fmt; 963 int len; 964 965 /* First: called with 0 length to calculate the needed length */ 966 len = __set_synth_event_print_fmt(event, NULL, 0); 967 968 print_fmt = kmalloc(len + 1, GFP_KERNEL); 969 if (!print_fmt) 970 return -ENOMEM; 971 972 /* Second: actually write the @print_fmt */ 973 __set_synth_event_print_fmt(event, print_fmt, len + 1); 974 call->print_fmt = print_fmt; 975 976 return 0; 977 } 978 979 static void free_synth_field(struct synth_field *field) 980 { 981 kfree(field->type); 982 kfree(field->name); 983 kfree(field); 984 } 985 986 static struct synth_field *parse_synth_field(int argc, const char **argv, 987 int *consumed) 988 { 989 struct synth_field *field; 990 const char *prefix = NULL, *field_type = argv[0], *field_name, *array; 991 int len, ret = 0; 992 993 if (field_type[0] == ';') 994 field_type++; 995 996 if (!strcmp(field_type, "unsigned")) { 997 if (argc < 3) 998 return ERR_PTR(-EINVAL); 999 prefix = "unsigned "; 1000 field_type = argv[1]; 1001 field_name = argv[2]; 1002 *consumed = 3; 1003 } else { 1004 field_name = argv[1]; 1005 *consumed = 2; 1006 } 1007 1008 field = kzalloc(sizeof(*field), GFP_KERNEL); 1009 if (!field) 1010 return ERR_PTR(-ENOMEM); 1011 1012 len = strlen(field_name); 1013 array = strchr(field_name, '['); 1014 if (array) 1015 len -= strlen(array); 1016 else if (field_name[len - 1] == ';') 1017 len--; 1018 1019 field->name = kmemdup_nul(field_name, len, GFP_KERNEL); 1020 if (!field->name) { 1021 ret = -ENOMEM; 1022 goto free; 1023 } 1024 1025 if (field_type[0] == ';') 1026 field_type++; 1027 len = strlen(field_type) + 1; 1028 if (array) 1029 len += strlen(array); 1030 if (prefix) 1031 len += strlen(prefix); 1032 1033 field->type = kzalloc(len, GFP_KERNEL); 1034 if (!field->type) { 1035 ret = -ENOMEM; 1036 goto free; 1037 } 1038 if (prefix) 1039 strcat(field->type, prefix); 1040 strcat(field->type, field_type); 1041 if (array) { 1042 strcat(field->type, array); 1043 if (field->type[len - 1] == ';') 1044 field->type[len - 1] = '\0'; 1045 } 1046 1047 field->size = synth_field_size(field->type); 1048 if (!field->size) { 1049 ret = -EINVAL; 1050 goto free; 1051 } 1052 1053 if (synth_field_is_string(field->type)) 1054 field->is_string = true; 1055 1056 field->is_signed = synth_field_signed(field->type); 1057 1058 out: 1059 return field; 1060 free: 1061 free_synth_field(field); 1062 field = ERR_PTR(ret); 1063 goto out; 1064 } 1065 1066 static void free_synth_tracepoint(struct tracepoint *tp) 1067 { 1068 if (!tp) 1069 return; 1070 1071 kfree(tp->name); 1072 kfree(tp); 1073 } 1074 1075 static struct tracepoint *alloc_synth_tracepoint(char *name) 1076 { 1077 struct tracepoint *tp; 1078 1079 tp = kzalloc(sizeof(*tp), GFP_KERNEL); 1080 if (!tp) 1081 return ERR_PTR(-ENOMEM); 1082 1083 tp->name = kstrdup(name, GFP_KERNEL); 1084 if (!tp->name) { 1085 kfree(tp); 1086 return ERR_PTR(-ENOMEM); 1087 } 1088 1089 return tp; 1090 } 1091 1092 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals, 1093 unsigned int var_ref_idx); 1094 1095 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals, 1096 unsigned int var_ref_idx) 1097 { 1098 struct tracepoint *tp = event->tp; 1099 1100 if (unlikely(atomic_read(&tp->key.enabled) > 0)) { 1101 struct tracepoint_func *probe_func_ptr; 1102 synth_probe_func_t probe_func; 1103 void *__data; 1104 1105 if (!(cpu_online(raw_smp_processor_id()))) 1106 return; 1107 1108 probe_func_ptr = rcu_dereference_sched((tp)->funcs); 1109 if (probe_func_ptr) { 1110 do { 1111 probe_func = probe_func_ptr->func; 1112 __data = probe_func_ptr->data; 1113 probe_func(__data, var_ref_vals, var_ref_idx); 1114 } while ((++probe_func_ptr)->func); 1115 } 1116 } 1117 } 1118 1119 static struct synth_event *find_synth_event(const char *name) 1120 { 1121 struct dyn_event *pos; 1122 struct synth_event *event; 1123 1124 for_each_dyn_event(pos) { 1125 if (!is_synth_event(pos)) 1126 continue; 1127 event = to_synth_event(pos); 1128 if (strcmp(event->name, name) == 0) 1129 return event; 1130 } 1131 1132 return NULL; 1133 } 1134 1135 static int register_synth_event(struct synth_event *event) 1136 { 1137 struct trace_event_call *call = &event->call; 1138 int ret = 0; 1139 1140 event->call.class = &event->class; 1141 event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL); 1142 if (!event->class.system) { 1143 ret = -ENOMEM; 1144 goto out; 1145 } 1146 1147 event->tp = alloc_synth_tracepoint(event->name); 1148 if (IS_ERR(event->tp)) { 1149 ret = PTR_ERR(event->tp); 1150 event->tp = NULL; 1151 goto out; 1152 } 1153 1154 INIT_LIST_HEAD(&call->class->fields); 1155 call->event.funcs = &synth_event_funcs; 1156 call->class->define_fields = synth_event_define_fields; 1157 1158 ret = register_trace_event(&call->event); 1159 if (!ret) { 1160 ret = -ENODEV; 1161 goto out; 1162 } 1163 call->flags = TRACE_EVENT_FL_TRACEPOINT; 1164 call->class->reg = trace_event_reg; 1165 call->class->probe = trace_event_raw_event_synth; 1166 call->data = event; 1167 call->tp = event->tp; 1168 1169 ret = trace_add_event_call(call); 1170 if (ret) { 1171 pr_warn("Failed to register synthetic event: %s\n", 1172 trace_event_name(call)); 1173 goto err; 1174 } 1175 1176 ret = set_synth_event_print_fmt(call); 1177 if (ret < 0) { 1178 trace_remove_event_call(call); 1179 goto err; 1180 } 1181 out: 1182 return ret; 1183 err: 1184 unregister_trace_event(&call->event); 1185 goto out; 1186 } 1187 1188 static int unregister_synth_event(struct synth_event *event) 1189 { 1190 struct trace_event_call *call = &event->call; 1191 int ret; 1192 1193 ret = trace_remove_event_call(call); 1194 1195 return ret; 1196 } 1197 1198 static void free_synth_event(struct synth_event *event) 1199 { 1200 unsigned int i; 1201 1202 if (!event) 1203 return; 1204 1205 for (i = 0; i < event->n_fields; i++) 1206 free_synth_field(event->fields[i]); 1207 1208 kfree(event->fields); 1209 kfree(event->name); 1210 kfree(event->class.system); 1211 free_synth_tracepoint(event->tp); 1212 free_synth_event_print_fmt(&event->call); 1213 kfree(event); 1214 } 1215 1216 static struct synth_event *alloc_synth_event(const char *name, int n_fields, 1217 struct synth_field **fields) 1218 { 1219 struct synth_event *event; 1220 unsigned int i; 1221 1222 event = kzalloc(sizeof(*event), GFP_KERNEL); 1223 if (!event) { 1224 event = ERR_PTR(-ENOMEM); 1225 goto out; 1226 } 1227 1228 event->name = kstrdup(name, GFP_KERNEL); 1229 if (!event->name) { 1230 kfree(event); 1231 event = ERR_PTR(-ENOMEM); 1232 goto out; 1233 } 1234 1235 event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL); 1236 if (!event->fields) { 1237 free_synth_event(event); 1238 event = ERR_PTR(-ENOMEM); 1239 goto out; 1240 } 1241 1242 dyn_event_init(&event->devent, &synth_event_ops); 1243 1244 for (i = 0; i < n_fields; i++) 1245 event->fields[i] = fields[i]; 1246 1247 event->n_fields = n_fields; 1248 out: 1249 return event; 1250 } 1251 1252 static void action_trace(struct hist_trigger_data *hist_data, 1253 struct tracing_map_elt *elt, void *rec, 1254 struct ring_buffer_event *rbe, void *key, 1255 struct action_data *data, u64 *var_ref_vals) 1256 { 1257 struct synth_event *event = data->synth_event; 1258 1259 trace_synth(event, var_ref_vals, data->var_ref_idx); 1260 } 1261 1262 struct hist_var_data { 1263 struct list_head list; 1264 struct hist_trigger_data *hist_data; 1265 }; 1266 1267 static int __create_synth_event(int argc, const char *name, const char **argv) 1268 { 1269 struct synth_field *field, *fields[SYNTH_FIELDS_MAX]; 1270 struct synth_event *event = NULL; 1271 int i, consumed = 0, n_fields = 0, ret = 0; 1272 1273 /* 1274 * Argument syntax: 1275 * - Add synthetic event: <event_name> field[;field] ... 1276 * - Remove synthetic event: !<event_name> field[;field] ... 1277 * where 'field' = type field_name 1278 */ 1279 1280 if (name[0] == '\0' || argc < 1) 1281 return -EINVAL; 1282 1283 mutex_lock(&event_mutex); 1284 1285 event = find_synth_event(name); 1286 if (event) { 1287 ret = -EEXIST; 1288 goto out; 1289 } 1290 1291 for (i = 0; i < argc - 1; i++) { 1292 if (strcmp(argv[i], ";") == 0) 1293 continue; 1294 if (n_fields == SYNTH_FIELDS_MAX) { 1295 ret = -EINVAL; 1296 goto err; 1297 } 1298 1299 field = parse_synth_field(argc - i, &argv[i], &consumed); 1300 if (IS_ERR(field)) { 1301 ret = PTR_ERR(field); 1302 goto err; 1303 } 1304 fields[n_fields++] = field; 1305 i += consumed - 1; 1306 } 1307 1308 if (i < argc && strcmp(argv[i], ";") != 0) { 1309 ret = -EINVAL; 1310 goto err; 1311 } 1312 1313 event = alloc_synth_event(name, n_fields, fields); 1314 if (IS_ERR(event)) { 1315 ret = PTR_ERR(event); 1316 event = NULL; 1317 goto err; 1318 } 1319 ret = register_synth_event(event); 1320 if (!ret) 1321 dyn_event_add(&event->devent); 1322 else 1323 free_synth_event(event); 1324 out: 1325 mutex_unlock(&event_mutex); 1326 1327 return ret; 1328 err: 1329 for (i = 0; i < n_fields; i++) 1330 free_synth_field(fields[i]); 1331 1332 goto out; 1333 } 1334 1335 static int create_or_delete_synth_event(int argc, char **argv) 1336 { 1337 const char *name = argv[0]; 1338 struct synth_event *event = NULL; 1339 int ret; 1340 1341 /* trace_run_command() ensures argc != 0 */ 1342 if (name[0] == '!') { 1343 mutex_lock(&event_mutex); 1344 event = find_synth_event(name + 1); 1345 if (event) { 1346 if (event->ref) 1347 ret = -EBUSY; 1348 else { 1349 ret = unregister_synth_event(event); 1350 if (!ret) { 1351 dyn_event_remove(&event->devent); 1352 free_synth_event(event); 1353 } 1354 } 1355 } else 1356 ret = -ENOENT; 1357 mutex_unlock(&event_mutex); 1358 return ret; 1359 } 1360 1361 ret = __create_synth_event(argc - 1, name, (const char **)argv + 1); 1362 return ret == -ECANCELED ? -EINVAL : ret; 1363 } 1364 1365 static int synth_event_create(int argc, const char **argv) 1366 { 1367 const char *name = argv[0]; 1368 int len; 1369 1370 if (name[0] != 's' || name[1] != ':') 1371 return -ECANCELED; 1372 name += 2; 1373 1374 /* This interface accepts group name prefix */ 1375 if (strchr(name, '/')) { 1376 len = str_has_prefix(name, SYNTH_SYSTEM "/"); 1377 if (len == 0) 1378 return -EINVAL; 1379 name += len; 1380 } 1381 return __create_synth_event(argc - 1, name, argv + 1); 1382 } 1383 1384 static int synth_event_release(struct dyn_event *ev) 1385 { 1386 struct synth_event *event = to_synth_event(ev); 1387 int ret; 1388 1389 if (event->ref) 1390 return -EBUSY; 1391 1392 ret = unregister_synth_event(event); 1393 if (ret) 1394 return ret; 1395 1396 dyn_event_remove(ev); 1397 free_synth_event(event); 1398 return 0; 1399 } 1400 1401 static int __synth_event_show(struct seq_file *m, struct synth_event *event) 1402 { 1403 struct synth_field *field; 1404 unsigned int i; 1405 1406 seq_printf(m, "%s\t", event->name); 1407 1408 for (i = 0; i < event->n_fields; i++) { 1409 field = event->fields[i]; 1410 1411 /* parameter values */ 1412 seq_printf(m, "%s %s%s", field->type, field->name, 1413 i == event->n_fields - 1 ? "" : "; "); 1414 } 1415 1416 seq_putc(m, '\n'); 1417 1418 return 0; 1419 } 1420 1421 static int synth_event_show(struct seq_file *m, struct dyn_event *ev) 1422 { 1423 struct synth_event *event = to_synth_event(ev); 1424 1425 seq_printf(m, "s:%s/", event->class.system); 1426 1427 return __synth_event_show(m, event); 1428 } 1429 1430 static int synth_events_seq_show(struct seq_file *m, void *v) 1431 { 1432 struct dyn_event *ev = v; 1433 1434 if (!is_synth_event(ev)) 1435 return 0; 1436 1437 return __synth_event_show(m, to_synth_event(ev)); 1438 } 1439 1440 static const struct seq_operations synth_events_seq_op = { 1441 .start = dyn_event_seq_start, 1442 .next = dyn_event_seq_next, 1443 .stop = dyn_event_seq_stop, 1444 .show = synth_events_seq_show, 1445 }; 1446 1447 static int synth_events_open(struct inode *inode, struct file *file) 1448 { 1449 int ret; 1450 1451 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 1452 ret = dyn_events_release_all(&synth_event_ops); 1453 if (ret < 0) 1454 return ret; 1455 } 1456 1457 return seq_open(file, &synth_events_seq_op); 1458 } 1459 1460 static ssize_t synth_events_write(struct file *file, 1461 const char __user *buffer, 1462 size_t count, loff_t *ppos) 1463 { 1464 return trace_parse_run_command(file, buffer, count, ppos, 1465 create_or_delete_synth_event); 1466 } 1467 1468 static const struct file_operations synth_events_fops = { 1469 .open = synth_events_open, 1470 .write = synth_events_write, 1471 .read = seq_read, 1472 .llseek = seq_lseek, 1473 .release = seq_release, 1474 }; 1475 1476 static u64 hist_field_timestamp(struct hist_field *hist_field, 1477 struct tracing_map_elt *elt, 1478 struct ring_buffer_event *rbe, 1479 void *event) 1480 { 1481 struct hist_trigger_data *hist_data = hist_field->hist_data; 1482 struct trace_array *tr = hist_data->event_file->tr; 1483 1484 u64 ts = ring_buffer_event_time_stamp(rbe); 1485 1486 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr)) 1487 ts = ns2usecs(ts); 1488 1489 return ts; 1490 } 1491 1492 static u64 hist_field_cpu(struct hist_field *hist_field, 1493 struct tracing_map_elt *elt, 1494 struct ring_buffer_event *rbe, 1495 void *event) 1496 { 1497 int cpu = smp_processor_id(); 1498 1499 return cpu; 1500 } 1501 1502 /** 1503 * check_field_for_var_ref - Check if a VAR_REF field references a variable 1504 * @hist_field: The VAR_REF field to check 1505 * @var_data: The hist trigger that owns the variable 1506 * @var_idx: The trigger variable identifier 1507 * 1508 * Check the given VAR_REF field to see whether or not it references 1509 * the given variable associated with the given trigger. 1510 * 1511 * Return: The VAR_REF field if it does reference the variable, NULL if not 1512 */ 1513 static struct hist_field * 1514 check_field_for_var_ref(struct hist_field *hist_field, 1515 struct hist_trigger_data *var_data, 1516 unsigned int var_idx) 1517 { 1518 WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF)); 1519 1520 if (hist_field && hist_field->var.idx == var_idx && 1521 hist_field->var.hist_data == var_data) 1522 return hist_field; 1523 1524 return NULL; 1525 } 1526 1527 /** 1528 * find_var_ref - Check if a trigger has a reference to a trigger variable 1529 * @hist_data: The hist trigger that might have a reference to the variable 1530 * @var_data: The hist trigger that owns the variable 1531 * @var_idx: The trigger variable identifier 1532 * 1533 * Check the list of var_refs[] on the first hist trigger to see 1534 * whether any of them are references to the variable on the second 1535 * trigger. 1536 * 1537 * Return: The VAR_REF field referencing the variable if so, NULL if not 1538 */ 1539 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data, 1540 struct hist_trigger_data *var_data, 1541 unsigned int var_idx) 1542 { 1543 struct hist_field *hist_field; 1544 unsigned int i; 1545 1546 for (i = 0; i < hist_data->n_var_refs; i++) { 1547 hist_field = hist_data->var_refs[i]; 1548 if (check_field_for_var_ref(hist_field, var_data, var_idx)) 1549 return hist_field; 1550 } 1551 1552 return NULL; 1553 } 1554 1555 /** 1556 * find_any_var_ref - Check if there is a reference to a given trigger variable 1557 * @hist_data: The hist trigger 1558 * @var_idx: The trigger variable identifier 1559 * 1560 * Check to see whether the given variable is currently referenced by 1561 * any other trigger. 1562 * 1563 * The trigger the variable is defined on is explicitly excluded - the 1564 * assumption being that a self-reference doesn't prevent a trigger 1565 * from being removed. 1566 * 1567 * Return: The VAR_REF field referencing the variable if so, NULL if not 1568 */ 1569 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data, 1570 unsigned int var_idx) 1571 { 1572 struct trace_array *tr = hist_data->event_file->tr; 1573 struct hist_field *found = NULL; 1574 struct hist_var_data *var_data; 1575 1576 list_for_each_entry(var_data, &tr->hist_vars, list) { 1577 if (var_data->hist_data == hist_data) 1578 continue; 1579 found = find_var_ref(var_data->hist_data, hist_data, var_idx); 1580 if (found) 1581 break; 1582 } 1583 1584 return found; 1585 } 1586 1587 /** 1588 * check_var_refs - Check if there is a reference to any of trigger's variables 1589 * @hist_data: The hist trigger 1590 * 1591 * A trigger can define one or more variables. If any one of them is 1592 * currently referenced by any other trigger, this function will 1593 * determine that. 1594 1595 * Typically used to determine whether or not a trigger can be removed 1596 * - if there are any references to a trigger's variables, it cannot. 1597 * 1598 * Return: True if there is a reference to any of trigger's variables 1599 */ 1600 static bool check_var_refs(struct hist_trigger_data *hist_data) 1601 { 1602 struct hist_field *field; 1603 bool found = false; 1604 int i; 1605 1606 for_each_hist_field(i, hist_data) { 1607 field = hist_data->fields[i]; 1608 if (field && field->flags & HIST_FIELD_FL_VAR) { 1609 if (find_any_var_ref(hist_data, field->var.idx)) { 1610 found = true; 1611 break; 1612 } 1613 } 1614 } 1615 1616 return found; 1617 } 1618 1619 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data) 1620 { 1621 struct trace_array *tr = hist_data->event_file->tr; 1622 struct hist_var_data *var_data, *found = NULL; 1623 1624 list_for_each_entry(var_data, &tr->hist_vars, list) { 1625 if (var_data->hist_data == hist_data) { 1626 found = var_data; 1627 break; 1628 } 1629 } 1630 1631 return found; 1632 } 1633 1634 static bool field_has_hist_vars(struct hist_field *hist_field, 1635 unsigned int level) 1636 { 1637 int i; 1638 1639 if (level > 3) 1640 return false; 1641 1642 if (!hist_field) 1643 return false; 1644 1645 if (hist_field->flags & HIST_FIELD_FL_VAR || 1646 hist_field->flags & HIST_FIELD_FL_VAR_REF) 1647 return true; 1648 1649 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) { 1650 struct hist_field *operand; 1651 1652 operand = hist_field->operands[i]; 1653 if (field_has_hist_vars(operand, level + 1)) 1654 return true; 1655 } 1656 1657 return false; 1658 } 1659 1660 static bool has_hist_vars(struct hist_trigger_data *hist_data) 1661 { 1662 struct hist_field *hist_field; 1663 int i; 1664 1665 for_each_hist_field(i, hist_data) { 1666 hist_field = hist_data->fields[i]; 1667 if (field_has_hist_vars(hist_field, 0)) 1668 return true; 1669 } 1670 1671 return false; 1672 } 1673 1674 static int save_hist_vars(struct hist_trigger_data *hist_data) 1675 { 1676 struct trace_array *tr = hist_data->event_file->tr; 1677 struct hist_var_data *var_data; 1678 1679 var_data = find_hist_vars(hist_data); 1680 if (var_data) 1681 return 0; 1682 1683 if (trace_array_get(tr) < 0) 1684 return -ENODEV; 1685 1686 var_data = kzalloc(sizeof(*var_data), GFP_KERNEL); 1687 if (!var_data) { 1688 trace_array_put(tr); 1689 return -ENOMEM; 1690 } 1691 1692 var_data->hist_data = hist_data; 1693 list_add(&var_data->list, &tr->hist_vars); 1694 1695 return 0; 1696 } 1697 1698 static void remove_hist_vars(struct hist_trigger_data *hist_data) 1699 { 1700 struct trace_array *tr = hist_data->event_file->tr; 1701 struct hist_var_data *var_data; 1702 1703 var_data = find_hist_vars(hist_data); 1704 if (!var_data) 1705 return; 1706 1707 if (WARN_ON(check_var_refs(hist_data))) 1708 return; 1709 1710 list_del(&var_data->list); 1711 1712 kfree(var_data); 1713 1714 trace_array_put(tr); 1715 } 1716 1717 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data, 1718 const char *var_name) 1719 { 1720 struct hist_field *hist_field, *found = NULL; 1721 int i; 1722 1723 for_each_hist_field(i, hist_data) { 1724 hist_field = hist_data->fields[i]; 1725 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR && 1726 strcmp(hist_field->var.name, var_name) == 0) { 1727 found = hist_field; 1728 break; 1729 } 1730 } 1731 1732 return found; 1733 } 1734 1735 static struct hist_field *find_var(struct hist_trigger_data *hist_data, 1736 struct trace_event_file *file, 1737 const char *var_name) 1738 { 1739 struct hist_trigger_data *test_data; 1740 struct event_trigger_data *test; 1741 struct hist_field *hist_field; 1742 1743 hist_field = find_var_field(hist_data, var_name); 1744 if (hist_field) 1745 return hist_field; 1746 1747 list_for_each_entry_rcu(test, &file->triggers, list) { 1748 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1749 test_data = test->private_data; 1750 hist_field = find_var_field(test_data, var_name); 1751 if (hist_field) 1752 return hist_field; 1753 } 1754 } 1755 1756 return NULL; 1757 } 1758 1759 static struct trace_event_file *find_var_file(struct trace_array *tr, 1760 char *system, 1761 char *event_name, 1762 char *var_name) 1763 { 1764 struct hist_trigger_data *var_hist_data; 1765 struct hist_var_data *var_data; 1766 struct trace_event_file *file, *found = NULL; 1767 1768 if (system) 1769 return find_event_file(tr, system, event_name); 1770 1771 list_for_each_entry(var_data, &tr->hist_vars, list) { 1772 var_hist_data = var_data->hist_data; 1773 file = var_hist_data->event_file; 1774 if (file == found) 1775 continue; 1776 1777 if (find_var_field(var_hist_data, var_name)) { 1778 if (found) { 1779 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name)); 1780 return NULL; 1781 } 1782 1783 found = file; 1784 } 1785 } 1786 1787 return found; 1788 } 1789 1790 static struct hist_field *find_file_var(struct trace_event_file *file, 1791 const char *var_name) 1792 { 1793 struct hist_trigger_data *test_data; 1794 struct event_trigger_data *test; 1795 struct hist_field *hist_field; 1796 1797 list_for_each_entry_rcu(test, &file->triggers, list) { 1798 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1799 test_data = test->private_data; 1800 hist_field = find_var_field(test_data, var_name); 1801 if (hist_field) 1802 return hist_field; 1803 } 1804 } 1805 1806 return NULL; 1807 } 1808 1809 static struct hist_field * 1810 find_match_var(struct hist_trigger_data *hist_data, char *var_name) 1811 { 1812 struct trace_array *tr = hist_data->event_file->tr; 1813 struct hist_field *hist_field, *found = NULL; 1814 struct trace_event_file *file; 1815 unsigned int i; 1816 1817 for (i = 0; i < hist_data->n_actions; i++) { 1818 struct action_data *data = hist_data->actions[i]; 1819 1820 if (data->handler == HANDLER_ONMATCH) { 1821 char *system = data->match_data.event_system; 1822 char *event_name = data->match_data.event; 1823 1824 file = find_var_file(tr, system, event_name, var_name); 1825 if (!file) 1826 continue; 1827 hist_field = find_file_var(file, var_name); 1828 if (hist_field) { 1829 if (found) { 1830 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, 1831 errpos(var_name)); 1832 return ERR_PTR(-EINVAL); 1833 } 1834 1835 found = hist_field; 1836 } 1837 } 1838 } 1839 return found; 1840 } 1841 1842 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data, 1843 char *system, 1844 char *event_name, 1845 char *var_name) 1846 { 1847 struct trace_array *tr = hist_data->event_file->tr; 1848 struct hist_field *hist_field = NULL; 1849 struct trace_event_file *file; 1850 1851 if (!system || !event_name) { 1852 hist_field = find_match_var(hist_data, var_name); 1853 if (IS_ERR(hist_field)) 1854 return NULL; 1855 if (hist_field) 1856 return hist_field; 1857 } 1858 1859 file = find_var_file(tr, system, event_name, var_name); 1860 if (!file) 1861 return NULL; 1862 1863 hist_field = find_file_var(file, var_name); 1864 1865 return hist_field; 1866 } 1867 1868 static u64 hist_field_var_ref(struct hist_field *hist_field, 1869 struct tracing_map_elt *elt, 1870 struct ring_buffer_event *rbe, 1871 void *event) 1872 { 1873 struct hist_elt_data *elt_data; 1874 u64 var_val = 0; 1875 1876 if (WARN_ON_ONCE(!elt)) 1877 return var_val; 1878 1879 elt_data = elt->private_data; 1880 var_val = elt_data->var_ref_vals[hist_field->var_ref_idx]; 1881 1882 return var_val; 1883 } 1884 1885 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key, 1886 u64 *var_ref_vals, bool self) 1887 { 1888 struct hist_trigger_data *var_data; 1889 struct tracing_map_elt *var_elt; 1890 struct hist_field *hist_field; 1891 unsigned int i, var_idx; 1892 bool resolved = true; 1893 u64 var_val = 0; 1894 1895 for (i = 0; i < hist_data->n_var_refs; i++) { 1896 hist_field = hist_data->var_refs[i]; 1897 var_idx = hist_field->var.idx; 1898 var_data = hist_field->var.hist_data; 1899 1900 if (var_data == NULL) { 1901 resolved = false; 1902 break; 1903 } 1904 1905 if ((self && var_data != hist_data) || 1906 (!self && var_data == hist_data)) 1907 continue; 1908 1909 var_elt = tracing_map_lookup(var_data->map, key); 1910 if (!var_elt) { 1911 resolved = false; 1912 break; 1913 } 1914 1915 if (!tracing_map_var_set(var_elt, var_idx)) { 1916 resolved = false; 1917 break; 1918 } 1919 1920 if (self || !hist_field->read_once) 1921 var_val = tracing_map_read_var(var_elt, var_idx); 1922 else 1923 var_val = tracing_map_read_var_once(var_elt, var_idx); 1924 1925 var_ref_vals[i] = var_val; 1926 } 1927 1928 return resolved; 1929 } 1930 1931 static const char *hist_field_name(struct hist_field *field, 1932 unsigned int level) 1933 { 1934 const char *field_name = ""; 1935 1936 if (level > 1) 1937 return field_name; 1938 1939 if (field->field) 1940 field_name = field->field->name; 1941 else if (field->flags & HIST_FIELD_FL_LOG2 || 1942 field->flags & HIST_FIELD_FL_ALIAS) 1943 field_name = hist_field_name(field->operands[0], ++level); 1944 else if (field->flags & HIST_FIELD_FL_CPU) 1945 field_name = "cpu"; 1946 else if (field->flags & HIST_FIELD_FL_EXPR || 1947 field->flags & HIST_FIELD_FL_VAR_REF) { 1948 if (field->system) { 1949 static char full_name[MAX_FILTER_STR_VAL]; 1950 1951 strcat(full_name, field->system); 1952 strcat(full_name, "."); 1953 strcat(full_name, field->event_name); 1954 strcat(full_name, "."); 1955 strcat(full_name, field->name); 1956 field_name = full_name; 1957 } else 1958 field_name = field->name; 1959 } else if (field->flags & HIST_FIELD_FL_TIMESTAMP) 1960 field_name = "common_timestamp"; 1961 1962 if (field_name == NULL) 1963 field_name = ""; 1964 1965 return field_name; 1966 } 1967 1968 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed) 1969 { 1970 hist_field_fn_t fn = NULL; 1971 1972 switch (field_size) { 1973 case 8: 1974 if (field_is_signed) 1975 fn = hist_field_s64; 1976 else 1977 fn = hist_field_u64; 1978 break; 1979 case 4: 1980 if (field_is_signed) 1981 fn = hist_field_s32; 1982 else 1983 fn = hist_field_u32; 1984 break; 1985 case 2: 1986 if (field_is_signed) 1987 fn = hist_field_s16; 1988 else 1989 fn = hist_field_u16; 1990 break; 1991 case 1: 1992 if (field_is_signed) 1993 fn = hist_field_s8; 1994 else 1995 fn = hist_field_u8; 1996 break; 1997 } 1998 1999 return fn; 2000 } 2001 2002 static int parse_map_size(char *str) 2003 { 2004 unsigned long size, map_bits; 2005 int ret; 2006 2007 strsep(&str, "="); 2008 if (!str) { 2009 ret = -EINVAL; 2010 goto out; 2011 } 2012 2013 ret = kstrtoul(str, 0, &size); 2014 if (ret) 2015 goto out; 2016 2017 map_bits = ilog2(roundup_pow_of_two(size)); 2018 if (map_bits < TRACING_MAP_BITS_MIN || 2019 map_bits > TRACING_MAP_BITS_MAX) 2020 ret = -EINVAL; 2021 else 2022 ret = map_bits; 2023 out: 2024 return ret; 2025 } 2026 2027 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs) 2028 { 2029 unsigned int i; 2030 2031 if (!attrs) 2032 return; 2033 2034 for (i = 0; i < attrs->n_assignments; i++) 2035 kfree(attrs->assignment_str[i]); 2036 2037 for (i = 0; i < attrs->n_actions; i++) 2038 kfree(attrs->action_str[i]); 2039 2040 kfree(attrs->name); 2041 kfree(attrs->sort_key_str); 2042 kfree(attrs->keys_str); 2043 kfree(attrs->vals_str); 2044 kfree(attrs->clock); 2045 kfree(attrs); 2046 } 2047 2048 static int parse_action(char *str, struct hist_trigger_attrs *attrs) 2049 { 2050 int ret = -EINVAL; 2051 2052 if (attrs->n_actions >= HIST_ACTIONS_MAX) 2053 return ret; 2054 2055 if ((str_has_prefix(str, "onmatch(")) || 2056 (str_has_prefix(str, "onmax(")) || 2057 (str_has_prefix(str, "onchange("))) { 2058 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL); 2059 if (!attrs->action_str[attrs->n_actions]) { 2060 ret = -ENOMEM; 2061 return ret; 2062 } 2063 attrs->n_actions++; 2064 ret = 0; 2065 } 2066 return ret; 2067 } 2068 2069 static int parse_assignment(struct trace_array *tr, 2070 char *str, struct hist_trigger_attrs *attrs) 2071 { 2072 int ret = 0; 2073 2074 if ((str_has_prefix(str, "key=")) || 2075 (str_has_prefix(str, "keys="))) { 2076 attrs->keys_str = kstrdup(str, GFP_KERNEL); 2077 if (!attrs->keys_str) { 2078 ret = -ENOMEM; 2079 goto out; 2080 } 2081 } else if ((str_has_prefix(str, "val=")) || 2082 (str_has_prefix(str, "vals=")) || 2083 (str_has_prefix(str, "values="))) { 2084 attrs->vals_str = kstrdup(str, GFP_KERNEL); 2085 if (!attrs->vals_str) { 2086 ret = -ENOMEM; 2087 goto out; 2088 } 2089 } else if (str_has_prefix(str, "sort=")) { 2090 attrs->sort_key_str = kstrdup(str, GFP_KERNEL); 2091 if (!attrs->sort_key_str) { 2092 ret = -ENOMEM; 2093 goto out; 2094 } 2095 } else if (str_has_prefix(str, "name=")) { 2096 attrs->name = kstrdup(str, GFP_KERNEL); 2097 if (!attrs->name) { 2098 ret = -ENOMEM; 2099 goto out; 2100 } 2101 } else if (str_has_prefix(str, "clock=")) { 2102 strsep(&str, "="); 2103 if (!str) { 2104 ret = -EINVAL; 2105 goto out; 2106 } 2107 2108 str = strstrip(str); 2109 attrs->clock = kstrdup(str, GFP_KERNEL); 2110 if (!attrs->clock) { 2111 ret = -ENOMEM; 2112 goto out; 2113 } 2114 } else if (str_has_prefix(str, "size=")) { 2115 int map_bits = parse_map_size(str); 2116 2117 if (map_bits < 0) { 2118 ret = map_bits; 2119 goto out; 2120 } 2121 attrs->map_bits = map_bits; 2122 } else { 2123 char *assignment; 2124 2125 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) { 2126 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str)); 2127 ret = -EINVAL; 2128 goto out; 2129 } 2130 2131 assignment = kstrdup(str, GFP_KERNEL); 2132 if (!assignment) { 2133 ret = -ENOMEM; 2134 goto out; 2135 } 2136 2137 attrs->assignment_str[attrs->n_assignments++] = assignment; 2138 } 2139 out: 2140 return ret; 2141 } 2142 2143 static struct hist_trigger_attrs * 2144 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str) 2145 { 2146 struct hist_trigger_attrs *attrs; 2147 int ret = 0; 2148 2149 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL); 2150 if (!attrs) 2151 return ERR_PTR(-ENOMEM); 2152 2153 while (trigger_str) { 2154 char *str = strsep(&trigger_str, ":"); 2155 2156 if (strchr(str, '=')) { 2157 ret = parse_assignment(tr, str, attrs); 2158 if (ret) 2159 goto free; 2160 } else if (strcmp(str, "pause") == 0) 2161 attrs->pause = true; 2162 else if ((strcmp(str, "cont") == 0) || 2163 (strcmp(str, "continue") == 0)) 2164 attrs->cont = true; 2165 else if (strcmp(str, "clear") == 0) 2166 attrs->clear = true; 2167 else { 2168 ret = parse_action(str, attrs); 2169 if (ret) 2170 goto free; 2171 } 2172 } 2173 2174 if (!attrs->keys_str) { 2175 ret = -EINVAL; 2176 goto free; 2177 } 2178 2179 if (!attrs->clock) { 2180 attrs->clock = kstrdup("global", GFP_KERNEL); 2181 if (!attrs->clock) { 2182 ret = -ENOMEM; 2183 goto free; 2184 } 2185 } 2186 2187 return attrs; 2188 free: 2189 destroy_hist_trigger_attrs(attrs); 2190 2191 return ERR_PTR(ret); 2192 } 2193 2194 static inline void save_comm(char *comm, struct task_struct *task) 2195 { 2196 if (!task->pid) { 2197 strcpy(comm, "<idle>"); 2198 return; 2199 } 2200 2201 if (WARN_ON_ONCE(task->pid < 0)) { 2202 strcpy(comm, "<XXX>"); 2203 return; 2204 } 2205 2206 strncpy(comm, task->comm, TASK_COMM_LEN); 2207 } 2208 2209 static void hist_elt_data_free(struct hist_elt_data *elt_data) 2210 { 2211 unsigned int i; 2212 2213 for (i = 0; i < SYNTH_FIELDS_MAX; i++) 2214 kfree(elt_data->field_var_str[i]); 2215 2216 kfree(elt_data->comm); 2217 kfree(elt_data); 2218 } 2219 2220 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt) 2221 { 2222 struct hist_elt_data *elt_data = elt->private_data; 2223 2224 hist_elt_data_free(elt_data); 2225 } 2226 2227 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt) 2228 { 2229 struct hist_trigger_data *hist_data = elt->map->private_data; 2230 unsigned int size = TASK_COMM_LEN; 2231 struct hist_elt_data *elt_data; 2232 struct hist_field *key_field; 2233 unsigned int i, n_str; 2234 2235 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL); 2236 if (!elt_data) 2237 return -ENOMEM; 2238 2239 for_each_hist_key_field(i, hist_data) { 2240 key_field = hist_data->fields[i]; 2241 2242 if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 2243 elt_data->comm = kzalloc(size, GFP_KERNEL); 2244 if (!elt_data->comm) { 2245 kfree(elt_data); 2246 return -ENOMEM; 2247 } 2248 break; 2249 } 2250 } 2251 2252 n_str = hist_data->n_field_var_str + hist_data->n_save_var_str; 2253 2254 size = STR_VAR_LEN_MAX; 2255 2256 for (i = 0; i < n_str; i++) { 2257 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL); 2258 if (!elt_data->field_var_str[i]) { 2259 hist_elt_data_free(elt_data); 2260 return -ENOMEM; 2261 } 2262 } 2263 2264 elt->private_data = elt_data; 2265 2266 return 0; 2267 } 2268 2269 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt) 2270 { 2271 struct hist_elt_data *elt_data = elt->private_data; 2272 2273 if (elt_data->comm) 2274 save_comm(elt_data->comm, current); 2275 } 2276 2277 static const struct tracing_map_ops hist_trigger_elt_data_ops = { 2278 .elt_alloc = hist_trigger_elt_data_alloc, 2279 .elt_free = hist_trigger_elt_data_free, 2280 .elt_init = hist_trigger_elt_data_init, 2281 }; 2282 2283 static const char *get_hist_field_flags(struct hist_field *hist_field) 2284 { 2285 const char *flags_str = NULL; 2286 2287 if (hist_field->flags & HIST_FIELD_FL_HEX) 2288 flags_str = "hex"; 2289 else if (hist_field->flags & HIST_FIELD_FL_SYM) 2290 flags_str = "sym"; 2291 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET) 2292 flags_str = "sym-offset"; 2293 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME) 2294 flags_str = "execname"; 2295 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL) 2296 flags_str = "syscall"; 2297 else if (hist_field->flags & HIST_FIELD_FL_LOG2) 2298 flags_str = "log2"; 2299 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS) 2300 flags_str = "usecs"; 2301 2302 return flags_str; 2303 } 2304 2305 static void expr_field_str(struct hist_field *field, char *expr) 2306 { 2307 if (field->flags & HIST_FIELD_FL_VAR_REF) 2308 strcat(expr, "$"); 2309 2310 strcat(expr, hist_field_name(field, 0)); 2311 2312 if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) { 2313 const char *flags_str = get_hist_field_flags(field); 2314 2315 if (flags_str) { 2316 strcat(expr, "."); 2317 strcat(expr, flags_str); 2318 } 2319 } 2320 } 2321 2322 static char *expr_str(struct hist_field *field, unsigned int level) 2323 { 2324 char *expr; 2325 2326 if (level > 1) 2327 return NULL; 2328 2329 expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2330 if (!expr) 2331 return NULL; 2332 2333 if (!field->operands[0]) { 2334 expr_field_str(field, expr); 2335 return expr; 2336 } 2337 2338 if (field->operator == FIELD_OP_UNARY_MINUS) { 2339 char *subexpr; 2340 2341 strcat(expr, "-("); 2342 subexpr = expr_str(field->operands[0], ++level); 2343 if (!subexpr) { 2344 kfree(expr); 2345 return NULL; 2346 } 2347 strcat(expr, subexpr); 2348 strcat(expr, ")"); 2349 2350 kfree(subexpr); 2351 2352 return expr; 2353 } 2354 2355 expr_field_str(field->operands[0], expr); 2356 2357 switch (field->operator) { 2358 case FIELD_OP_MINUS: 2359 strcat(expr, "-"); 2360 break; 2361 case FIELD_OP_PLUS: 2362 strcat(expr, "+"); 2363 break; 2364 default: 2365 kfree(expr); 2366 return NULL; 2367 } 2368 2369 expr_field_str(field->operands[1], expr); 2370 2371 return expr; 2372 } 2373 2374 static int contains_operator(char *str) 2375 { 2376 enum field_op_id field_op = FIELD_OP_NONE; 2377 char *op; 2378 2379 op = strpbrk(str, "+-"); 2380 if (!op) 2381 return FIELD_OP_NONE; 2382 2383 switch (*op) { 2384 case '-': 2385 if (*str == '-') 2386 field_op = FIELD_OP_UNARY_MINUS; 2387 else 2388 field_op = FIELD_OP_MINUS; 2389 break; 2390 case '+': 2391 field_op = FIELD_OP_PLUS; 2392 break; 2393 default: 2394 break; 2395 } 2396 2397 return field_op; 2398 } 2399 2400 static void __destroy_hist_field(struct hist_field *hist_field) 2401 { 2402 kfree(hist_field->var.name); 2403 kfree(hist_field->name); 2404 kfree(hist_field->type); 2405 2406 kfree(hist_field); 2407 } 2408 2409 static void destroy_hist_field(struct hist_field *hist_field, 2410 unsigned int level) 2411 { 2412 unsigned int i; 2413 2414 if (level > 3) 2415 return; 2416 2417 if (!hist_field) 2418 return; 2419 2420 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) 2421 return; /* var refs will be destroyed separately */ 2422 2423 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) 2424 destroy_hist_field(hist_field->operands[i], level + 1); 2425 2426 __destroy_hist_field(hist_field); 2427 } 2428 2429 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, 2430 struct ftrace_event_field *field, 2431 unsigned long flags, 2432 char *var_name) 2433 { 2434 struct hist_field *hist_field; 2435 2436 if (field && is_function_field(field)) 2437 return NULL; 2438 2439 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL); 2440 if (!hist_field) 2441 return NULL; 2442 2443 hist_field->hist_data = hist_data; 2444 2445 if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS) 2446 goto out; /* caller will populate */ 2447 2448 if (flags & HIST_FIELD_FL_VAR_REF) { 2449 hist_field->fn = hist_field_var_ref; 2450 goto out; 2451 } 2452 2453 if (flags & HIST_FIELD_FL_HITCOUNT) { 2454 hist_field->fn = hist_field_counter; 2455 hist_field->size = sizeof(u64); 2456 hist_field->type = kstrdup("u64", GFP_KERNEL); 2457 if (!hist_field->type) 2458 goto free; 2459 goto out; 2460 } 2461 2462 if (flags & HIST_FIELD_FL_STACKTRACE) { 2463 hist_field->fn = hist_field_none; 2464 goto out; 2465 } 2466 2467 if (flags & HIST_FIELD_FL_LOG2) { 2468 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2; 2469 hist_field->fn = hist_field_log2; 2470 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL); 2471 hist_field->size = hist_field->operands[0]->size; 2472 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL); 2473 if (!hist_field->type) 2474 goto free; 2475 goto out; 2476 } 2477 2478 if (flags & HIST_FIELD_FL_TIMESTAMP) { 2479 hist_field->fn = hist_field_timestamp; 2480 hist_field->size = sizeof(u64); 2481 hist_field->type = kstrdup("u64", GFP_KERNEL); 2482 if (!hist_field->type) 2483 goto free; 2484 goto out; 2485 } 2486 2487 if (flags & HIST_FIELD_FL_CPU) { 2488 hist_field->fn = hist_field_cpu; 2489 hist_field->size = sizeof(int); 2490 hist_field->type = kstrdup("unsigned int", GFP_KERNEL); 2491 if (!hist_field->type) 2492 goto free; 2493 goto out; 2494 } 2495 2496 if (WARN_ON_ONCE(!field)) 2497 goto out; 2498 2499 if (is_string_field(field)) { 2500 flags |= HIST_FIELD_FL_STRING; 2501 2502 hist_field->size = MAX_FILTER_STR_VAL; 2503 hist_field->type = kstrdup(field->type, GFP_KERNEL); 2504 if (!hist_field->type) 2505 goto free; 2506 2507 if (field->filter_type == FILTER_STATIC_STRING) 2508 hist_field->fn = hist_field_string; 2509 else if (field->filter_type == FILTER_DYN_STRING) 2510 hist_field->fn = hist_field_dynstring; 2511 else 2512 hist_field->fn = hist_field_pstring; 2513 } else { 2514 hist_field->size = field->size; 2515 hist_field->is_signed = field->is_signed; 2516 hist_field->type = kstrdup(field->type, GFP_KERNEL); 2517 if (!hist_field->type) 2518 goto free; 2519 2520 hist_field->fn = select_value_fn(field->size, 2521 field->is_signed); 2522 if (!hist_field->fn) { 2523 destroy_hist_field(hist_field, 0); 2524 return NULL; 2525 } 2526 } 2527 out: 2528 hist_field->field = field; 2529 hist_field->flags = flags; 2530 2531 if (var_name) { 2532 hist_field->var.name = kstrdup(var_name, GFP_KERNEL); 2533 if (!hist_field->var.name) 2534 goto free; 2535 } 2536 2537 return hist_field; 2538 free: 2539 destroy_hist_field(hist_field, 0); 2540 return NULL; 2541 } 2542 2543 static void destroy_hist_fields(struct hist_trigger_data *hist_data) 2544 { 2545 unsigned int i; 2546 2547 for (i = 0; i < HIST_FIELDS_MAX; i++) { 2548 if (hist_data->fields[i]) { 2549 destroy_hist_field(hist_data->fields[i], 0); 2550 hist_data->fields[i] = NULL; 2551 } 2552 } 2553 2554 for (i = 0; i < hist_data->n_var_refs; i++) { 2555 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF)); 2556 __destroy_hist_field(hist_data->var_refs[i]); 2557 hist_data->var_refs[i] = NULL; 2558 } 2559 } 2560 2561 static int init_var_ref(struct hist_field *ref_field, 2562 struct hist_field *var_field, 2563 char *system, char *event_name) 2564 { 2565 int err = 0; 2566 2567 ref_field->var.idx = var_field->var.idx; 2568 ref_field->var.hist_data = var_field->hist_data; 2569 ref_field->size = var_field->size; 2570 ref_field->is_signed = var_field->is_signed; 2571 ref_field->flags |= var_field->flags & 2572 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2573 2574 if (system) { 2575 ref_field->system = kstrdup(system, GFP_KERNEL); 2576 if (!ref_field->system) 2577 return -ENOMEM; 2578 } 2579 2580 if (event_name) { 2581 ref_field->event_name = kstrdup(event_name, GFP_KERNEL); 2582 if (!ref_field->event_name) { 2583 err = -ENOMEM; 2584 goto free; 2585 } 2586 } 2587 2588 if (var_field->var.name) { 2589 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL); 2590 if (!ref_field->name) { 2591 err = -ENOMEM; 2592 goto free; 2593 } 2594 } else if (var_field->name) { 2595 ref_field->name = kstrdup(var_field->name, GFP_KERNEL); 2596 if (!ref_field->name) { 2597 err = -ENOMEM; 2598 goto free; 2599 } 2600 } 2601 2602 ref_field->type = kstrdup(var_field->type, GFP_KERNEL); 2603 if (!ref_field->type) { 2604 err = -ENOMEM; 2605 goto free; 2606 } 2607 out: 2608 return err; 2609 free: 2610 kfree(ref_field->system); 2611 kfree(ref_field->event_name); 2612 kfree(ref_field->name); 2613 2614 goto out; 2615 } 2616 2617 /** 2618 * create_var_ref - Create a variable reference and attach it to trigger 2619 * @hist_data: The trigger that will be referencing the variable 2620 * @var_field: The VAR field to create a reference to 2621 * @system: The optional system string 2622 * @event_name: The optional event_name string 2623 * 2624 * Given a variable hist_field, create a VAR_REF hist_field that 2625 * represents a reference to it. 2626 * 2627 * This function also adds the reference to the trigger that 2628 * now references the variable. 2629 * 2630 * Return: The VAR_REF field if successful, NULL if not 2631 */ 2632 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data, 2633 struct hist_field *var_field, 2634 char *system, char *event_name) 2635 { 2636 unsigned long flags = HIST_FIELD_FL_VAR_REF; 2637 struct hist_field *ref_field; 2638 2639 ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL); 2640 if (ref_field) { 2641 if (init_var_ref(ref_field, var_field, system, event_name)) { 2642 destroy_hist_field(ref_field, 0); 2643 return NULL; 2644 } 2645 2646 hist_data->var_refs[hist_data->n_var_refs] = ref_field; 2647 ref_field->var_ref_idx = hist_data->n_var_refs++; 2648 } 2649 2650 return ref_field; 2651 } 2652 2653 static bool is_var_ref(char *var_name) 2654 { 2655 if (!var_name || strlen(var_name) < 2 || var_name[0] != '$') 2656 return false; 2657 2658 return true; 2659 } 2660 2661 static char *field_name_from_var(struct hist_trigger_data *hist_data, 2662 char *var_name) 2663 { 2664 char *name, *field; 2665 unsigned int i; 2666 2667 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 2668 name = hist_data->attrs->var_defs.name[i]; 2669 2670 if (strcmp(var_name, name) == 0) { 2671 field = hist_data->attrs->var_defs.expr[i]; 2672 if (contains_operator(field) || is_var_ref(field)) 2673 continue; 2674 return field; 2675 } 2676 } 2677 2678 return NULL; 2679 } 2680 2681 static char *local_field_var_ref(struct hist_trigger_data *hist_data, 2682 char *system, char *event_name, 2683 char *var_name) 2684 { 2685 struct trace_event_call *call; 2686 2687 if (system && event_name) { 2688 call = hist_data->event_file->event_call; 2689 2690 if (strcmp(system, call->class->system) != 0) 2691 return NULL; 2692 2693 if (strcmp(event_name, trace_event_name(call)) != 0) 2694 return NULL; 2695 } 2696 2697 if (!!system != !!event_name) 2698 return NULL; 2699 2700 if (!is_var_ref(var_name)) 2701 return NULL; 2702 2703 var_name++; 2704 2705 return field_name_from_var(hist_data, var_name); 2706 } 2707 2708 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data, 2709 char *system, char *event_name, 2710 char *var_name) 2711 { 2712 struct hist_field *var_field = NULL, *ref_field = NULL; 2713 struct trace_array *tr = hist_data->event_file->tr; 2714 2715 if (!is_var_ref(var_name)) 2716 return NULL; 2717 2718 var_name++; 2719 2720 var_field = find_event_var(hist_data, system, event_name, var_name); 2721 if (var_field) 2722 ref_field = create_var_ref(hist_data, var_field, 2723 system, event_name); 2724 2725 if (!ref_field) 2726 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name)); 2727 2728 return ref_field; 2729 } 2730 2731 static struct ftrace_event_field * 2732 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, 2733 char *field_str, unsigned long *flags) 2734 { 2735 struct ftrace_event_field *field = NULL; 2736 char *field_name, *modifier, *str; 2737 struct trace_array *tr = file->tr; 2738 2739 modifier = str = kstrdup(field_str, GFP_KERNEL); 2740 if (!modifier) 2741 return ERR_PTR(-ENOMEM); 2742 2743 field_name = strsep(&modifier, "."); 2744 if (modifier) { 2745 if (strcmp(modifier, "hex") == 0) 2746 *flags |= HIST_FIELD_FL_HEX; 2747 else if (strcmp(modifier, "sym") == 0) 2748 *flags |= HIST_FIELD_FL_SYM; 2749 else if (strcmp(modifier, "sym-offset") == 0) 2750 *flags |= HIST_FIELD_FL_SYM_OFFSET; 2751 else if ((strcmp(modifier, "execname") == 0) && 2752 (strcmp(field_name, "common_pid") == 0)) 2753 *flags |= HIST_FIELD_FL_EXECNAME; 2754 else if (strcmp(modifier, "syscall") == 0) 2755 *flags |= HIST_FIELD_FL_SYSCALL; 2756 else if (strcmp(modifier, "log2") == 0) 2757 *flags |= HIST_FIELD_FL_LOG2; 2758 else if (strcmp(modifier, "usecs") == 0) 2759 *flags |= HIST_FIELD_FL_TIMESTAMP_USECS; 2760 else { 2761 hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier)); 2762 field = ERR_PTR(-EINVAL); 2763 goto out; 2764 } 2765 } 2766 2767 if (strcmp(field_name, "common_timestamp") == 0) { 2768 *flags |= HIST_FIELD_FL_TIMESTAMP; 2769 hist_data->enable_timestamps = true; 2770 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS) 2771 hist_data->attrs->ts_in_usecs = true; 2772 } else if (strcmp(field_name, "cpu") == 0) 2773 *flags |= HIST_FIELD_FL_CPU; 2774 else { 2775 field = trace_find_event_field(file->event_call, field_name); 2776 if (!field || !field->size) { 2777 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name)); 2778 field = ERR_PTR(-EINVAL); 2779 goto out; 2780 } 2781 } 2782 out: 2783 kfree(str); 2784 2785 return field; 2786 } 2787 2788 static struct hist_field *create_alias(struct hist_trigger_data *hist_data, 2789 struct hist_field *var_ref, 2790 char *var_name) 2791 { 2792 struct hist_field *alias = NULL; 2793 unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR; 2794 2795 alias = create_hist_field(hist_data, NULL, flags, var_name); 2796 if (!alias) 2797 return NULL; 2798 2799 alias->fn = var_ref->fn; 2800 alias->operands[0] = var_ref; 2801 2802 if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) { 2803 destroy_hist_field(alias, 0); 2804 return NULL; 2805 } 2806 2807 alias->var_ref_idx = var_ref->var_ref_idx; 2808 2809 return alias; 2810 } 2811 2812 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data, 2813 struct trace_event_file *file, char *str, 2814 unsigned long *flags, char *var_name) 2815 { 2816 char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str; 2817 struct ftrace_event_field *field = NULL; 2818 struct hist_field *hist_field = NULL; 2819 int ret = 0; 2820 2821 s = strchr(str, '.'); 2822 if (s) { 2823 s = strchr(++s, '.'); 2824 if (s) { 2825 ref_system = strsep(&str, "."); 2826 if (!str) { 2827 ret = -EINVAL; 2828 goto out; 2829 } 2830 ref_event = strsep(&str, "."); 2831 if (!str) { 2832 ret = -EINVAL; 2833 goto out; 2834 } 2835 ref_var = str; 2836 } 2837 } 2838 2839 s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var); 2840 if (!s) { 2841 hist_field = parse_var_ref(hist_data, ref_system, 2842 ref_event, ref_var); 2843 if (hist_field) { 2844 if (var_name) { 2845 hist_field = create_alias(hist_data, hist_field, var_name); 2846 if (!hist_field) { 2847 ret = -ENOMEM; 2848 goto out; 2849 } 2850 } 2851 return hist_field; 2852 } 2853 } else 2854 str = s; 2855 2856 field = parse_field(hist_data, file, str, flags); 2857 if (IS_ERR(field)) { 2858 ret = PTR_ERR(field); 2859 goto out; 2860 } 2861 2862 hist_field = create_hist_field(hist_data, field, *flags, var_name); 2863 if (!hist_field) { 2864 ret = -ENOMEM; 2865 goto out; 2866 } 2867 2868 return hist_field; 2869 out: 2870 return ERR_PTR(ret); 2871 } 2872 2873 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2874 struct trace_event_file *file, 2875 char *str, unsigned long flags, 2876 char *var_name, unsigned int level); 2877 2878 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data, 2879 struct trace_event_file *file, 2880 char *str, unsigned long flags, 2881 char *var_name, unsigned int level) 2882 { 2883 struct hist_field *operand1, *expr = NULL; 2884 unsigned long operand_flags; 2885 int ret = 0; 2886 char *s; 2887 2888 /* we support only -(xxx) i.e. explicit parens required */ 2889 2890 if (level > 3) { 2891 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str)); 2892 ret = -EINVAL; 2893 goto free; 2894 } 2895 2896 str++; /* skip leading '-' */ 2897 2898 s = strchr(str, '('); 2899 if (s) 2900 str++; 2901 else { 2902 ret = -EINVAL; 2903 goto free; 2904 } 2905 2906 s = strrchr(str, ')'); 2907 if (s) 2908 *s = '\0'; 2909 else { 2910 ret = -EINVAL; /* no closing ')' */ 2911 goto free; 2912 } 2913 2914 flags |= HIST_FIELD_FL_EXPR; 2915 expr = create_hist_field(hist_data, NULL, flags, var_name); 2916 if (!expr) { 2917 ret = -ENOMEM; 2918 goto free; 2919 } 2920 2921 operand_flags = 0; 2922 operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level); 2923 if (IS_ERR(operand1)) { 2924 ret = PTR_ERR(operand1); 2925 goto free; 2926 } 2927 2928 expr->flags |= operand1->flags & 2929 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2930 expr->fn = hist_field_unary_minus; 2931 expr->operands[0] = operand1; 2932 expr->operator = FIELD_OP_UNARY_MINUS; 2933 expr->name = expr_str(expr, 0); 2934 expr->type = kstrdup(operand1->type, GFP_KERNEL); 2935 if (!expr->type) { 2936 ret = -ENOMEM; 2937 goto free; 2938 } 2939 2940 return expr; 2941 free: 2942 destroy_hist_field(expr, 0); 2943 return ERR_PTR(ret); 2944 } 2945 2946 static int check_expr_operands(struct trace_array *tr, 2947 struct hist_field *operand1, 2948 struct hist_field *operand2) 2949 { 2950 unsigned long operand1_flags = operand1->flags; 2951 unsigned long operand2_flags = operand2->flags; 2952 2953 if ((operand1_flags & HIST_FIELD_FL_VAR_REF) || 2954 (operand1_flags & HIST_FIELD_FL_ALIAS)) { 2955 struct hist_field *var; 2956 2957 var = find_var_field(operand1->var.hist_data, operand1->name); 2958 if (!var) 2959 return -EINVAL; 2960 operand1_flags = var->flags; 2961 } 2962 2963 if ((operand2_flags & HIST_FIELD_FL_VAR_REF) || 2964 (operand2_flags & HIST_FIELD_FL_ALIAS)) { 2965 struct hist_field *var; 2966 2967 var = find_var_field(operand2->var.hist_data, operand2->name); 2968 if (!var) 2969 return -EINVAL; 2970 operand2_flags = var->flags; 2971 } 2972 2973 if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) != 2974 (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) { 2975 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0); 2976 return -EINVAL; 2977 } 2978 2979 return 0; 2980 } 2981 2982 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2983 struct trace_event_file *file, 2984 char *str, unsigned long flags, 2985 char *var_name, unsigned int level) 2986 { 2987 struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL; 2988 unsigned long operand_flags; 2989 int field_op, ret = -EINVAL; 2990 char *sep, *operand1_str; 2991 2992 if (level > 3) { 2993 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str)); 2994 return ERR_PTR(-EINVAL); 2995 } 2996 2997 field_op = contains_operator(str); 2998 2999 if (field_op == FIELD_OP_NONE) 3000 return parse_atom(hist_data, file, str, &flags, var_name); 3001 3002 if (field_op == FIELD_OP_UNARY_MINUS) 3003 return parse_unary(hist_data, file, str, flags, var_name, ++level); 3004 3005 switch (field_op) { 3006 case FIELD_OP_MINUS: 3007 sep = "-"; 3008 break; 3009 case FIELD_OP_PLUS: 3010 sep = "+"; 3011 break; 3012 default: 3013 goto free; 3014 } 3015 3016 operand1_str = strsep(&str, sep); 3017 if (!operand1_str || !str) 3018 goto free; 3019 3020 operand_flags = 0; 3021 operand1 = parse_atom(hist_data, file, operand1_str, 3022 &operand_flags, NULL); 3023 if (IS_ERR(operand1)) { 3024 ret = PTR_ERR(operand1); 3025 operand1 = NULL; 3026 goto free; 3027 } 3028 3029 /* rest of string could be another expression e.g. b+c in a+b+c */ 3030 operand_flags = 0; 3031 operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level); 3032 if (IS_ERR(operand2)) { 3033 ret = PTR_ERR(operand2); 3034 operand2 = NULL; 3035 goto free; 3036 } 3037 3038 ret = check_expr_operands(file->tr, operand1, operand2); 3039 if (ret) 3040 goto free; 3041 3042 flags |= HIST_FIELD_FL_EXPR; 3043 3044 flags |= operand1->flags & 3045 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 3046 3047 expr = create_hist_field(hist_data, NULL, flags, var_name); 3048 if (!expr) { 3049 ret = -ENOMEM; 3050 goto free; 3051 } 3052 3053 operand1->read_once = true; 3054 operand2->read_once = true; 3055 3056 expr->operands[0] = operand1; 3057 expr->operands[1] = operand2; 3058 expr->operator = field_op; 3059 expr->name = expr_str(expr, 0); 3060 expr->type = kstrdup(operand1->type, GFP_KERNEL); 3061 if (!expr->type) { 3062 ret = -ENOMEM; 3063 goto free; 3064 } 3065 3066 switch (field_op) { 3067 case FIELD_OP_MINUS: 3068 expr->fn = hist_field_minus; 3069 break; 3070 case FIELD_OP_PLUS: 3071 expr->fn = hist_field_plus; 3072 break; 3073 default: 3074 ret = -EINVAL; 3075 goto free; 3076 } 3077 3078 return expr; 3079 free: 3080 destroy_hist_field(operand1, 0); 3081 destroy_hist_field(operand2, 0); 3082 destroy_hist_field(expr, 0); 3083 3084 return ERR_PTR(ret); 3085 } 3086 3087 static char *find_trigger_filter(struct hist_trigger_data *hist_data, 3088 struct trace_event_file *file) 3089 { 3090 struct event_trigger_data *test; 3091 3092 list_for_each_entry_rcu(test, &file->triggers, list) { 3093 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 3094 if (test->private_data == hist_data) 3095 return test->filter_str; 3096 } 3097 } 3098 3099 return NULL; 3100 } 3101 3102 static struct event_command trigger_hist_cmd; 3103 static int event_hist_trigger_func(struct event_command *cmd_ops, 3104 struct trace_event_file *file, 3105 char *glob, char *cmd, char *param); 3106 3107 static bool compatible_keys(struct hist_trigger_data *target_hist_data, 3108 struct hist_trigger_data *hist_data, 3109 unsigned int n_keys) 3110 { 3111 struct hist_field *target_hist_field, *hist_field; 3112 unsigned int n, i, j; 3113 3114 if (hist_data->n_fields - hist_data->n_vals != n_keys) 3115 return false; 3116 3117 i = hist_data->n_vals; 3118 j = target_hist_data->n_vals; 3119 3120 for (n = 0; n < n_keys; n++) { 3121 hist_field = hist_data->fields[i + n]; 3122 target_hist_field = target_hist_data->fields[j + n]; 3123 3124 if (strcmp(hist_field->type, target_hist_field->type) != 0) 3125 return false; 3126 if (hist_field->size != target_hist_field->size) 3127 return false; 3128 if (hist_field->is_signed != target_hist_field->is_signed) 3129 return false; 3130 } 3131 3132 return true; 3133 } 3134 3135 static struct hist_trigger_data * 3136 find_compatible_hist(struct hist_trigger_data *target_hist_data, 3137 struct trace_event_file *file) 3138 { 3139 struct hist_trigger_data *hist_data; 3140 struct event_trigger_data *test; 3141 unsigned int n_keys; 3142 3143 n_keys = target_hist_data->n_fields - target_hist_data->n_vals; 3144 3145 list_for_each_entry_rcu(test, &file->triggers, list) { 3146 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 3147 hist_data = test->private_data; 3148 3149 if (compatible_keys(target_hist_data, hist_data, n_keys)) 3150 return hist_data; 3151 } 3152 } 3153 3154 return NULL; 3155 } 3156 3157 static struct trace_event_file *event_file(struct trace_array *tr, 3158 char *system, char *event_name) 3159 { 3160 struct trace_event_file *file; 3161 3162 file = __find_event_file(tr, system, event_name); 3163 if (!file) 3164 return ERR_PTR(-EINVAL); 3165 3166 return file; 3167 } 3168 3169 static struct hist_field * 3170 find_synthetic_field_var(struct hist_trigger_data *target_hist_data, 3171 char *system, char *event_name, char *field_name) 3172 { 3173 struct hist_field *event_var; 3174 char *synthetic_name; 3175 3176 synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 3177 if (!synthetic_name) 3178 return ERR_PTR(-ENOMEM); 3179 3180 strcpy(synthetic_name, "synthetic_"); 3181 strcat(synthetic_name, field_name); 3182 3183 event_var = find_event_var(target_hist_data, system, event_name, synthetic_name); 3184 3185 kfree(synthetic_name); 3186 3187 return event_var; 3188 } 3189 3190 /** 3191 * create_field_var_hist - Automatically create a histogram and var for a field 3192 * @target_hist_data: The target hist trigger 3193 * @subsys_name: Optional subsystem name 3194 * @event_name: Optional event name 3195 * @field_name: The name of the field (and the resulting variable) 3196 * 3197 * Hist trigger actions fetch data from variables, not directly from 3198 * events. However, for convenience, users are allowed to directly 3199 * specify an event field in an action, which will be automatically 3200 * converted into a variable on their behalf. 3201 3202 * If a user specifies a field on an event that isn't the event the 3203 * histogram currently being defined (the target event histogram), the 3204 * only way that can be accomplished is if a new hist trigger is 3205 * created and the field variable defined on that. 3206 * 3207 * This function creates a new histogram compatible with the target 3208 * event (meaning a histogram with the same key as the target 3209 * histogram), and creates a variable for the specified field, but 3210 * with 'synthetic_' prepended to the variable name in order to avoid 3211 * collision with normal field variables. 3212 * 3213 * Return: The variable created for the field. 3214 */ 3215 static struct hist_field * 3216 create_field_var_hist(struct hist_trigger_data *target_hist_data, 3217 char *subsys_name, char *event_name, char *field_name) 3218 { 3219 struct trace_array *tr = target_hist_data->event_file->tr; 3220 struct hist_field *event_var = ERR_PTR(-EINVAL); 3221 struct hist_trigger_data *hist_data; 3222 unsigned int i, n, first = true; 3223 struct field_var_hist *var_hist; 3224 struct trace_event_file *file; 3225 struct hist_field *key_field; 3226 char *saved_filter; 3227 char *cmd; 3228 int ret; 3229 3230 if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) { 3231 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name)); 3232 return ERR_PTR(-EINVAL); 3233 } 3234 3235 file = event_file(tr, subsys_name, event_name); 3236 3237 if (IS_ERR(file)) { 3238 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name)); 3239 ret = PTR_ERR(file); 3240 return ERR_PTR(ret); 3241 } 3242 3243 /* 3244 * Look for a histogram compatible with target. We'll use the 3245 * found histogram specification to create a new matching 3246 * histogram with our variable on it. target_hist_data is not 3247 * yet a registered histogram so we can't use that. 3248 */ 3249 hist_data = find_compatible_hist(target_hist_data, file); 3250 if (!hist_data) { 3251 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name)); 3252 return ERR_PTR(-EINVAL); 3253 } 3254 3255 /* See if a synthetic field variable has already been created */ 3256 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 3257 event_name, field_name); 3258 if (!IS_ERR_OR_NULL(event_var)) 3259 return event_var; 3260 3261 var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL); 3262 if (!var_hist) 3263 return ERR_PTR(-ENOMEM); 3264 3265 cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 3266 if (!cmd) { 3267 kfree(var_hist); 3268 return ERR_PTR(-ENOMEM); 3269 } 3270 3271 /* Use the same keys as the compatible histogram */ 3272 strcat(cmd, "keys="); 3273 3274 for_each_hist_key_field(i, hist_data) { 3275 key_field = hist_data->fields[i]; 3276 if (!first) 3277 strcat(cmd, ","); 3278 strcat(cmd, key_field->field->name); 3279 first = false; 3280 } 3281 3282 /* Create the synthetic field variable specification */ 3283 strcat(cmd, ":synthetic_"); 3284 strcat(cmd, field_name); 3285 strcat(cmd, "="); 3286 strcat(cmd, field_name); 3287 3288 /* Use the same filter as the compatible histogram */ 3289 saved_filter = find_trigger_filter(hist_data, file); 3290 if (saved_filter) { 3291 strcat(cmd, " if "); 3292 strcat(cmd, saved_filter); 3293 } 3294 3295 var_hist->cmd = kstrdup(cmd, GFP_KERNEL); 3296 if (!var_hist->cmd) { 3297 kfree(cmd); 3298 kfree(var_hist); 3299 return ERR_PTR(-ENOMEM); 3300 } 3301 3302 /* Save the compatible histogram information */ 3303 var_hist->hist_data = hist_data; 3304 3305 /* Create the new histogram with our variable */ 3306 ret = event_hist_trigger_func(&trigger_hist_cmd, file, 3307 "", "hist", cmd); 3308 if (ret) { 3309 kfree(cmd); 3310 kfree(var_hist->cmd); 3311 kfree(var_hist); 3312 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name)); 3313 return ERR_PTR(ret); 3314 } 3315 3316 kfree(cmd); 3317 3318 /* If we can't find the variable, something went wrong */ 3319 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 3320 event_name, field_name); 3321 if (IS_ERR_OR_NULL(event_var)) { 3322 kfree(var_hist->cmd); 3323 kfree(var_hist); 3324 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name)); 3325 return ERR_PTR(-EINVAL); 3326 } 3327 3328 n = target_hist_data->n_field_var_hists; 3329 target_hist_data->field_var_hists[n] = var_hist; 3330 target_hist_data->n_field_var_hists++; 3331 3332 return event_var; 3333 } 3334 3335 static struct hist_field * 3336 find_target_event_var(struct hist_trigger_data *hist_data, 3337 char *subsys_name, char *event_name, char *var_name) 3338 { 3339 struct trace_event_file *file = hist_data->event_file; 3340 struct hist_field *hist_field = NULL; 3341 3342 if (subsys_name) { 3343 struct trace_event_call *call; 3344 3345 if (!event_name) 3346 return NULL; 3347 3348 call = file->event_call; 3349 3350 if (strcmp(subsys_name, call->class->system) != 0) 3351 return NULL; 3352 3353 if (strcmp(event_name, trace_event_name(call)) != 0) 3354 return NULL; 3355 } 3356 3357 hist_field = find_var_field(hist_data, var_name); 3358 3359 return hist_field; 3360 } 3361 3362 static inline void __update_field_vars(struct tracing_map_elt *elt, 3363 struct ring_buffer_event *rbe, 3364 void *rec, 3365 struct field_var **field_vars, 3366 unsigned int n_field_vars, 3367 unsigned int field_var_str_start) 3368 { 3369 struct hist_elt_data *elt_data = elt->private_data; 3370 unsigned int i, j, var_idx; 3371 u64 var_val; 3372 3373 for (i = 0, j = field_var_str_start; i < n_field_vars; i++) { 3374 struct field_var *field_var = field_vars[i]; 3375 struct hist_field *var = field_var->var; 3376 struct hist_field *val = field_var->val; 3377 3378 var_val = val->fn(val, elt, rbe, rec); 3379 var_idx = var->var.idx; 3380 3381 if (val->flags & HIST_FIELD_FL_STRING) { 3382 char *str = elt_data->field_var_str[j++]; 3383 char *val_str = (char *)(uintptr_t)var_val; 3384 3385 strscpy(str, val_str, STR_VAR_LEN_MAX); 3386 var_val = (u64)(uintptr_t)str; 3387 } 3388 tracing_map_set_var(elt, var_idx, var_val); 3389 } 3390 } 3391 3392 static void update_field_vars(struct hist_trigger_data *hist_data, 3393 struct tracing_map_elt *elt, 3394 struct ring_buffer_event *rbe, 3395 void *rec) 3396 { 3397 __update_field_vars(elt, rbe, rec, hist_data->field_vars, 3398 hist_data->n_field_vars, 0); 3399 } 3400 3401 static void save_track_data_vars(struct hist_trigger_data *hist_data, 3402 struct tracing_map_elt *elt, void *rec, 3403 struct ring_buffer_event *rbe, void *key, 3404 struct action_data *data, u64 *var_ref_vals) 3405 { 3406 __update_field_vars(elt, rbe, rec, hist_data->save_vars, 3407 hist_data->n_save_vars, hist_data->n_field_var_str); 3408 } 3409 3410 static struct hist_field *create_var(struct hist_trigger_data *hist_data, 3411 struct trace_event_file *file, 3412 char *name, int size, const char *type) 3413 { 3414 struct hist_field *var; 3415 int idx; 3416 3417 if (find_var(hist_data, file, name) && !hist_data->remove) { 3418 var = ERR_PTR(-EINVAL); 3419 goto out; 3420 } 3421 3422 var = kzalloc(sizeof(struct hist_field), GFP_KERNEL); 3423 if (!var) { 3424 var = ERR_PTR(-ENOMEM); 3425 goto out; 3426 } 3427 3428 idx = tracing_map_add_var(hist_data->map); 3429 if (idx < 0) { 3430 kfree(var); 3431 var = ERR_PTR(-EINVAL); 3432 goto out; 3433 } 3434 3435 var->flags = HIST_FIELD_FL_VAR; 3436 var->var.idx = idx; 3437 var->var.hist_data = var->hist_data = hist_data; 3438 var->size = size; 3439 var->var.name = kstrdup(name, GFP_KERNEL); 3440 var->type = kstrdup(type, GFP_KERNEL); 3441 if (!var->var.name || !var->type) { 3442 kfree(var->var.name); 3443 kfree(var->type); 3444 kfree(var); 3445 var = ERR_PTR(-ENOMEM); 3446 } 3447 out: 3448 return var; 3449 } 3450 3451 static struct field_var *create_field_var(struct hist_trigger_data *hist_data, 3452 struct trace_event_file *file, 3453 char *field_name) 3454 { 3455 struct hist_field *val = NULL, *var = NULL; 3456 unsigned long flags = HIST_FIELD_FL_VAR; 3457 struct trace_array *tr = file->tr; 3458 struct field_var *field_var; 3459 int ret = 0; 3460 3461 if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) { 3462 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name)); 3463 ret = -EINVAL; 3464 goto err; 3465 } 3466 3467 val = parse_atom(hist_data, file, field_name, &flags, NULL); 3468 if (IS_ERR(val)) { 3469 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name)); 3470 ret = PTR_ERR(val); 3471 goto err; 3472 } 3473 3474 var = create_var(hist_data, file, field_name, val->size, val->type); 3475 if (IS_ERR(var)) { 3476 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name)); 3477 kfree(val); 3478 ret = PTR_ERR(var); 3479 goto err; 3480 } 3481 3482 field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL); 3483 if (!field_var) { 3484 kfree(val); 3485 kfree(var); 3486 ret = -ENOMEM; 3487 goto err; 3488 } 3489 3490 field_var->var = var; 3491 field_var->val = val; 3492 out: 3493 return field_var; 3494 err: 3495 field_var = ERR_PTR(ret); 3496 goto out; 3497 } 3498 3499 /** 3500 * create_target_field_var - Automatically create a variable for a field 3501 * @target_hist_data: The target hist trigger 3502 * @subsys_name: Optional subsystem name 3503 * @event_name: Optional event name 3504 * @var_name: The name of the field (and the resulting variable) 3505 * 3506 * Hist trigger actions fetch data from variables, not directly from 3507 * events. However, for convenience, users are allowed to directly 3508 * specify an event field in an action, which will be automatically 3509 * converted into a variable on their behalf. 3510 3511 * This function creates a field variable with the name var_name on 3512 * the hist trigger currently being defined on the target event. If 3513 * subsys_name and event_name are specified, this function simply 3514 * verifies that they do in fact match the target event subsystem and 3515 * event name. 3516 * 3517 * Return: The variable created for the field. 3518 */ 3519 static struct field_var * 3520 create_target_field_var(struct hist_trigger_data *target_hist_data, 3521 char *subsys_name, char *event_name, char *var_name) 3522 { 3523 struct trace_event_file *file = target_hist_data->event_file; 3524 3525 if (subsys_name) { 3526 struct trace_event_call *call; 3527 3528 if (!event_name) 3529 return NULL; 3530 3531 call = file->event_call; 3532 3533 if (strcmp(subsys_name, call->class->system) != 0) 3534 return NULL; 3535 3536 if (strcmp(event_name, trace_event_name(call)) != 0) 3537 return NULL; 3538 } 3539 3540 return create_field_var(target_hist_data, file, var_name); 3541 } 3542 3543 static bool check_track_val_max(u64 track_val, u64 var_val) 3544 { 3545 if (var_val <= track_val) 3546 return false; 3547 3548 return true; 3549 } 3550 3551 static bool check_track_val_changed(u64 track_val, u64 var_val) 3552 { 3553 if (var_val == track_val) 3554 return false; 3555 3556 return true; 3557 } 3558 3559 static u64 get_track_val(struct hist_trigger_data *hist_data, 3560 struct tracing_map_elt *elt, 3561 struct action_data *data) 3562 { 3563 unsigned int track_var_idx = data->track_data.track_var->var.idx; 3564 u64 track_val; 3565 3566 track_val = tracing_map_read_var(elt, track_var_idx); 3567 3568 return track_val; 3569 } 3570 3571 static void save_track_val(struct hist_trigger_data *hist_data, 3572 struct tracing_map_elt *elt, 3573 struct action_data *data, u64 var_val) 3574 { 3575 unsigned int track_var_idx = data->track_data.track_var->var.idx; 3576 3577 tracing_map_set_var(elt, track_var_idx, var_val); 3578 } 3579 3580 static void save_track_data(struct hist_trigger_data *hist_data, 3581 struct tracing_map_elt *elt, void *rec, 3582 struct ring_buffer_event *rbe, void *key, 3583 struct action_data *data, u64 *var_ref_vals) 3584 { 3585 if (data->track_data.save_data) 3586 data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals); 3587 } 3588 3589 static bool check_track_val(struct tracing_map_elt *elt, 3590 struct action_data *data, 3591 u64 var_val) 3592 { 3593 struct hist_trigger_data *hist_data; 3594 u64 track_val; 3595 3596 hist_data = data->track_data.track_var->hist_data; 3597 track_val = get_track_val(hist_data, elt, data); 3598 3599 return data->track_data.check_val(track_val, var_val); 3600 } 3601 3602 #ifdef CONFIG_TRACER_SNAPSHOT 3603 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data) 3604 { 3605 /* called with tr->max_lock held */ 3606 struct track_data *track_data = tr->cond_snapshot->cond_data; 3607 struct hist_elt_data *elt_data, *track_elt_data; 3608 struct snapshot_context *context = cond_data; 3609 struct action_data *action; 3610 u64 track_val; 3611 3612 if (!track_data) 3613 return false; 3614 3615 action = track_data->action_data; 3616 3617 track_val = get_track_val(track_data->hist_data, context->elt, 3618 track_data->action_data); 3619 3620 if (!action->track_data.check_val(track_data->track_val, track_val)) 3621 return false; 3622 3623 track_data->track_val = track_val; 3624 memcpy(track_data->key, context->key, track_data->key_len); 3625 3626 elt_data = context->elt->private_data; 3627 track_elt_data = track_data->elt.private_data; 3628 if (elt_data->comm) 3629 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN); 3630 3631 track_data->updated = true; 3632 3633 return true; 3634 } 3635 3636 static void save_track_data_snapshot(struct hist_trigger_data *hist_data, 3637 struct tracing_map_elt *elt, void *rec, 3638 struct ring_buffer_event *rbe, void *key, 3639 struct action_data *data, 3640 u64 *var_ref_vals) 3641 { 3642 struct trace_event_file *file = hist_data->event_file; 3643 struct snapshot_context context; 3644 3645 context.elt = elt; 3646 context.key = key; 3647 3648 tracing_snapshot_cond(file->tr, &context); 3649 } 3650 3651 static void hist_trigger_print_key(struct seq_file *m, 3652 struct hist_trigger_data *hist_data, 3653 void *key, 3654 struct tracing_map_elt *elt); 3655 3656 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data) 3657 { 3658 unsigned int i; 3659 3660 if (!hist_data->n_actions) 3661 return NULL; 3662 3663 for (i = 0; i < hist_data->n_actions; i++) { 3664 struct action_data *data = hist_data->actions[i]; 3665 3666 if (data->action == ACTION_SNAPSHOT) 3667 return data; 3668 } 3669 3670 return NULL; 3671 } 3672 3673 static void track_data_snapshot_print(struct seq_file *m, 3674 struct hist_trigger_data *hist_data) 3675 { 3676 struct trace_event_file *file = hist_data->event_file; 3677 struct track_data *track_data; 3678 struct action_data *action; 3679 3680 track_data = tracing_cond_snapshot_data(file->tr); 3681 if (!track_data) 3682 return; 3683 3684 if (!track_data->updated) 3685 return; 3686 3687 action = snapshot_action(hist_data); 3688 if (!action) 3689 return; 3690 3691 seq_puts(m, "\nSnapshot taken (see tracing/snapshot). Details:\n"); 3692 seq_printf(m, "\ttriggering value { %s(%s) }: %10llu", 3693 action->handler == HANDLER_ONMAX ? "onmax" : "onchange", 3694 action->track_data.var_str, track_data->track_val); 3695 3696 seq_puts(m, "\ttriggered by event with key: "); 3697 hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt); 3698 seq_putc(m, '\n'); 3699 } 3700 #else 3701 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data) 3702 { 3703 return false; 3704 } 3705 static void save_track_data_snapshot(struct hist_trigger_data *hist_data, 3706 struct tracing_map_elt *elt, void *rec, 3707 struct ring_buffer_event *rbe, void *key, 3708 struct action_data *data, 3709 u64 *var_ref_vals) {} 3710 static void track_data_snapshot_print(struct seq_file *m, 3711 struct hist_trigger_data *hist_data) {} 3712 #endif /* CONFIG_TRACER_SNAPSHOT */ 3713 3714 static void track_data_print(struct seq_file *m, 3715 struct hist_trigger_data *hist_data, 3716 struct tracing_map_elt *elt, 3717 struct action_data *data) 3718 { 3719 u64 track_val = get_track_val(hist_data, elt, data); 3720 unsigned int i, save_var_idx; 3721 3722 if (data->handler == HANDLER_ONMAX) 3723 seq_printf(m, "\n\tmax: %10llu", track_val); 3724 else if (data->handler == HANDLER_ONCHANGE) 3725 seq_printf(m, "\n\tchanged: %10llu", track_val); 3726 3727 if (data->action == ACTION_SNAPSHOT) 3728 return; 3729 3730 for (i = 0; i < hist_data->n_save_vars; i++) { 3731 struct hist_field *save_val = hist_data->save_vars[i]->val; 3732 struct hist_field *save_var = hist_data->save_vars[i]->var; 3733 u64 val; 3734 3735 save_var_idx = save_var->var.idx; 3736 3737 val = tracing_map_read_var(elt, save_var_idx); 3738 3739 if (save_val->flags & HIST_FIELD_FL_STRING) { 3740 seq_printf(m, " %s: %-32s", save_var->var.name, 3741 (char *)(uintptr_t)(val)); 3742 } else 3743 seq_printf(m, " %s: %10llu", save_var->var.name, val); 3744 } 3745 } 3746 3747 static void ontrack_action(struct hist_trigger_data *hist_data, 3748 struct tracing_map_elt *elt, void *rec, 3749 struct ring_buffer_event *rbe, void *key, 3750 struct action_data *data, u64 *var_ref_vals) 3751 { 3752 u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx]; 3753 3754 if (check_track_val(elt, data, var_val)) { 3755 save_track_val(hist_data, elt, data, var_val); 3756 save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals); 3757 } 3758 } 3759 3760 static void action_data_destroy(struct action_data *data) 3761 { 3762 unsigned int i; 3763 3764 lockdep_assert_held(&event_mutex); 3765 3766 kfree(data->action_name); 3767 3768 for (i = 0; i < data->n_params; i++) 3769 kfree(data->params[i]); 3770 3771 if (data->synth_event) 3772 data->synth_event->ref--; 3773 3774 kfree(data->synth_event_name); 3775 3776 kfree(data); 3777 } 3778 3779 static void track_data_destroy(struct hist_trigger_data *hist_data, 3780 struct action_data *data) 3781 { 3782 struct trace_event_file *file = hist_data->event_file; 3783 3784 destroy_hist_field(data->track_data.track_var, 0); 3785 3786 if (data->action == ACTION_SNAPSHOT) { 3787 struct track_data *track_data; 3788 3789 track_data = tracing_cond_snapshot_data(file->tr); 3790 if (track_data && track_data->hist_data == hist_data) { 3791 tracing_snapshot_cond_disable(file->tr); 3792 track_data_free(track_data); 3793 } 3794 } 3795 3796 kfree(data->track_data.var_str); 3797 3798 action_data_destroy(data); 3799 } 3800 3801 static int action_create(struct hist_trigger_data *hist_data, 3802 struct action_data *data); 3803 3804 static int track_data_create(struct hist_trigger_data *hist_data, 3805 struct action_data *data) 3806 { 3807 struct hist_field *var_field, *ref_field, *track_var = NULL; 3808 struct trace_event_file *file = hist_data->event_file; 3809 struct trace_array *tr = file->tr; 3810 char *track_data_var_str; 3811 int ret = 0; 3812 3813 track_data_var_str = data->track_data.var_str; 3814 if (track_data_var_str[0] != '$') { 3815 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str)); 3816 return -EINVAL; 3817 } 3818 track_data_var_str++; 3819 3820 var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str); 3821 if (!var_field) { 3822 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str)); 3823 return -EINVAL; 3824 } 3825 3826 ref_field = create_var_ref(hist_data, var_field, NULL, NULL); 3827 if (!ref_field) 3828 return -ENOMEM; 3829 3830 data->track_data.var_ref = ref_field; 3831 3832 if (data->handler == HANDLER_ONMAX) 3833 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64"); 3834 if (IS_ERR(track_var)) { 3835 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0); 3836 ret = PTR_ERR(track_var); 3837 goto out; 3838 } 3839 3840 if (data->handler == HANDLER_ONCHANGE) 3841 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64"); 3842 if (IS_ERR(track_var)) { 3843 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0); 3844 ret = PTR_ERR(track_var); 3845 goto out; 3846 } 3847 data->track_data.track_var = track_var; 3848 3849 ret = action_create(hist_data, data); 3850 out: 3851 return ret; 3852 } 3853 3854 static int parse_action_params(struct trace_array *tr, char *params, 3855 struct action_data *data) 3856 { 3857 char *param, *saved_param; 3858 bool first_param = true; 3859 int ret = 0; 3860 3861 while (params) { 3862 if (data->n_params >= SYNTH_FIELDS_MAX) { 3863 hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0); 3864 goto out; 3865 } 3866 3867 param = strsep(¶ms, ","); 3868 if (!param) { 3869 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0); 3870 ret = -EINVAL; 3871 goto out; 3872 } 3873 3874 param = strstrip(param); 3875 if (strlen(param) < 2) { 3876 hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param)); 3877 ret = -EINVAL; 3878 goto out; 3879 } 3880 3881 saved_param = kstrdup(param, GFP_KERNEL); 3882 if (!saved_param) { 3883 ret = -ENOMEM; 3884 goto out; 3885 } 3886 3887 if (first_param && data->use_trace_keyword) { 3888 data->synth_event_name = saved_param; 3889 first_param = false; 3890 continue; 3891 } 3892 first_param = false; 3893 3894 data->params[data->n_params++] = saved_param; 3895 } 3896 out: 3897 return ret; 3898 } 3899 3900 static int action_parse(struct trace_array *tr, char *str, struct action_data *data, 3901 enum handler_id handler) 3902 { 3903 char *action_name; 3904 int ret = 0; 3905 3906 strsep(&str, "."); 3907 if (!str) { 3908 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0); 3909 ret = -EINVAL; 3910 goto out; 3911 } 3912 3913 action_name = strsep(&str, "("); 3914 if (!action_name || !str) { 3915 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0); 3916 ret = -EINVAL; 3917 goto out; 3918 } 3919 3920 if (str_has_prefix(action_name, "save")) { 3921 char *params = strsep(&str, ")"); 3922 3923 if (!params) { 3924 hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0); 3925 ret = -EINVAL; 3926 goto out; 3927 } 3928 3929 ret = parse_action_params(tr, params, data); 3930 if (ret) 3931 goto out; 3932 3933 if (handler == HANDLER_ONMAX) 3934 data->track_data.check_val = check_track_val_max; 3935 else if (handler == HANDLER_ONCHANGE) 3936 data->track_data.check_val = check_track_val_changed; 3937 else { 3938 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name)); 3939 ret = -EINVAL; 3940 goto out; 3941 } 3942 3943 data->track_data.save_data = save_track_data_vars; 3944 data->fn = ontrack_action; 3945 data->action = ACTION_SAVE; 3946 } else if (str_has_prefix(action_name, "snapshot")) { 3947 char *params = strsep(&str, ")"); 3948 3949 if (!str) { 3950 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params)); 3951 ret = -EINVAL; 3952 goto out; 3953 } 3954 3955 if (handler == HANDLER_ONMAX) 3956 data->track_data.check_val = check_track_val_max; 3957 else if (handler == HANDLER_ONCHANGE) 3958 data->track_data.check_val = check_track_val_changed; 3959 else { 3960 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name)); 3961 ret = -EINVAL; 3962 goto out; 3963 } 3964 3965 data->track_data.save_data = save_track_data_snapshot; 3966 data->fn = ontrack_action; 3967 data->action = ACTION_SNAPSHOT; 3968 } else { 3969 char *params = strsep(&str, ")"); 3970 3971 if (str_has_prefix(action_name, "trace")) 3972 data->use_trace_keyword = true; 3973 3974 if (params) { 3975 ret = parse_action_params(tr, params, data); 3976 if (ret) 3977 goto out; 3978 } 3979 3980 if (handler == HANDLER_ONMAX) 3981 data->track_data.check_val = check_track_val_max; 3982 else if (handler == HANDLER_ONCHANGE) 3983 data->track_data.check_val = check_track_val_changed; 3984 3985 if (handler != HANDLER_ONMATCH) { 3986 data->track_data.save_data = action_trace; 3987 data->fn = ontrack_action; 3988 } else 3989 data->fn = action_trace; 3990 3991 data->action = ACTION_TRACE; 3992 } 3993 3994 data->action_name = kstrdup(action_name, GFP_KERNEL); 3995 if (!data->action_name) { 3996 ret = -ENOMEM; 3997 goto out; 3998 } 3999 4000 data->handler = handler; 4001 out: 4002 return ret; 4003 } 4004 4005 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data, 4006 char *str, enum handler_id handler) 4007 { 4008 struct action_data *data; 4009 int ret = -EINVAL; 4010 char *var_str; 4011 4012 data = kzalloc(sizeof(*data), GFP_KERNEL); 4013 if (!data) 4014 return ERR_PTR(-ENOMEM); 4015 4016 var_str = strsep(&str, ")"); 4017 if (!var_str || !str) { 4018 ret = -EINVAL; 4019 goto free; 4020 } 4021 4022 data->track_data.var_str = kstrdup(var_str, GFP_KERNEL); 4023 if (!data->track_data.var_str) { 4024 ret = -ENOMEM; 4025 goto free; 4026 } 4027 4028 ret = action_parse(hist_data->event_file->tr, str, data, handler); 4029 if (ret) 4030 goto free; 4031 out: 4032 return data; 4033 free: 4034 track_data_destroy(hist_data, data); 4035 data = ERR_PTR(ret); 4036 goto out; 4037 } 4038 4039 static void onmatch_destroy(struct action_data *data) 4040 { 4041 kfree(data->match_data.event); 4042 kfree(data->match_data.event_system); 4043 4044 action_data_destroy(data); 4045 } 4046 4047 static void destroy_field_var(struct field_var *field_var) 4048 { 4049 if (!field_var) 4050 return; 4051 4052 destroy_hist_field(field_var->var, 0); 4053 destroy_hist_field(field_var->val, 0); 4054 4055 kfree(field_var); 4056 } 4057 4058 static void destroy_field_vars(struct hist_trigger_data *hist_data) 4059 { 4060 unsigned int i; 4061 4062 for (i = 0; i < hist_data->n_field_vars; i++) 4063 destroy_field_var(hist_data->field_vars[i]); 4064 } 4065 4066 static void save_field_var(struct hist_trigger_data *hist_data, 4067 struct field_var *field_var) 4068 { 4069 hist_data->field_vars[hist_data->n_field_vars++] = field_var; 4070 4071 if (field_var->val->flags & HIST_FIELD_FL_STRING) 4072 hist_data->n_field_var_str++; 4073 } 4074 4075 4076 static int check_synth_field(struct synth_event *event, 4077 struct hist_field *hist_field, 4078 unsigned int field_pos) 4079 { 4080 struct synth_field *field; 4081 4082 if (field_pos >= event->n_fields) 4083 return -EINVAL; 4084 4085 field = event->fields[field_pos]; 4086 4087 if (strcmp(field->type, hist_field->type) != 0) 4088 return -EINVAL; 4089 4090 return 0; 4091 } 4092 4093 static struct hist_field * 4094 trace_action_find_var(struct hist_trigger_data *hist_data, 4095 struct action_data *data, 4096 char *system, char *event, char *var) 4097 { 4098 struct trace_array *tr = hist_data->event_file->tr; 4099 struct hist_field *hist_field; 4100 4101 var++; /* skip '$' */ 4102 4103 hist_field = find_target_event_var(hist_data, system, event, var); 4104 if (!hist_field) { 4105 if (!system && data->handler == HANDLER_ONMATCH) { 4106 system = data->match_data.event_system; 4107 event = data->match_data.event; 4108 } 4109 4110 hist_field = find_event_var(hist_data, system, event, var); 4111 } 4112 4113 if (!hist_field) 4114 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var)); 4115 4116 return hist_field; 4117 } 4118 4119 static struct hist_field * 4120 trace_action_create_field_var(struct hist_trigger_data *hist_data, 4121 struct action_data *data, char *system, 4122 char *event, char *var) 4123 { 4124 struct hist_field *hist_field = NULL; 4125 struct field_var *field_var; 4126 4127 /* 4128 * First try to create a field var on the target event (the 4129 * currently being defined). This will create a variable for 4130 * unqualified fields on the target event, or if qualified, 4131 * target fields that have qualified names matching the target. 4132 */ 4133 field_var = create_target_field_var(hist_data, system, event, var); 4134 4135 if (field_var && !IS_ERR(field_var)) { 4136 save_field_var(hist_data, field_var); 4137 hist_field = field_var->var; 4138 } else { 4139 field_var = NULL; 4140 /* 4141 * If no explicit system.event is specfied, default to 4142 * looking for fields on the onmatch(system.event.xxx) 4143 * event. 4144 */ 4145 if (!system && data->handler == HANDLER_ONMATCH) { 4146 system = data->match_data.event_system; 4147 event = data->match_data.event; 4148 } 4149 4150 /* 4151 * At this point, we're looking at a field on another 4152 * event. Because we can't modify a hist trigger on 4153 * another event to add a variable for a field, we need 4154 * to create a new trigger on that event and create the 4155 * variable at the same time. 4156 */ 4157 hist_field = create_field_var_hist(hist_data, system, event, var); 4158 if (IS_ERR(hist_field)) 4159 goto free; 4160 } 4161 out: 4162 return hist_field; 4163 free: 4164 destroy_field_var(field_var); 4165 hist_field = NULL; 4166 goto out; 4167 } 4168 4169 static int trace_action_create(struct hist_trigger_data *hist_data, 4170 struct action_data *data) 4171 { 4172 struct trace_array *tr = hist_data->event_file->tr; 4173 char *event_name, *param, *system = NULL; 4174 struct hist_field *hist_field, *var_ref; 4175 unsigned int i, var_ref_idx; 4176 unsigned int field_pos = 0; 4177 struct synth_event *event; 4178 char *synth_event_name; 4179 int ret = 0; 4180 4181 lockdep_assert_held(&event_mutex); 4182 4183 if (data->use_trace_keyword) 4184 synth_event_name = data->synth_event_name; 4185 else 4186 synth_event_name = data->action_name; 4187 4188 event = find_synth_event(synth_event_name); 4189 if (!event) { 4190 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name)); 4191 return -EINVAL; 4192 } 4193 4194 event->ref++; 4195 4196 var_ref_idx = hist_data->n_var_refs; 4197 4198 for (i = 0; i < data->n_params; i++) { 4199 char *p; 4200 4201 p = param = kstrdup(data->params[i], GFP_KERNEL); 4202 if (!param) { 4203 ret = -ENOMEM; 4204 goto err; 4205 } 4206 4207 system = strsep(¶m, "."); 4208 if (!param) { 4209 param = (char *)system; 4210 system = event_name = NULL; 4211 } else { 4212 event_name = strsep(¶m, "."); 4213 if (!param) { 4214 kfree(p); 4215 ret = -EINVAL; 4216 goto err; 4217 } 4218 } 4219 4220 if (param[0] == '$') 4221 hist_field = trace_action_find_var(hist_data, data, 4222 system, event_name, 4223 param); 4224 else 4225 hist_field = trace_action_create_field_var(hist_data, 4226 data, 4227 system, 4228 event_name, 4229 param); 4230 4231 if (!hist_field) { 4232 kfree(p); 4233 ret = -EINVAL; 4234 goto err; 4235 } 4236 4237 if (check_synth_field(event, hist_field, field_pos) == 0) { 4238 var_ref = create_var_ref(hist_data, hist_field, 4239 system, event_name); 4240 if (!var_ref) { 4241 kfree(p); 4242 ret = -ENOMEM; 4243 goto err; 4244 } 4245 4246 field_pos++; 4247 kfree(p); 4248 continue; 4249 } 4250 4251 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param)); 4252 kfree(p); 4253 ret = -EINVAL; 4254 goto err; 4255 } 4256 4257 if (field_pos != event->n_fields) { 4258 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name)); 4259 ret = -EINVAL; 4260 goto err; 4261 } 4262 4263 data->synth_event = event; 4264 data->var_ref_idx = var_ref_idx; 4265 out: 4266 return ret; 4267 err: 4268 event->ref--; 4269 4270 goto out; 4271 } 4272 4273 static int action_create(struct hist_trigger_data *hist_data, 4274 struct action_data *data) 4275 { 4276 struct trace_event_file *file = hist_data->event_file; 4277 struct trace_array *tr = file->tr; 4278 struct track_data *track_data; 4279 struct field_var *field_var; 4280 unsigned int i; 4281 char *param; 4282 int ret = 0; 4283 4284 if (data->action == ACTION_TRACE) 4285 return trace_action_create(hist_data, data); 4286 4287 if (data->action == ACTION_SNAPSHOT) { 4288 track_data = track_data_alloc(hist_data->key_size, data, hist_data); 4289 if (IS_ERR(track_data)) { 4290 ret = PTR_ERR(track_data); 4291 goto out; 4292 } 4293 4294 ret = tracing_snapshot_cond_enable(file->tr, track_data, 4295 cond_snapshot_update); 4296 if (ret) 4297 track_data_free(track_data); 4298 4299 goto out; 4300 } 4301 4302 if (data->action == ACTION_SAVE) { 4303 if (hist_data->n_save_vars) { 4304 ret = -EEXIST; 4305 hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0); 4306 goto out; 4307 } 4308 4309 for (i = 0; i < data->n_params; i++) { 4310 param = kstrdup(data->params[i], GFP_KERNEL); 4311 if (!param) { 4312 ret = -ENOMEM; 4313 goto out; 4314 } 4315 4316 field_var = create_target_field_var(hist_data, NULL, NULL, param); 4317 if (IS_ERR(field_var)) { 4318 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL, 4319 errpos(param)); 4320 ret = PTR_ERR(field_var); 4321 kfree(param); 4322 goto out; 4323 } 4324 4325 hist_data->save_vars[hist_data->n_save_vars++] = field_var; 4326 if (field_var->val->flags & HIST_FIELD_FL_STRING) 4327 hist_data->n_save_var_str++; 4328 kfree(param); 4329 } 4330 } 4331 out: 4332 return ret; 4333 } 4334 4335 static int onmatch_create(struct hist_trigger_data *hist_data, 4336 struct action_data *data) 4337 { 4338 return action_create(hist_data, data); 4339 } 4340 4341 static struct action_data *onmatch_parse(struct trace_array *tr, char *str) 4342 { 4343 char *match_event, *match_event_system; 4344 struct action_data *data; 4345 int ret = -EINVAL; 4346 4347 data = kzalloc(sizeof(*data), GFP_KERNEL); 4348 if (!data) 4349 return ERR_PTR(-ENOMEM); 4350 4351 match_event = strsep(&str, ")"); 4352 if (!match_event || !str) { 4353 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event)); 4354 goto free; 4355 } 4356 4357 match_event_system = strsep(&match_event, "."); 4358 if (!match_event) { 4359 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system)); 4360 goto free; 4361 } 4362 4363 if (IS_ERR(event_file(tr, match_event_system, match_event))) { 4364 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event)); 4365 goto free; 4366 } 4367 4368 data->match_data.event = kstrdup(match_event, GFP_KERNEL); 4369 if (!data->match_data.event) { 4370 ret = -ENOMEM; 4371 goto free; 4372 } 4373 4374 data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL); 4375 if (!data->match_data.event_system) { 4376 ret = -ENOMEM; 4377 goto free; 4378 } 4379 4380 ret = action_parse(tr, str, data, HANDLER_ONMATCH); 4381 if (ret) 4382 goto free; 4383 out: 4384 return data; 4385 free: 4386 onmatch_destroy(data); 4387 data = ERR_PTR(ret); 4388 goto out; 4389 } 4390 4391 static int create_hitcount_val(struct hist_trigger_data *hist_data) 4392 { 4393 hist_data->fields[HITCOUNT_IDX] = 4394 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL); 4395 if (!hist_data->fields[HITCOUNT_IDX]) 4396 return -ENOMEM; 4397 4398 hist_data->n_vals++; 4399 hist_data->n_fields++; 4400 4401 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX)) 4402 return -EINVAL; 4403 4404 return 0; 4405 } 4406 4407 static int __create_val_field(struct hist_trigger_data *hist_data, 4408 unsigned int val_idx, 4409 struct trace_event_file *file, 4410 char *var_name, char *field_str, 4411 unsigned long flags) 4412 { 4413 struct hist_field *hist_field; 4414 int ret = 0; 4415 4416 hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0); 4417 if (IS_ERR(hist_field)) { 4418 ret = PTR_ERR(hist_field); 4419 goto out; 4420 } 4421 4422 hist_data->fields[val_idx] = hist_field; 4423 4424 ++hist_data->n_vals; 4425 ++hist_data->n_fields; 4426 4427 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 4428 ret = -EINVAL; 4429 out: 4430 return ret; 4431 } 4432 4433 static int create_val_field(struct hist_trigger_data *hist_data, 4434 unsigned int val_idx, 4435 struct trace_event_file *file, 4436 char *field_str) 4437 { 4438 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX)) 4439 return -EINVAL; 4440 4441 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0); 4442 } 4443 4444 static int create_var_field(struct hist_trigger_data *hist_data, 4445 unsigned int val_idx, 4446 struct trace_event_file *file, 4447 char *var_name, char *expr_str) 4448 { 4449 struct trace_array *tr = hist_data->event_file->tr; 4450 unsigned long flags = 0; 4451 4452 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 4453 return -EINVAL; 4454 4455 if (find_var(hist_data, file, var_name) && !hist_data->remove) { 4456 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name)); 4457 return -EINVAL; 4458 } 4459 4460 flags |= HIST_FIELD_FL_VAR; 4461 hist_data->n_vars++; 4462 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX)) 4463 return -EINVAL; 4464 4465 return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags); 4466 } 4467 4468 static int create_val_fields(struct hist_trigger_data *hist_data, 4469 struct trace_event_file *file) 4470 { 4471 char *fields_str, *field_str; 4472 unsigned int i, j = 1; 4473 int ret; 4474 4475 ret = create_hitcount_val(hist_data); 4476 if (ret) 4477 goto out; 4478 4479 fields_str = hist_data->attrs->vals_str; 4480 if (!fields_str) 4481 goto out; 4482 4483 strsep(&fields_str, "="); 4484 if (!fields_str) 4485 goto out; 4486 4487 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX && 4488 j < TRACING_MAP_VALS_MAX; i++) { 4489 field_str = strsep(&fields_str, ","); 4490 if (!field_str) 4491 break; 4492 4493 if (strcmp(field_str, "hitcount") == 0) 4494 continue; 4495 4496 ret = create_val_field(hist_data, j++, file, field_str); 4497 if (ret) 4498 goto out; 4499 } 4500 4501 if (fields_str && (strcmp(fields_str, "hitcount") != 0)) 4502 ret = -EINVAL; 4503 out: 4504 return ret; 4505 } 4506 4507 static int create_key_field(struct hist_trigger_data *hist_data, 4508 unsigned int key_idx, 4509 unsigned int key_offset, 4510 struct trace_event_file *file, 4511 char *field_str) 4512 { 4513 struct trace_array *tr = hist_data->event_file->tr; 4514 struct hist_field *hist_field = NULL; 4515 unsigned long flags = 0; 4516 unsigned int key_size; 4517 int ret = 0; 4518 4519 if (WARN_ON(key_idx >= HIST_FIELDS_MAX)) 4520 return -EINVAL; 4521 4522 flags |= HIST_FIELD_FL_KEY; 4523 4524 if (strcmp(field_str, "stacktrace") == 0) { 4525 flags |= HIST_FIELD_FL_STACKTRACE; 4526 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH; 4527 hist_field = create_hist_field(hist_data, NULL, flags, NULL); 4528 } else { 4529 hist_field = parse_expr(hist_data, file, field_str, flags, 4530 NULL, 0); 4531 if (IS_ERR(hist_field)) { 4532 ret = PTR_ERR(hist_field); 4533 goto out; 4534 } 4535 4536 if (field_has_hist_vars(hist_field, 0)) { 4537 hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str)); 4538 destroy_hist_field(hist_field, 0); 4539 ret = -EINVAL; 4540 goto out; 4541 } 4542 4543 key_size = hist_field->size; 4544 } 4545 4546 hist_data->fields[key_idx] = hist_field; 4547 4548 key_size = ALIGN(key_size, sizeof(u64)); 4549 hist_data->fields[key_idx]->size = key_size; 4550 hist_data->fields[key_idx]->offset = key_offset; 4551 4552 hist_data->key_size += key_size; 4553 4554 if (hist_data->key_size > HIST_KEY_SIZE_MAX) { 4555 ret = -EINVAL; 4556 goto out; 4557 } 4558 4559 hist_data->n_keys++; 4560 hist_data->n_fields++; 4561 4562 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX)) 4563 return -EINVAL; 4564 4565 ret = key_size; 4566 out: 4567 return ret; 4568 } 4569 4570 static int create_key_fields(struct hist_trigger_data *hist_data, 4571 struct trace_event_file *file) 4572 { 4573 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals; 4574 char *fields_str, *field_str; 4575 int ret = -EINVAL; 4576 4577 fields_str = hist_data->attrs->keys_str; 4578 if (!fields_str) 4579 goto out; 4580 4581 strsep(&fields_str, "="); 4582 if (!fields_str) 4583 goto out; 4584 4585 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) { 4586 field_str = strsep(&fields_str, ","); 4587 if (!field_str) 4588 break; 4589 ret = create_key_field(hist_data, i, key_offset, 4590 file, field_str); 4591 if (ret < 0) 4592 goto out; 4593 key_offset += ret; 4594 } 4595 if (fields_str) { 4596 ret = -EINVAL; 4597 goto out; 4598 } 4599 ret = 0; 4600 out: 4601 return ret; 4602 } 4603 4604 static int create_var_fields(struct hist_trigger_data *hist_data, 4605 struct trace_event_file *file) 4606 { 4607 unsigned int i, j = hist_data->n_vals; 4608 int ret = 0; 4609 4610 unsigned int n_vars = hist_data->attrs->var_defs.n_vars; 4611 4612 for (i = 0; i < n_vars; i++) { 4613 char *var_name = hist_data->attrs->var_defs.name[i]; 4614 char *expr = hist_data->attrs->var_defs.expr[i]; 4615 4616 ret = create_var_field(hist_data, j++, file, var_name, expr); 4617 if (ret) 4618 goto out; 4619 } 4620 out: 4621 return ret; 4622 } 4623 4624 static void free_var_defs(struct hist_trigger_data *hist_data) 4625 { 4626 unsigned int i; 4627 4628 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 4629 kfree(hist_data->attrs->var_defs.name[i]); 4630 kfree(hist_data->attrs->var_defs.expr[i]); 4631 } 4632 4633 hist_data->attrs->var_defs.n_vars = 0; 4634 } 4635 4636 static int parse_var_defs(struct hist_trigger_data *hist_data) 4637 { 4638 struct trace_array *tr = hist_data->event_file->tr; 4639 char *s, *str, *var_name, *field_str; 4640 unsigned int i, j, n_vars = 0; 4641 int ret = 0; 4642 4643 for (i = 0; i < hist_data->attrs->n_assignments; i++) { 4644 str = hist_data->attrs->assignment_str[i]; 4645 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) { 4646 field_str = strsep(&str, ","); 4647 if (!field_str) 4648 break; 4649 4650 var_name = strsep(&field_str, "="); 4651 if (!var_name || !field_str) { 4652 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT, 4653 errpos(var_name)); 4654 ret = -EINVAL; 4655 goto free; 4656 } 4657 4658 if (n_vars == TRACING_MAP_VARS_MAX) { 4659 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name)); 4660 ret = -EINVAL; 4661 goto free; 4662 } 4663 4664 s = kstrdup(var_name, GFP_KERNEL); 4665 if (!s) { 4666 ret = -ENOMEM; 4667 goto free; 4668 } 4669 hist_data->attrs->var_defs.name[n_vars] = s; 4670 4671 s = kstrdup(field_str, GFP_KERNEL); 4672 if (!s) { 4673 kfree(hist_data->attrs->var_defs.name[n_vars]); 4674 ret = -ENOMEM; 4675 goto free; 4676 } 4677 hist_data->attrs->var_defs.expr[n_vars++] = s; 4678 4679 hist_data->attrs->var_defs.n_vars = n_vars; 4680 } 4681 } 4682 4683 return ret; 4684 free: 4685 free_var_defs(hist_data); 4686 4687 return ret; 4688 } 4689 4690 static int create_hist_fields(struct hist_trigger_data *hist_data, 4691 struct trace_event_file *file) 4692 { 4693 int ret; 4694 4695 ret = parse_var_defs(hist_data); 4696 if (ret) 4697 goto out; 4698 4699 ret = create_val_fields(hist_data, file); 4700 if (ret) 4701 goto out; 4702 4703 ret = create_var_fields(hist_data, file); 4704 if (ret) 4705 goto out; 4706 4707 ret = create_key_fields(hist_data, file); 4708 if (ret) 4709 goto out; 4710 out: 4711 free_var_defs(hist_data); 4712 4713 return ret; 4714 } 4715 4716 static int is_descending(const char *str) 4717 { 4718 if (!str) 4719 return 0; 4720 4721 if (strcmp(str, "descending") == 0) 4722 return 1; 4723 4724 if (strcmp(str, "ascending") == 0) 4725 return 0; 4726 4727 return -EINVAL; 4728 } 4729 4730 static int create_sort_keys(struct hist_trigger_data *hist_data) 4731 { 4732 char *fields_str = hist_data->attrs->sort_key_str; 4733 struct tracing_map_sort_key *sort_key; 4734 int descending, ret = 0; 4735 unsigned int i, j, k; 4736 4737 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */ 4738 4739 if (!fields_str) 4740 goto out; 4741 4742 strsep(&fields_str, "="); 4743 if (!fields_str) { 4744 ret = -EINVAL; 4745 goto out; 4746 } 4747 4748 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) { 4749 struct hist_field *hist_field; 4750 char *field_str, *field_name; 4751 const char *test_name; 4752 4753 sort_key = &hist_data->sort_keys[i]; 4754 4755 field_str = strsep(&fields_str, ","); 4756 if (!field_str) { 4757 if (i == 0) 4758 ret = -EINVAL; 4759 break; 4760 } 4761 4762 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) { 4763 ret = -EINVAL; 4764 break; 4765 } 4766 4767 field_name = strsep(&field_str, "."); 4768 if (!field_name) { 4769 ret = -EINVAL; 4770 break; 4771 } 4772 4773 if (strcmp(field_name, "hitcount") == 0) { 4774 descending = is_descending(field_str); 4775 if (descending < 0) { 4776 ret = descending; 4777 break; 4778 } 4779 sort_key->descending = descending; 4780 continue; 4781 } 4782 4783 for (j = 1, k = 1; j < hist_data->n_fields; j++) { 4784 unsigned int idx; 4785 4786 hist_field = hist_data->fields[j]; 4787 if (hist_field->flags & HIST_FIELD_FL_VAR) 4788 continue; 4789 4790 idx = k++; 4791 4792 test_name = hist_field_name(hist_field, 0); 4793 4794 if (strcmp(field_name, test_name) == 0) { 4795 sort_key->field_idx = idx; 4796 descending = is_descending(field_str); 4797 if (descending < 0) { 4798 ret = descending; 4799 goto out; 4800 } 4801 sort_key->descending = descending; 4802 break; 4803 } 4804 } 4805 if (j == hist_data->n_fields) { 4806 ret = -EINVAL; 4807 break; 4808 } 4809 } 4810 4811 hist_data->n_sort_keys = i; 4812 out: 4813 return ret; 4814 } 4815 4816 static void destroy_actions(struct hist_trigger_data *hist_data) 4817 { 4818 unsigned int i; 4819 4820 for (i = 0; i < hist_data->n_actions; i++) { 4821 struct action_data *data = hist_data->actions[i]; 4822 4823 if (data->handler == HANDLER_ONMATCH) 4824 onmatch_destroy(data); 4825 else if (data->handler == HANDLER_ONMAX || 4826 data->handler == HANDLER_ONCHANGE) 4827 track_data_destroy(hist_data, data); 4828 else 4829 kfree(data); 4830 } 4831 } 4832 4833 static int parse_actions(struct hist_trigger_data *hist_data) 4834 { 4835 struct trace_array *tr = hist_data->event_file->tr; 4836 struct action_data *data; 4837 unsigned int i; 4838 int ret = 0; 4839 char *str; 4840 int len; 4841 4842 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4843 str = hist_data->attrs->action_str[i]; 4844 4845 if ((len = str_has_prefix(str, "onmatch("))) { 4846 char *action_str = str + len; 4847 4848 data = onmatch_parse(tr, action_str); 4849 if (IS_ERR(data)) { 4850 ret = PTR_ERR(data); 4851 break; 4852 } 4853 } else if ((len = str_has_prefix(str, "onmax("))) { 4854 char *action_str = str + len; 4855 4856 data = track_data_parse(hist_data, action_str, 4857 HANDLER_ONMAX); 4858 if (IS_ERR(data)) { 4859 ret = PTR_ERR(data); 4860 break; 4861 } 4862 } else if ((len = str_has_prefix(str, "onchange("))) { 4863 char *action_str = str + len; 4864 4865 data = track_data_parse(hist_data, action_str, 4866 HANDLER_ONCHANGE); 4867 if (IS_ERR(data)) { 4868 ret = PTR_ERR(data); 4869 break; 4870 } 4871 } else { 4872 ret = -EINVAL; 4873 break; 4874 } 4875 4876 hist_data->actions[hist_data->n_actions++] = data; 4877 } 4878 4879 return ret; 4880 } 4881 4882 static int create_actions(struct hist_trigger_data *hist_data) 4883 { 4884 struct action_data *data; 4885 unsigned int i; 4886 int ret = 0; 4887 4888 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4889 data = hist_data->actions[i]; 4890 4891 if (data->handler == HANDLER_ONMATCH) { 4892 ret = onmatch_create(hist_data, data); 4893 if (ret) 4894 break; 4895 } else if (data->handler == HANDLER_ONMAX || 4896 data->handler == HANDLER_ONCHANGE) { 4897 ret = track_data_create(hist_data, data); 4898 if (ret) 4899 break; 4900 } else { 4901 ret = -EINVAL; 4902 break; 4903 } 4904 } 4905 4906 return ret; 4907 } 4908 4909 static void print_actions(struct seq_file *m, 4910 struct hist_trigger_data *hist_data, 4911 struct tracing_map_elt *elt) 4912 { 4913 unsigned int i; 4914 4915 for (i = 0; i < hist_data->n_actions; i++) { 4916 struct action_data *data = hist_data->actions[i]; 4917 4918 if (data->action == ACTION_SNAPSHOT) 4919 continue; 4920 4921 if (data->handler == HANDLER_ONMAX || 4922 data->handler == HANDLER_ONCHANGE) 4923 track_data_print(m, hist_data, elt, data); 4924 } 4925 } 4926 4927 static void print_action_spec(struct seq_file *m, 4928 struct hist_trigger_data *hist_data, 4929 struct action_data *data) 4930 { 4931 unsigned int i; 4932 4933 if (data->action == ACTION_SAVE) { 4934 for (i = 0; i < hist_data->n_save_vars; i++) { 4935 seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name); 4936 if (i < hist_data->n_save_vars - 1) 4937 seq_puts(m, ","); 4938 } 4939 } else if (data->action == ACTION_TRACE) { 4940 if (data->use_trace_keyword) 4941 seq_printf(m, "%s", data->synth_event_name); 4942 for (i = 0; i < data->n_params; i++) { 4943 if (i || data->use_trace_keyword) 4944 seq_puts(m, ","); 4945 seq_printf(m, "%s", data->params[i]); 4946 } 4947 } 4948 } 4949 4950 static void print_track_data_spec(struct seq_file *m, 4951 struct hist_trigger_data *hist_data, 4952 struct action_data *data) 4953 { 4954 if (data->handler == HANDLER_ONMAX) 4955 seq_puts(m, ":onmax("); 4956 else if (data->handler == HANDLER_ONCHANGE) 4957 seq_puts(m, ":onchange("); 4958 seq_printf(m, "%s", data->track_data.var_str); 4959 seq_printf(m, ").%s(", data->action_name); 4960 4961 print_action_spec(m, hist_data, data); 4962 4963 seq_puts(m, ")"); 4964 } 4965 4966 static void print_onmatch_spec(struct seq_file *m, 4967 struct hist_trigger_data *hist_data, 4968 struct action_data *data) 4969 { 4970 seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system, 4971 data->match_data.event); 4972 4973 seq_printf(m, "%s(", data->action_name); 4974 4975 print_action_spec(m, hist_data, data); 4976 4977 seq_puts(m, ")"); 4978 } 4979 4980 static bool actions_match(struct hist_trigger_data *hist_data, 4981 struct hist_trigger_data *hist_data_test) 4982 { 4983 unsigned int i, j; 4984 4985 if (hist_data->n_actions != hist_data_test->n_actions) 4986 return false; 4987 4988 for (i = 0; i < hist_data->n_actions; i++) { 4989 struct action_data *data = hist_data->actions[i]; 4990 struct action_data *data_test = hist_data_test->actions[i]; 4991 char *action_name, *action_name_test; 4992 4993 if (data->handler != data_test->handler) 4994 return false; 4995 if (data->action != data_test->action) 4996 return false; 4997 4998 if (data->n_params != data_test->n_params) 4999 return false; 5000 5001 for (j = 0; j < data->n_params; j++) { 5002 if (strcmp(data->params[j], data_test->params[j]) != 0) 5003 return false; 5004 } 5005 5006 if (data->use_trace_keyword) 5007 action_name = data->synth_event_name; 5008 else 5009 action_name = data->action_name; 5010 5011 if (data_test->use_trace_keyword) 5012 action_name_test = data_test->synth_event_name; 5013 else 5014 action_name_test = data_test->action_name; 5015 5016 if (strcmp(action_name, action_name_test) != 0) 5017 return false; 5018 5019 if (data->handler == HANDLER_ONMATCH) { 5020 if (strcmp(data->match_data.event_system, 5021 data_test->match_data.event_system) != 0) 5022 return false; 5023 if (strcmp(data->match_data.event, 5024 data_test->match_data.event) != 0) 5025 return false; 5026 } else if (data->handler == HANDLER_ONMAX || 5027 data->handler == HANDLER_ONCHANGE) { 5028 if (strcmp(data->track_data.var_str, 5029 data_test->track_data.var_str) != 0) 5030 return false; 5031 } 5032 } 5033 5034 return true; 5035 } 5036 5037 5038 static void print_actions_spec(struct seq_file *m, 5039 struct hist_trigger_data *hist_data) 5040 { 5041 unsigned int i; 5042 5043 for (i = 0; i < hist_data->n_actions; i++) { 5044 struct action_data *data = hist_data->actions[i]; 5045 5046 if (data->handler == HANDLER_ONMATCH) 5047 print_onmatch_spec(m, hist_data, data); 5048 else if (data->handler == HANDLER_ONMAX || 5049 data->handler == HANDLER_ONCHANGE) 5050 print_track_data_spec(m, hist_data, data); 5051 } 5052 } 5053 5054 static void destroy_field_var_hists(struct hist_trigger_data *hist_data) 5055 { 5056 unsigned int i; 5057 5058 for (i = 0; i < hist_data->n_field_var_hists; i++) { 5059 kfree(hist_data->field_var_hists[i]->cmd); 5060 kfree(hist_data->field_var_hists[i]); 5061 } 5062 } 5063 5064 static void destroy_hist_data(struct hist_trigger_data *hist_data) 5065 { 5066 if (!hist_data) 5067 return; 5068 5069 destroy_hist_trigger_attrs(hist_data->attrs); 5070 destroy_hist_fields(hist_data); 5071 tracing_map_destroy(hist_data->map); 5072 5073 destroy_actions(hist_data); 5074 destroy_field_vars(hist_data); 5075 destroy_field_var_hists(hist_data); 5076 5077 kfree(hist_data); 5078 } 5079 5080 static int create_tracing_map_fields(struct hist_trigger_data *hist_data) 5081 { 5082 struct tracing_map *map = hist_data->map; 5083 struct ftrace_event_field *field; 5084 struct hist_field *hist_field; 5085 int i, idx = 0; 5086 5087 for_each_hist_field(i, hist_data) { 5088 hist_field = hist_data->fields[i]; 5089 if (hist_field->flags & HIST_FIELD_FL_KEY) { 5090 tracing_map_cmp_fn_t cmp_fn; 5091 5092 field = hist_field->field; 5093 5094 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE) 5095 cmp_fn = tracing_map_cmp_none; 5096 else if (!field) 5097 cmp_fn = tracing_map_cmp_num(hist_field->size, 5098 hist_field->is_signed); 5099 else if (is_string_field(field)) 5100 cmp_fn = tracing_map_cmp_string; 5101 else 5102 cmp_fn = tracing_map_cmp_num(field->size, 5103 field->is_signed); 5104 idx = tracing_map_add_key_field(map, 5105 hist_field->offset, 5106 cmp_fn); 5107 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR)) 5108 idx = tracing_map_add_sum_field(map); 5109 5110 if (idx < 0) 5111 return idx; 5112 5113 if (hist_field->flags & HIST_FIELD_FL_VAR) { 5114 idx = tracing_map_add_var(map); 5115 if (idx < 0) 5116 return idx; 5117 hist_field->var.idx = idx; 5118 hist_field->var.hist_data = hist_data; 5119 } 5120 } 5121 5122 return 0; 5123 } 5124 5125 static struct hist_trigger_data * 5126 create_hist_data(unsigned int map_bits, 5127 struct hist_trigger_attrs *attrs, 5128 struct trace_event_file *file, 5129 bool remove) 5130 { 5131 const struct tracing_map_ops *map_ops = NULL; 5132 struct hist_trigger_data *hist_data; 5133 int ret = 0; 5134 5135 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL); 5136 if (!hist_data) 5137 return ERR_PTR(-ENOMEM); 5138 5139 hist_data->attrs = attrs; 5140 hist_data->remove = remove; 5141 hist_data->event_file = file; 5142 5143 ret = parse_actions(hist_data); 5144 if (ret) 5145 goto free; 5146 5147 ret = create_hist_fields(hist_data, file); 5148 if (ret) 5149 goto free; 5150 5151 ret = create_sort_keys(hist_data); 5152 if (ret) 5153 goto free; 5154 5155 map_ops = &hist_trigger_elt_data_ops; 5156 5157 hist_data->map = tracing_map_create(map_bits, hist_data->key_size, 5158 map_ops, hist_data); 5159 if (IS_ERR(hist_data->map)) { 5160 ret = PTR_ERR(hist_data->map); 5161 hist_data->map = NULL; 5162 goto free; 5163 } 5164 5165 ret = create_tracing_map_fields(hist_data); 5166 if (ret) 5167 goto free; 5168 out: 5169 return hist_data; 5170 free: 5171 hist_data->attrs = NULL; 5172 5173 destroy_hist_data(hist_data); 5174 5175 hist_data = ERR_PTR(ret); 5176 5177 goto out; 5178 } 5179 5180 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, 5181 struct tracing_map_elt *elt, void *rec, 5182 struct ring_buffer_event *rbe, 5183 u64 *var_ref_vals) 5184 { 5185 struct hist_elt_data *elt_data; 5186 struct hist_field *hist_field; 5187 unsigned int i, var_idx; 5188 u64 hist_val; 5189 5190 elt_data = elt->private_data; 5191 elt_data->var_ref_vals = var_ref_vals; 5192 5193 for_each_hist_val_field(i, hist_data) { 5194 hist_field = hist_data->fields[i]; 5195 hist_val = hist_field->fn(hist_field, elt, rbe, rec); 5196 if (hist_field->flags & HIST_FIELD_FL_VAR) { 5197 var_idx = hist_field->var.idx; 5198 tracing_map_set_var(elt, var_idx, hist_val); 5199 continue; 5200 } 5201 tracing_map_update_sum(elt, i, hist_val); 5202 } 5203 5204 for_each_hist_key_field(i, hist_data) { 5205 hist_field = hist_data->fields[i]; 5206 if (hist_field->flags & HIST_FIELD_FL_VAR) { 5207 hist_val = hist_field->fn(hist_field, elt, rbe, rec); 5208 var_idx = hist_field->var.idx; 5209 tracing_map_set_var(elt, var_idx, hist_val); 5210 } 5211 } 5212 5213 update_field_vars(hist_data, elt, rbe, rec); 5214 } 5215 5216 static inline void add_to_key(char *compound_key, void *key, 5217 struct hist_field *key_field, void *rec) 5218 { 5219 size_t size = key_field->size; 5220 5221 if (key_field->flags & HIST_FIELD_FL_STRING) { 5222 struct ftrace_event_field *field; 5223 5224 field = key_field->field; 5225 if (field->filter_type == FILTER_DYN_STRING) 5226 size = *(u32 *)(rec + field->offset) >> 16; 5227 else if (field->filter_type == FILTER_PTR_STRING) 5228 size = strlen(key); 5229 else if (field->filter_type == FILTER_STATIC_STRING) 5230 size = field->size; 5231 5232 /* ensure NULL-termination */ 5233 if (size > key_field->size - 1) 5234 size = key_field->size - 1; 5235 5236 strncpy(compound_key + key_field->offset, (char *)key, size); 5237 } else 5238 memcpy(compound_key + key_field->offset, key, size); 5239 } 5240 5241 static void 5242 hist_trigger_actions(struct hist_trigger_data *hist_data, 5243 struct tracing_map_elt *elt, void *rec, 5244 struct ring_buffer_event *rbe, void *key, 5245 u64 *var_ref_vals) 5246 { 5247 struct action_data *data; 5248 unsigned int i; 5249 5250 for (i = 0; i < hist_data->n_actions; i++) { 5251 data = hist_data->actions[i]; 5252 data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals); 5253 } 5254 } 5255 5256 static void event_hist_trigger(struct event_trigger_data *data, void *rec, 5257 struct ring_buffer_event *rbe) 5258 { 5259 struct hist_trigger_data *hist_data = data->private_data; 5260 bool use_compound_key = (hist_data->n_keys > 1); 5261 unsigned long entries[HIST_STACKTRACE_DEPTH]; 5262 u64 var_ref_vals[TRACING_MAP_VARS_MAX]; 5263 char compound_key[HIST_KEY_SIZE_MAX]; 5264 struct tracing_map_elt *elt = NULL; 5265 struct hist_field *key_field; 5266 u64 field_contents; 5267 void *key = NULL; 5268 unsigned int i; 5269 5270 memset(compound_key, 0, hist_data->key_size); 5271 5272 for_each_hist_key_field(i, hist_data) { 5273 key_field = hist_data->fields[i]; 5274 5275 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 5276 memset(entries, 0, HIST_STACKTRACE_SIZE); 5277 stack_trace_save(entries, HIST_STACKTRACE_DEPTH, 5278 HIST_STACKTRACE_SKIP); 5279 key = entries; 5280 } else { 5281 field_contents = key_field->fn(key_field, elt, rbe, rec); 5282 if (key_field->flags & HIST_FIELD_FL_STRING) { 5283 key = (void *)(unsigned long)field_contents; 5284 use_compound_key = true; 5285 } else 5286 key = (void *)&field_contents; 5287 } 5288 5289 if (use_compound_key) 5290 add_to_key(compound_key, key, key_field, rec); 5291 } 5292 5293 if (use_compound_key) 5294 key = compound_key; 5295 5296 if (hist_data->n_var_refs && 5297 !resolve_var_refs(hist_data, key, var_ref_vals, false)) 5298 return; 5299 5300 elt = tracing_map_insert(hist_data->map, key); 5301 if (!elt) 5302 return; 5303 5304 hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals); 5305 5306 if (resolve_var_refs(hist_data, key, var_ref_vals, true)) 5307 hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals); 5308 } 5309 5310 static void hist_trigger_stacktrace_print(struct seq_file *m, 5311 unsigned long *stacktrace_entries, 5312 unsigned int max_entries) 5313 { 5314 char str[KSYM_SYMBOL_LEN]; 5315 unsigned int spaces = 8; 5316 unsigned int i; 5317 5318 for (i = 0; i < max_entries; i++) { 5319 if (!stacktrace_entries[i]) 5320 return; 5321 5322 seq_printf(m, "%*c", 1 + spaces, ' '); 5323 sprint_symbol(str, stacktrace_entries[i]); 5324 seq_printf(m, "%s\n", str); 5325 } 5326 } 5327 5328 static void hist_trigger_print_key(struct seq_file *m, 5329 struct hist_trigger_data *hist_data, 5330 void *key, 5331 struct tracing_map_elt *elt) 5332 { 5333 struct hist_field *key_field; 5334 char str[KSYM_SYMBOL_LEN]; 5335 bool multiline = false; 5336 const char *field_name; 5337 unsigned int i; 5338 u64 uval; 5339 5340 seq_puts(m, "{ "); 5341 5342 for_each_hist_key_field(i, hist_data) { 5343 key_field = hist_data->fields[i]; 5344 5345 if (i > hist_data->n_vals) 5346 seq_puts(m, ", "); 5347 5348 field_name = hist_field_name(key_field, 0); 5349 5350 if (key_field->flags & HIST_FIELD_FL_HEX) { 5351 uval = *(u64 *)(key + key_field->offset); 5352 seq_printf(m, "%s: %llx", field_name, uval); 5353 } else if (key_field->flags & HIST_FIELD_FL_SYM) { 5354 uval = *(u64 *)(key + key_field->offset); 5355 sprint_symbol_no_offset(str, uval); 5356 seq_printf(m, "%s: [%llx] %-45s", field_name, 5357 uval, str); 5358 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) { 5359 uval = *(u64 *)(key + key_field->offset); 5360 sprint_symbol(str, uval); 5361 seq_printf(m, "%s: [%llx] %-55s", field_name, 5362 uval, str); 5363 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 5364 struct hist_elt_data *elt_data = elt->private_data; 5365 char *comm; 5366 5367 if (WARN_ON_ONCE(!elt_data)) 5368 return; 5369 5370 comm = elt_data->comm; 5371 5372 uval = *(u64 *)(key + key_field->offset); 5373 seq_printf(m, "%s: %-16s[%10llu]", field_name, 5374 comm, uval); 5375 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) { 5376 const char *syscall_name; 5377 5378 uval = *(u64 *)(key + key_field->offset); 5379 syscall_name = get_syscall_name(uval); 5380 if (!syscall_name) 5381 syscall_name = "unknown_syscall"; 5382 5383 seq_printf(m, "%s: %-30s[%3llu]", field_name, 5384 syscall_name, uval); 5385 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 5386 seq_puts(m, "stacktrace:\n"); 5387 hist_trigger_stacktrace_print(m, 5388 key + key_field->offset, 5389 HIST_STACKTRACE_DEPTH); 5390 multiline = true; 5391 } else if (key_field->flags & HIST_FIELD_FL_LOG2) { 5392 seq_printf(m, "%s: ~ 2^%-2llu", field_name, 5393 *(u64 *)(key + key_field->offset)); 5394 } else if (key_field->flags & HIST_FIELD_FL_STRING) { 5395 seq_printf(m, "%s: %-50s", field_name, 5396 (char *)(key + key_field->offset)); 5397 } else { 5398 uval = *(u64 *)(key + key_field->offset); 5399 seq_printf(m, "%s: %10llu", field_name, uval); 5400 } 5401 } 5402 5403 if (!multiline) 5404 seq_puts(m, " "); 5405 5406 seq_puts(m, "}"); 5407 } 5408 5409 static void hist_trigger_entry_print(struct seq_file *m, 5410 struct hist_trigger_data *hist_data, 5411 void *key, 5412 struct tracing_map_elt *elt) 5413 { 5414 const char *field_name; 5415 unsigned int i; 5416 5417 hist_trigger_print_key(m, hist_data, key, elt); 5418 5419 seq_printf(m, " hitcount: %10llu", 5420 tracing_map_read_sum(elt, HITCOUNT_IDX)); 5421 5422 for (i = 1; i < hist_data->n_vals; i++) { 5423 field_name = hist_field_name(hist_data->fields[i], 0); 5424 5425 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR || 5426 hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR) 5427 continue; 5428 5429 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) { 5430 seq_printf(m, " %s: %10llx", field_name, 5431 tracing_map_read_sum(elt, i)); 5432 } else { 5433 seq_printf(m, " %s: %10llu", field_name, 5434 tracing_map_read_sum(elt, i)); 5435 } 5436 } 5437 5438 print_actions(m, hist_data, elt); 5439 5440 seq_puts(m, "\n"); 5441 } 5442 5443 static int print_entries(struct seq_file *m, 5444 struct hist_trigger_data *hist_data) 5445 { 5446 struct tracing_map_sort_entry **sort_entries = NULL; 5447 struct tracing_map *map = hist_data->map; 5448 int i, n_entries; 5449 5450 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys, 5451 hist_data->n_sort_keys, 5452 &sort_entries); 5453 if (n_entries < 0) 5454 return n_entries; 5455 5456 for (i = 0; i < n_entries; i++) 5457 hist_trigger_entry_print(m, hist_data, 5458 sort_entries[i]->key, 5459 sort_entries[i]->elt); 5460 5461 tracing_map_destroy_sort_entries(sort_entries, n_entries); 5462 5463 return n_entries; 5464 } 5465 5466 static void hist_trigger_show(struct seq_file *m, 5467 struct event_trigger_data *data, int n) 5468 { 5469 struct hist_trigger_data *hist_data; 5470 int n_entries; 5471 5472 if (n > 0) 5473 seq_puts(m, "\n\n"); 5474 5475 seq_puts(m, "# event histogram\n#\n# trigger info: "); 5476 data->ops->print(m, data->ops, data); 5477 seq_puts(m, "#\n\n"); 5478 5479 hist_data = data->private_data; 5480 n_entries = print_entries(m, hist_data); 5481 if (n_entries < 0) 5482 n_entries = 0; 5483 5484 track_data_snapshot_print(m, hist_data); 5485 5486 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n", 5487 (u64)atomic64_read(&hist_data->map->hits), 5488 n_entries, (u64)atomic64_read(&hist_data->map->drops)); 5489 } 5490 5491 static int hist_show(struct seq_file *m, void *v) 5492 { 5493 struct event_trigger_data *data; 5494 struct trace_event_file *event_file; 5495 int n = 0, ret = 0; 5496 5497 mutex_lock(&event_mutex); 5498 5499 event_file = event_file_data(m->private); 5500 if (unlikely(!event_file)) { 5501 ret = -ENODEV; 5502 goto out_unlock; 5503 } 5504 5505 list_for_each_entry_rcu(data, &event_file->triggers, list) { 5506 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 5507 hist_trigger_show(m, data, n++); 5508 } 5509 5510 out_unlock: 5511 mutex_unlock(&event_mutex); 5512 5513 return ret; 5514 } 5515 5516 static int event_hist_open(struct inode *inode, struct file *file) 5517 { 5518 return single_open(file, hist_show, file); 5519 } 5520 5521 const struct file_operations event_hist_fops = { 5522 .open = event_hist_open, 5523 .read = seq_read, 5524 .llseek = seq_lseek, 5525 .release = single_release, 5526 }; 5527 5528 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field) 5529 { 5530 const char *field_name = hist_field_name(hist_field, 0); 5531 5532 if (hist_field->var.name) 5533 seq_printf(m, "%s=", hist_field->var.name); 5534 5535 if (hist_field->flags & HIST_FIELD_FL_CPU) 5536 seq_puts(m, "cpu"); 5537 else if (field_name) { 5538 if (hist_field->flags & HIST_FIELD_FL_VAR_REF || 5539 hist_field->flags & HIST_FIELD_FL_ALIAS) 5540 seq_putc(m, '$'); 5541 seq_printf(m, "%s", field_name); 5542 } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP) 5543 seq_puts(m, "common_timestamp"); 5544 5545 if (hist_field->flags) { 5546 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) && 5547 !(hist_field->flags & HIST_FIELD_FL_EXPR)) { 5548 const char *flags = get_hist_field_flags(hist_field); 5549 5550 if (flags) 5551 seq_printf(m, ".%s", flags); 5552 } 5553 } 5554 } 5555 5556 static int event_hist_trigger_print(struct seq_file *m, 5557 struct event_trigger_ops *ops, 5558 struct event_trigger_data *data) 5559 { 5560 struct hist_trigger_data *hist_data = data->private_data; 5561 struct hist_field *field; 5562 bool have_var = false; 5563 unsigned int i; 5564 5565 seq_puts(m, "hist:"); 5566 5567 if (data->name) 5568 seq_printf(m, "%s:", data->name); 5569 5570 seq_puts(m, "keys="); 5571 5572 for_each_hist_key_field(i, hist_data) { 5573 field = hist_data->fields[i]; 5574 5575 if (i > hist_data->n_vals) 5576 seq_puts(m, ","); 5577 5578 if (field->flags & HIST_FIELD_FL_STACKTRACE) 5579 seq_puts(m, "stacktrace"); 5580 else 5581 hist_field_print(m, field); 5582 } 5583 5584 seq_puts(m, ":vals="); 5585 5586 for_each_hist_val_field(i, hist_data) { 5587 field = hist_data->fields[i]; 5588 if (field->flags & HIST_FIELD_FL_VAR) { 5589 have_var = true; 5590 continue; 5591 } 5592 5593 if (i == HITCOUNT_IDX) 5594 seq_puts(m, "hitcount"); 5595 else { 5596 seq_puts(m, ","); 5597 hist_field_print(m, field); 5598 } 5599 } 5600 5601 if (have_var) { 5602 unsigned int n = 0; 5603 5604 seq_puts(m, ":"); 5605 5606 for_each_hist_val_field(i, hist_data) { 5607 field = hist_data->fields[i]; 5608 5609 if (field->flags & HIST_FIELD_FL_VAR) { 5610 if (n++) 5611 seq_puts(m, ","); 5612 hist_field_print(m, field); 5613 } 5614 } 5615 } 5616 5617 seq_puts(m, ":sort="); 5618 5619 for (i = 0; i < hist_data->n_sort_keys; i++) { 5620 struct tracing_map_sort_key *sort_key; 5621 unsigned int idx, first_key_idx; 5622 5623 /* skip VAR vals */ 5624 first_key_idx = hist_data->n_vals - hist_data->n_vars; 5625 5626 sort_key = &hist_data->sort_keys[i]; 5627 idx = sort_key->field_idx; 5628 5629 if (WARN_ON(idx >= HIST_FIELDS_MAX)) 5630 return -EINVAL; 5631 5632 if (i > 0) 5633 seq_puts(m, ","); 5634 5635 if (idx == HITCOUNT_IDX) 5636 seq_puts(m, "hitcount"); 5637 else { 5638 if (idx >= first_key_idx) 5639 idx += hist_data->n_vars; 5640 hist_field_print(m, hist_data->fields[idx]); 5641 } 5642 5643 if (sort_key->descending) 5644 seq_puts(m, ".descending"); 5645 } 5646 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits)); 5647 if (hist_data->enable_timestamps) 5648 seq_printf(m, ":clock=%s", hist_data->attrs->clock); 5649 5650 print_actions_spec(m, hist_data); 5651 5652 if (data->filter_str) 5653 seq_printf(m, " if %s", data->filter_str); 5654 5655 if (data->paused) 5656 seq_puts(m, " [paused]"); 5657 else 5658 seq_puts(m, " [active]"); 5659 5660 seq_putc(m, '\n'); 5661 5662 return 0; 5663 } 5664 5665 static int event_hist_trigger_init(struct event_trigger_ops *ops, 5666 struct event_trigger_data *data) 5667 { 5668 struct hist_trigger_data *hist_data = data->private_data; 5669 5670 if (!data->ref && hist_data->attrs->name) 5671 save_named_trigger(hist_data->attrs->name, data); 5672 5673 data->ref++; 5674 5675 return 0; 5676 } 5677 5678 static void unregister_field_var_hists(struct hist_trigger_data *hist_data) 5679 { 5680 struct trace_event_file *file; 5681 unsigned int i; 5682 char *cmd; 5683 int ret; 5684 5685 for (i = 0; i < hist_data->n_field_var_hists; i++) { 5686 file = hist_data->field_var_hists[i]->hist_data->event_file; 5687 cmd = hist_data->field_var_hists[i]->cmd; 5688 ret = event_hist_trigger_func(&trigger_hist_cmd, file, 5689 "!hist", "hist", cmd); 5690 } 5691 } 5692 5693 static void event_hist_trigger_free(struct event_trigger_ops *ops, 5694 struct event_trigger_data *data) 5695 { 5696 struct hist_trigger_data *hist_data = data->private_data; 5697 5698 if (WARN_ON_ONCE(data->ref <= 0)) 5699 return; 5700 5701 data->ref--; 5702 if (!data->ref) { 5703 if (data->name) 5704 del_named_trigger(data); 5705 5706 trigger_data_free(data); 5707 5708 remove_hist_vars(hist_data); 5709 5710 unregister_field_var_hists(hist_data); 5711 5712 destroy_hist_data(hist_data); 5713 } 5714 } 5715 5716 static struct event_trigger_ops event_hist_trigger_ops = { 5717 .func = event_hist_trigger, 5718 .print = event_hist_trigger_print, 5719 .init = event_hist_trigger_init, 5720 .free = event_hist_trigger_free, 5721 }; 5722 5723 static int event_hist_trigger_named_init(struct event_trigger_ops *ops, 5724 struct event_trigger_data *data) 5725 { 5726 data->ref++; 5727 5728 save_named_trigger(data->named_data->name, data); 5729 5730 event_hist_trigger_init(ops, data->named_data); 5731 5732 return 0; 5733 } 5734 5735 static void event_hist_trigger_named_free(struct event_trigger_ops *ops, 5736 struct event_trigger_data *data) 5737 { 5738 if (WARN_ON_ONCE(data->ref <= 0)) 5739 return; 5740 5741 event_hist_trigger_free(ops, data->named_data); 5742 5743 data->ref--; 5744 if (!data->ref) { 5745 del_named_trigger(data); 5746 trigger_data_free(data); 5747 } 5748 } 5749 5750 static struct event_trigger_ops event_hist_trigger_named_ops = { 5751 .func = event_hist_trigger, 5752 .print = event_hist_trigger_print, 5753 .init = event_hist_trigger_named_init, 5754 .free = event_hist_trigger_named_free, 5755 }; 5756 5757 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd, 5758 char *param) 5759 { 5760 return &event_hist_trigger_ops; 5761 } 5762 5763 static void hist_clear(struct event_trigger_data *data) 5764 { 5765 struct hist_trigger_data *hist_data = data->private_data; 5766 5767 if (data->name) 5768 pause_named_trigger(data); 5769 5770 tracepoint_synchronize_unregister(); 5771 5772 tracing_map_clear(hist_data->map); 5773 5774 if (data->name) 5775 unpause_named_trigger(data); 5776 } 5777 5778 static bool compatible_field(struct ftrace_event_field *field, 5779 struct ftrace_event_field *test_field) 5780 { 5781 if (field == test_field) 5782 return true; 5783 if (field == NULL || test_field == NULL) 5784 return false; 5785 if (strcmp(field->name, test_field->name) != 0) 5786 return false; 5787 if (strcmp(field->type, test_field->type) != 0) 5788 return false; 5789 if (field->size != test_field->size) 5790 return false; 5791 if (field->is_signed != test_field->is_signed) 5792 return false; 5793 5794 return true; 5795 } 5796 5797 static bool hist_trigger_match(struct event_trigger_data *data, 5798 struct event_trigger_data *data_test, 5799 struct event_trigger_data *named_data, 5800 bool ignore_filter) 5801 { 5802 struct tracing_map_sort_key *sort_key, *sort_key_test; 5803 struct hist_trigger_data *hist_data, *hist_data_test; 5804 struct hist_field *key_field, *key_field_test; 5805 unsigned int i; 5806 5807 if (named_data && (named_data != data_test) && 5808 (named_data != data_test->named_data)) 5809 return false; 5810 5811 if (!named_data && is_named_trigger(data_test)) 5812 return false; 5813 5814 hist_data = data->private_data; 5815 hist_data_test = data_test->private_data; 5816 5817 if (hist_data->n_vals != hist_data_test->n_vals || 5818 hist_data->n_fields != hist_data_test->n_fields || 5819 hist_data->n_sort_keys != hist_data_test->n_sort_keys) 5820 return false; 5821 5822 if (!ignore_filter) { 5823 if ((data->filter_str && !data_test->filter_str) || 5824 (!data->filter_str && data_test->filter_str)) 5825 return false; 5826 } 5827 5828 for_each_hist_field(i, hist_data) { 5829 key_field = hist_data->fields[i]; 5830 key_field_test = hist_data_test->fields[i]; 5831 5832 if (key_field->flags != key_field_test->flags) 5833 return false; 5834 if (!compatible_field(key_field->field, key_field_test->field)) 5835 return false; 5836 if (key_field->offset != key_field_test->offset) 5837 return false; 5838 if (key_field->size != key_field_test->size) 5839 return false; 5840 if (key_field->is_signed != key_field_test->is_signed) 5841 return false; 5842 if (!!key_field->var.name != !!key_field_test->var.name) 5843 return false; 5844 if (key_field->var.name && 5845 strcmp(key_field->var.name, key_field_test->var.name) != 0) 5846 return false; 5847 } 5848 5849 for (i = 0; i < hist_data->n_sort_keys; i++) { 5850 sort_key = &hist_data->sort_keys[i]; 5851 sort_key_test = &hist_data_test->sort_keys[i]; 5852 5853 if (sort_key->field_idx != sort_key_test->field_idx || 5854 sort_key->descending != sort_key_test->descending) 5855 return false; 5856 } 5857 5858 if (!ignore_filter && data->filter_str && 5859 (strcmp(data->filter_str, data_test->filter_str) != 0)) 5860 return false; 5861 5862 if (!actions_match(hist_data, hist_data_test)) 5863 return false; 5864 5865 return true; 5866 } 5867 5868 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops, 5869 struct event_trigger_data *data, 5870 struct trace_event_file *file) 5871 { 5872 struct hist_trigger_data *hist_data = data->private_data; 5873 struct event_trigger_data *test, *named_data = NULL; 5874 struct trace_array *tr = file->tr; 5875 int ret = 0; 5876 5877 if (hist_data->attrs->name) { 5878 named_data = find_named_trigger(hist_data->attrs->name); 5879 if (named_data) { 5880 if (!hist_trigger_match(data, named_data, named_data, 5881 true)) { 5882 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name)); 5883 ret = -EINVAL; 5884 goto out; 5885 } 5886 } 5887 } 5888 5889 if (hist_data->attrs->name && !named_data) 5890 goto new; 5891 5892 list_for_each_entry_rcu(test, &file->triggers, list) { 5893 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5894 if (!hist_trigger_match(data, test, named_data, false)) 5895 continue; 5896 if (hist_data->attrs->pause) 5897 test->paused = true; 5898 else if (hist_data->attrs->cont) 5899 test->paused = false; 5900 else if (hist_data->attrs->clear) 5901 hist_clear(test); 5902 else { 5903 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0); 5904 ret = -EEXIST; 5905 } 5906 goto out; 5907 } 5908 } 5909 new: 5910 if (hist_data->attrs->cont || hist_data->attrs->clear) { 5911 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0); 5912 ret = -ENOENT; 5913 goto out; 5914 } 5915 5916 if (hist_data->attrs->pause) 5917 data->paused = true; 5918 5919 if (named_data) { 5920 data->private_data = named_data->private_data; 5921 set_named_trigger_data(data, named_data); 5922 data->ops = &event_hist_trigger_named_ops; 5923 } 5924 5925 if (data->ops->init) { 5926 ret = data->ops->init(data->ops, data); 5927 if (ret < 0) 5928 goto out; 5929 } 5930 5931 if (hist_data->enable_timestamps) { 5932 char *clock = hist_data->attrs->clock; 5933 5934 ret = tracing_set_clock(file->tr, hist_data->attrs->clock); 5935 if (ret) { 5936 hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock)); 5937 goto out; 5938 } 5939 5940 tracing_set_time_stamp_abs(file->tr, true); 5941 } 5942 5943 if (named_data) 5944 destroy_hist_data(hist_data); 5945 5946 ret++; 5947 out: 5948 return ret; 5949 } 5950 5951 static int hist_trigger_enable(struct event_trigger_data *data, 5952 struct trace_event_file *file) 5953 { 5954 int ret = 0; 5955 5956 list_add_tail_rcu(&data->list, &file->triggers); 5957 5958 update_cond_flag(file); 5959 5960 if (trace_event_trigger_enable_disable(file, 1) < 0) { 5961 list_del_rcu(&data->list); 5962 update_cond_flag(file); 5963 ret--; 5964 } 5965 5966 return ret; 5967 } 5968 5969 static bool have_hist_trigger_match(struct event_trigger_data *data, 5970 struct trace_event_file *file) 5971 { 5972 struct hist_trigger_data *hist_data = data->private_data; 5973 struct event_trigger_data *test, *named_data = NULL; 5974 bool match = false; 5975 5976 if (hist_data->attrs->name) 5977 named_data = find_named_trigger(hist_data->attrs->name); 5978 5979 list_for_each_entry_rcu(test, &file->triggers, list) { 5980 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5981 if (hist_trigger_match(data, test, named_data, false)) { 5982 match = true; 5983 break; 5984 } 5985 } 5986 } 5987 5988 return match; 5989 } 5990 5991 static bool hist_trigger_check_refs(struct event_trigger_data *data, 5992 struct trace_event_file *file) 5993 { 5994 struct hist_trigger_data *hist_data = data->private_data; 5995 struct event_trigger_data *test, *named_data = NULL; 5996 5997 if (hist_data->attrs->name) 5998 named_data = find_named_trigger(hist_data->attrs->name); 5999 6000 list_for_each_entry_rcu(test, &file->triggers, list) { 6001 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6002 if (!hist_trigger_match(data, test, named_data, false)) 6003 continue; 6004 hist_data = test->private_data; 6005 if (check_var_refs(hist_data)) 6006 return true; 6007 break; 6008 } 6009 } 6010 6011 return false; 6012 } 6013 6014 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops, 6015 struct event_trigger_data *data, 6016 struct trace_event_file *file) 6017 { 6018 struct hist_trigger_data *hist_data = data->private_data; 6019 struct event_trigger_data *test, *named_data = NULL; 6020 bool unregistered = false; 6021 6022 if (hist_data->attrs->name) 6023 named_data = find_named_trigger(hist_data->attrs->name); 6024 6025 list_for_each_entry_rcu(test, &file->triggers, list) { 6026 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6027 if (!hist_trigger_match(data, test, named_data, false)) 6028 continue; 6029 unregistered = true; 6030 list_del_rcu(&test->list); 6031 trace_event_trigger_enable_disable(file, 0); 6032 update_cond_flag(file); 6033 break; 6034 } 6035 } 6036 6037 if (unregistered && test->ops->free) 6038 test->ops->free(test->ops, test); 6039 6040 if (hist_data->enable_timestamps) { 6041 if (!hist_data->remove || unregistered) 6042 tracing_set_time_stamp_abs(file->tr, false); 6043 } 6044 } 6045 6046 static bool hist_file_check_refs(struct trace_event_file *file) 6047 { 6048 struct hist_trigger_data *hist_data; 6049 struct event_trigger_data *test; 6050 6051 list_for_each_entry_rcu(test, &file->triggers, list) { 6052 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6053 hist_data = test->private_data; 6054 if (check_var_refs(hist_data)) 6055 return true; 6056 } 6057 } 6058 6059 return false; 6060 } 6061 6062 static void hist_unreg_all(struct trace_event_file *file) 6063 { 6064 struct event_trigger_data *test, *n; 6065 struct hist_trigger_data *hist_data; 6066 struct synth_event *se; 6067 const char *se_name; 6068 6069 lockdep_assert_held(&event_mutex); 6070 6071 if (hist_file_check_refs(file)) 6072 return; 6073 6074 list_for_each_entry_safe(test, n, &file->triggers, list) { 6075 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6076 hist_data = test->private_data; 6077 list_del_rcu(&test->list); 6078 trace_event_trigger_enable_disable(file, 0); 6079 6080 se_name = trace_event_name(file->event_call); 6081 se = find_synth_event(se_name); 6082 if (se) 6083 se->ref--; 6084 6085 update_cond_flag(file); 6086 if (hist_data->enable_timestamps) 6087 tracing_set_time_stamp_abs(file->tr, false); 6088 if (test->ops->free) 6089 test->ops->free(test->ops, test); 6090 } 6091 } 6092 } 6093 6094 static int event_hist_trigger_func(struct event_command *cmd_ops, 6095 struct trace_event_file *file, 6096 char *glob, char *cmd, char *param) 6097 { 6098 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT; 6099 struct event_trigger_data *trigger_data; 6100 struct hist_trigger_attrs *attrs; 6101 struct event_trigger_ops *trigger_ops; 6102 struct hist_trigger_data *hist_data; 6103 struct synth_event *se; 6104 const char *se_name; 6105 bool remove = false; 6106 char *trigger, *p; 6107 int ret = 0; 6108 6109 lockdep_assert_held(&event_mutex); 6110 6111 if (glob && strlen(glob)) { 6112 hist_err_clear(); 6113 last_cmd_set(file, param); 6114 } 6115 6116 if (!param) 6117 return -EINVAL; 6118 6119 if (glob[0] == '!') 6120 remove = true; 6121 6122 /* 6123 * separate the trigger from the filter (k:v [if filter]) 6124 * allowing for whitespace in the trigger 6125 */ 6126 p = trigger = param; 6127 do { 6128 p = strstr(p, "if"); 6129 if (!p) 6130 break; 6131 if (p == param) 6132 return -EINVAL; 6133 if (*(p - 1) != ' ' && *(p - 1) != '\t') { 6134 p++; 6135 continue; 6136 } 6137 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1) 6138 return -EINVAL; 6139 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') { 6140 p++; 6141 continue; 6142 } 6143 break; 6144 } while (p); 6145 6146 if (!p) 6147 param = NULL; 6148 else { 6149 *(p - 1) = '\0'; 6150 param = strstrip(p); 6151 trigger = strstrip(trigger); 6152 } 6153 6154 attrs = parse_hist_trigger_attrs(file->tr, trigger); 6155 if (IS_ERR(attrs)) 6156 return PTR_ERR(attrs); 6157 6158 if (attrs->map_bits) 6159 hist_trigger_bits = attrs->map_bits; 6160 6161 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove); 6162 if (IS_ERR(hist_data)) { 6163 destroy_hist_trigger_attrs(attrs); 6164 return PTR_ERR(hist_data); 6165 } 6166 6167 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger); 6168 6169 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL); 6170 if (!trigger_data) { 6171 ret = -ENOMEM; 6172 goto out_free; 6173 } 6174 6175 trigger_data->count = -1; 6176 trigger_data->ops = trigger_ops; 6177 trigger_data->cmd_ops = cmd_ops; 6178 6179 INIT_LIST_HEAD(&trigger_data->list); 6180 RCU_INIT_POINTER(trigger_data->filter, NULL); 6181 6182 trigger_data->private_data = hist_data; 6183 6184 /* if param is non-empty, it's supposed to be a filter */ 6185 if (param && cmd_ops->set_filter) { 6186 ret = cmd_ops->set_filter(param, trigger_data, file); 6187 if (ret < 0) 6188 goto out_free; 6189 } 6190 6191 if (remove) { 6192 if (!have_hist_trigger_match(trigger_data, file)) 6193 goto out_free; 6194 6195 if (hist_trigger_check_refs(trigger_data, file)) { 6196 ret = -EBUSY; 6197 goto out_free; 6198 } 6199 6200 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file); 6201 se_name = trace_event_name(file->event_call); 6202 se = find_synth_event(se_name); 6203 if (se) 6204 se->ref--; 6205 ret = 0; 6206 goto out_free; 6207 } 6208 6209 ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file); 6210 /* 6211 * The above returns on success the # of triggers registered, 6212 * but if it didn't register any it returns zero. Consider no 6213 * triggers registered a failure too. 6214 */ 6215 if (!ret) { 6216 if (!(attrs->pause || attrs->cont || attrs->clear)) 6217 ret = -ENOENT; 6218 goto out_free; 6219 } else if (ret < 0) 6220 goto out_free; 6221 6222 if (get_named_trigger_data(trigger_data)) 6223 goto enable; 6224 6225 if (has_hist_vars(hist_data)) 6226 save_hist_vars(hist_data); 6227 6228 ret = create_actions(hist_data); 6229 if (ret) 6230 goto out_unreg; 6231 6232 ret = tracing_map_init(hist_data->map); 6233 if (ret) 6234 goto out_unreg; 6235 enable: 6236 ret = hist_trigger_enable(trigger_data, file); 6237 if (ret) 6238 goto out_unreg; 6239 6240 se_name = trace_event_name(file->event_call); 6241 se = find_synth_event(se_name); 6242 if (se) 6243 se->ref++; 6244 /* Just return zero, not the number of registered triggers */ 6245 ret = 0; 6246 out: 6247 if (ret == 0) 6248 hist_err_clear(); 6249 6250 return ret; 6251 out_unreg: 6252 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file); 6253 out_free: 6254 if (cmd_ops->set_filter) 6255 cmd_ops->set_filter(NULL, trigger_data, NULL); 6256 6257 remove_hist_vars(hist_data); 6258 6259 kfree(trigger_data); 6260 6261 destroy_hist_data(hist_data); 6262 goto out; 6263 } 6264 6265 static struct event_command trigger_hist_cmd = { 6266 .name = "hist", 6267 .trigger_type = ETT_EVENT_HIST, 6268 .flags = EVENT_CMD_FL_NEEDS_REC, 6269 .func = event_hist_trigger_func, 6270 .reg = hist_register_trigger, 6271 .unreg = hist_unregister_trigger, 6272 .unreg_all = hist_unreg_all, 6273 .get_trigger_ops = event_hist_get_trigger_ops, 6274 .set_filter = set_trigger_filter, 6275 }; 6276 6277 __init int register_trigger_hist_cmd(void) 6278 { 6279 int ret; 6280 6281 ret = register_event_command(&trigger_hist_cmd); 6282 WARN_ON(ret < 0); 6283 6284 return ret; 6285 } 6286 6287 static void 6288 hist_enable_trigger(struct event_trigger_data *data, void *rec, 6289 struct ring_buffer_event *event) 6290 { 6291 struct enable_trigger_data *enable_data = data->private_data; 6292 struct event_trigger_data *test; 6293 6294 list_for_each_entry_rcu(test, &enable_data->file->triggers, list) { 6295 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6296 if (enable_data->enable) 6297 test->paused = false; 6298 else 6299 test->paused = true; 6300 } 6301 } 6302 } 6303 6304 static void 6305 hist_enable_count_trigger(struct event_trigger_data *data, void *rec, 6306 struct ring_buffer_event *event) 6307 { 6308 if (!data->count) 6309 return; 6310 6311 if (data->count != -1) 6312 (data->count)--; 6313 6314 hist_enable_trigger(data, rec, event); 6315 } 6316 6317 static struct event_trigger_ops hist_enable_trigger_ops = { 6318 .func = hist_enable_trigger, 6319 .print = event_enable_trigger_print, 6320 .init = event_trigger_init, 6321 .free = event_enable_trigger_free, 6322 }; 6323 6324 static struct event_trigger_ops hist_enable_count_trigger_ops = { 6325 .func = hist_enable_count_trigger, 6326 .print = event_enable_trigger_print, 6327 .init = event_trigger_init, 6328 .free = event_enable_trigger_free, 6329 }; 6330 6331 static struct event_trigger_ops hist_disable_trigger_ops = { 6332 .func = hist_enable_trigger, 6333 .print = event_enable_trigger_print, 6334 .init = event_trigger_init, 6335 .free = event_enable_trigger_free, 6336 }; 6337 6338 static struct event_trigger_ops hist_disable_count_trigger_ops = { 6339 .func = hist_enable_count_trigger, 6340 .print = event_enable_trigger_print, 6341 .init = event_trigger_init, 6342 .free = event_enable_trigger_free, 6343 }; 6344 6345 static struct event_trigger_ops * 6346 hist_enable_get_trigger_ops(char *cmd, char *param) 6347 { 6348 struct event_trigger_ops *ops; 6349 bool enable; 6350 6351 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0); 6352 6353 if (enable) 6354 ops = param ? &hist_enable_count_trigger_ops : 6355 &hist_enable_trigger_ops; 6356 else 6357 ops = param ? &hist_disable_count_trigger_ops : 6358 &hist_disable_trigger_ops; 6359 6360 return ops; 6361 } 6362 6363 static void hist_enable_unreg_all(struct trace_event_file *file) 6364 { 6365 struct event_trigger_data *test, *n; 6366 6367 list_for_each_entry_safe(test, n, &file->triggers, list) { 6368 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) { 6369 list_del_rcu(&test->list); 6370 update_cond_flag(file); 6371 trace_event_trigger_enable_disable(file, 0); 6372 if (test->ops->free) 6373 test->ops->free(test->ops, test); 6374 } 6375 } 6376 } 6377 6378 static struct event_command trigger_hist_enable_cmd = { 6379 .name = ENABLE_HIST_STR, 6380 .trigger_type = ETT_HIST_ENABLE, 6381 .func = event_enable_trigger_func, 6382 .reg = event_enable_register_trigger, 6383 .unreg = event_enable_unregister_trigger, 6384 .unreg_all = hist_enable_unreg_all, 6385 .get_trigger_ops = hist_enable_get_trigger_ops, 6386 .set_filter = set_trigger_filter, 6387 }; 6388 6389 static struct event_command trigger_hist_disable_cmd = { 6390 .name = DISABLE_HIST_STR, 6391 .trigger_type = ETT_HIST_ENABLE, 6392 .func = event_enable_trigger_func, 6393 .reg = event_enable_register_trigger, 6394 .unreg = event_enable_unregister_trigger, 6395 .unreg_all = hist_enable_unreg_all, 6396 .get_trigger_ops = hist_enable_get_trigger_ops, 6397 .set_filter = set_trigger_filter, 6398 }; 6399 6400 static __init void unregister_trigger_hist_enable_disable_cmds(void) 6401 { 6402 unregister_event_command(&trigger_hist_enable_cmd); 6403 unregister_event_command(&trigger_hist_disable_cmd); 6404 } 6405 6406 __init int register_trigger_hist_enable_disable_cmds(void) 6407 { 6408 int ret; 6409 6410 ret = register_event_command(&trigger_hist_enable_cmd); 6411 if (WARN_ON(ret < 0)) 6412 return ret; 6413 ret = register_event_command(&trigger_hist_disable_cmd); 6414 if (WARN_ON(ret < 0)) 6415 unregister_trigger_hist_enable_disable_cmds(); 6416 6417 return ret; 6418 } 6419 6420 static __init int trace_events_hist_init(void) 6421 { 6422 struct dentry *entry = NULL; 6423 struct dentry *d_tracer; 6424 int err = 0; 6425 6426 err = dyn_event_register(&synth_event_ops); 6427 if (err) { 6428 pr_warn("Could not register synth_event_ops\n"); 6429 return err; 6430 } 6431 6432 d_tracer = tracing_init_dentry(); 6433 if (IS_ERR(d_tracer)) { 6434 err = PTR_ERR(d_tracer); 6435 goto err; 6436 } 6437 6438 entry = tracefs_create_file("synthetic_events", 0644, d_tracer, 6439 NULL, &synth_events_fops); 6440 if (!entry) { 6441 err = -ENODEV; 6442 goto err; 6443 } 6444 6445 return err; 6446 err: 6447 pr_warn("Could not create tracefs 'synthetic_events' entry\n"); 6448 6449 return err; 6450 } 6451 6452 fs_initcall(trace_events_hist_init); 6453