1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * trace_events_hist - trace event hist triggers 4 * 5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com> 6 */ 7 8 #include <linux/module.h> 9 #include <linux/kallsyms.h> 10 #include <linux/mutex.h> 11 #include <linux/slab.h> 12 #include <linux/stacktrace.h> 13 #include <linux/rculist.h> 14 #include <linux/tracefs.h> 15 16 #include "tracing_map.h" 17 #include "trace.h" 18 #include "trace_dynevent.h" 19 20 #define SYNTH_SYSTEM "synthetic" 21 #define SYNTH_FIELDS_MAX 16 22 23 #define STR_VAR_LEN_MAX 32 /* must be multiple of sizeof(u64) */ 24 25 #define ERRORS \ 26 C(NONE, "No error"), \ 27 C(DUPLICATE_VAR, "Variable already defined"), \ 28 C(VAR_NOT_UNIQUE, "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \ 29 C(TOO_MANY_VARS, "Too many variables defined"), \ 30 C(MALFORMED_ASSIGNMENT, "Malformed assignment"), \ 31 C(NAMED_MISMATCH, "Named hist trigger doesn't match existing named trigger (includes variables)"), \ 32 C(TRIGGER_EEXIST, "Hist trigger already exists"), \ 33 C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \ 34 C(SET_CLOCK_FAIL, "Couldn't set trace_clock"), \ 35 C(BAD_FIELD_MODIFIER, "Invalid field modifier"), \ 36 C(TOO_MANY_SUBEXPR, "Too many subexpressions (3 max)"), \ 37 C(TIMESTAMP_MISMATCH, "Timestamp units in expression don't match"), \ 38 C(TOO_MANY_FIELD_VARS, "Too many field variables defined"), \ 39 C(EVENT_FILE_NOT_FOUND, "Event file not found"), \ 40 C(HIST_NOT_FOUND, "Matching event histogram not found"), \ 41 C(HIST_CREATE_FAIL, "Couldn't create histogram for field"), \ 42 C(SYNTH_VAR_NOT_FOUND, "Couldn't find synthetic variable"), \ 43 C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"), \ 44 C(SYNTH_TYPE_MISMATCH, "Param type doesn't match synthetic event field type"), \ 45 C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \ 46 C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"), \ 47 C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"), \ 48 C(ONX_NOT_VAR, "For onmax(x) or onchange(x), x must be a variable"), \ 49 C(ONX_VAR_NOT_FOUND, "Couldn't find onmax or onchange variable"), \ 50 C(ONX_VAR_CREATE_FAIL, "Couldn't create onmax or onchange variable"), \ 51 C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"), \ 52 C(TOO_MANY_PARAMS, "Too many action params"), \ 53 C(PARAM_NOT_FOUND, "Couldn't find param"), \ 54 C(INVALID_PARAM, "Invalid action param"), \ 55 C(ACTION_NOT_FOUND, "No action found"), \ 56 C(NO_SAVE_PARAMS, "No params found for save()"), \ 57 C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \ 58 C(ACTION_MISMATCH, "Handler doesn't support action"), \ 59 C(NO_CLOSING_PAREN, "No closing paren found"), \ 60 C(SUBSYS_NOT_FOUND, "Missing subsystem"), \ 61 C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"), \ 62 C(INVALID_REF_KEY, "Using variable references as keys not supported"), \ 63 C(VAR_NOT_FOUND, "Couldn't find variable"), \ 64 C(FIELD_NOT_FOUND, "Couldn't find field"), 65 66 #undef C 67 #define C(a, b) HIST_ERR_##a 68 69 enum { ERRORS }; 70 71 #undef C 72 #define C(a, b) b 73 74 static const char *err_text[] = { ERRORS }; 75 76 struct hist_field; 77 78 typedef u64 (*hist_field_fn_t) (struct hist_field *field, 79 struct tracing_map_elt *elt, 80 struct ring_buffer_event *rbe, 81 void *event); 82 83 #define HIST_FIELD_OPERANDS_MAX 2 84 #define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX) 85 #define HIST_ACTIONS_MAX 8 86 87 enum field_op_id { 88 FIELD_OP_NONE, 89 FIELD_OP_PLUS, 90 FIELD_OP_MINUS, 91 FIELD_OP_UNARY_MINUS, 92 }; 93 94 /* 95 * A hist_var (histogram variable) contains variable information for 96 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF 97 * flag set. A hist_var has a variable name e.g. ts0, and is 98 * associated with a given histogram trigger, as specified by 99 * hist_data. The hist_var idx is the unique index assigned to the 100 * variable by the hist trigger's tracing_map. The idx is what is 101 * used to set a variable's value and, by a variable reference, to 102 * retrieve it. 103 */ 104 struct hist_var { 105 char *name; 106 struct hist_trigger_data *hist_data; 107 unsigned int idx; 108 }; 109 110 struct hist_field { 111 struct ftrace_event_field *field; 112 unsigned long flags; 113 hist_field_fn_t fn; 114 unsigned int size; 115 unsigned int offset; 116 unsigned int is_signed; 117 const char *type; 118 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX]; 119 struct hist_trigger_data *hist_data; 120 121 /* 122 * Variable fields contain variable-specific info in var. 123 */ 124 struct hist_var var; 125 enum field_op_id operator; 126 char *system; 127 char *event_name; 128 129 /* 130 * The name field is used for EXPR and VAR_REF fields. VAR 131 * fields contain the variable name in var.name. 132 */ 133 char *name; 134 135 /* 136 * When a histogram trigger is hit, if it has any references 137 * to variables, the values of those variables are collected 138 * into a var_ref_vals array by resolve_var_refs(). The 139 * current value of each variable is read from the tracing_map 140 * using the hist field's hist_var.idx and entered into the 141 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx]. 142 */ 143 unsigned int var_ref_idx; 144 bool read_once; 145 }; 146 147 static u64 hist_field_none(struct hist_field *field, 148 struct tracing_map_elt *elt, 149 struct ring_buffer_event *rbe, 150 void *event) 151 { 152 return 0; 153 } 154 155 static u64 hist_field_counter(struct hist_field *field, 156 struct tracing_map_elt *elt, 157 struct ring_buffer_event *rbe, 158 void *event) 159 { 160 return 1; 161 } 162 163 static u64 hist_field_string(struct hist_field *hist_field, 164 struct tracing_map_elt *elt, 165 struct ring_buffer_event *rbe, 166 void *event) 167 { 168 char *addr = (char *)(event + hist_field->field->offset); 169 170 return (u64)(unsigned long)addr; 171 } 172 173 static u64 hist_field_dynstring(struct hist_field *hist_field, 174 struct tracing_map_elt *elt, 175 struct ring_buffer_event *rbe, 176 void *event) 177 { 178 u32 str_item = *(u32 *)(event + hist_field->field->offset); 179 int str_loc = str_item & 0xffff; 180 char *addr = (char *)(event + str_loc); 181 182 return (u64)(unsigned long)addr; 183 } 184 185 static u64 hist_field_pstring(struct hist_field *hist_field, 186 struct tracing_map_elt *elt, 187 struct ring_buffer_event *rbe, 188 void *event) 189 { 190 char **addr = (char **)(event + hist_field->field->offset); 191 192 return (u64)(unsigned long)*addr; 193 } 194 195 static u64 hist_field_log2(struct hist_field *hist_field, 196 struct tracing_map_elt *elt, 197 struct ring_buffer_event *rbe, 198 void *event) 199 { 200 struct hist_field *operand = hist_field->operands[0]; 201 202 u64 val = operand->fn(operand, elt, rbe, event); 203 204 return (u64) ilog2(roundup_pow_of_two(val)); 205 } 206 207 static u64 hist_field_plus(struct hist_field *hist_field, 208 struct tracing_map_elt *elt, 209 struct ring_buffer_event *rbe, 210 void *event) 211 { 212 struct hist_field *operand1 = hist_field->operands[0]; 213 struct hist_field *operand2 = hist_field->operands[1]; 214 215 u64 val1 = operand1->fn(operand1, elt, rbe, event); 216 u64 val2 = operand2->fn(operand2, elt, rbe, event); 217 218 return val1 + val2; 219 } 220 221 static u64 hist_field_minus(struct hist_field *hist_field, 222 struct tracing_map_elt *elt, 223 struct ring_buffer_event *rbe, 224 void *event) 225 { 226 struct hist_field *operand1 = hist_field->operands[0]; 227 struct hist_field *operand2 = hist_field->operands[1]; 228 229 u64 val1 = operand1->fn(operand1, elt, rbe, event); 230 u64 val2 = operand2->fn(operand2, elt, rbe, event); 231 232 return val1 - val2; 233 } 234 235 static u64 hist_field_unary_minus(struct hist_field *hist_field, 236 struct tracing_map_elt *elt, 237 struct ring_buffer_event *rbe, 238 void *event) 239 { 240 struct hist_field *operand = hist_field->operands[0]; 241 242 s64 sval = (s64)operand->fn(operand, elt, rbe, event); 243 u64 val = (u64)-sval; 244 245 return val; 246 } 247 248 #define DEFINE_HIST_FIELD_FN(type) \ 249 static u64 hist_field_##type(struct hist_field *hist_field, \ 250 struct tracing_map_elt *elt, \ 251 struct ring_buffer_event *rbe, \ 252 void *event) \ 253 { \ 254 type *addr = (type *)(event + hist_field->field->offset); \ 255 \ 256 return (u64)(unsigned long)*addr; \ 257 } 258 259 DEFINE_HIST_FIELD_FN(s64); 260 DEFINE_HIST_FIELD_FN(u64); 261 DEFINE_HIST_FIELD_FN(s32); 262 DEFINE_HIST_FIELD_FN(u32); 263 DEFINE_HIST_FIELD_FN(s16); 264 DEFINE_HIST_FIELD_FN(u16); 265 DEFINE_HIST_FIELD_FN(s8); 266 DEFINE_HIST_FIELD_FN(u8); 267 268 #define for_each_hist_field(i, hist_data) \ 269 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++) 270 271 #define for_each_hist_val_field(i, hist_data) \ 272 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++) 273 274 #define for_each_hist_key_field(i, hist_data) \ 275 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++) 276 277 #define HIST_STACKTRACE_DEPTH 16 278 #define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long)) 279 #define HIST_STACKTRACE_SKIP 5 280 281 #define HITCOUNT_IDX 0 282 #define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE) 283 284 enum hist_field_flags { 285 HIST_FIELD_FL_HITCOUNT = 1 << 0, 286 HIST_FIELD_FL_KEY = 1 << 1, 287 HIST_FIELD_FL_STRING = 1 << 2, 288 HIST_FIELD_FL_HEX = 1 << 3, 289 HIST_FIELD_FL_SYM = 1 << 4, 290 HIST_FIELD_FL_SYM_OFFSET = 1 << 5, 291 HIST_FIELD_FL_EXECNAME = 1 << 6, 292 HIST_FIELD_FL_SYSCALL = 1 << 7, 293 HIST_FIELD_FL_STACKTRACE = 1 << 8, 294 HIST_FIELD_FL_LOG2 = 1 << 9, 295 HIST_FIELD_FL_TIMESTAMP = 1 << 10, 296 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11, 297 HIST_FIELD_FL_VAR = 1 << 12, 298 HIST_FIELD_FL_EXPR = 1 << 13, 299 HIST_FIELD_FL_VAR_REF = 1 << 14, 300 HIST_FIELD_FL_CPU = 1 << 15, 301 HIST_FIELD_FL_ALIAS = 1 << 16, 302 }; 303 304 struct var_defs { 305 unsigned int n_vars; 306 char *name[TRACING_MAP_VARS_MAX]; 307 char *expr[TRACING_MAP_VARS_MAX]; 308 }; 309 310 struct hist_trigger_attrs { 311 char *keys_str; 312 char *vals_str; 313 char *sort_key_str; 314 char *name; 315 char *clock; 316 bool pause; 317 bool cont; 318 bool clear; 319 bool ts_in_usecs; 320 unsigned int map_bits; 321 322 char *assignment_str[TRACING_MAP_VARS_MAX]; 323 unsigned int n_assignments; 324 325 char *action_str[HIST_ACTIONS_MAX]; 326 unsigned int n_actions; 327 328 struct var_defs var_defs; 329 }; 330 331 struct field_var { 332 struct hist_field *var; 333 struct hist_field *val; 334 }; 335 336 struct field_var_hist { 337 struct hist_trigger_data *hist_data; 338 char *cmd; 339 }; 340 341 struct hist_trigger_data { 342 struct hist_field *fields[HIST_FIELDS_MAX]; 343 unsigned int n_vals; 344 unsigned int n_keys; 345 unsigned int n_fields; 346 unsigned int n_vars; 347 unsigned int key_size; 348 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX]; 349 unsigned int n_sort_keys; 350 struct trace_event_file *event_file; 351 struct hist_trigger_attrs *attrs; 352 struct tracing_map *map; 353 bool enable_timestamps; 354 bool remove; 355 struct hist_field *var_refs[TRACING_MAP_VARS_MAX]; 356 unsigned int n_var_refs; 357 358 struct action_data *actions[HIST_ACTIONS_MAX]; 359 unsigned int n_actions; 360 361 struct field_var *field_vars[SYNTH_FIELDS_MAX]; 362 unsigned int n_field_vars; 363 unsigned int n_field_var_str; 364 struct field_var_hist *field_var_hists[SYNTH_FIELDS_MAX]; 365 unsigned int n_field_var_hists; 366 367 struct field_var *save_vars[SYNTH_FIELDS_MAX]; 368 unsigned int n_save_vars; 369 unsigned int n_save_var_str; 370 }; 371 372 static int synth_event_create(int argc, const char **argv); 373 static int synth_event_show(struct seq_file *m, struct dyn_event *ev); 374 static int synth_event_release(struct dyn_event *ev); 375 static bool synth_event_is_busy(struct dyn_event *ev); 376 static bool synth_event_match(const char *system, const char *event, 377 struct dyn_event *ev); 378 379 static struct dyn_event_operations synth_event_ops = { 380 .create = synth_event_create, 381 .show = synth_event_show, 382 .is_busy = synth_event_is_busy, 383 .free = synth_event_release, 384 .match = synth_event_match, 385 }; 386 387 struct synth_field { 388 char *type; 389 char *name; 390 size_t size; 391 bool is_signed; 392 bool is_string; 393 }; 394 395 struct synth_event { 396 struct dyn_event devent; 397 int ref; 398 char *name; 399 struct synth_field **fields; 400 unsigned int n_fields; 401 unsigned int n_u64; 402 struct trace_event_class class; 403 struct trace_event_call call; 404 struct tracepoint *tp; 405 }; 406 407 static bool is_synth_event(struct dyn_event *ev) 408 { 409 return ev->ops == &synth_event_ops; 410 } 411 412 static struct synth_event *to_synth_event(struct dyn_event *ev) 413 { 414 return container_of(ev, struct synth_event, devent); 415 } 416 417 static bool synth_event_is_busy(struct dyn_event *ev) 418 { 419 struct synth_event *event = to_synth_event(ev); 420 421 return event->ref != 0; 422 } 423 424 static bool synth_event_match(const char *system, const char *event, 425 struct dyn_event *ev) 426 { 427 struct synth_event *sev = to_synth_event(ev); 428 429 return strcmp(sev->name, event) == 0 && 430 (!system || strcmp(system, SYNTH_SYSTEM) == 0); 431 } 432 433 struct action_data; 434 435 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data, 436 struct tracing_map_elt *elt, void *rec, 437 struct ring_buffer_event *rbe, void *key, 438 struct action_data *data, u64 *var_ref_vals); 439 440 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val); 441 442 enum handler_id { 443 HANDLER_ONMATCH = 1, 444 HANDLER_ONMAX, 445 HANDLER_ONCHANGE, 446 }; 447 448 enum action_id { 449 ACTION_SAVE = 1, 450 ACTION_TRACE, 451 ACTION_SNAPSHOT, 452 }; 453 454 struct action_data { 455 enum handler_id handler; 456 enum action_id action; 457 char *action_name; 458 action_fn_t fn; 459 460 unsigned int n_params; 461 char *params[SYNTH_FIELDS_MAX]; 462 463 /* 464 * When a histogram trigger is hit, the values of any 465 * references to variables, including variables being passed 466 * as parameters to synthetic events, are collected into a 467 * var_ref_vals array. This var_ref_idx is the index of the 468 * first param in the array to be passed to the synthetic 469 * event invocation. 470 */ 471 unsigned int var_ref_idx; 472 struct synth_event *synth_event; 473 bool use_trace_keyword; 474 char *synth_event_name; 475 476 union { 477 struct { 478 char *event; 479 char *event_system; 480 } match_data; 481 482 struct { 483 /* 484 * var_str contains the $-unstripped variable 485 * name referenced by var_ref, and used when 486 * printing the action. Because var_ref 487 * creation is deferred to create_actions(), 488 * we need a per-action way to save it until 489 * then, thus var_str. 490 */ 491 char *var_str; 492 493 /* 494 * var_ref refers to the variable being 495 * tracked e.g onmax($var). 496 */ 497 struct hist_field *var_ref; 498 499 /* 500 * track_var contains the 'invisible' tracking 501 * variable created to keep the current 502 * e.g. max value. 503 */ 504 struct hist_field *track_var; 505 506 check_track_val_fn_t check_val; 507 action_fn_t save_data; 508 } track_data; 509 }; 510 }; 511 512 struct track_data { 513 u64 track_val; 514 bool updated; 515 516 unsigned int key_len; 517 void *key; 518 struct tracing_map_elt elt; 519 520 struct action_data *action_data; 521 struct hist_trigger_data *hist_data; 522 }; 523 524 struct hist_elt_data { 525 char *comm; 526 u64 *var_ref_vals; 527 char *field_var_str[SYNTH_FIELDS_MAX]; 528 }; 529 530 struct snapshot_context { 531 struct tracing_map_elt *elt; 532 void *key; 533 }; 534 535 static void track_data_free(struct track_data *track_data) 536 { 537 struct hist_elt_data *elt_data; 538 539 if (!track_data) 540 return; 541 542 kfree(track_data->key); 543 544 elt_data = track_data->elt.private_data; 545 if (elt_data) { 546 kfree(elt_data->comm); 547 kfree(elt_data); 548 } 549 550 kfree(track_data); 551 } 552 553 static struct track_data *track_data_alloc(unsigned int key_len, 554 struct action_data *action_data, 555 struct hist_trigger_data *hist_data) 556 { 557 struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL); 558 struct hist_elt_data *elt_data; 559 560 if (!data) 561 return ERR_PTR(-ENOMEM); 562 563 data->key = kzalloc(key_len, GFP_KERNEL); 564 if (!data->key) { 565 track_data_free(data); 566 return ERR_PTR(-ENOMEM); 567 } 568 569 data->key_len = key_len; 570 data->action_data = action_data; 571 data->hist_data = hist_data; 572 573 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL); 574 if (!elt_data) { 575 track_data_free(data); 576 return ERR_PTR(-ENOMEM); 577 } 578 data->elt.private_data = elt_data; 579 580 elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL); 581 if (!elt_data->comm) { 582 track_data_free(data); 583 return ERR_PTR(-ENOMEM); 584 } 585 586 return data; 587 } 588 589 static char last_cmd[MAX_FILTER_STR_VAL]; 590 static char last_cmd_loc[MAX_FILTER_STR_VAL]; 591 592 static int errpos(char *str) 593 { 594 return err_pos(last_cmd, str); 595 } 596 597 static void last_cmd_set(struct trace_event_file *file, char *str) 598 { 599 const char *system = NULL, *name = NULL; 600 struct trace_event_call *call; 601 602 if (!str) 603 return; 604 605 strncpy(last_cmd, str, MAX_FILTER_STR_VAL - 1); 606 607 if (file) { 608 call = file->event_call; 609 610 system = call->class->system; 611 if (system) { 612 name = trace_event_name(call); 613 if (!name) 614 system = NULL; 615 } 616 } 617 618 if (system) 619 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name); 620 } 621 622 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos) 623 { 624 tracing_log_err(tr, last_cmd_loc, last_cmd, err_text, 625 err_type, err_pos); 626 } 627 628 static void hist_err_clear(void) 629 { 630 last_cmd[0] = '\0'; 631 last_cmd_loc[0] = '\0'; 632 } 633 634 struct synth_trace_event { 635 struct trace_entry ent; 636 u64 fields[]; 637 }; 638 639 static int synth_event_define_fields(struct trace_event_call *call) 640 { 641 struct synth_trace_event trace; 642 int offset = offsetof(typeof(trace), fields); 643 struct synth_event *event = call->data; 644 unsigned int i, size, n_u64; 645 char *name, *type; 646 bool is_signed; 647 int ret = 0; 648 649 for (i = 0, n_u64 = 0; i < event->n_fields; i++) { 650 size = event->fields[i]->size; 651 is_signed = event->fields[i]->is_signed; 652 type = event->fields[i]->type; 653 name = event->fields[i]->name; 654 ret = trace_define_field(call, type, name, offset, size, 655 is_signed, FILTER_OTHER); 656 if (ret) 657 break; 658 659 if (event->fields[i]->is_string) { 660 offset += STR_VAR_LEN_MAX; 661 n_u64 += STR_VAR_LEN_MAX / sizeof(u64); 662 } else { 663 offset += sizeof(u64); 664 n_u64++; 665 } 666 } 667 668 event->n_u64 = n_u64; 669 670 return ret; 671 } 672 673 static bool synth_field_signed(char *type) 674 { 675 if (str_has_prefix(type, "u")) 676 return false; 677 678 return true; 679 } 680 681 static int synth_field_is_string(char *type) 682 { 683 if (strstr(type, "char[") != NULL) 684 return true; 685 686 return false; 687 } 688 689 static int synth_field_string_size(char *type) 690 { 691 char buf[4], *end, *start; 692 unsigned int len; 693 int size, err; 694 695 start = strstr(type, "char["); 696 if (start == NULL) 697 return -EINVAL; 698 start += sizeof("char[") - 1; 699 700 end = strchr(type, ']'); 701 if (!end || end < start) 702 return -EINVAL; 703 704 len = end - start; 705 if (len > 3) 706 return -EINVAL; 707 708 strncpy(buf, start, len); 709 buf[len] = '\0'; 710 711 err = kstrtouint(buf, 0, &size); 712 if (err) 713 return err; 714 715 if (size > STR_VAR_LEN_MAX) 716 return -EINVAL; 717 718 return size; 719 } 720 721 static int synth_field_size(char *type) 722 { 723 int size = 0; 724 725 if (strcmp(type, "s64") == 0) 726 size = sizeof(s64); 727 else if (strcmp(type, "u64") == 0) 728 size = sizeof(u64); 729 else if (strcmp(type, "s32") == 0) 730 size = sizeof(s32); 731 else if (strcmp(type, "u32") == 0) 732 size = sizeof(u32); 733 else if (strcmp(type, "s16") == 0) 734 size = sizeof(s16); 735 else if (strcmp(type, "u16") == 0) 736 size = sizeof(u16); 737 else if (strcmp(type, "s8") == 0) 738 size = sizeof(s8); 739 else if (strcmp(type, "u8") == 0) 740 size = sizeof(u8); 741 else if (strcmp(type, "char") == 0) 742 size = sizeof(char); 743 else if (strcmp(type, "unsigned char") == 0) 744 size = sizeof(unsigned char); 745 else if (strcmp(type, "int") == 0) 746 size = sizeof(int); 747 else if (strcmp(type, "unsigned int") == 0) 748 size = sizeof(unsigned int); 749 else if (strcmp(type, "long") == 0) 750 size = sizeof(long); 751 else if (strcmp(type, "unsigned long") == 0) 752 size = sizeof(unsigned long); 753 else if (strcmp(type, "pid_t") == 0) 754 size = sizeof(pid_t); 755 else if (synth_field_is_string(type)) 756 size = synth_field_string_size(type); 757 758 return size; 759 } 760 761 static const char *synth_field_fmt(char *type) 762 { 763 const char *fmt = "%llu"; 764 765 if (strcmp(type, "s64") == 0) 766 fmt = "%lld"; 767 else if (strcmp(type, "u64") == 0) 768 fmt = "%llu"; 769 else if (strcmp(type, "s32") == 0) 770 fmt = "%d"; 771 else if (strcmp(type, "u32") == 0) 772 fmt = "%u"; 773 else if (strcmp(type, "s16") == 0) 774 fmt = "%d"; 775 else if (strcmp(type, "u16") == 0) 776 fmt = "%u"; 777 else if (strcmp(type, "s8") == 0) 778 fmt = "%d"; 779 else if (strcmp(type, "u8") == 0) 780 fmt = "%u"; 781 else if (strcmp(type, "char") == 0) 782 fmt = "%d"; 783 else if (strcmp(type, "unsigned char") == 0) 784 fmt = "%u"; 785 else if (strcmp(type, "int") == 0) 786 fmt = "%d"; 787 else if (strcmp(type, "unsigned int") == 0) 788 fmt = "%u"; 789 else if (strcmp(type, "long") == 0) 790 fmt = "%ld"; 791 else if (strcmp(type, "unsigned long") == 0) 792 fmt = "%lu"; 793 else if (strcmp(type, "pid_t") == 0) 794 fmt = "%d"; 795 else if (synth_field_is_string(type)) 796 fmt = "%s"; 797 798 return fmt; 799 } 800 801 static enum print_line_t print_synth_event(struct trace_iterator *iter, 802 int flags, 803 struct trace_event *event) 804 { 805 struct trace_array *tr = iter->tr; 806 struct trace_seq *s = &iter->seq; 807 struct synth_trace_event *entry; 808 struct synth_event *se; 809 unsigned int i, n_u64; 810 char print_fmt[32]; 811 const char *fmt; 812 813 entry = (struct synth_trace_event *)iter->ent; 814 se = container_of(event, struct synth_event, call.event); 815 816 trace_seq_printf(s, "%s: ", se->name); 817 818 for (i = 0, n_u64 = 0; i < se->n_fields; i++) { 819 if (trace_seq_has_overflowed(s)) 820 goto end; 821 822 fmt = synth_field_fmt(se->fields[i]->type); 823 824 /* parameter types */ 825 if (tr->trace_flags & TRACE_ITER_VERBOSE) 826 trace_seq_printf(s, "%s ", fmt); 827 828 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt); 829 830 /* parameter values */ 831 if (se->fields[i]->is_string) { 832 trace_seq_printf(s, print_fmt, se->fields[i]->name, 833 (char *)&entry->fields[n_u64], 834 i == se->n_fields - 1 ? "" : " "); 835 n_u64 += STR_VAR_LEN_MAX / sizeof(u64); 836 } else { 837 trace_seq_printf(s, print_fmt, se->fields[i]->name, 838 entry->fields[n_u64], 839 i == se->n_fields - 1 ? "" : " "); 840 n_u64++; 841 } 842 } 843 end: 844 trace_seq_putc(s, '\n'); 845 846 return trace_handle_return(s); 847 } 848 849 static struct trace_event_functions synth_event_funcs = { 850 .trace = print_synth_event 851 }; 852 853 static notrace void trace_event_raw_event_synth(void *__data, 854 u64 *var_ref_vals, 855 unsigned int var_ref_idx) 856 { 857 struct trace_event_file *trace_file = __data; 858 struct synth_trace_event *entry; 859 struct trace_event_buffer fbuffer; 860 struct ring_buffer *buffer; 861 struct synth_event *event; 862 unsigned int i, n_u64; 863 int fields_size = 0; 864 865 event = trace_file->event_call->data; 866 867 if (trace_trigger_soft_disabled(trace_file)) 868 return; 869 870 fields_size = event->n_u64 * sizeof(u64); 871 872 /* 873 * Avoid ring buffer recursion detection, as this event 874 * is being performed within another event. 875 */ 876 buffer = trace_file->tr->trace_buffer.buffer; 877 ring_buffer_nest_start(buffer); 878 879 entry = trace_event_buffer_reserve(&fbuffer, trace_file, 880 sizeof(*entry) + fields_size); 881 if (!entry) 882 goto out; 883 884 for (i = 0, n_u64 = 0; i < event->n_fields; i++) { 885 if (event->fields[i]->is_string) { 886 char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i]; 887 char *str_field = (char *)&entry->fields[n_u64]; 888 889 strscpy(str_field, str_val, STR_VAR_LEN_MAX); 890 n_u64 += STR_VAR_LEN_MAX / sizeof(u64); 891 } else { 892 entry->fields[n_u64] = var_ref_vals[var_ref_idx + i]; 893 n_u64++; 894 } 895 } 896 897 trace_event_buffer_commit(&fbuffer); 898 out: 899 ring_buffer_nest_end(buffer); 900 } 901 902 static void free_synth_event_print_fmt(struct trace_event_call *call) 903 { 904 if (call) { 905 kfree(call->print_fmt); 906 call->print_fmt = NULL; 907 } 908 } 909 910 static int __set_synth_event_print_fmt(struct synth_event *event, 911 char *buf, int len) 912 { 913 const char *fmt; 914 int pos = 0; 915 int i; 916 917 /* When len=0, we just calculate the needed length */ 918 #define LEN_OR_ZERO (len ? len - pos : 0) 919 920 pos += snprintf(buf + pos, LEN_OR_ZERO, "\""); 921 for (i = 0; i < event->n_fields; i++) { 922 fmt = synth_field_fmt(event->fields[i]->type); 923 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s", 924 event->fields[i]->name, fmt, 925 i == event->n_fields - 1 ? "" : ", "); 926 } 927 pos += snprintf(buf + pos, LEN_OR_ZERO, "\""); 928 929 for (i = 0; i < event->n_fields; i++) { 930 pos += snprintf(buf + pos, LEN_OR_ZERO, 931 ", REC->%s", event->fields[i]->name); 932 } 933 934 #undef LEN_OR_ZERO 935 936 /* return the length of print_fmt */ 937 return pos; 938 } 939 940 static int set_synth_event_print_fmt(struct trace_event_call *call) 941 { 942 struct synth_event *event = call->data; 943 char *print_fmt; 944 int len; 945 946 /* First: called with 0 length to calculate the needed length */ 947 len = __set_synth_event_print_fmt(event, NULL, 0); 948 949 print_fmt = kmalloc(len + 1, GFP_KERNEL); 950 if (!print_fmt) 951 return -ENOMEM; 952 953 /* Second: actually write the @print_fmt */ 954 __set_synth_event_print_fmt(event, print_fmt, len + 1); 955 call->print_fmt = print_fmt; 956 957 return 0; 958 } 959 960 static void free_synth_field(struct synth_field *field) 961 { 962 kfree(field->type); 963 kfree(field->name); 964 kfree(field); 965 } 966 967 static struct synth_field *parse_synth_field(int argc, const char **argv, 968 int *consumed) 969 { 970 struct synth_field *field; 971 const char *prefix = NULL, *field_type = argv[0], *field_name, *array; 972 int len, ret = 0; 973 974 if (field_type[0] == ';') 975 field_type++; 976 977 if (!strcmp(field_type, "unsigned")) { 978 if (argc < 3) 979 return ERR_PTR(-EINVAL); 980 prefix = "unsigned "; 981 field_type = argv[1]; 982 field_name = argv[2]; 983 *consumed = 3; 984 } else { 985 field_name = argv[1]; 986 *consumed = 2; 987 } 988 989 field = kzalloc(sizeof(*field), GFP_KERNEL); 990 if (!field) 991 return ERR_PTR(-ENOMEM); 992 993 len = strlen(field_name); 994 array = strchr(field_name, '['); 995 if (array) 996 len -= strlen(array); 997 else if (field_name[len - 1] == ';') 998 len--; 999 1000 field->name = kmemdup_nul(field_name, len, GFP_KERNEL); 1001 if (!field->name) { 1002 ret = -ENOMEM; 1003 goto free; 1004 } 1005 1006 if (field_type[0] == ';') 1007 field_type++; 1008 len = strlen(field_type) + 1; 1009 if (array) 1010 len += strlen(array); 1011 if (prefix) 1012 len += strlen(prefix); 1013 1014 field->type = kzalloc(len, GFP_KERNEL); 1015 if (!field->type) { 1016 ret = -ENOMEM; 1017 goto free; 1018 } 1019 if (prefix) 1020 strcat(field->type, prefix); 1021 strcat(field->type, field_type); 1022 if (array) { 1023 strcat(field->type, array); 1024 if (field->type[len - 1] == ';') 1025 field->type[len - 1] = '\0'; 1026 } 1027 1028 field->size = synth_field_size(field->type); 1029 if (!field->size) { 1030 ret = -EINVAL; 1031 goto free; 1032 } 1033 1034 if (synth_field_is_string(field->type)) 1035 field->is_string = true; 1036 1037 field->is_signed = synth_field_signed(field->type); 1038 1039 out: 1040 return field; 1041 free: 1042 free_synth_field(field); 1043 field = ERR_PTR(ret); 1044 goto out; 1045 } 1046 1047 static void free_synth_tracepoint(struct tracepoint *tp) 1048 { 1049 if (!tp) 1050 return; 1051 1052 kfree(tp->name); 1053 kfree(tp); 1054 } 1055 1056 static struct tracepoint *alloc_synth_tracepoint(char *name) 1057 { 1058 struct tracepoint *tp; 1059 1060 tp = kzalloc(sizeof(*tp), GFP_KERNEL); 1061 if (!tp) 1062 return ERR_PTR(-ENOMEM); 1063 1064 tp->name = kstrdup(name, GFP_KERNEL); 1065 if (!tp->name) { 1066 kfree(tp); 1067 return ERR_PTR(-ENOMEM); 1068 } 1069 1070 return tp; 1071 } 1072 1073 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals, 1074 unsigned int var_ref_idx); 1075 1076 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals, 1077 unsigned int var_ref_idx) 1078 { 1079 struct tracepoint *tp = event->tp; 1080 1081 if (unlikely(atomic_read(&tp->key.enabled) > 0)) { 1082 struct tracepoint_func *probe_func_ptr; 1083 synth_probe_func_t probe_func; 1084 void *__data; 1085 1086 if (!(cpu_online(raw_smp_processor_id()))) 1087 return; 1088 1089 probe_func_ptr = rcu_dereference_sched((tp)->funcs); 1090 if (probe_func_ptr) { 1091 do { 1092 probe_func = probe_func_ptr->func; 1093 __data = probe_func_ptr->data; 1094 probe_func(__data, var_ref_vals, var_ref_idx); 1095 } while ((++probe_func_ptr)->func); 1096 } 1097 } 1098 } 1099 1100 static struct synth_event *find_synth_event(const char *name) 1101 { 1102 struct dyn_event *pos; 1103 struct synth_event *event; 1104 1105 for_each_dyn_event(pos) { 1106 if (!is_synth_event(pos)) 1107 continue; 1108 event = to_synth_event(pos); 1109 if (strcmp(event->name, name) == 0) 1110 return event; 1111 } 1112 1113 return NULL; 1114 } 1115 1116 static int register_synth_event(struct synth_event *event) 1117 { 1118 struct trace_event_call *call = &event->call; 1119 int ret = 0; 1120 1121 event->call.class = &event->class; 1122 event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL); 1123 if (!event->class.system) { 1124 ret = -ENOMEM; 1125 goto out; 1126 } 1127 1128 event->tp = alloc_synth_tracepoint(event->name); 1129 if (IS_ERR(event->tp)) { 1130 ret = PTR_ERR(event->tp); 1131 event->tp = NULL; 1132 goto out; 1133 } 1134 1135 INIT_LIST_HEAD(&call->class->fields); 1136 call->event.funcs = &synth_event_funcs; 1137 call->class->define_fields = synth_event_define_fields; 1138 1139 ret = register_trace_event(&call->event); 1140 if (!ret) { 1141 ret = -ENODEV; 1142 goto out; 1143 } 1144 call->flags = TRACE_EVENT_FL_TRACEPOINT; 1145 call->class->reg = trace_event_reg; 1146 call->class->probe = trace_event_raw_event_synth; 1147 call->data = event; 1148 call->tp = event->tp; 1149 1150 ret = trace_add_event_call(call); 1151 if (ret) { 1152 pr_warn("Failed to register synthetic event: %s\n", 1153 trace_event_name(call)); 1154 goto err; 1155 } 1156 1157 ret = set_synth_event_print_fmt(call); 1158 if (ret < 0) { 1159 trace_remove_event_call(call); 1160 goto err; 1161 } 1162 out: 1163 return ret; 1164 err: 1165 unregister_trace_event(&call->event); 1166 goto out; 1167 } 1168 1169 static int unregister_synth_event(struct synth_event *event) 1170 { 1171 struct trace_event_call *call = &event->call; 1172 int ret; 1173 1174 ret = trace_remove_event_call(call); 1175 1176 return ret; 1177 } 1178 1179 static void free_synth_event(struct synth_event *event) 1180 { 1181 unsigned int i; 1182 1183 if (!event) 1184 return; 1185 1186 for (i = 0; i < event->n_fields; i++) 1187 free_synth_field(event->fields[i]); 1188 1189 kfree(event->fields); 1190 kfree(event->name); 1191 kfree(event->class.system); 1192 free_synth_tracepoint(event->tp); 1193 free_synth_event_print_fmt(&event->call); 1194 kfree(event); 1195 } 1196 1197 static struct synth_event *alloc_synth_event(const char *name, int n_fields, 1198 struct synth_field **fields) 1199 { 1200 struct synth_event *event; 1201 unsigned int i; 1202 1203 event = kzalloc(sizeof(*event), GFP_KERNEL); 1204 if (!event) { 1205 event = ERR_PTR(-ENOMEM); 1206 goto out; 1207 } 1208 1209 event->name = kstrdup(name, GFP_KERNEL); 1210 if (!event->name) { 1211 kfree(event); 1212 event = ERR_PTR(-ENOMEM); 1213 goto out; 1214 } 1215 1216 event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL); 1217 if (!event->fields) { 1218 free_synth_event(event); 1219 event = ERR_PTR(-ENOMEM); 1220 goto out; 1221 } 1222 1223 dyn_event_init(&event->devent, &synth_event_ops); 1224 1225 for (i = 0; i < n_fields; i++) 1226 event->fields[i] = fields[i]; 1227 1228 event->n_fields = n_fields; 1229 out: 1230 return event; 1231 } 1232 1233 static void action_trace(struct hist_trigger_data *hist_data, 1234 struct tracing_map_elt *elt, void *rec, 1235 struct ring_buffer_event *rbe, void *key, 1236 struct action_data *data, u64 *var_ref_vals) 1237 { 1238 struct synth_event *event = data->synth_event; 1239 1240 trace_synth(event, var_ref_vals, data->var_ref_idx); 1241 } 1242 1243 struct hist_var_data { 1244 struct list_head list; 1245 struct hist_trigger_data *hist_data; 1246 }; 1247 1248 static int __create_synth_event(int argc, const char *name, const char **argv) 1249 { 1250 struct synth_field *field, *fields[SYNTH_FIELDS_MAX]; 1251 struct synth_event *event = NULL; 1252 int i, consumed = 0, n_fields = 0, ret = 0; 1253 1254 /* 1255 * Argument syntax: 1256 * - Add synthetic event: <event_name> field[;field] ... 1257 * - Remove synthetic event: !<event_name> field[;field] ... 1258 * where 'field' = type field_name 1259 */ 1260 1261 if (name[0] == '\0' || argc < 1) 1262 return -EINVAL; 1263 1264 mutex_lock(&event_mutex); 1265 1266 event = find_synth_event(name); 1267 if (event) { 1268 ret = -EEXIST; 1269 goto out; 1270 } 1271 1272 for (i = 0; i < argc - 1; i++) { 1273 if (strcmp(argv[i], ";") == 0) 1274 continue; 1275 if (n_fields == SYNTH_FIELDS_MAX) { 1276 ret = -EINVAL; 1277 goto err; 1278 } 1279 1280 field = parse_synth_field(argc - i, &argv[i], &consumed); 1281 if (IS_ERR(field)) { 1282 ret = PTR_ERR(field); 1283 goto err; 1284 } 1285 fields[n_fields++] = field; 1286 i += consumed - 1; 1287 } 1288 1289 if (i < argc && strcmp(argv[i], ";") != 0) { 1290 ret = -EINVAL; 1291 goto err; 1292 } 1293 1294 event = alloc_synth_event(name, n_fields, fields); 1295 if (IS_ERR(event)) { 1296 ret = PTR_ERR(event); 1297 event = NULL; 1298 goto err; 1299 } 1300 ret = register_synth_event(event); 1301 if (!ret) 1302 dyn_event_add(&event->devent); 1303 else 1304 free_synth_event(event); 1305 out: 1306 mutex_unlock(&event_mutex); 1307 1308 return ret; 1309 err: 1310 for (i = 0; i < n_fields; i++) 1311 free_synth_field(fields[i]); 1312 1313 goto out; 1314 } 1315 1316 static int create_or_delete_synth_event(int argc, char **argv) 1317 { 1318 const char *name = argv[0]; 1319 struct synth_event *event = NULL; 1320 int ret; 1321 1322 /* trace_run_command() ensures argc != 0 */ 1323 if (name[0] == '!') { 1324 mutex_lock(&event_mutex); 1325 event = find_synth_event(name + 1); 1326 if (event) { 1327 if (event->ref) 1328 ret = -EBUSY; 1329 else { 1330 ret = unregister_synth_event(event); 1331 if (!ret) { 1332 dyn_event_remove(&event->devent); 1333 free_synth_event(event); 1334 } 1335 } 1336 } else 1337 ret = -ENOENT; 1338 mutex_unlock(&event_mutex); 1339 return ret; 1340 } 1341 1342 ret = __create_synth_event(argc - 1, name, (const char **)argv + 1); 1343 return ret == -ECANCELED ? -EINVAL : ret; 1344 } 1345 1346 static int synth_event_create(int argc, const char **argv) 1347 { 1348 const char *name = argv[0]; 1349 int len; 1350 1351 if (name[0] != 's' || name[1] != ':') 1352 return -ECANCELED; 1353 name += 2; 1354 1355 /* This interface accepts group name prefix */ 1356 if (strchr(name, '/')) { 1357 len = str_has_prefix(name, SYNTH_SYSTEM "/"); 1358 if (len == 0) 1359 return -EINVAL; 1360 name += len; 1361 } 1362 return __create_synth_event(argc - 1, name, argv + 1); 1363 } 1364 1365 static int synth_event_release(struct dyn_event *ev) 1366 { 1367 struct synth_event *event = to_synth_event(ev); 1368 int ret; 1369 1370 if (event->ref) 1371 return -EBUSY; 1372 1373 ret = unregister_synth_event(event); 1374 if (ret) 1375 return ret; 1376 1377 dyn_event_remove(ev); 1378 free_synth_event(event); 1379 return 0; 1380 } 1381 1382 static int __synth_event_show(struct seq_file *m, struct synth_event *event) 1383 { 1384 struct synth_field *field; 1385 unsigned int i; 1386 1387 seq_printf(m, "%s\t", event->name); 1388 1389 for (i = 0; i < event->n_fields; i++) { 1390 field = event->fields[i]; 1391 1392 /* parameter values */ 1393 seq_printf(m, "%s %s%s", field->type, field->name, 1394 i == event->n_fields - 1 ? "" : "; "); 1395 } 1396 1397 seq_putc(m, '\n'); 1398 1399 return 0; 1400 } 1401 1402 static int synth_event_show(struct seq_file *m, struct dyn_event *ev) 1403 { 1404 struct synth_event *event = to_synth_event(ev); 1405 1406 seq_printf(m, "s:%s/", event->class.system); 1407 1408 return __synth_event_show(m, event); 1409 } 1410 1411 static int synth_events_seq_show(struct seq_file *m, void *v) 1412 { 1413 struct dyn_event *ev = v; 1414 1415 if (!is_synth_event(ev)) 1416 return 0; 1417 1418 return __synth_event_show(m, to_synth_event(ev)); 1419 } 1420 1421 static const struct seq_operations synth_events_seq_op = { 1422 .start = dyn_event_seq_start, 1423 .next = dyn_event_seq_next, 1424 .stop = dyn_event_seq_stop, 1425 .show = synth_events_seq_show, 1426 }; 1427 1428 static int synth_events_open(struct inode *inode, struct file *file) 1429 { 1430 int ret; 1431 1432 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 1433 ret = dyn_events_release_all(&synth_event_ops); 1434 if (ret < 0) 1435 return ret; 1436 } 1437 1438 return seq_open(file, &synth_events_seq_op); 1439 } 1440 1441 static ssize_t synth_events_write(struct file *file, 1442 const char __user *buffer, 1443 size_t count, loff_t *ppos) 1444 { 1445 return trace_parse_run_command(file, buffer, count, ppos, 1446 create_or_delete_synth_event); 1447 } 1448 1449 static const struct file_operations synth_events_fops = { 1450 .open = synth_events_open, 1451 .write = synth_events_write, 1452 .read = seq_read, 1453 .llseek = seq_lseek, 1454 .release = seq_release, 1455 }; 1456 1457 static u64 hist_field_timestamp(struct hist_field *hist_field, 1458 struct tracing_map_elt *elt, 1459 struct ring_buffer_event *rbe, 1460 void *event) 1461 { 1462 struct hist_trigger_data *hist_data = hist_field->hist_data; 1463 struct trace_array *tr = hist_data->event_file->tr; 1464 1465 u64 ts = ring_buffer_event_time_stamp(rbe); 1466 1467 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr)) 1468 ts = ns2usecs(ts); 1469 1470 return ts; 1471 } 1472 1473 static u64 hist_field_cpu(struct hist_field *hist_field, 1474 struct tracing_map_elt *elt, 1475 struct ring_buffer_event *rbe, 1476 void *event) 1477 { 1478 int cpu = smp_processor_id(); 1479 1480 return cpu; 1481 } 1482 1483 /** 1484 * check_field_for_var_ref - Check if a VAR_REF field references a variable 1485 * @hist_field: The VAR_REF field to check 1486 * @var_data: The hist trigger that owns the variable 1487 * @var_idx: The trigger variable identifier 1488 * 1489 * Check the given VAR_REF field to see whether or not it references 1490 * the given variable associated with the given trigger. 1491 * 1492 * Return: The VAR_REF field if it does reference the variable, NULL if not 1493 */ 1494 static struct hist_field * 1495 check_field_for_var_ref(struct hist_field *hist_field, 1496 struct hist_trigger_data *var_data, 1497 unsigned int var_idx) 1498 { 1499 WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF)); 1500 1501 if (hist_field && hist_field->var.idx == var_idx && 1502 hist_field->var.hist_data == var_data) 1503 return hist_field; 1504 1505 return NULL; 1506 } 1507 1508 /** 1509 * find_var_ref - Check if a trigger has a reference to a trigger variable 1510 * @hist_data: The hist trigger that might have a reference to the variable 1511 * @var_data: The hist trigger that owns the variable 1512 * @var_idx: The trigger variable identifier 1513 * 1514 * Check the list of var_refs[] on the first hist trigger to see 1515 * whether any of them are references to the variable on the second 1516 * trigger. 1517 * 1518 * Return: The VAR_REF field referencing the variable if so, NULL if not 1519 */ 1520 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data, 1521 struct hist_trigger_data *var_data, 1522 unsigned int var_idx) 1523 { 1524 struct hist_field *hist_field; 1525 unsigned int i; 1526 1527 for (i = 0; i < hist_data->n_var_refs; i++) { 1528 hist_field = hist_data->var_refs[i]; 1529 if (check_field_for_var_ref(hist_field, var_data, var_idx)) 1530 return hist_field; 1531 } 1532 1533 return NULL; 1534 } 1535 1536 /** 1537 * find_any_var_ref - Check if there is a reference to a given trigger variable 1538 * @hist_data: The hist trigger 1539 * @var_idx: The trigger variable identifier 1540 * 1541 * Check to see whether the given variable is currently referenced by 1542 * any other trigger. 1543 * 1544 * The trigger the variable is defined on is explicitly excluded - the 1545 * assumption being that a self-reference doesn't prevent a trigger 1546 * from being removed. 1547 * 1548 * Return: The VAR_REF field referencing the variable if so, NULL if not 1549 */ 1550 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data, 1551 unsigned int var_idx) 1552 { 1553 struct trace_array *tr = hist_data->event_file->tr; 1554 struct hist_field *found = NULL; 1555 struct hist_var_data *var_data; 1556 1557 list_for_each_entry(var_data, &tr->hist_vars, list) { 1558 if (var_data->hist_data == hist_data) 1559 continue; 1560 found = find_var_ref(var_data->hist_data, hist_data, var_idx); 1561 if (found) 1562 break; 1563 } 1564 1565 return found; 1566 } 1567 1568 /** 1569 * check_var_refs - Check if there is a reference to any of trigger's variables 1570 * @hist_data: The hist trigger 1571 * 1572 * A trigger can define one or more variables. If any one of them is 1573 * currently referenced by any other trigger, this function will 1574 * determine that. 1575 1576 * Typically used to determine whether or not a trigger can be removed 1577 * - if there are any references to a trigger's variables, it cannot. 1578 * 1579 * Return: True if there is a reference to any of trigger's variables 1580 */ 1581 static bool check_var_refs(struct hist_trigger_data *hist_data) 1582 { 1583 struct hist_field *field; 1584 bool found = false; 1585 int i; 1586 1587 for_each_hist_field(i, hist_data) { 1588 field = hist_data->fields[i]; 1589 if (field && field->flags & HIST_FIELD_FL_VAR) { 1590 if (find_any_var_ref(hist_data, field->var.idx)) { 1591 found = true; 1592 break; 1593 } 1594 } 1595 } 1596 1597 return found; 1598 } 1599 1600 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data) 1601 { 1602 struct trace_array *tr = hist_data->event_file->tr; 1603 struct hist_var_data *var_data, *found = NULL; 1604 1605 list_for_each_entry(var_data, &tr->hist_vars, list) { 1606 if (var_data->hist_data == hist_data) { 1607 found = var_data; 1608 break; 1609 } 1610 } 1611 1612 return found; 1613 } 1614 1615 static bool field_has_hist_vars(struct hist_field *hist_field, 1616 unsigned int level) 1617 { 1618 int i; 1619 1620 if (level > 3) 1621 return false; 1622 1623 if (!hist_field) 1624 return false; 1625 1626 if (hist_field->flags & HIST_FIELD_FL_VAR || 1627 hist_field->flags & HIST_FIELD_FL_VAR_REF) 1628 return true; 1629 1630 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) { 1631 struct hist_field *operand; 1632 1633 operand = hist_field->operands[i]; 1634 if (field_has_hist_vars(operand, level + 1)) 1635 return true; 1636 } 1637 1638 return false; 1639 } 1640 1641 static bool has_hist_vars(struct hist_trigger_data *hist_data) 1642 { 1643 struct hist_field *hist_field; 1644 int i; 1645 1646 for_each_hist_field(i, hist_data) { 1647 hist_field = hist_data->fields[i]; 1648 if (field_has_hist_vars(hist_field, 0)) 1649 return true; 1650 } 1651 1652 return false; 1653 } 1654 1655 static int save_hist_vars(struct hist_trigger_data *hist_data) 1656 { 1657 struct trace_array *tr = hist_data->event_file->tr; 1658 struct hist_var_data *var_data; 1659 1660 var_data = find_hist_vars(hist_data); 1661 if (var_data) 1662 return 0; 1663 1664 if (trace_array_get(tr) < 0) 1665 return -ENODEV; 1666 1667 var_data = kzalloc(sizeof(*var_data), GFP_KERNEL); 1668 if (!var_data) { 1669 trace_array_put(tr); 1670 return -ENOMEM; 1671 } 1672 1673 var_data->hist_data = hist_data; 1674 list_add(&var_data->list, &tr->hist_vars); 1675 1676 return 0; 1677 } 1678 1679 static void remove_hist_vars(struct hist_trigger_data *hist_data) 1680 { 1681 struct trace_array *tr = hist_data->event_file->tr; 1682 struct hist_var_data *var_data; 1683 1684 var_data = find_hist_vars(hist_data); 1685 if (!var_data) 1686 return; 1687 1688 if (WARN_ON(check_var_refs(hist_data))) 1689 return; 1690 1691 list_del(&var_data->list); 1692 1693 kfree(var_data); 1694 1695 trace_array_put(tr); 1696 } 1697 1698 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data, 1699 const char *var_name) 1700 { 1701 struct hist_field *hist_field, *found = NULL; 1702 int i; 1703 1704 for_each_hist_field(i, hist_data) { 1705 hist_field = hist_data->fields[i]; 1706 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR && 1707 strcmp(hist_field->var.name, var_name) == 0) { 1708 found = hist_field; 1709 break; 1710 } 1711 } 1712 1713 return found; 1714 } 1715 1716 static struct hist_field *find_var(struct hist_trigger_data *hist_data, 1717 struct trace_event_file *file, 1718 const char *var_name) 1719 { 1720 struct hist_trigger_data *test_data; 1721 struct event_trigger_data *test; 1722 struct hist_field *hist_field; 1723 1724 hist_field = find_var_field(hist_data, var_name); 1725 if (hist_field) 1726 return hist_field; 1727 1728 list_for_each_entry_rcu(test, &file->triggers, list) { 1729 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1730 test_data = test->private_data; 1731 hist_field = find_var_field(test_data, var_name); 1732 if (hist_field) 1733 return hist_field; 1734 } 1735 } 1736 1737 return NULL; 1738 } 1739 1740 static struct trace_event_file *find_var_file(struct trace_array *tr, 1741 char *system, 1742 char *event_name, 1743 char *var_name) 1744 { 1745 struct hist_trigger_data *var_hist_data; 1746 struct hist_var_data *var_data; 1747 struct trace_event_file *file, *found = NULL; 1748 1749 if (system) 1750 return find_event_file(tr, system, event_name); 1751 1752 list_for_each_entry(var_data, &tr->hist_vars, list) { 1753 var_hist_data = var_data->hist_data; 1754 file = var_hist_data->event_file; 1755 if (file == found) 1756 continue; 1757 1758 if (find_var_field(var_hist_data, var_name)) { 1759 if (found) { 1760 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name)); 1761 return NULL; 1762 } 1763 1764 found = file; 1765 } 1766 } 1767 1768 return found; 1769 } 1770 1771 static struct hist_field *find_file_var(struct trace_event_file *file, 1772 const char *var_name) 1773 { 1774 struct hist_trigger_data *test_data; 1775 struct event_trigger_data *test; 1776 struct hist_field *hist_field; 1777 1778 list_for_each_entry_rcu(test, &file->triggers, list) { 1779 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1780 test_data = test->private_data; 1781 hist_field = find_var_field(test_data, var_name); 1782 if (hist_field) 1783 return hist_field; 1784 } 1785 } 1786 1787 return NULL; 1788 } 1789 1790 static struct hist_field * 1791 find_match_var(struct hist_trigger_data *hist_data, char *var_name) 1792 { 1793 struct trace_array *tr = hist_data->event_file->tr; 1794 struct hist_field *hist_field, *found = NULL; 1795 struct trace_event_file *file; 1796 unsigned int i; 1797 1798 for (i = 0; i < hist_data->n_actions; i++) { 1799 struct action_data *data = hist_data->actions[i]; 1800 1801 if (data->handler == HANDLER_ONMATCH) { 1802 char *system = data->match_data.event_system; 1803 char *event_name = data->match_data.event; 1804 1805 file = find_var_file(tr, system, event_name, var_name); 1806 if (!file) 1807 continue; 1808 hist_field = find_file_var(file, var_name); 1809 if (hist_field) { 1810 if (found) { 1811 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, 1812 errpos(var_name)); 1813 return ERR_PTR(-EINVAL); 1814 } 1815 1816 found = hist_field; 1817 } 1818 } 1819 } 1820 return found; 1821 } 1822 1823 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data, 1824 char *system, 1825 char *event_name, 1826 char *var_name) 1827 { 1828 struct trace_array *tr = hist_data->event_file->tr; 1829 struct hist_field *hist_field = NULL; 1830 struct trace_event_file *file; 1831 1832 if (!system || !event_name) { 1833 hist_field = find_match_var(hist_data, var_name); 1834 if (IS_ERR(hist_field)) 1835 return NULL; 1836 if (hist_field) 1837 return hist_field; 1838 } 1839 1840 file = find_var_file(tr, system, event_name, var_name); 1841 if (!file) 1842 return NULL; 1843 1844 hist_field = find_file_var(file, var_name); 1845 1846 return hist_field; 1847 } 1848 1849 static u64 hist_field_var_ref(struct hist_field *hist_field, 1850 struct tracing_map_elt *elt, 1851 struct ring_buffer_event *rbe, 1852 void *event) 1853 { 1854 struct hist_elt_data *elt_data; 1855 u64 var_val = 0; 1856 1857 elt_data = elt->private_data; 1858 var_val = elt_data->var_ref_vals[hist_field->var_ref_idx]; 1859 1860 return var_val; 1861 } 1862 1863 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key, 1864 u64 *var_ref_vals, bool self) 1865 { 1866 struct hist_trigger_data *var_data; 1867 struct tracing_map_elt *var_elt; 1868 struct hist_field *hist_field; 1869 unsigned int i, var_idx; 1870 bool resolved = true; 1871 u64 var_val = 0; 1872 1873 for (i = 0; i < hist_data->n_var_refs; i++) { 1874 hist_field = hist_data->var_refs[i]; 1875 var_idx = hist_field->var.idx; 1876 var_data = hist_field->var.hist_data; 1877 1878 if (var_data == NULL) { 1879 resolved = false; 1880 break; 1881 } 1882 1883 if ((self && var_data != hist_data) || 1884 (!self && var_data == hist_data)) 1885 continue; 1886 1887 var_elt = tracing_map_lookup(var_data->map, key); 1888 if (!var_elt) { 1889 resolved = false; 1890 break; 1891 } 1892 1893 if (!tracing_map_var_set(var_elt, var_idx)) { 1894 resolved = false; 1895 break; 1896 } 1897 1898 if (self || !hist_field->read_once) 1899 var_val = tracing_map_read_var(var_elt, var_idx); 1900 else 1901 var_val = tracing_map_read_var_once(var_elt, var_idx); 1902 1903 var_ref_vals[i] = var_val; 1904 } 1905 1906 return resolved; 1907 } 1908 1909 static const char *hist_field_name(struct hist_field *field, 1910 unsigned int level) 1911 { 1912 const char *field_name = ""; 1913 1914 if (level > 1) 1915 return field_name; 1916 1917 if (field->field) 1918 field_name = field->field->name; 1919 else if (field->flags & HIST_FIELD_FL_LOG2 || 1920 field->flags & HIST_FIELD_FL_ALIAS) 1921 field_name = hist_field_name(field->operands[0], ++level); 1922 else if (field->flags & HIST_FIELD_FL_CPU) 1923 field_name = "cpu"; 1924 else if (field->flags & HIST_FIELD_FL_EXPR || 1925 field->flags & HIST_FIELD_FL_VAR_REF) { 1926 if (field->system) { 1927 static char full_name[MAX_FILTER_STR_VAL]; 1928 1929 strcat(full_name, field->system); 1930 strcat(full_name, "."); 1931 strcat(full_name, field->event_name); 1932 strcat(full_name, "."); 1933 strcat(full_name, field->name); 1934 field_name = full_name; 1935 } else 1936 field_name = field->name; 1937 } else if (field->flags & HIST_FIELD_FL_TIMESTAMP) 1938 field_name = "common_timestamp"; 1939 1940 if (field_name == NULL) 1941 field_name = ""; 1942 1943 return field_name; 1944 } 1945 1946 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed) 1947 { 1948 hist_field_fn_t fn = NULL; 1949 1950 switch (field_size) { 1951 case 8: 1952 if (field_is_signed) 1953 fn = hist_field_s64; 1954 else 1955 fn = hist_field_u64; 1956 break; 1957 case 4: 1958 if (field_is_signed) 1959 fn = hist_field_s32; 1960 else 1961 fn = hist_field_u32; 1962 break; 1963 case 2: 1964 if (field_is_signed) 1965 fn = hist_field_s16; 1966 else 1967 fn = hist_field_u16; 1968 break; 1969 case 1: 1970 if (field_is_signed) 1971 fn = hist_field_s8; 1972 else 1973 fn = hist_field_u8; 1974 break; 1975 } 1976 1977 return fn; 1978 } 1979 1980 static int parse_map_size(char *str) 1981 { 1982 unsigned long size, map_bits; 1983 int ret; 1984 1985 strsep(&str, "="); 1986 if (!str) { 1987 ret = -EINVAL; 1988 goto out; 1989 } 1990 1991 ret = kstrtoul(str, 0, &size); 1992 if (ret) 1993 goto out; 1994 1995 map_bits = ilog2(roundup_pow_of_two(size)); 1996 if (map_bits < TRACING_MAP_BITS_MIN || 1997 map_bits > TRACING_MAP_BITS_MAX) 1998 ret = -EINVAL; 1999 else 2000 ret = map_bits; 2001 out: 2002 return ret; 2003 } 2004 2005 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs) 2006 { 2007 unsigned int i; 2008 2009 if (!attrs) 2010 return; 2011 2012 for (i = 0; i < attrs->n_assignments; i++) 2013 kfree(attrs->assignment_str[i]); 2014 2015 for (i = 0; i < attrs->n_actions; i++) 2016 kfree(attrs->action_str[i]); 2017 2018 kfree(attrs->name); 2019 kfree(attrs->sort_key_str); 2020 kfree(attrs->keys_str); 2021 kfree(attrs->vals_str); 2022 kfree(attrs->clock); 2023 kfree(attrs); 2024 } 2025 2026 static int parse_action(char *str, struct hist_trigger_attrs *attrs) 2027 { 2028 int ret = -EINVAL; 2029 2030 if (attrs->n_actions >= HIST_ACTIONS_MAX) 2031 return ret; 2032 2033 if ((str_has_prefix(str, "onmatch(")) || 2034 (str_has_prefix(str, "onmax(")) || 2035 (str_has_prefix(str, "onchange("))) { 2036 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL); 2037 if (!attrs->action_str[attrs->n_actions]) { 2038 ret = -ENOMEM; 2039 return ret; 2040 } 2041 attrs->n_actions++; 2042 ret = 0; 2043 } 2044 return ret; 2045 } 2046 2047 static int parse_assignment(struct trace_array *tr, 2048 char *str, struct hist_trigger_attrs *attrs) 2049 { 2050 int ret = 0; 2051 2052 if ((str_has_prefix(str, "key=")) || 2053 (str_has_prefix(str, "keys="))) { 2054 attrs->keys_str = kstrdup(str, GFP_KERNEL); 2055 if (!attrs->keys_str) { 2056 ret = -ENOMEM; 2057 goto out; 2058 } 2059 } else if ((str_has_prefix(str, "val=")) || 2060 (str_has_prefix(str, "vals=")) || 2061 (str_has_prefix(str, "values="))) { 2062 attrs->vals_str = kstrdup(str, GFP_KERNEL); 2063 if (!attrs->vals_str) { 2064 ret = -ENOMEM; 2065 goto out; 2066 } 2067 } else if (str_has_prefix(str, "sort=")) { 2068 attrs->sort_key_str = kstrdup(str, GFP_KERNEL); 2069 if (!attrs->sort_key_str) { 2070 ret = -ENOMEM; 2071 goto out; 2072 } 2073 } else if (str_has_prefix(str, "name=")) { 2074 attrs->name = kstrdup(str, GFP_KERNEL); 2075 if (!attrs->name) { 2076 ret = -ENOMEM; 2077 goto out; 2078 } 2079 } else if (str_has_prefix(str, "clock=")) { 2080 strsep(&str, "="); 2081 if (!str) { 2082 ret = -EINVAL; 2083 goto out; 2084 } 2085 2086 str = strstrip(str); 2087 attrs->clock = kstrdup(str, GFP_KERNEL); 2088 if (!attrs->clock) { 2089 ret = -ENOMEM; 2090 goto out; 2091 } 2092 } else if (str_has_prefix(str, "size=")) { 2093 int map_bits = parse_map_size(str); 2094 2095 if (map_bits < 0) { 2096 ret = map_bits; 2097 goto out; 2098 } 2099 attrs->map_bits = map_bits; 2100 } else { 2101 char *assignment; 2102 2103 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) { 2104 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str)); 2105 ret = -EINVAL; 2106 goto out; 2107 } 2108 2109 assignment = kstrdup(str, GFP_KERNEL); 2110 if (!assignment) { 2111 ret = -ENOMEM; 2112 goto out; 2113 } 2114 2115 attrs->assignment_str[attrs->n_assignments++] = assignment; 2116 } 2117 out: 2118 return ret; 2119 } 2120 2121 static struct hist_trigger_attrs * 2122 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str) 2123 { 2124 struct hist_trigger_attrs *attrs; 2125 int ret = 0; 2126 2127 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL); 2128 if (!attrs) 2129 return ERR_PTR(-ENOMEM); 2130 2131 while (trigger_str) { 2132 char *str = strsep(&trigger_str, ":"); 2133 2134 if (strchr(str, '=')) { 2135 ret = parse_assignment(tr, str, attrs); 2136 if (ret) 2137 goto free; 2138 } else if (strcmp(str, "pause") == 0) 2139 attrs->pause = true; 2140 else if ((strcmp(str, "cont") == 0) || 2141 (strcmp(str, "continue") == 0)) 2142 attrs->cont = true; 2143 else if (strcmp(str, "clear") == 0) 2144 attrs->clear = true; 2145 else { 2146 ret = parse_action(str, attrs); 2147 if (ret) 2148 goto free; 2149 } 2150 } 2151 2152 if (!attrs->keys_str) { 2153 ret = -EINVAL; 2154 goto free; 2155 } 2156 2157 if (!attrs->clock) { 2158 attrs->clock = kstrdup("global", GFP_KERNEL); 2159 if (!attrs->clock) { 2160 ret = -ENOMEM; 2161 goto free; 2162 } 2163 } 2164 2165 return attrs; 2166 free: 2167 destroy_hist_trigger_attrs(attrs); 2168 2169 return ERR_PTR(ret); 2170 } 2171 2172 static inline void save_comm(char *comm, struct task_struct *task) 2173 { 2174 if (!task->pid) { 2175 strcpy(comm, "<idle>"); 2176 return; 2177 } 2178 2179 if (WARN_ON_ONCE(task->pid < 0)) { 2180 strcpy(comm, "<XXX>"); 2181 return; 2182 } 2183 2184 strncpy(comm, task->comm, TASK_COMM_LEN); 2185 } 2186 2187 static void hist_elt_data_free(struct hist_elt_data *elt_data) 2188 { 2189 unsigned int i; 2190 2191 for (i = 0; i < SYNTH_FIELDS_MAX; i++) 2192 kfree(elt_data->field_var_str[i]); 2193 2194 kfree(elt_data->comm); 2195 kfree(elt_data); 2196 } 2197 2198 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt) 2199 { 2200 struct hist_elt_data *elt_data = elt->private_data; 2201 2202 hist_elt_data_free(elt_data); 2203 } 2204 2205 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt) 2206 { 2207 struct hist_trigger_data *hist_data = elt->map->private_data; 2208 unsigned int size = TASK_COMM_LEN; 2209 struct hist_elt_data *elt_data; 2210 struct hist_field *key_field; 2211 unsigned int i, n_str; 2212 2213 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL); 2214 if (!elt_data) 2215 return -ENOMEM; 2216 2217 for_each_hist_key_field(i, hist_data) { 2218 key_field = hist_data->fields[i]; 2219 2220 if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 2221 elt_data->comm = kzalloc(size, GFP_KERNEL); 2222 if (!elt_data->comm) { 2223 kfree(elt_data); 2224 return -ENOMEM; 2225 } 2226 break; 2227 } 2228 } 2229 2230 n_str = hist_data->n_field_var_str + hist_data->n_save_var_str; 2231 2232 size = STR_VAR_LEN_MAX; 2233 2234 for (i = 0; i < n_str; i++) { 2235 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL); 2236 if (!elt_data->field_var_str[i]) { 2237 hist_elt_data_free(elt_data); 2238 return -ENOMEM; 2239 } 2240 } 2241 2242 elt->private_data = elt_data; 2243 2244 return 0; 2245 } 2246 2247 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt) 2248 { 2249 struct hist_elt_data *elt_data = elt->private_data; 2250 2251 if (elt_data->comm) 2252 save_comm(elt_data->comm, current); 2253 } 2254 2255 static const struct tracing_map_ops hist_trigger_elt_data_ops = { 2256 .elt_alloc = hist_trigger_elt_data_alloc, 2257 .elt_free = hist_trigger_elt_data_free, 2258 .elt_init = hist_trigger_elt_data_init, 2259 }; 2260 2261 static const char *get_hist_field_flags(struct hist_field *hist_field) 2262 { 2263 const char *flags_str = NULL; 2264 2265 if (hist_field->flags & HIST_FIELD_FL_HEX) 2266 flags_str = "hex"; 2267 else if (hist_field->flags & HIST_FIELD_FL_SYM) 2268 flags_str = "sym"; 2269 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET) 2270 flags_str = "sym-offset"; 2271 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME) 2272 flags_str = "execname"; 2273 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL) 2274 flags_str = "syscall"; 2275 else if (hist_field->flags & HIST_FIELD_FL_LOG2) 2276 flags_str = "log2"; 2277 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS) 2278 flags_str = "usecs"; 2279 2280 return flags_str; 2281 } 2282 2283 static void expr_field_str(struct hist_field *field, char *expr) 2284 { 2285 if (field->flags & HIST_FIELD_FL_VAR_REF) 2286 strcat(expr, "$"); 2287 2288 strcat(expr, hist_field_name(field, 0)); 2289 2290 if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) { 2291 const char *flags_str = get_hist_field_flags(field); 2292 2293 if (flags_str) { 2294 strcat(expr, "."); 2295 strcat(expr, flags_str); 2296 } 2297 } 2298 } 2299 2300 static char *expr_str(struct hist_field *field, unsigned int level) 2301 { 2302 char *expr; 2303 2304 if (level > 1) 2305 return NULL; 2306 2307 expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2308 if (!expr) 2309 return NULL; 2310 2311 if (!field->operands[0]) { 2312 expr_field_str(field, expr); 2313 return expr; 2314 } 2315 2316 if (field->operator == FIELD_OP_UNARY_MINUS) { 2317 char *subexpr; 2318 2319 strcat(expr, "-("); 2320 subexpr = expr_str(field->operands[0], ++level); 2321 if (!subexpr) { 2322 kfree(expr); 2323 return NULL; 2324 } 2325 strcat(expr, subexpr); 2326 strcat(expr, ")"); 2327 2328 kfree(subexpr); 2329 2330 return expr; 2331 } 2332 2333 expr_field_str(field->operands[0], expr); 2334 2335 switch (field->operator) { 2336 case FIELD_OP_MINUS: 2337 strcat(expr, "-"); 2338 break; 2339 case FIELD_OP_PLUS: 2340 strcat(expr, "+"); 2341 break; 2342 default: 2343 kfree(expr); 2344 return NULL; 2345 } 2346 2347 expr_field_str(field->operands[1], expr); 2348 2349 return expr; 2350 } 2351 2352 static int contains_operator(char *str) 2353 { 2354 enum field_op_id field_op = FIELD_OP_NONE; 2355 char *op; 2356 2357 op = strpbrk(str, "+-"); 2358 if (!op) 2359 return FIELD_OP_NONE; 2360 2361 switch (*op) { 2362 case '-': 2363 if (*str == '-') 2364 field_op = FIELD_OP_UNARY_MINUS; 2365 else 2366 field_op = FIELD_OP_MINUS; 2367 break; 2368 case '+': 2369 field_op = FIELD_OP_PLUS; 2370 break; 2371 default: 2372 break; 2373 } 2374 2375 return field_op; 2376 } 2377 2378 static void __destroy_hist_field(struct hist_field *hist_field) 2379 { 2380 kfree(hist_field->var.name); 2381 kfree(hist_field->name); 2382 kfree(hist_field->type); 2383 2384 kfree(hist_field); 2385 } 2386 2387 static void destroy_hist_field(struct hist_field *hist_field, 2388 unsigned int level) 2389 { 2390 unsigned int i; 2391 2392 if (level > 3) 2393 return; 2394 2395 if (!hist_field) 2396 return; 2397 2398 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) 2399 return; /* var refs will be destroyed separately */ 2400 2401 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) 2402 destroy_hist_field(hist_field->operands[i], level + 1); 2403 2404 __destroy_hist_field(hist_field); 2405 } 2406 2407 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, 2408 struct ftrace_event_field *field, 2409 unsigned long flags, 2410 char *var_name) 2411 { 2412 struct hist_field *hist_field; 2413 2414 if (field && is_function_field(field)) 2415 return NULL; 2416 2417 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL); 2418 if (!hist_field) 2419 return NULL; 2420 2421 hist_field->hist_data = hist_data; 2422 2423 if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS) 2424 goto out; /* caller will populate */ 2425 2426 if (flags & HIST_FIELD_FL_VAR_REF) { 2427 hist_field->fn = hist_field_var_ref; 2428 goto out; 2429 } 2430 2431 if (flags & HIST_FIELD_FL_HITCOUNT) { 2432 hist_field->fn = hist_field_counter; 2433 hist_field->size = sizeof(u64); 2434 hist_field->type = kstrdup("u64", GFP_KERNEL); 2435 if (!hist_field->type) 2436 goto free; 2437 goto out; 2438 } 2439 2440 if (flags & HIST_FIELD_FL_STACKTRACE) { 2441 hist_field->fn = hist_field_none; 2442 goto out; 2443 } 2444 2445 if (flags & HIST_FIELD_FL_LOG2) { 2446 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2; 2447 hist_field->fn = hist_field_log2; 2448 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL); 2449 hist_field->size = hist_field->operands[0]->size; 2450 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL); 2451 if (!hist_field->type) 2452 goto free; 2453 goto out; 2454 } 2455 2456 if (flags & HIST_FIELD_FL_TIMESTAMP) { 2457 hist_field->fn = hist_field_timestamp; 2458 hist_field->size = sizeof(u64); 2459 hist_field->type = kstrdup("u64", GFP_KERNEL); 2460 if (!hist_field->type) 2461 goto free; 2462 goto out; 2463 } 2464 2465 if (flags & HIST_FIELD_FL_CPU) { 2466 hist_field->fn = hist_field_cpu; 2467 hist_field->size = sizeof(int); 2468 hist_field->type = kstrdup("unsigned int", GFP_KERNEL); 2469 if (!hist_field->type) 2470 goto free; 2471 goto out; 2472 } 2473 2474 if (WARN_ON_ONCE(!field)) 2475 goto out; 2476 2477 if (is_string_field(field)) { 2478 flags |= HIST_FIELD_FL_STRING; 2479 2480 hist_field->size = MAX_FILTER_STR_VAL; 2481 hist_field->type = kstrdup(field->type, GFP_KERNEL); 2482 if (!hist_field->type) 2483 goto free; 2484 2485 if (field->filter_type == FILTER_STATIC_STRING) 2486 hist_field->fn = hist_field_string; 2487 else if (field->filter_type == FILTER_DYN_STRING) 2488 hist_field->fn = hist_field_dynstring; 2489 else 2490 hist_field->fn = hist_field_pstring; 2491 } else { 2492 hist_field->size = field->size; 2493 hist_field->is_signed = field->is_signed; 2494 hist_field->type = kstrdup(field->type, GFP_KERNEL); 2495 if (!hist_field->type) 2496 goto free; 2497 2498 hist_field->fn = select_value_fn(field->size, 2499 field->is_signed); 2500 if (!hist_field->fn) { 2501 destroy_hist_field(hist_field, 0); 2502 return NULL; 2503 } 2504 } 2505 out: 2506 hist_field->field = field; 2507 hist_field->flags = flags; 2508 2509 if (var_name) { 2510 hist_field->var.name = kstrdup(var_name, GFP_KERNEL); 2511 if (!hist_field->var.name) 2512 goto free; 2513 } 2514 2515 return hist_field; 2516 free: 2517 destroy_hist_field(hist_field, 0); 2518 return NULL; 2519 } 2520 2521 static void destroy_hist_fields(struct hist_trigger_data *hist_data) 2522 { 2523 unsigned int i; 2524 2525 for (i = 0; i < HIST_FIELDS_MAX; i++) { 2526 if (hist_data->fields[i]) { 2527 destroy_hist_field(hist_data->fields[i], 0); 2528 hist_data->fields[i] = NULL; 2529 } 2530 } 2531 2532 for (i = 0; i < hist_data->n_var_refs; i++) { 2533 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF)); 2534 __destroy_hist_field(hist_data->var_refs[i]); 2535 hist_data->var_refs[i] = NULL; 2536 } 2537 } 2538 2539 static int init_var_ref(struct hist_field *ref_field, 2540 struct hist_field *var_field, 2541 char *system, char *event_name) 2542 { 2543 int err = 0; 2544 2545 ref_field->var.idx = var_field->var.idx; 2546 ref_field->var.hist_data = var_field->hist_data; 2547 ref_field->size = var_field->size; 2548 ref_field->is_signed = var_field->is_signed; 2549 ref_field->flags |= var_field->flags & 2550 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2551 2552 if (system) { 2553 ref_field->system = kstrdup(system, GFP_KERNEL); 2554 if (!ref_field->system) 2555 return -ENOMEM; 2556 } 2557 2558 if (event_name) { 2559 ref_field->event_name = kstrdup(event_name, GFP_KERNEL); 2560 if (!ref_field->event_name) { 2561 err = -ENOMEM; 2562 goto free; 2563 } 2564 } 2565 2566 if (var_field->var.name) { 2567 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL); 2568 if (!ref_field->name) { 2569 err = -ENOMEM; 2570 goto free; 2571 } 2572 } else if (var_field->name) { 2573 ref_field->name = kstrdup(var_field->name, GFP_KERNEL); 2574 if (!ref_field->name) { 2575 err = -ENOMEM; 2576 goto free; 2577 } 2578 } 2579 2580 ref_field->type = kstrdup(var_field->type, GFP_KERNEL); 2581 if (!ref_field->type) { 2582 err = -ENOMEM; 2583 goto free; 2584 } 2585 out: 2586 return err; 2587 free: 2588 kfree(ref_field->system); 2589 kfree(ref_field->event_name); 2590 kfree(ref_field->name); 2591 2592 goto out; 2593 } 2594 2595 /** 2596 * create_var_ref - Create a variable reference and attach it to trigger 2597 * @hist_data: The trigger that will be referencing the variable 2598 * @var_field: The VAR field to create a reference to 2599 * @system: The optional system string 2600 * @event_name: The optional event_name string 2601 * 2602 * Given a variable hist_field, create a VAR_REF hist_field that 2603 * represents a reference to it. 2604 * 2605 * This function also adds the reference to the trigger that 2606 * now references the variable. 2607 * 2608 * Return: The VAR_REF field if successful, NULL if not 2609 */ 2610 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data, 2611 struct hist_field *var_field, 2612 char *system, char *event_name) 2613 { 2614 unsigned long flags = HIST_FIELD_FL_VAR_REF; 2615 struct hist_field *ref_field; 2616 2617 ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL); 2618 if (ref_field) { 2619 if (init_var_ref(ref_field, var_field, system, event_name)) { 2620 destroy_hist_field(ref_field, 0); 2621 return NULL; 2622 } 2623 2624 hist_data->var_refs[hist_data->n_var_refs] = ref_field; 2625 ref_field->var_ref_idx = hist_data->n_var_refs++; 2626 } 2627 2628 return ref_field; 2629 } 2630 2631 static bool is_var_ref(char *var_name) 2632 { 2633 if (!var_name || strlen(var_name) < 2 || var_name[0] != '$') 2634 return false; 2635 2636 return true; 2637 } 2638 2639 static char *field_name_from_var(struct hist_trigger_data *hist_data, 2640 char *var_name) 2641 { 2642 char *name, *field; 2643 unsigned int i; 2644 2645 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 2646 name = hist_data->attrs->var_defs.name[i]; 2647 2648 if (strcmp(var_name, name) == 0) { 2649 field = hist_data->attrs->var_defs.expr[i]; 2650 if (contains_operator(field) || is_var_ref(field)) 2651 continue; 2652 return field; 2653 } 2654 } 2655 2656 return NULL; 2657 } 2658 2659 static char *local_field_var_ref(struct hist_trigger_data *hist_data, 2660 char *system, char *event_name, 2661 char *var_name) 2662 { 2663 struct trace_event_call *call; 2664 2665 if (system && event_name) { 2666 call = hist_data->event_file->event_call; 2667 2668 if (strcmp(system, call->class->system) != 0) 2669 return NULL; 2670 2671 if (strcmp(event_name, trace_event_name(call)) != 0) 2672 return NULL; 2673 } 2674 2675 if (!!system != !!event_name) 2676 return NULL; 2677 2678 if (!is_var_ref(var_name)) 2679 return NULL; 2680 2681 var_name++; 2682 2683 return field_name_from_var(hist_data, var_name); 2684 } 2685 2686 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data, 2687 char *system, char *event_name, 2688 char *var_name) 2689 { 2690 struct hist_field *var_field = NULL, *ref_field = NULL; 2691 struct trace_array *tr = hist_data->event_file->tr; 2692 2693 if (!is_var_ref(var_name)) 2694 return NULL; 2695 2696 var_name++; 2697 2698 var_field = find_event_var(hist_data, system, event_name, var_name); 2699 if (var_field) 2700 ref_field = create_var_ref(hist_data, var_field, 2701 system, event_name); 2702 2703 if (!ref_field) 2704 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name)); 2705 2706 return ref_field; 2707 } 2708 2709 static struct ftrace_event_field * 2710 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, 2711 char *field_str, unsigned long *flags) 2712 { 2713 struct ftrace_event_field *field = NULL; 2714 char *field_name, *modifier, *str; 2715 struct trace_array *tr = file->tr; 2716 2717 modifier = str = kstrdup(field_str, GFP_KERNEL); 2718 if (!modifier) 2719 return ERR_PTR(-ENOMEM); 2720 2721 field_name = strsep(&modifier, "."); 2722 if (modifier) { 2723 if (strcmp(modifier, "hex") == 0) 2724 *flags |= HIST_FIELD_FL_HEX; 2725 else if (strcmp(modifier, "sym") == 0) 2726 *flags |= HIST_FIELD_FL_SYM; 2727 else if (strcmp(modifier, "sym-offset") == 0) 2728 *flags |= HIST_FIELD_FL_SYM_OFFSET; 2729 else if ((strcmp(modifier, "execname") == 0) && 2730 (strcmp(field_name, "common_pid") == 0)) 2731 *flags |= HIST_FIELD_FL_EXECNAME; 2732 else if (strcmp(modifier, "syscall") == 0) 2733 *flags |= HIST_FIELD_FL_SYSCALL; 2734 else if (strcmp(modifier, "log2") == 0) 2735 *flags |= HIST_FIELD_FL_LOG2; 2736 else if (strcmp(modifier, "usecs") == 0) 2737 *flags |= HIST_FIELD_FL_TIMESTAMP_USECS; 2738 else { 2739 hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier)); 2740 field = ERR_PTR(-EINVAL); 2741 goto out; 2742 } 2743 } 2744 2745 if (strcmp(field_name, "common_timestamp") == 0) { 2746 *flags |= HIST_FIELD_FL_TIMESTAMP; 2747 hist_data->enable_timestamps = true; 2748 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS) 2749 hist_data->attrs->ts_in_usecs = true; 2750 } else if (strcmp(field_name, "cpu") == 0) 2751 *flags |= HIST_FIELD_FL_CPU; 2752 else { 2753 field = trace_find_event_field(file->event_call, field_name); 2754 if (!field || !field->size) { 2755 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name)); 2756 field = ERR_PTR(-EINVAL); 2757 goto out; 2758 } 2759 } 2760 out: 2761 kfree(str); 2762 2763 return field; 2764 } 2765 2766 static struct hist_field *create_alias(struct hist_trigger_data *hist_data, 2767 struct hist_field *var_ref, 2768 char *var_name) 2769 { 2770 struct hist_field *alias = NULL; 2771 unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR; 2772 2773 alias = create_hist_field(hist_data, NULL, flags, var_name); 2774 if (!alias) 2775 return NULL; 2776 2777 alias->fn = var_ref->fn; 2778 alias->operands[0] = var_ref; 2779 2780 if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) { 2781 destroy_hist_field(alias, 0); 2782 return NULL; 2783 } 2784 2785 return alias; 2786 } 2787 2788 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data, 2789 struct trace_event_file *file, char *str, 2790 unsigned long *flags, char *var_name) 2791 { 2792 char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str; 2793 struct ftrace_event_field *field = NULL; 2794 struct hist_field *hist_field = NULL; 2795 int ret = 0; 2796 2797 s = strchr(str, '.'); 2798 if (s) { 2799 s = strchr(++s, '.'); 2800 if (s) { 2801 ref_system = strsep(&str, "."); 2802 if (!str) { 2803 ret = -EINVAL; 2804 goto out; 2805 } 2806 ref_event = strsep(&str, "."); 2807 if (!str) { 2808 ret = -EINVAL; 2809 goto out; 2810 } 2811 ref_var = str; 2812 } 2813 } 2814 2815 s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var); 2816 if (!s) { 2817 hist_field = parse_var_ref(hist_data, ref_system, 2818 ref_event, ref_var); 2819 if (hist_field) { 2820 if (var_name) { 2821 hist_field = create_alias(hist_data, hist_field, var_name); 2822 if (!hist_field) { 2823 ret = -ENOMEM; 2824 goto out; 2825 } 2826 } 2827 return hist_field; 2828 } 2829 } else 2830 str = s; 2831 2832 field = parse_field(hist_data, file, str, flags); 2833 if (IS_ERR(field)) { 2834 ret = PTR_ERR(field); 2835 goto out; 2836 } 2837 2838 hist_field = create_hist_field(hist_data, field, *flags, var_name); 2839 if (!hist_field) { 2840 ret = -ENOMEM; 2841 goto out; 2842 } 2843 2844 return hist_field; 2845 out: 2846 return ERR_PTR(ret); 2847 } 2848 2849 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2850 struct trace_event_file *file, 2851 char *str, unsigned long flags, 2852 char *var_name, unsigned int level); 2853 2854 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data, 2855 struct trace_event_file *file, 2856 char *str, unsigned long flags, 2857 char *var_name, unsigned int level) 2858 { 2859 struct hist_field *operand1, *expr = NULL; 2860 unsigned long operand_flags; 2861 int ret = 0; 2862 char *s; 2863 2864 /* we support only -(xxx) i.e. explicit parens required */ 2865 2866 if (level > 3) { 2867 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str)); 2868 ret = -EINVAL; 2869 goto free; 2870 } 2871 2872 str++; /* skip leading '-' */ 2873 2874 s = strchr(str, '('); 2875 if (s) 2876 str++; 2877 else { 2878 ret = -EINVAL; 2879 goto free; 2880 } 2881 2882 s = strrchr(str, ')'); 2883 if (s) 2884 *s = '\0'; 2885 else { 2886 ret = -EINVAL; /* no closing ')' */ 2887 goto free; 2888 } 2889 2890 flags |= HIST_FIELD_FL_EXPR; 2891 expr = create_hist_field(hist_data, NULL, flags, var_name); 2892 if (!expr) { 2893 ret = -ENOMEM; 2894 goto free; 2895 } 2896 2897 operand_flags = 0; 2898 operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level); 2899 if (IS_ERR(operand1)) { 2900 ret = PTR_ERR(operand1); 2901 goto free; 2902 } 2903 2904 expr->flags |= operand1->flags & 2905 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2906 expr->fn = hist_field_unary_minus; 2907 expr->operands[0] = operand1; 2908 expr->operator = FIELD_OP_UNARY_MINUS; 2909 expr->name = expr_str(expr, 0); 2910 expr->type = kstrdup(operand1->type, GFP_KERNEL); 2911 if (!expr->type) { 2912 ret = -ENOMEM; 2913 goto free; 2914 } 2915 2916 return expr; 2917 free: 2918 destroy_hist_field(expr, 0); 2919 return ERR_PTR(ret); 2920 } 2921 2922 static int check_expr_operands(struct trace_array *tr, 2923 struct hist_field *operand1, 2924 struct hist_field *operand2) 2925 { 2926 unsigned long operand1_flags = operand1->flags; 2927 unsigned long operand2_flags = operand2->flags; 2928 2929 if ((operand1_flags & HIST_FIELD_FL_VAR_REF) || 2930 (operand1_flags & HIST_FIELD_FL_ALIAS)) { 2931 struct hist_field *var; 2932 2933 var = find_var_field(operand1->var.hist_data, operand1->name); 2934 if (!var) 2935 return -EINVAL; 2936 operand1_flags = var->flags; 2937 } 2938 2939 if ((operand2_flags & HIST_FIELD_FL_VAR_REF) || 2940 (operand2_flags & HIST_FIELD_FL_ALIAS)) { 2941 struct hist_field *var; 2942 2943 var = find_var_field(operand2->var.hist_data, operand2->name); 2944 if (!var) 2945 return -EINVAL; 2946 operand2_flags = var->flags; 2947 } 2948 2949 if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) != 2950 (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) { 2951 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0); 2952 return -EINVAL; 2953 } 2954 2955 return 0; 2956 } 2957 2958 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2959 struct trace_event_file *file, 2960 char *str, unsigned long flags, 2961 char *var_name, unsigned int level) 2962 { 2963 struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL; 2964 unsigned long operand_flags; 2965 int field_op, ret = -EINVAL; 2966 char *sep, *operand1_str; 2967 2968 if (level > 3) { 2969 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str)); 2970 return ERR_PTR(-EINVAL); 2971 } 2972 2973 field_op = contains_operator(str); 2974 2975 if (field_op == FIELD_OP_NONE) 2976 return parse_atom(hist_data, file, str, &flags, var_name); 2977 2978 if (field_op == FIELD_OP_UNARY_MINUS) 2979 return parse_unary(hist_data, file, str, flags, var_name, ++level); 2980 2981 switch (field_op) { 2982 case FIELD_OP_MINUS: 2983 sep = "-"; 2984 break; 2985 case FIELD_OP_PLUS: 2986 sep = "+"; 2987 break; 2988 default: 2989 goto free; 2990 } 2991 2992 operand1_str = strsep(&str, sep); 2993 if (!operand1_str || !str) 2994 goto free; 2995 2996 operand_flags = 0; 2997 operand1 = parse_atom(hist_data, file, operand1_str, 2998 &operand_flags, NULL); 2999 if (IS_ERR(operand1)) { 3000 ret = PTR_ERR(operand1); 3001 operand1 = NULL; 3002 goto free; 3003 } 3004 3005 /* rest of string could be another expression e.g. b+c in a+b+c */ 3006 operand_flags = 0; 3007 operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level); 3008 if (IS_ERR(operand2)) { 3009 ret = PTR_ERR(operand2); 3010 operand2 = NULL; 3011 goto free; 3012 } 3013 3014 ret = check_expr_operands(file->tr, operand1, operand2); 3015 if (ret) 3016 goto free; 3017 3018 flags |= HIST_FIELD_FL_EXPR; 3019 3020 flags |= operand1->flags & 3021 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 3022 3023 expr = create_hist_field(hist_data, NULL, flags, var_name); 3024 if (!expr) { 3025 ret = -ENOMEM; 3026 goto free; 3027 } 3028 3029 operand1->read_once = true; 3030 operand2->read_once = true; 3031 3032 expr->operands[0] = operand1; 3033 expr->operands[1] = operand2; 3034 expr->operator = field_op; 3035 expr->name = expr_str(expr, 0); 3036 expr->type = kstrdup(operand1->type, GFP_KERNEL); 3037 if (!expr->type) { 3038 ret = -ENOMEM; 3039 goto free; 3040 } 3041 3042 switch (field_op) { 3043 case FIELD_OP_MINUS: 3044 expr->fn = hist_field_minus; 3045 break; 3046 case FIELD_OP_PLUS: 3047 expr->fn = hist_field_plus; 3048 break; 3049 default: 3050 ret = -EINVAL; 3051 goto free; 3052 } 3053 3054 return expr; 3055 free: 3056 destroy_hist_field(operand1, 0); 3057 destroy_hist_field(operand2, 0); 3058 destroy_hist_field(expr, 0); 3059 3060 return ERR_PTR(ret); 3061 } 3062 3063 static char *find_trigger_filter(struct hist_trigger_data *hist_data, 3064 struct trace_event_file *file) 3065 { 3066 struct event_trigger_data *test; 3067 3068 list_for_each_entry_rcu(test, &file->triggers, list) { 3069 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 3070 if (test->private_data == hist_data) 3071 return test->filter_str; 3072 } 3073 } 3074 3075 return NULL; 3076 } 3077 3078 static struct event_command trigger_hist_cmd; 3079 static int event_hist_trigger_func(struct event_command *cmd_ops, 3080 struct trace_event_file *file, 3081 char *glob, char *cmd, char *param); 3082 3083 static bool compatible_keys(struct hist_trigger_data *target_hist_data, 3084 struct hist_trigger_data *hist_data, 3085 unsigned int n_keys) 3086 { 3087 struct hist_field *target_hist_field, *hist_field; 3088 unsigned int n, i, j; 3089 3090 if (hist_data->n_fields - hist_data->n_vals != n_keys) 3091 return false; 3092 3093 i = hist_data->n_vals; 3094 j = target_hist_data->n_vals; 3095 3096 for (n = 0; n < n_keys; n++) { 3097 hist_field = hist_data->fields[i + n]; 3098 target_hist_field = target_hist_data->fields[j + n]; 3099 3100 if (strcmp(hist_field->type, target_hist_field->type) != 0) 3101 return false; 3102 if (hist_field->size != target_hist_field->size) 3103 return false; 3104 if (hist_field->is_signed != target_hist_field->is_signed) 3105 return false; 3106 } 3107 3108 return true; 3109 } 3110 3111 static struct hist_trigger_data * 3112 find_compatible_hist(struct hist_trigger_data *target_hist_data, 3113 struct trace_event_file *file) 3114 { 3115 struct hist_trigger_data *hist_data; 3116 struct event_trigger_data *test; 3117 unsigned int n_keys; 3118 3119 n_keys = target_hist_data->n_fields - target_hist_data->n_vals; 3120 3121 list_for_each_entry_rcu(test, &file->triggers, list) { 3122 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 3123 hist_data = test->private_data; 3124 3125 if (compatible_keys(target_hist_data, hist_data, n_keys)) 3126 return hist_data; 3127 } 3128 } 3129 3130 return NULL; 3131 } 3132 3133 static struct trace_event_file *event_file(struct trace_array *tr, 3134 char *system, char *event_name) 3135 { 3136 struct trace_event_file *file; 3137 3138 file = __find_event_file(tr, system, event_name); 3139 if (!file) 3140 return ERR_PTR(-EINVAL); 3141 3142 return file; 3143 } 3144 3145 static struct hist_field * 3146 find_synthetic_field_var(struct hist_trigger_data *target_hist_data, 3147 char *system, char *event_name, char *field_name) 3148 { 3149 struct hist_field *event_var; 3150 char *synthetic_name; 3151 3152 synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 3153 if (!synthetic_name) 3154 return ERR_PTR(-ENOMEM); 3155 3156 strcpy(synthetic_name, "synthetic_"); 3157 strcat(synthetic_name, field_name); 3158 3159 event_var = find_event_var(target_hist_data, system, event_name, synthetic_name); 3160 3161 kfree(synthetic_name); 3162 3163 return event_var; 3164 } 3165 3166 /** 3167 * create_field_var_hist - Automatically create a histogram and var for a field 3168 * @target_hist_data: The target hist trigger 3169 * @subsys_name: Optional subsystem name 3170 * @event_name: Optional event name 3171 * @field_name: The name of the field (and the resulting variable) 3172 * 3173 * Hist trigger actions fetch data from variables, not directly from 3174 * events. However, for convenience, users are allowed to directly 3175 * specify an event field in an action, which will be automatically 3176 * converted into a variable on their behalf. 3177 3178 * If a user specifies a field on an event that isn't the event the 3179 * histogram currently being defined (the target event histogram), the 3180 * only way that can be accomplished is if a new hist trigger is 3181 * created and the field variable defined on that. 3182 * 3183 * This function creates a new histogram compatible with the target 3184 * event (meaning a histogram with the same key as the target 3185 * histogram), and creates a variable for the specified field, but 3186 * with 'synthetic_' prepended to the variable name in order to avoid 3187 * collision with normal field variables. 3188 * 3189 * Return: The variable created for the field. 3190 */ 3191 static struct hist_field * 3192 create_field_var_hist(struct hist_trigger_data *target_hist_data, 3193 char *subsys_name, char *event_name, char *field_name) 3194 { 3195 struct trace_array *tr = target_hist_data->event_file->tr; 3196 struct hist_field *event_var = ERR_PTR(-EINVAL); 3197 struct hist_trigger_data *hist_data; 3198 unsigned int i, n, first = true; 3199 struct field_var_hist *var_hist; 3200 struct trace_event_file *file; 3201 struct hist_field *key_field; 3202 char *saved_filter; 3203 char *cmd; 3204 int ret; 3205 3206 if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) { 3207 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name)); 3208 return ERR_PTR(-EINVAL); 3209 } 3210 3211 file = event_file(tr, subsys_name, event_name); 3212 3213 if (IS_ERR(file)) { 3214 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name)); 3215 ret = PTR_ERR(file); 3216 return ERR_PTR(ret); 3217 } 3218 3219 /* 3220 * Look for a histogram compatible with target. We'll use the 3221 * found histogram specification to create a new matching 3222 * histogram with our variable on it. target_hist_data is not 3223 * yet a registered histogram so we can't use that. 3224 */ 3225 hist_data = find_compatible_hist(target_hist_data, file); 3226 if (!hist_data) { 3227 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name)); 3228 return ERR_PTR(-EINVAL); 3229 } 3230 3231 /* See if a synthetic field variable has already been created */ 3232 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 3233 event_name, field_name); 3234 if (!IS_ERR_OR_NULL(event_var)) 3235 return event_var; 3236 3237 var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL); 3238 if (!var_hist) 3239 return ERR_PTR(-ENOMEM); 3240 3241 cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 3242 if (!cmd) { 3243 kfree(var_hist); 3244 return ERR_PTR(-ENOMEM); 3245 } 3246 3247 /* Use the same keys as the compatible histogram */ 3248 strcat(cmd, "keys="); 3249 3250 for_each_hist_key_field(i, hist_data) { 3251 key_field = hist_data->fields[i]; 3252 if (!first) 3253 strcat(cmd, ","); 3254 strcat(cmd, key_field->field->name); 3255 first = false; 3256 } 3257 3258 /* Create the synthetic field variable specification */ 3259 strcat(cmd, ":synthetic_"); 3260 strcat(cmd, field_name); 3261 strcat(cmd, "="); 3262 strcat(cmd, field_name); 3263 3264 /* Use the same filter as the compatible histogram */ 3265 saved_filter = find_trigger_filter(hist_data, file); 3266 if (saved_filter) { 3267 strcat(cmd, " if "); 3268 strcat(cmd, saved_filter); 3269 } 3270 3271 var_hist->cmd = kstrdup(cmd, GFP_KERNEL); 3272 if (!var_hist->cmd) { 3273 kfree(cmd); 3274 kfree(var_hist); 3275 return ERR_PTR(-ENOMEM); 3276 } 3277 3278 /* Save the compatible histogram information */ 3279 var_hist->hist_data = hist_data; 3280 3281 /* Create the new histogram with our variable */ 3282 ret = event_hist_trigger_func(&trigger_hist_cmd, file, 3283 "", "hist", cmd); 3284 if (ret) { 3285 kfree(cmd); 3286 kfree(var_hist->cmd); 3287 kfree(var_hist); 3288 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name)); 3289 return ERR_PTR(ret); 3290 } 3291 3292 kfree(cmd); 3293 3294 /* If we can't find the variable, something went wrong */ 3295 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 3296 event_name, field_name); 3297 if (IS_ERR_OR_NULL(event_var)) { 3298 kfree(var_hist->cmd); 3299 kfree(var_hist); 3300 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name)); 3301 return ERR_PTR(-EINVAL); 3302 } 3303 3304 n = target_hist_data->n_field_var_hists; 3305 target_hist_data->field_var_hists[n] = var_hist; 3306 target_hist_data->n_field_var_hists++; 3307 3308 return event_var; 3309 } 3310 3311 static struct hist_field * 3312 find_target_event_var(struct hist_trigger_data *hist_data, 3313 char *subsys_name, char *event_name, char *var_name) 3314 { 3315 struct trace_event_file *file = hist_data->event_file; 3316 struct hist_field *hist_field = NULL; 3317 3318 if (subsys_name) { 3319 struct trace_event_call *call; 3320 3321 if (!event_name) 3322 return NULL; 3323 3324 call = file->event_call; 3325 3326 if (strcmp(subsys_name, call->class->system) != 0) 3327 return NULL; 3328 3329 if (strcmp(event_name, trace_event_name(call)) != 0) 3330 return NULL; 3331 } 3332 3333 hist_field = find_var_field(hist_data, var_name); 3334 3335 return hist_field; 3336 } 3337 3338 static inline void __update_field_vars(struct tracing_map_elt *elt, 3339 struct ring_buffer_event *rbe, 3340 void *rec, 3341 struct field_var **field_vars, 3342 unsigned int n_field_vars, 3343 unsigned int field_var_str_start) 3344 { 3345 struct hist_elt_data *elt_data = elt->private_data; 3346 unsigned int i, j, var_idx; 3347 u64 var_val; 3348 3349 for (i = 0, j = field_var_str_start; i < n_field_vars; i++) { 3350 struct field_var *field_var = field_vars[i]; 3351 struct hist_field *var = field_var->var; 3352 struct hist_field *val = field_var->val; 3353 3354 var_val = val->fn(val, elt, rbe, rec); 3355 var_idx = var->var.idx; 3356 3357 if (val->flags & HIST_FIELD_FL_STRING) { 3358 char *str = elt_data->field_var_str[j++]; 3359 char *val_str = (char *)(uintptr_t)var_val; 3360 3361 strscpy(str, val_str, STR_VAR_LEN_MAX); 3362 var_val = (u64)(uintptr_t)str; 3363 } 3364 tracing_map_set_var(elt, var_idx, var_val); 3365 } 3366 } 3367 3368 static void update_field_vars(struct hist_trigger_data *hist_data, 3369 struct tracing_map_elt *elt, 3370 struct ring_buffer_event *rbe, 3371 void *rec) 3372 { 3373 __update_field_vars(elt, rbe, rec, hist_data->field_vars, 3374 hist_data->n_field_vars, 0); 3375 } 3376 3377 static void save_track_data_vars(struct hist_trigger_data *hist_data, 3378 struct tracing_map_elt *elt, void *rec, 3379 struct ring_buffer_event *rbe, void *key, 3380 struct action_data *data, u64 *var_ref_vals) 3381 { 3382 __update_field_vars(elt, rbe, rec, hist_data->save_vars, 3383 hist_data->n_save_vars, hist_data->n_field_var_str); 3384 } 3385 3386 static struct hist_field *create_var(struct hist_trigger_data *hist_data, 3387 struct trace_event_file *file, 3388 char *name, int size, const char *type) 3389 { 3390 struct hist_field *var; 3391 int idx; 3392 3393 if (find_var(hist_data, file, name) && !hist_data->remove) { 3394 var = ERR_PTR(-EINVAL); 3395 goto out; 3396 } 3397 3398 var = kzalloc(sizeof(struct hist_field), GFP_KERNEL); 3399 if (!var) { 3400 var = ERR_PTR(-ENOMEM); 3401 goto out; 3402 } 3403 3404 idx = tracing_map_add_var(hist_data->map); 3405 if (idx < 0) { 3406 kfree(var); 3407 var = ERR_PTR(-EINVAL); 3408 goto out; 3409 } 3410 3411 var->flags = HIST_FIELD_FL_VAR; 3412 var->var.idx = idx; 3413 var->var.hist_data = var->hist_data = hist_data; 3414 var->size = size; 3415 var->var.name = kstrdup(name, GFP_KERNEL); 3416 var->type = kstrdup(type, GFP_KERNEL); 3417 if (!var->var.name || !var->type) { 3418 kfree(var->var.name); 3419 kfree(var->type); 3420 kfree(var); 3421 var = ERR_PTR(-ENOMEM); 3422 } 3423 out: 3424 return var; 3425 } 3426 3427 static struct field_var *create_field_var(struct hist_trigger_data *hist_data, 3428 struct trace_event_file *file, 3429 char *field_name) 3430 { 3431 struct hist_field *val = NULL, *var = NULL; 3432 unsigned long flags = HIST_FIELD_FL_VAR; 3433 struct trace_array *tr = file->tr; 3434 struct field_var *field_var; 3435 int ret = 0; 3436 3437 if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) { 3438 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name)); 3439 ret = -EINVAL; 3440 goto err; 3441 } 3442 3443 val = parse_atom(hist_data, file, field_name, &flags, NULL); 3444 if (IS_ERR(val)) { 3445 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name)); 3446 ret = PTR_ERR(val); 3447 goto err; 3448 } 3449 3450 var = create_var(hist_data, file, field_name, val->size, val->type); 3451 if (IS_ERR(var)) { 3452 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name)); 3453 kfree(val); 3454 ret = PTR_ERR(var); 3455 goto err; 3456 } 3457 3458 field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL); 3459 if (!field_var) { 3460 kfree(val); 3461 kfree(var); 3462 ret = -ENOMEM; 3463 goto err; 3464 } 3465 3466 field_var->var = var; 3467 field_var->val = val; 3468 out: 3469 return field_var; 3470 err: 3471 field_var = ERR_PTR(ret); 3472 goto out; 3473 } 3474 3475 /** 3476 * create_target_field_var - Automatically create a variable for a field 3477 * @target_hist_data: The target hist trigger 3478 * @subsys_name: Optional subsystem name 3479 * @event_name: Optional event name 3480 * @var_name: The name of the field (and the resulting variable) 3481 * 3482 * Hist trigger actions fetch data from variables, not directly from 3483 * events. However, for convenience, users are allowed to directly 3484 * specify an event field in an action, which will be automatically 3485 * converted into a variable on their behalf. 3486 3487 * This function creates a field variable with the name var_name on 3488 * the hist trigger currently being defined on the target event. If 3489 * subsys_name and event_name are specified, this function simply 3490 * verifies that they do in fact match the target event subsystem and 3491 * event name. 3492 * 3493 * Return: The variable created for the field. 3494 */ 3495 static struct field_var * 3496 create_target_field_var(struct hist_trigger_data *target_hist_data, 3497 char *subsys_name, char *event_name, char *var_name) 3498 { 3499 struct trace_event_file *file = target_hist_data->event_file; 3500 3501 if (subsys_name) { 3502 struct trace_event_call *call; 3503 3504 if (!event_name) 3505 return NULL; 3506 3507 call = file->event_call; 3508 3509 if (strcmp(subsys_name, call->class->system) != 0) 3510 return NULL; 3511 3512 if (strcmp(event_name, trace_event_name(call)) != 0) 3513 return NULL; 3514 } 3515 3516 return create_field_var(target_hist_data, file, var_name); 3517 } 3518 3519 static bool check_track_val_max(u64 track_val, u64 var_val) 3520 { 3521 if (var_val <= track_val) 3522 return false; 3523 3524 return true; 3525 } 3526 3527 static bool check_track_val_changed(u64 track_val, u64 var_val) 3528 { 3529 if (var_val == track_val) 3530 return false; 3531 3532 return true; 3533 } 3534 3535 static u64 get_track_val(struct hist_trigger_data *hist_data, 3536 struct tracing_map_elt *elt, 3537 struct action_data *data) 3538 { 3539 unsigned int track_var_idx = data->track_data.track_var->var.idx; 3540 u64 track_val; 3541 3542 track_val = tracing_map_read_var(elt, track_var_idx); 3543 3544 return track_val; 3545 } 3546 3547 static void save_track_val(struct hist_trigger_data *hist_data, 3548 struct tracing_map_elt *elt, 3549 struct action_data *data, u64 var_val) 3550 { 3551 unsigned int track_var_idx = data->track_data.track_var->var.idx; 3552 3553 tracing_map_set_var(elt, track_var_idx, var_val); 3554 } 3555 3556 static void save_track_data(struct hist_trigger_data *hist_data, 3557 struct tracing_map_elt *elt, void *rec, 3558 struct ring_buffer_event *rbe, void *key, 3559 struct action_data *data, u64 *var_ref_vals) 3560 { 3561 if (data->track_data.save_data) 3562 data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals); 3563 } 3564 3565 static bool check_track_val(struct tracing_map_elt *elt, 3566 struct action_data *data, 3567 u64 var_val) 3568 { 3569 struct hist_trigger_data *hist_data; 3570 u64 track_val; 3571 3572 hist_data = data->track_data.track_var->hist_data; 3573 track_val = get_track_val(hist_data, elt, data); 3574 3575 return data->track_data.check_val(track_val, var_val); 3576 } 3577 3578 #ifdef CONFIG_TRACER_SNAPSHOT 3579 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data) 3580 { 3581 /* called with tr->max_lock held */ 3582 struct track_data *track_data = tr->cond_snapshot->cond_data; 3583 struct hist_elt_data *elt_data, *track_elt_data; 3584 struct snapshot_context *context = cond_data; 3585 u64 track_val; 3586 3587 if (!track_data) 3588 return false; 3589 3590 track_val = get_track_val(track_data->hist_data, context->elt, 3591 track_data->action_data); 3592 3593 track_data->track_val = track_val; 3594 memcpy(track_data->key, context->key, track_data->key_len); 3595 3596 elt_data = context->elt->private_data; 3597 track_elt_data = track_data->elt.private_data; 3598 if (elt_data->comm) 3599 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN); 3600 3601 track_data->updated = true; 3602 3603 return true; 3604 } 3605 3606 static void save_track_data_snapshot(struct hist_trigger_data *hist_data, 3607 struct tracing_map_elt *elt, void *rec, 3608 struct ring_buffer_event *rbe, void *key, 3609 struct action_data *data, 3610 u64 *var_ref_vals) 3611 { 3612 struct trace_event_file *file = hist_data->event_file; 3613 struct snapshot_context context; 3614 3615 context.elt = elt; 3616 context.key = key; 3617 3618 tracing_snapshot_cond(file->tr, &context); 3619 } 3620 3621 static void hist_trigger_print_key(struct seq_file *m, 3622 struct hist_trigger_data *hist_data, 3623 void *key, 3624 struct tracing_map_elt *elt); 3625 3626 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data) 3627 { 3628 unsigned int i; 3629 3630 if (!hist_data->n_actions) 3631 return NULL; 3632 3633 for (i = 0; i < hist_data->n_actions; i++) { 3634 struct action_data *data = hist_data->actions[i]; 3635 3636 if (data->action == ACTION_SNAPSHOT) 3637 return data; 3638 } 3639 3640 return NULL; 3641 } 3642 3643 static void track_data_snapshot_print(struct seq_file *m, 3644 struct hist_trigger_data *hist_data) 3645 { 3646 struct trace_event_file *file = hist_data->event_file; 3647 struct track_data *track_data; 3648 struct action_data *action; 3649 3650 track_data = tracing_cond_snapshot_data(file->tr); 3651 if (!track_data) 3652 return; 3653 3654 if (!track_data->updated) 3655 return; 3656 3657 action = snapshot_action(hist_data); 3658 if (!action) 3659 return; 3660 3661 seq_puts(m, "\nSnapshot taken (see tracing/snapshot). Details:\n"); 3662 seq_printf(m, "\ttriggering value { %s(%s) }: %10llu", 3663 action->handler == HANDLER_ONMAX ? "onmax" : "onchange", 3664 action->track_data.var_str, track_data->track_val); 3665 3666 seq_puts(m, "\ttriggered by event with key: "); 3667 hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt); 3668 seq_putc(m, '\n'); 3669 } 3670 #else 3671 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data) 3672 { 3673 return false; 3674 } 3675 static void save_track_data_snapshot(struct hist_trigger_data *hist_data, 3676 struct tracing_map_elt *elt, void *rec, 3677 struct ring_buffer_event *rbe, void *key, 3678 struct action_data *data, 3679 u64 *var_ref_vals) {} 3680 static void track_data_snapshot_print(struct seq_file *m, 3681 struct hist_trigger_data *hist_data) {} 3682 #endif /* CONFIG_TRACER_SNAPSHOT */ 3683 3684 static void track_data_print(struct seq_file *m, 3685 struct hist_trigger_data *hist_data, 3686 struct tracing_map_elt *elt, 3687 struct action_data *data) 3688 { 3689 u64 track_val = get_track_val(hist_data, elt, data); 3690 unsigned int i, save_var_idx; 3691 3692 if (data->handler == HANDLER_ONMAX) 3693 seq_printf(m, "\n\tmax: %10llu", track_val); 3694 else if (data->handler == HANDLER_ONCHANGE) 3695 seq_printf(m, "\n\tchanged: %10llu", track_val); 3696 3697 if (data->action == ACTION_SNAPSHOT) 3698 return; 3699 3700 for (i = 0; i < hist_data->n_save_vars; i++) { 3701 struct hist_field *save_val = hist_data->save_vars[i]->val; 3702 struct hist_field *save_var = hist_data->save_vars[i]->var; 3703 u64 val; 3704 3705 save_var_idx = save_var->var.idx; 3706 3707 val = tracing_map_read_var(elt, save_var_idx); 3708 3709 if (save_val->flags & HIST_FIELD_FL_STRING) { 3710 seq_printf(m, " %s: %-32s", save_var->var.name, 3711 (char *)(uintptr_t)(val)); 3712 } else 3713 seq_printf(m, " %s: %10llu", save_var->var.name, val); 3714 } 3715 } 3716 3717 static void ontrack_action(struct hist_trigger_data *hist_data, 3718 struct tracing_map_elt *elt, void *rec, 3719 struct ring_buffer_event *rbe, void *key, 3720 struct action_data *data, u64 *var_ref_vals) 3721 { 3722 u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx]; 3723 3724 if (check_track_val(elt, data, var_val)) { 3725 save_track_val(hist_data, elt, data, var_val); 3726 save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals); 3727 } 3728 } 3729 3730 static void action_data_destroy(struct action_data *data) 3731 { 3732 unsigned int i; 3733 3734 lockdep_assert_held(&event_mutex); 3735 3736 kfree(data->action_name); 3737 3738 for (i = 0; i < data->n_params; i++) 3739 kfree(data->params[i]); 3740 3741 if (data->synth_event) 3742 data->synth_event->ref--; 3743 3744 kfree(data->synth_event_name); 3745 3746 kfree(data); 3747 } 3748 3749 static void track_data_destroy(struct hist_trigger_data *hist_data, 3750 struct action_data *data) 3751 { 3752 struct trace_event_file *file = hist_data->event_file; 3753 3754 destroy_hist_field(data->track_data.track_var, 0); 3755 3756 if (data->action == ACTION_SNAPSHOT) { 3757 struct track_data *track_data; 3758 3759 track_data = tracing_cond_snapshot_data(file->tr); 3760 if (track_data && track_data->hist_data == hist_data) { 3761 tracing_snapshot_cond_disable(file->tr); 3762 track_data_free(track_data); 3763 } 3764 } 3765 3766 kfree(data->track_data.var_str); 3767 3768 action_data_destroy(data); 3769 } 3770 3771 static int action_create(struct hist_trigger_data *hist_data, 3772 struct action_data *data); 3773 3774 static int track_data_create(struct hist_trigger_data *hist_data, 3775 struct action_data *data) 3776 { 3777 struct hist_field *var_field, *ref_field, *track_var = NULL; 3778 struct trace_event_file *file = hist_data->event_file; 3779 struct trace_array *tr = file->tr; 3780 char *track_data_var_str; 3781 int ret = 0; 3782 3783 track_data_var_str = data->track_data.var_str; 3784 if (track_data_var_str[0] != '$') { 3785 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str)); 3786 return -EINVAL; 3787 } 3788 track_data_var_str++; 3789 3790 var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str); 3791 if (!var_field) { 3792 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str)); 3793 return -EINVAL; 3794 } 3795 3796 ref_field = create_var_ref(hist_data, var_field, NULL, NULL); 3797 if (!ref_field) 3798 return -ENOMEM; 3799 3800 data->track_data.var_ref = ref_field; 3801 3802 if (data->handler == HANDLER_ONMAX) 3803 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64"); 3804 if (IS_ERR(track_var)) { 3805 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0); 3806 ret = PTR_ERR(track_var); 3807 goto out; 3808 } 3809 3810 if (data->handler == HANDLER_ONCHANGE) 3811 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64"); 3812 if (IS_ERR(track_var)) { 3813 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0); 3814 ret = PTR_ERR(track_var); 3815 goto out; 3816 } 3817 data->track_data.track_var = track_var; 3818 3819 ret = action_create(hist_data, data); 3820 out: 3821 return ret; 3822 } 3823 3824 static int parse_action_params(struct trace_array *tr, char *params, 3825 struct action_data *data) 3826 { 3827 char *param, *saved_param; 3828 bool first_param = true; 3829 int ret = 0; 3830 3831 while (params) { 3832 if (data->n_params >= SYNTH_FIELDS_MAX) { 3833 hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0); 3834 goto out; 3835 } 3836 3837 param = strsep(¶ms, ","); 3838 if (!param) { 3839 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0); 3840 ret = -EINVAL; 3841 goto out; 3842 } 3843 3844 param = strstrip(param); 3845 if (strlen(param) < 2) { 3846 hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param)); 3847 ret = -EINVAL; 3848 goto out; 3849 } 3850 3851 saved_param = kstrdup(param, GFP_KERNEL); 3852 if (!saved_param) { 3853 ret = -ENOMEM; 3854 goto out; 3855 } 3856 3857 if (first_param && data->use_trace_keyword) { 3858 data->synth_event_name = saved_param; 3859 first_param = false; 3860 continue; 3861 } 3862 first_param = false; 3863 3864 data->params[data->n_params++] = saved_param; 3865 } 3866 out: 3867 return ret; 3868 } 3869 3870 static int action_parse(struct trace_array *tr, char *str, struct action_data *data, 3871 enum handler_id handler) 3872 { 3873 char *action_name; 3874 int ret = 0; 3875 3876 strsep(&str, "."); 3877 if (!str) { 3878 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0); 3879 ret = -EINVAL; 3880 goto out; 3881 } 3882 3883 action_name = strsep(&str, "("); 3884 if (!action_name || !str) { 3885 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0); 3886 ret = -EINVAL; 3887 goto out; 3888 } 3889 3890 if (str_has_prefix(action_name, "save")) { 3891 char *params = strsep(&str, ")"); 3892 3893 if (!params) { 3894 hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0); 3895 ret = -EINVAL; 3896 goto out; 3897 } 3898 3899 ret = parse_action_params(tr, params, data); 3900 if (ret) 3901 goto out; 3902 3903 if (handler == HANDLER_ONMAX) 3904 data->track_data.check_val = check_track_val_max; 3905 else if (handler == HANDLER_ONCHANGE) 3906 data->track_data.check_val = check_track_val_changed; 3907 else { 3908 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name)); 3909 ret = -EINVAL; 3910 goto out; 3911 } 3912 3913 data->track_data.save_data = save_track_data_vars; 3914 data->fn = ontrack_action; 3915 data->action = ACTION_SAVE; 3916 } else if (str_has_prefix(action_name, "snapshot")) { 3917 char *params = strsep(&str, ")"); 3918 3919 if (!str) { 3920 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params)); 3921 ret = -EINVAL; 3922 goto out; 3923 } 3924 3925 if (handler == HANDLER_ONMAX) 3926 data->track_data.check_val = check_track_val_max; 3927 else if (handler == HANDLER_ONCHANGE) 3928 data->track_data.check_val = check_track_val_changed; 3929 else { 3930 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name)); 3931 ret = -EINVAL; 3932 goto out; 3933 } 3934 3935 data->track_data.save_data = save_track_data_snapshot; 3936 data->fn = ontrack_action; 3937 data->action = ACTION_SNAPSHOT; 3938 } else { 3939 char *params = strsep(&str, ")"); 3940 3941 if (str_has_prefix(action_name, "trace")) 3942 data->use_trace_keyword = true; 3943 3944 if (params) { 3945 ret = parse_action_params(tr, params, data); 3946 if (ret) 3947 goto out; 3948 } 3949 3950 if (handler == HANDLER_ONMAX) 3951 data->track_data.check_val = check_track_val_max; 3952 else if (handler == HANDLER_ONCHANGE) 3953 data->track_data.check_val = check_track_val_changed; 3954 3955 if (handler != HANDLER_ONMATCH) { 3956 data->track_data.save_data = action_trace; 3957 data->fn = ontrack_action; 3958 } else 3959 data->fn = action_trace; 3960 3961 data->action = ACTION_TRACE; 3962 } 3963 3964 data->action_name = kstrdup(action_name, GFP_KERNEL); 3965 if (!data->action_name) { 3966 ret = -ENOMEM; 3967 goto out; 3968 } 3969 3970 data->handler = handler; 3971 out: 3972 return ret; 3973 } 3974 3975 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data, 3976 char *str, enum handler_id handler) 3977 { 3978 struct action_data *data; 3979 int ret = -EINVAL; 3980 char *var_str; 3981 3982 data = kzalloc(sizeof(*data), GFP_KERNEL); 3983 if (!data) 3984 return ERR_PTR(-ENOMEM); 3985 3986 var_str = strsep(&str, ")"); 3987 if (!var_str || !str) { 3988 ret = -EINVAL; 3989 goto free; 3990 } 3991 3992 data->track_data.var_str = kstrdup(var_str, GFP_KERNEL); 3993 if (!data->track_data.var_str) { 3994 ret = -ENOMEM; 3995 goto free; 3996 } 3997 3998 ret = action_parse(hist_data->event_file->tr, str, data, handler); 3999 if (ret) 4000 goto free; 4001 out: 4002 return data; 4003 free: 4004 track_data_destroy(hist_data, data); 4005 data = ERR_PTR(ret); 4006 goto out; 4007 } 4008 4009 static void onmatch_destroy(struct action_data *data) 4010 { 4011 kfree(data->match_data.event); 4012 kfree(data->match_data.event_system); 4013 4014 action_data_destroy(data); 4015 } 4016 4017 static void destroy_field_var(struct field_var *field_var) 4018 { 4019 if (!field_var) 4020 return; 4021 4022 destroy_hist_field(field_var->var, 0); 4023 destroy_hist_field(field_var->val, 0); 4024 4025 kfree(field_var); 4026 } 4027 4028 static void destroy_field_vars(struct hist_trigger_data *hist_data) 4029 { 4030 unsigned int i; 4031 4032 for (i = 0; i < hist_data->n_field_vars; i++) 4033 destroy_field_var(hist_data->field_vars[i]); 4034 } 4035 4036 static void save_field_var(struct hist_trigger_data *hist_data, 4037 struct field_var *field_var) 4038 { 4039 hist_data->field_vars[hist_data->n_field_vars++] = field_var; 4040 4041 if (field_var->val->flags & HIST_FIELD_FL_STRING) 4042 hist_data->n_field_var_str++; 4043 } 4044 4045 4046 static int check_synth_field(struct synth_event *event, 4047 struct hist_field *hist_field, 4048 unsigned int field_pos) 4049 { 4050 struct synth_field *field; 4051 4052 if (field_pos >= event->n_fields) 4053 return -EINVAL; 4054 4055 field = event->fields[field_pos]; 4056 4057 if (strcmp(field->type, hist_field->type) != 0) 4058 return -EINVAL; 4059 4060 return 0; 4061 } 4062 4063 static struct hist_field * 4064 trace_action_find_var(struct hist_trigger_data *hist_data, 4065 struct action_data *data, 4066 char *system, char *event, char *var) 4067 { 4068 struct trace_array *tr = hist_data->event_file->tr; 4069 struct hist_field *hist_field; 4070 4071 var++; /* skip '$' */ 4072 4073 hist_field = find_target_event_var(hist_data, system, event, var); 4074 if (!hist_field) { 4075 if (!system && data->handler == HANDLER_ONMATCH) { 4076 system = data->match_data.event_system; 4077 event = data->match_data.event; 4078 } 4079 4080 hist_field = find_event_var(hist_data, system, event, var); 4081 } 4082 4083 if (!hist_field) 4084 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var)); 4085 4086 return hist_field; 4087 } 4088 4089 static struct hist_field * 4090 trace_action_create_field_var(struct hist_trigger_data *hist_data, 4091 struct action_data *data, char *system, 4092 char *event, char *var) 4093 { 4094 struct hist_field *hist_field = NULL; 4095 struct field_var *field_var; 4096 4097 /* 4098 * First try to create a field var on the target event (the 4099 * currently being defined). This will create a variable for 4100 * unqualified fields on the target event, or if qualified, 4101 * target fields that have qualified names matching the target. 4102 */ 4103 field_var = create_target_field_var(hist_data, system, event, var); 4104 4105 if (field_var && !IS_ERR(field_var)) { 4106 save_field_var(hist_data, field_var); 4107 hist_field = field_var->var; 4108 } else { 4109 field_var = NULL; 4110 /* 4111 * If no explicit system.event is specfied, default to 4112 * looking for fields on the onmatch(system.event.xxx) 4113 * event. 4114 */ 4115 if (!system && data->handler == HANDLER_ONMATCH) { 4116 system = data->match_data.event_system; 4117 event = data->match_data.event; 4118 } 4119 4120 /* 4121 * At this point, we're looking at a field on another 4122 * event. Because we can't modify a hist trigger on 4123 * another event to add a variable for a field, we need 4124 * to create a new trigger on that event and create the 4125 * variable at the same time. 4126 */ 4127 hist_field = create_field_var_hist(hist_data, system, event, var); 4128 if (IS_ERR(hist_field)) 4129 goto free; 4130 } 4131 out: 4132 return hist_field; 4133 free: 4134 destroy_field_var(field_var); 4135 hist_field = NULL; 4136 goto out; 4137 } 4138 4139 static int trace_action_create(struct hist_trigger_data *hist_data, 4140 struct action_data *data) 4141 { 4142 struct trace_array *tr = hist_data->event_file->tr; 4143 char *event_name, *param, *system = NULL; 4144 struct hist_field *hist_field, *var_ref; 4145 unsigned int i, var_ref_idx; 4146 unsigned int field_pos = 0; 4147 struct synth_event *event; 4148 char *synth_event_name; 4149 int ret = 0; 4150 4151 lockdep_assert_held(&event_mutex); 4152 4153 if (data->use_trace_keyword) 4154 synth_event_name = data->synth_event_name; 4155 else 4156 synth_event_name = data->action_name; 4157 4158 event = find_synth_event(synth_event_name); 4159 if (!event) { 4160 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name)); 4161 return -EINVAL; 4162 } 4163 4164 event->ref++; 4165 4166 var_ref_idx = hist_data->n_var_refs; 4167 4168 for (i = 0; i < data->n_params; i++) { 4169 char *p; 4170 4171 p = param = kstrdup(data->params[i], GFP_KERNEL); 4172 if (!param) { 4173 ret = -ENOMEM; 4174 goto err; 4175 } 4176 4177 system = strsep(¶m, "."); 4178 if (!param) { 4179 param = (char *)system; 4180 system = event_name = NULL; 4181 } else { 4182 event_name = strsep(¶m, "."); 4183 if (!param) { 4184 kfree(p); 4185 ret = -EINVAL; 4186 goto err; 4187 } 4188 } 4189 4190 if (param[0] == '$') 4191 hist_field = trace_action_find_var(hist_data, data, 4192 system, event_name, 4193 param); 4194 else 4195 hist_field = trace_action_create_field_var(hist_data, 4196 data, 4197 system, 4198 event_name, 4199 param); 4200 4201 if (!hist_field) { 4202 kfree(p); 4203 ret = -EINVAL; 4204 goto err; 4205 } 4206 4207 if (check_synth_field(event, hist_field, field_pos) == 0) { 4208 var_ref = create_var_ref(hist_data, hist_field, 4209 system, event_name); 4210 if (!var_ref) { 4211 kfree(p); 4212 ret = -ENOMEM; 4213 goto err; 4214 } 4215 4216 field_pos++; 4217 kfree(p); 4218 continue; 4219 } 4220 4221 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param)); 4222 kfree(p); 4223 ret = -EINVAL; 4224 goto err; 4225 } 4226 4227 if (field_pos != event->n_fields) { 4228 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name)); 4229 ret = -EINVAL; 4230 goto err; 4231 } 4232 4233 data->synth_event = event; 4234 data->var_ref_idx = var_ref_idx; 4235 out: 4236 return ret; 4237 err: 4238 event->ref--; 4239 4240 goto out; 4241 } 4242 4243 static int action_create(struct hist_trigger_data *hist_data, 4244 struct action_data *data) 4245 { 4246 struct trace_event_file *file = hist_data->event_file; 4247 struct trace_array *tr = file->tr; 4248 struct track_data *track_data; 4249 struct field_var *field_var; 4250 unsigned int i; 4251 char *param; 4252 int ret = 0; 4253 4254 if (data->action == ACTION_TRACE) 4255 return trace_action_create(hist_data, data); 4256 4257 if (data->action == ACTION_SNAPSHOT) { 4258 track_data = track_data_alloc(hist_data->key_size, data, hist_data); 4259 if (IS_ERR(track_data)) { 4260 ret = PTR_ERR(track_data); 4261 goto out; 4262 } 4263 4264 ret = tracing_snapshot_cond_enable(file->tr, track_data, 4265 cond_snapshot_update); 4266 if (ret) 4267 track_data_free(track_data); 4268 4269 goto out; 4270 } 4271 4272 if (data->action == ACTION_SAVE) { 4273 if (hist_data->n_save_vars) { 4274 ret = -EEXIST; 4275 hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0); 4276 goto out; 4277 } 4278 4279 for (i = 0; i < data->n_params; i++) { 4280 param = kstrdup(data->params[i], GFP_KERNEL); 4281 if (!param) { 4282 ret = -ENOMEM; 4283 goto out; 4284 } 4285 4286 field_var = create_target_field_var(hist_data, NULL, NULL, param); 4287 if (IS_ERR(field_var)) { 4288 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL, 4289 errpos(param)); 4290 ret = PTR_ERR(field_var); 4291 kfree(param); 4292 goto out; 4293 } 4294 4295 hist_data->save_vars[hist_data->n_save_vars++] = field_var; 4296 if (field_var->val->flags & HIST_FIELD_FL_STRING) 4297 hist_data->n_save_var_str++; 4298 kfree(param); 4299 } 4300 } 4301 out: 4302 return ret; 4303 } 4304 4305 static int onmatch_create(struct hist_trigger_data *hist_data, 4306 struct action_data *data) 4307 { 4308 return action_create(hist_data, data); 4309 } 4310 4311 static struct action_data *onmatch_parse(struct trace_array *tr, char *str) 4312 { 4313 char *match_event, *match_event_system; 4314 struct action_data *data; 4315 int ret = -EINVAL; 4316 4317 data = kzalloc(sizeof(*data), GFP_KERNEL); 4318 if (!data) 4319 return ERR_PTR(-ENOMEM); 4320 4321 match_event = strsep(&str, ")"); 4322 if (!match_event || !str) { 4323 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event)); 4324 goto free; 4325 } 4326 4327 match_event_system = strsep(&match_event, "."); 4328 if (!match_event) { 4329 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system)); 4330 goto free; 4331 } 4332 4333 if (IS_ERR(event_file(tr, match_event_system, match_event))) { 4334 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event)); 4335 goto free; 4336 } 4337 4338 data->match_data.event = kstrdup(match_event, GFP_KERNEL); 4339 if (!data->match_data.event) { 4340 ret = -ENOMEM; 4341 goto free; 4342 } 4343 4344 data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL); 4345 if (!data->match_data.event_system) { 4346 ret = -ENOMEM; 4347 goto free; 4348 } 4349 4350 ret = action_parse(tr, str, data, HANDLER_ONMATCH); 4351 if (ret) 4352 goto free; 4353 out: 4354 return data; 4355 free: 4356 onmatch_destroy(data); 4357 data = ERR_PTR(ret); 4358 goto out; 4359 } 4360 4361 static int create_hitcount_val(struct hist_trigger_data *hist_data) 4362 { 4363 hist_data->fields[HITCOUNT_IDX] = 4364 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL); 4365 if (!hist_data->fields[HITCOUNT_IDX]) 4366 return -ENOMEM; 4367 4368 hist_data->n_vals++; 4369 hist_data->n_fields++; 4370 4371 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX)) 4372 return -EINVAL; 4373 4374 return 0; 4375 } 4376 4377 static int __create_val_field(struct hist_trigger_data *hist_data, 4378 unsigned int val_idx, 4379 struct trace_event_file *file, 4380 char *var_name, char *field_str, 4381 unsigned long flags) 4382 { 4383 struct hist_field *hist_field; 4384 int ret = 0; 4385 4386 hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0); 4387 if (IS_ERR(hist_field)) { 4388 ret = PTR_ERR(hist_field); 4389 goto out; 4390 } 4391 4392 hist_data->fields[val_idx] = hist_field; 4393 4394 ++hist_data->n_vals; 4395 ++hist_data->n_fields; 4396 4397 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 4398 ret = -EINVAL; 4399 out: 4400 return ret; 4401 } 4402 4403 static int create_val_field(struct hist_trigger_data *hist_data, 4404 unsigned int val_idx, 4405 struct trace_event_file *file, 4406 char *field_str) 4407 { 4408 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX)) 4409 return -EINVAL; 4410 4411 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0); 4412 } 4413 4414 static int create_var_field(struct hist_trigger_data *hist_data, 4415 unsigned int val_idx, 4416 struct trace_event_file *file, 4417 char *var_name, char *expr_str) 4418 { 4419 struct trace_array *tr = hist_data->event_file->tr; 4420 unsigned long flags = 0; 4421 4422 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 4423 return -EINVAL; 4424 4425 if (find_var(hist_data, file, var_name) && !hist_data->remove) { 4426 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name)); 4427 return -EINVAL; 4428 } 4429 4430 flags |= HIST_FIELD_FL_VAR; 4431 hist_data->n_vars++; 4432 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX)) 4433 return -EINVAL; 4434 4435 return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags); 4436 } 4437 4438 static int create_val_fields(struct hist_trigger_data *hist_data, 4439 struct trace_event_file *file) 4440 { 4441 char *fields_str, *field_str; 4442 unsigned int i, j = 1; 4443 int ret; 4444 4445 ret = create_hitcount_val(hist_data); 4446 if (ret) 4447 goto out; 4448 4449 fields_str = hist_data->attrs->vals_str; 4450 if (!fields_str) 4451 goto out; 4452 4453 strsep(&fields_str, "="); 4454 if (!fields_str) 4455 goto out; 4456 4457 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX && 4458 j < TRACING_MAP_VALS_MAX; i++) { 4459 field_str = strsep(&fields_str, ","); 4460 if (!field_str) 4461 break; 4462 4463 if (strcmp(field_str, "hitcount") == 0) 4464 continue; 4465 4466 ret = create_val_field(hist_data, j++, file, field_str); 4467 if (ret) 4468 goto out; 4469 } 4470 4471 if (fields_str && (strcmp(fields_str, "hitcount") != 0)) 4472 ret = -EINVAL; 4473 out: 4474 return ret; 4475 } 4476 4477 static int create_key_field(struct hist_trigger_data *hist_data, 4478 unsigned int key_idx, 4479 unsigned int key_offset, 4480 struct trace_event_file *file, 4481 char *field_str) 4482 { 4483 struct trace_array *tr = hist_data->event_file->tr; 4484 struct hist_field *hist_field = NULL; 4485 unsigned long flags = 0; 4486 unsigned int key_size; 4487 int ret = 0; 4488 4489 if (WARN_ON(key_idx >= HIST_FIELDS_MAX)) 4490 return -EINVAL; 4491 4492 flags |= HIST_FIELD_FL_KEY; 4493 4494 if (strcmp(field_str, "stacktrace") == 0) { 4495 flags |= HIST_FIELD_FL_STACKTRACE; 4496 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH; 4497 hist_field = create_hist_field(hist_data, NULL, flags, NULL); 4498 } else { 4499 hist_field = parse_expr(hist_data, file, field_str, flags, 4500 NULL, 0); 4501 if (IS_ERR(hist_field)) { 4502 ret = PTR_ERR(hist_field); 4503 goto out; 4504 } 4505 4506 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) { 4507 hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str)); 4508 destroy_hist_field(hist_field, 0); 4509 ret = -EINVAL; 4510 goto out; 4511 } 4512 4513 key_size = hist_field->size; 4514 } 4515 4516 hist_data->fields[key_idx] = hist_field; 4517 4518 key_size = ALIGN(key_size, sizeof(u64)); 4519 hist_data->fields[key_idx]->size = key_size; 4520 hist_data->fields[key_idx]->offset = key_offset; 4521 4522 hist_data->key_size += key_size; 4523 4524 if (hist_data->key_size > HIST_KEY_SIZE_MAX) { 4525 ret = -EINVAL; 4526 goto out; 4527 } 4528 4529 hist_data->n_keys++; 4530 hist_data->n_fields++; 4531 4532 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX)) 4533 return -EINVAL; 4534 4535 ret = key_size; 4536 out: 4537 return ret; 4538 } 4539 4540 static int create_key_fields(struct hist_trigger_data *hist_data, 4541 struct trace_event_file *file) 4542 { 4543 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals; 4544 char *fields_str, *field_str; 4545 int ret = -EINVAL; 4546 4547 fields_str = hist_data->attrs->keys_str; 4548 if (!fields_str) 4549 goto out; 4550 4551 strsep(&fields_str, "="); 4552 if (!fields_str) 4553 goto out; 4554 4555 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) { 4556 field_str = strsep(&fields_str, ","); 4557 if (!field_str) 4558 break; 4559 ret = create_key_field(hist_data, i, key_offset, 4560 file, field_str); 4561 if (ret < 0) 4562 goto out; 4563 key_offset += ret; 4564 } 4565 if (fields_str) { 4566 ret = -EINVAL; 4567 goto out; 4568 } 4569 ret = 0; 4570 out: 4571 return ret; 4572 } 4573 4574 static int create_var_fields(struct hist_trigger_data *hist_data, 4575 struct trace_event_file *file) 4576 { 4577 unsigned int i, j = hist_data->n_vals; 4578 int ret = 0; 4579 4580 unsigned int n_vars = hist_data->attrs->var_defs.n_vars; 4581 4582 for (i = 0; i < n_vars; i++) { 4583 char *var_name = hist_data->attrs->var_defs.name[i]; 4584 char *expr = hist_data->attrs->var_defs.expr[i]; 4585 4586 ret = create_var_field(hist_data, j++, file, var_name, expr); 4587 if (ret) 4588 goto out; 4589 } 4590 out: 4591 return ret; 4592 } 4593 4594 static void free_var_defs(struct hist_trigger_data *hist_data) 4595 { 4596 unsigned int i; 4597 4598 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 4599 kfree(hist_data->attrs->var_defs.name[i]); 4600 kfree(hist_data->attrs->var_defs.expr[i]); 4601 } 4602 4603 hist_data->attrs->var_defs.n_vars = 0; 4604 } 4605 4606 static int parse_var_defs(struct hist_trigger_data *hist_data) 4607 { 4608 struct trace_array *tr = hist_data->event_file->tr; 4609 char *s, *str, *var_name, *field_str; 4610 unsigned int i, j, n_vars = 0; 4611 int ret = 0; 4612 4613 for (i = 0; i < hist_data->attrs->n_assignments; i++) { 4614 str = hist_data->attrs->assignment_str[i]; 4615 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) { 4616 field_str = strsep(&str, ","); 4617 if (!field_str) 4618 break; 4619 4620 var_name = strsep(&field_str, "="); 4621 if (!var_name || !field_str) { 4622 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT, 4623 errpos(var_name)); 4624 ret = -EINVAL; 4625 goto free; 4626 } 4627 4628 if (n_vars == TRACING_MAP_VARS_MAX) { 4629 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name)); 4630 ret = -EINVAL; 4631 goto free; 4632 } 4633 4634 s = kstrdup(var_name, GFP_KERNEL); 4635 if (!s) { 4636 ret = -ENOMEM; 4637 goto free; 4638 } 4639 hist_data->attrs->var_defs.name[n_vars] = s; 4640 4641 s = kstrdup(field_str, GFP_KERNEL); 4642 if (!s) { 4643 kfree(hist_data->attrs->var_defs.name[n_vars]); 4644 ret = -ENOMEM; 4645 goto free; 4646 } 4647 hist_data->attrs->var_defs.expr[n_vars++] = s; 4648 4649 hist_data->attrs->var_defs.n_vars = n_vars; 4650 } 4651 } 4652 4653 return ret; 4654 free: 4655 free_var_defs(hist_data); 4656 4657 return ret; 4658 } 4659 4660 static int create_hist_fields(struct hist_trigger_data *hist_data, 4661 struct trace_event_file *file) 4662 { 4663 int ret; 4664 4665 ret = parse_var_defs(hist_data); 4666 if (ret) 4667 goto out; 4668 4669 ret = create_val_fields(hist_data, file); 4670 if (ret) 4671 goto out; 4672 4673 ret = create_var_fields(hist_data, file); 4674 if (ret) 4675 goto out; 4676 4677 ret = create_key_fields(hist_data, file); 4678 if (ret) 4679 goto out; 4680 out: 4681 free_var_defs(hist_data); 4682 4683 return ret; 4684 } 4685 4686 static int is_descending(const char *str) 4687 { 4688 if (!str) 4689 return 0; 4690 4691 if (strcmp(str, "descending") == 0) 4692 return 1; 4693 4694 if (strcmp(str, "ascending") == 0) 4695 return 0; 4696 4697 return -EINVAL; 4698 } 4699 4700 static int create_sort_keys(struct hist_trigger_data *hist_data) 4701 { 4702 char *fields_str = hist_data->attrs->sort_key_str; 4703 struct tracing_map_sort_key *sort_key; 4704 int descending, ret = 0; 4705 unsigned int i, j, k; 4706 4707 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */ 4708 4709 if (!fields_str) 4710 goto out; 4711 4712 strsep(&fields_str, "="); 4713 if (!fields_str) { 4714 ret = -EINVAL; 4715 goto out; 4716 } 4717 4718 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) { 4719 struct hist_field *hist_field; 4720 char *field_str, *field_name; 4721 const char *test_name; 4722 4723 sort_key = &hist_data->sort_keys[i]; 4724 4725 field_str = strsep(&fields_str, ","); 4726 if (!field_str) { 4727 if (i == 0) 4728 ret = -EINVAL; 4729 break; 4730 } 4731 4732 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) { 4733 ret = -EINVAL; 4734 break; 4735 } 4736 4737 field_name = strsep(&field_str, "."); 4738 if (!field_name) { 4739 ret = -EINVAL; 4740 break; 4741 } 4742 4743 if (strcmp(field_name, "hitcount") == 0) { 4744 descending = is_descending(field_str); 4745 if (descending < 0) { 4746 ret = descending; 4747 break; 4748 } 4749 sort_key->descending = descending; 4750 continue; 4751 } 4752 4753 for (j = 1, k = 1; j < hist_data->n_fields; j++) { 4754 unsigned int idx; 4755 4756 hist_field = hist_data->fields[j]; 4757 if (hist_field->flags & HIST_FIELD_FL_VAR) 4758 continue; 4759 4760 idx = k++; 4761 4762 test_name = hist_field_name(hist_field, 0); 4763 4764 if (strcmp(field_name, test_name) == 0) { 4765 sort_key->field_idx = idx; 4766 descending = is_descending(field_str); 4767 if (descending < 0) { 4768 ret = descending; 4769 goto out; 4770 } 4771 sort_key->descending = descending; 4772 break; 4773 } 4774 } 4775 if (j == hist_data->n_fields) { 4776 ret = -EINVAL; 4777 break; 4778 } 4779 } 4780 4781 hist_data->n_sort_keys = i; 4782 out: 4783 return ret; 4784 } 4785 4786 static void destroy_actions(struct hist_trigger_data *hist_data) 4787 { 4788 unsigned int i; 4789 4790 for (i = 0; i < hist_data->n_actions; i++) { 4791 struct action_data *data = hist_data->actions[i]; 4792 4793 if (data->handler == HANDLER_ONMATCH) 4794 onmatch_destroy(data); 4795 else if (data->handler == HANDLER_ONMAX || 4796 data->handler == HANDLER_ONCHANGE) 4797 track_data_destroy(hist_data, data); 4798 else 4799 kfree(data); 4800 } 4801 } 4802 4803 static int parse_actions(struct hist_trigger_data *hist_data) 4804 { 4805 struct trace_array *tr = hist_data->event_file->tr; 4806 struct action_data *data; 4807 unsigned int i; 4808 int ret = 0; 4809 char *str; 4810 int len; 4811 4812 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4813 str = hist_data->attrs->action_str[i]; 4814 4815 if ((len = str_has_prefix(str, "onmatch("))) { 4816 char *action_str = str + len; 4817 4818 data = onmatch_parse(tr, action_str); 4819 if (IS_ERR(data)) { 4820 ret = PTR_ERR(data); 4821 break; 4822 } 4823 } else if ((len = str_has_prefix(str, "onmax("))) { 4824 char *action_str = str + len; 4825 4826 data = track_data_parse(hist_data, action_str, 4827 HANDLER_ONMAX); 4828 if (IS_ERR(data)) { 4829 ret = PTR_ERR(data); 4830 break; 4831 } 4832 } else if ((len = str_has_prefix(str, "onchange("))) { 4833 char *action_str = str + len; 4834 4835 data = track_data_parse(hist_data, action_str, 4836 HANDLER_ONCHANGE); 4837 if (IS_ERR(data)) { 4838 ret = PTR_ERR(data); 4839 break; 4840 } 4841 } else { 4842 ret = -EINVAL; 4843 break; 4844 } 4845 4846 hist_data->actions[hist_data->n_actions++] = data; 4847 } 4848 4849 return ret; 4850 } 4851 4852 static int create_actions(struct hist_trigger_data *hist_data) 4853 { 4854 struct action_data *data; 4855 unsigned int i; 4856 int ret = 0; 4857 4858 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4859 data = hist_data->actions[i]; 4860 4861 if (data->handler == HANDLER_ONMATCH) { 4862 ret = onmatch_create(hist_data, data); 4863 if (ret) 4864 break; 4865 } else if (data->handler == HANDLER_ONMAX || 4866 data->handler == HANDLER_ONCHANGE) { 4867 ret = track_data_create(hist_data, data); 4868 if (ret) 4869 break; 4870 } else { 4871 ret = -EINVAL; 4872 break; 4873 } 4874 } 4875 4876 return ret; 4877 } 4878 4879 static void print_actions(struct seq_file *m, 4880 struct hist_trigger_data *hist_data, 4881 struct tracing_map_elt *elt) 4882 { 4883 unsigned int i; 4884 4885 for (i = 0; i < hist_data->n_actions; i++) { 4886 struct action_data *data = hist_data->actions[i]; 4887 4888 if (data->action == ACTION_SNAPSHOT) 4889 continue; 4890 4891 if (data->handler == HANDLER_ONMAX || 4892 data->handler == HANDLER_ONCHANGE) 4893 track_data_print(m, hist_data, elt, data); 4894 } 4895 } 4896 4897 static void print_action_spec(struct seq_file *m, 4898 struct hist_trigger_data *hist_data, 4899 struct action_data *data) 4900 { 4901 unsigned int i; 4902 4903 if (data->action == ACTION_SAVE) { 4904 for (i = 0; i < hist_data->n_save_vars; i++) { 4905 seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name); 4906 if (i < hist_data->n_save_vars - 1) 4907 seq_puts(m, ","); 4908 } 4909 } else if (data->action == ACTION_TRACE) { 4910 if (data->use_trace_keyword) 4911 seq_printf(m, "%s", data->synth_event_name); 4912 for (i = 0; i < data->n_params; i++) { 4913 if (i || data->use_trace_keyword) 4914 seq_puts(m, ","); 4915 seq_printf(m, "%s", data->params[i]); 4916 } 4917 } 4918 } 4919 4920 static void print_track_data_spec(struct seq_file *m, 4921 struct hist_trigger_data *hist_data, 4922 struct action_data *data) 4923 { 4924 if (data->handler == HANDLER_ONMAX) 4925 seq_puts(m, ":onmax("); 4926 else if (data->handler == HANDLER_ONCHANGE) 4927 seq_puts(m, ":onchange("); 4928 seq_printf(m, "%s", data->track_data.var_str); 4929 seq_printf(m, ").%s(", data->action_name); 4930 4931 print_action_spec(m, hist_data, data); 4932 4933 seq_puts(m, ")"); 4934 } 4935 4936 static void print_onmatch_spec(struct seq_file *m, 4937 struct hist_trigger_data *hist_data, 4938 struct action_data *data) 4939 { 4940 seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system, 4941 data->match_data.event); 4942 4943 seq_printf(m, "%s(", data->action_name); 4944 4945 print_action_spec(m, hist_data, data); 4946 4947 seq_puts(m, ")"); 4948 } 4949 4950 static bool actions_match(struct hist_trigger_data *hist_data, 4951 struct hist_trigger_data *hist_data_test) 4952 { 4953 unsigned int i, j; 4954 4955 if (hist_data->n_actions != hist_data_test->n_actions) 4956 return false; 4957 4958 for (i = 0; i < hist_data->n_actions; i++) { 4959 struct action_data *data = hist_data->actions[i]; 4960 struct action_data *data_test = hist_data_test->actions[i]; 4961 char *action_name, *action_name_test; 4962 4963 if (data->handler != data_test->handler) 4964 return false; 4965 if (data->action != data_test->action) 4966 return false; 4967 4968 if (data->n_params != data_test->n_params) 4969 return false; 4970 4971 for (j = 0; j < data->n_params; j++) { 4972 if (strcmp(data->params[j], data_test->params[j]) != 0) 4973 return false; 4974 } 4975 4976 if (data->use_trace_keyword) 4977 action_name = data->synth_event_name; 4978 else 4979 action_name = data->action_name; 4980 4981 if (data_test->use_trace_keyword) 4982 action_name_test = data_test->synth_event_name; 4983 else 4984 action_name_test = data_test->action_name; 4985 4986 if (strcmp(action_name, action_name_test) != 0) 4987 return false; 4988 4989 if (data->handler == HANDLER_ONMATCH) { 4990 if (strcmp(data->match_data.event_system, 4991 data_test->match_data.event_system) != 0) 4992 return false; 4993 if (strcmp(data->match_data.event, 4994 data_test->match_data.event) != 0) 4995 return false; 4996 } else if (data->handler == HANDLER_ONMAX || 4997 data->handler == HANDLER_ONCHANGE) { 4998 if (strcmp(data->track_data.var_str, 4999 data_test->track_data.var_str) != 0) 5000 return false; 5001 } 5002 } 5003 5004 return true; 5005 } 5006 5007 5008 static void print_actions_spec(struct seq_file *m, 5009 struct hist_trigger_data *hist_data) 5010 { 5011 unsigned int i; 5012 5013 for (i = 0; i < hist_data->n_actions; i++) { 5014 struct action_data *data = hist_data->actions[i]; 5015 5016 if (data->handler == HANDLER_ONMATCH) 5017 print_onmatch_spec(m, hist_data, data); 5018 else if (data->handler == HANDLER_ONMAX || 5019 data->handler == HANDLER_ONCHANGE) 5020 print_track_data_spec(m, hist_data, data); 5021 } 5022 } 5023 5024 static void destroy_field_var_hists(struct hist_trigger_data *hist_data) 5025 { 5026 unsigned int i; 5027 5028 for (i = 0; i < hist_data->n_field_var_hists; i++) { 5029 kfree(hist_data->field_var_hists[i]->cmd); 5030 kfree(hist_data->field_var_hists[i]); 5031 } 5032 } 5033 5034 static void destroy_hist_data(struct hist_trigger_data *hist_data) 5035 { 5036 if (!hist_data) 5037 return; 5038 5039 destroy_hist_trigger_attrs(hist_data->attrs); 5040 destroy_hist_fields(hist_data); 5041 tracing_map_destroy(hist_data->map); 5042 5043 destroy_actions(hist_data); 5044 destroy_field_vars(hist_data); 5045 destroy_field_var_hists(hist_data); 5046 5047 kfree(hist_data); 5048 } 5049 5050 static int create_tracing_map_fields(struct hist_trigger_data *hist_data) 5051 { 5052 struct tracing_map *map = hist_data->map; 5053 struct ftrace_event_field *field; 5054 struct hist_field *hist_field; 5055 int i, idx = 0; 5056 5057 for_each_hist_field(i, hist_data) { 5058 hist_field = hist_data->fields[i]; 5059 if (hist_field->flags & HIST_FIELD_FL_KEY) { 5060 tracing_map_cmp_fn_t cmp_fn; 5061 5062 field = hist_field->field; 5063 5064 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE) 5065 cmp_fn = tracing_map_cmp_none; 5066 else if (!field) 5067 cmp_fn = tracing_map_cmp_num(hist_field->size, 5068 hist_field->is_signed); 5069 else if (is_string_field(field)) 5070 cmp_fn = tracing_map_cmp_string; 5071 else 5072 cmp_fn = tracing_map_cmp_num(field->size, 5073 field->is_signed); 5074 idx = tracing_map_add_key_field(map, 5075 hist_field->offset, 5076 cmp_fn); 5077 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR)) 5078 idx = tracing_map_add_sum_field(map); 5079 5080 if (idx < 0) 5081 return idx; 5082 5083 if (hist_field->flags & HIST_FIELD_FL_VAR) { 5084 idx = tracing_map_add_var(map); 5085 if (idx < 0) 5086 return idx; 5087 hist_field->var.idx = idx; 5088 hist_field->var.hist_data = hist_data; 5089 } 5090 } 5091 5092 return 0; 5093 } 5094 5095 static struct hist_trigger_data * 5096 create_hist_data(unsigned int map_bits, 5097 struct hist_trigger_attrs *attrs, 5098 struct trace_event_file *file, 5099 bool remove) 5100 { 5101 const struct tracing_map_ops *map_ops = NULL; 5102 struct hist_trigger_data *hist_data; 5103 int ret = 0; 5104 5105 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL); 5106 if (!hist_data) 5107 return ERR_PTR(-ENOMEM); 5108 5109 hist_data->attrs = attrs; 5110 hist_data->remove = remove; 5111 hist_data->event_file = file; 5112 5113 ret = parse_actions(hist_data); 5114 if (ret) 5115 goto free; 5116 5117 ret = create_hist_fields(hist_data, file); 5118 if (ret) 5119 goto free; 5120 5121 ret = create_sort_keys(hist_data); 5122 if (ret) 5123 goto free; 5124 5125 map_ops = &hist_trigger_elt_data_ops; 5126 5127 hist_data->map = tracing_map_create(map_bits, hist_data->key_size, 5128 map_ops, hist_data); 5129 if (IS_ERR(hist_data->map)) { 5130 ret = PTR_ERR(hist_data->map); 5131 hist_data->map = NULL; 5132 goto free; 5133 } 5134 5135 ret = create_tracing_map_fields(hist_data); 5136 if (ret) 5137 goto free; 5138 out: 5139 return hist_data; 5140 free: 5141 hist_data->attrs = NULL; 5142 5143 destroy_hist_data(hist_data); 5144 5145 hist_data = ERR_PTR(ret); 5146 5147 goto out; 5148 } 5149 5150 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, 5151 struct tracing_map_elt *elt, void *rec, 5152 struct ring_buffer_event *rbe, 5153 u64 *var_ref_vals) 5154 { 5155 struct hist_elt_data *elt_data; 5156 struct hist_field *hist_field; 5157 unsigned int i, var_idx; 5158 u64 hist_val; 5159 5160 elt_data = elt->private_data; 5161 elt_data->var_ref_vals = var_ref_vals; 5162 5163 for_each_hist_val_field(i, hist_data) { 5164 hist_field = hist_data->fields[i]; 5165 hist_val = hist_field->fn(hist_field, elt, rbe, rec); 5166 if (hist_field->flags & HIST_FIELD_FL_VAR) { 5167 var_idx = hist_field->var.idx; 5168 tracing_map_set_var(elt, var_idx, hist_val); 5169 continue; 5170 } 5171 tracing_map_update_sum(elt, i, hist_val); 5172 } 5173 5174 for_each_hist_key_field(i, hist_data) { 5175 hist_field = hist_data->fields[i]; 5176 if (hist_field->flags & HIST_FIELD_FL_VAR) { 5177 hist_val = hist_field->fn(hist_field, elt, rbe, rec); 5178 var_idx = hist_field->var.idx; 5179 tracing_map_set_var(elt, var_idx, hist_val); 5180 } 5181 } 5182 5183 update_field_vars(hist_data, elt, rbe, rec); 5184 } 5185 5186 static inline void add_to_key(char *compound_key, void *key, 5187 struct hist_field *key_field, void *rec) 5188 { 5189 size_t size = key_field->size; 5190 5191 if (key_field->flags & HIST_FIELD_FL_STRING) { 5192 struct ftrace_event_field *field; 5193 5194 field = key_field->field; 5195 if (field->filter_type == FILTER_DYN_STRING) 5196 size = *(u32 *)(rec + field->offset) >> 16; 5197 else if (field->filter_type == FILTER_PTR_STRING) 5198 size = strlen(key); 5199 else if (field->filter_type == FILTER_STATIC_STRING) 5200 size = field->size; 5201 5202 /* ensure NULL-termination */ 5203 if (size > key_field->size - 1) 5204 size = key_field->size - 1; 5205 5206 strncpy(compound_key + key_field->offset, (char *)key, size); 5207 } else 5208 memcpy(compound_key + key_field->offset, key, size); 5209 } 5210 5211 static void 5212 hist_trigger_actions(struct hist_trigger_data *hist_data, 5213 struct tracing_map_elt *elt, void *rec, 5214 struct ring_buffer_event *rbe, void *key, 5215 u64 *var_ref_vals) 5216 { 5217 struct action_data *data; 5218 unsigned int i; 5219 5220 for (i = 0; i < hist_data->n_actions; i++) { 5221 data = hist_data->actions[i]; 5222 data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals); 5223 } 5224 } 5225 5226 static void event_hist_trigger(struct event_trigger_data *data, void *rec, 5227 struct ring_buffer_event *rbe) 5228 { 5229 struct hist_trigger_data *hist_data = data->private_data; 5230 bool use_compound_key = (hist_data->n_keys > 1); 5231 unsigned long entries[HIST_STACKTRACE_DEPTH]; 5232 u64 var_ref_vals[TRACING_MAP_VARS_MAX]; 5233 char compound_key[HIST_KEY_SIZE_MAX]; 5234 struct tracing_map_elt *elt = NULL; 5235 struct hist_field *key_field; 5236 u64 field_contents; 5237 void *key = NULL; 5238 unsigned int i; 5239 5240 memset(compound_key, 0, hist_data->key_size); 5241 5242 for_each_hist_key_field(i, hist_data) { 5243 key_field = hist_data->fields[i]; 5244 5245 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 5246 memset(entries, 0, HIST_STACKTRACE_SIZE); 5247 stack_trace_save(entries, HIST_STACKTRACE_DEPTH, 5248 HIST_STACKTRACE_SKIP); 5249 key = entries; 5250 } else { 5251 field_contents = key_field->fn(key_field, elt, rbe, rec); 5252 if (key_field->flags & HIST_FIELD_FL_STRING) { 5253 key = (void *)(unsigned long)field_contents; 5254 use_compound_key = true; 5255 } else 5256 key = (void *)&field_contents; 5257 } 5258 5259 if (use_compound_key) 5260 add_to_key(compound_key, key, key_field, rec); 5261 } 5262 5263 if (use_compound_key) 5264 key = compound_key; 5265 5266 if (hist_data->n_var_refs && 5267 !resolve_var_refs(hist_data, key, var_ref_vals, false)) 5268 return; 5269 5270 elt = tracing_map_insert(hist_data->map, key); 5271 if (!elt) 5272 return; 5273 5274 hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals); 5275 5276 if (resolve_var_refs(hist_data, key, var_ref_vals, true)) 5277 hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals); 5278 } 5279 5280 static void hist_trigger_stacktrace_print(struct seq_file *m, 5281 unsigned long *stacktrace_entries, 5282 unsigned int max_entries) 5283 { 5284 char str[KSYM_SYMBOL_LEN]; 5285 unsigned int spaces = 8; 5286 unsigned int i; 5287 5288 for (i = 0; i < max_entries; i++) { 5289 if (!stacktrace_entries[i]) 5290 return; 5291 5292 seq_printf(m, "%*c", 1 + spaces, ' '); 5293 sprint_symbol(str, stacktrace_entries[i]); 5294 seq_printf(m, "%s\n", str); 5295 } 5296 } 5297 5298 static void hist_trigger_print_key(struct seq_file *m, 5299 struct hist_trigger_data *hist_data, 5300 void *key, 5301 struct tracing_map_elt *elt) 5302 { 5303 struct hist_field *key_field; 5304 char str[KSYM_SYMBOL_LEN]; 5305 bool multiline = false; 5306 const char *field_name; 5307 unsigned int i; 5308 u64 uval; 5309 5310 seq_puts(m, "{ "); 5311 5312 for_each_hist_key_field(i, hist_data) { 5313 key_field = hist_data->fields[i]; 5314 5315 if (i > hist_data->n_vals) 5316 seq_puts(m, ", "); 5317 5318 field_name = hist_field_name(key_field, 0); 5319 5320 if (key_field->flags & HIST_FIELD_FL_HEX) { 5321 uval = *(u64 *)(key + key_field->offset); 5322 seq_printf(m, "%s: %llx", field_name, uval); 5323 } else if (key_field->flags & HIST_FIELD_FL_SYM) { 5324 uval = *(u64 *)(key + key_field->offset); 5325 sprint_symbol_no_offset(str, uval); 5326 seq_printf(m, "%s: [%llx] %-45s", field_name, 5327 uval, str); 5328 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) { 5329 uval = *(u64 *)(key + key_field->offset); 5330 sprint_symbol(str, uval); 5331 seq_printf(m, "%s: [%llx] %-55s", field_name, 5332 uval, str); 5333 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 5334 struct hist_elt_data *elt_data = elt->private_data; 5335 char *comm; 5336 5337 if (WARN_ON_ONCE(!elt_data)) 5338 return; 5339 5340 comm = elt_data->comm; 5341 5342 uval = *(u64 *)(key + key_field->offset); 5343 seq_printf(m, "%s: %-16s[%10llu]", field_name, 5344 comm, uval); 5345 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) { 5346 const char *syscall_name; 5347 5348 uval = *(u64 *)(key + key_field->offset); 5349 syscall_name = get_syscall_name(uval); 5350 if (!syscall_name) 5351 syscall_name = "unknown_syscall"; 5352 5353 seq_printf(m, "%s: %-30s[%3llu]", field_name, 5354 syscall_name, uval); 5355 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 5356 seq_puts(m, "stacktrace:\n"); 5357 hist_trigger_stacktrace_print(m, 5358 key + key_field->offset, 5359 HIST_STACKTRACE_DEPTH); 5360 multiline = true; 5361 } else if (key_field->flags & HIST_FIELD_FL_LOG2) { 5362 seq_printf(m, "%s: ~ 2^%-2llu", field_name, 5363 *(u64 *)(key + key_field->offset)); 5364 } else if (key_field->flags & HIST_FIELD_FL_STRING) { 5365 seq_printf(m, "%s: %-50s", field_name, 5366 (char *)(key + key_field->offset)); 5367 } else { 5368 uval = *(u64 *)(key + key_field->offset); 5369 seq_printf(m, "%s: %10llu", field_name, uval); 5370 } 5371 } 5372 5373 if (!multiline) 5374 seq_puts(m, " "); 5375 5376 seq_puts(m, "}"); 5377 } 5378 5379 static void hist_trigger_entry_print(struct seq_file *m, 5380 struct hist_trigger_data *hist_data, 5381 void *key, 5382 struct tracing_map_elt *elt) 5383 { 5384 const char *field_name; 5385 unsigned int i; 5386 5387 hist_trigger_print_key(m, hist_data, key, elt); 5388 5389 seq_printf(m, " hitcount: %10llu", 5390 tracing_map_read_sum(elt, HITCOUNT_IDX)); 5391 5392 for (i = 1; i < hist_data->n_vals; i++) { 5393 field_name = hist_field_name(hist_data->fields[i], 0); 5394 5395 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR || 5396 hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR) 5397 continue; 5398 5399 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) { 5400 seq_printf(m, " %s: %10llx", field_name, 5401 tracing_map_read_sum(elt, i)); 5402 } else { 5403 seq_printf(m, " %s: %10llu", field_name, 5404 tracing_map_read_sum(elt, i)); 5405 } 5406 } 5407 5408 print_actions(m, hist_data, elt); 5409 5410 seq_puts(m, "\n"); 5411 } 5412 5413 static int print_entries(struct seq_file *m, 5414 struct hist_trigger_data *hist_data) 5415 { 5416 struct tracing_map_sort_entry **sort_entries = NULL; 5417 struct tracing_map *map = hist_data->map; 5418 int i, n_entries; 5419 5420 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys, 5421 hist_data->n_sort_keys, 5422 &sort_entries); 5423 if (n_entries < 0) 5424 return n_entries; 5425 5426 for (i = 0; i < n_entries; i++) 5427 hist_trigger_entry_print(m, hist_data, 5428 sort_entries[i]->key, 5429 sort_entries[i]->elt); 5430 5431 tracing_map_destroy_sort_entries(sort_entries, n_entries); 5432 5433 return n_entries; 5434 } 5435 5436 static void hist_trigger_show(struct seq_file *m, 5437 struct event_trigger_data *data, int n) 5438 { 5439 struct hist_trigger_data *hist_data; 5440 int n_entries; 5441 5442 if (n > 0) 5443 seq_puts(m, "\n\n"); 5444 5445 seq_puts(m, "# event histogram\n#\n# trigger info: "); 5446 data->ops->print(m, data->ops, data); 5447 seq_puts(m, "#\n\n"); 5448 5449 hist_data = data->private_data; 5450 n_entries = print_entries(m, hist_data); 5451 if (n_entries < 0) 5452 n_entries = 0; 5453 5454 track_data_snapshot_print(m, hist_data); 5455 5456 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n", 5457 (u64)atomic64_read(&hist_data->map->hits), 5458 n_entries, (u64)atomic64_read(&hist_data->map->drops)); 5459 } 5460 5461 static int hist_show(struct seq_file *m, void *v) 5462 { 5463 struct event_trigger_data *data; 5464 struct trace_event_file *event_file; 5465 int n = 0, ret = 0; 5466 5467 mutex_lock(&event_mutex); 5468 5469 event_file = event_file_data(m->private); 5470 if (unlikely(!event_file)) { 5471 ret = -ENODEV; 5472 goto out_unlock; 5473 } 5474 5475 list_for_each_entry_rcu(data, &event_file->triggers, list) { 5476 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 5477 hist_trigger_show(m, data, n++); 5478 } 5479 5480 out_unlock: 5481 mutex_unlock(&event_mutex); 5482 5483 return ret; 5484 } 5485 5486 static int event_hist_open(struct inode *inode, struct file *file) 5487 { 5488 return single_open(file, hist_show, file); 5489 } 5490 5491 const struct file_operations event_hist_fops = { 5492 .open = event_hist_open, 5493 .read = seq_read, 5494 .llseek = seq_lseek, 5495 .release = single_release, 5496 }; 5497 5498 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field) 5499 { 5500 const char *field_name = hist_field_name(hist_field, 0); 5501 5502 if (hist_field->var.name) 5503 seq_printf(m, "%s=", hist_field->var.name); 5504 5505 if (hist_field->flags & HIST_FIELD_FL_CPU) 5506 seq_puts(m, "cpu"); 5507 else if (field_name) { 5508 if (hist_field->flags & HIST_FIELD_FL_VAR_REF || 5509 hist_field->flags & HIST_FIELD_FL_ALIAS) 5510 seq_putc(m, '$'); 5511 seq_printf(m, "%s", field_name); 5512 } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP) 5513 seq_puts(m, "common_timestamp"); 5514 5515 if (hist_field->flags) { 5516 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) && 5517 !(hist_field->flags & HIST_FIELD_FL_EXPR)) { 5518 const char *flags = get_hist_field_flags(hist_field); 5519 5520 if (flags) 5521 seq_printf(m, ".%s", flags); 5522 } 5523 } 5524 } 5525 5526 static int event_hist_trigger_print(struct seq_file *m, 5527 struct event_trigger_ops *ops, 5528 struct event_trigger_data *data) 5529 { 5530 struct hist_trigger_data *hist_data = data->private_data; 5531 struct hist_field *field; 5532 bool have_var = false; 5533 unsigned int i; 5534 5535 seq_puts(m, "hist:"); 5536 5537 if (data->name) 5538 seq_printf(m, "%s:", data->name); 5539 5540 seq_puts(m, "keys="); 5541 5542 for_each_hist_key_field(i, hist_data) { 5543 field = hist_data->fields[i]; 5544 5545 if (i > hist_data->n_vals) 5546 seq_puts(m, ","); 5547 5548 if (field->flags & HIST_FIELD_FL_STACKTRACE) 5549 seq_puts(m, "stacktrace"); 5550 else 5551 hist_field_print(m, field); 5552 } 5553 5554 seq_puts(m, ":vals="); 5555 5556 for_each_hist_val_field(i, hist_data) { 5557 field = hist_data->fields[i]; 5558 if (field->flags & HIST_FIELD_FL_VAR) { 5559 have_var = true; 5560 continue; 5561 } 5562 5563 if (i == HITCOUNT_IDX) 5564 seq_puts(m, "hitcount"); 5565 else { 5566 seq_puts(m, ","); 5567 hist_field_print(m, field); 5568 } 5569 } 5570 5571 if (have_var) { 5572 unsigned int n = 0; 5573 5574 seq_puts(m, ":"); 5575 5576 for_each_hist_val_field(i, hist_data) { 5577 field = hist_data->fields[i]; 5578 5579 if (field->flags & HIST_FIELD_FL_VAR) { 5580 if (n++) 5581 seq_puts(m, ","); 5582 hist_field_print(m, field); 5583 } 5584 } 5585 } 5586 5587 seq_puts(m, ":sort="); 5588 5589 for (i = 0; i < hist_data->n_sort_keys; i++) { 5590 struct tracing_map_sort_key *sort_key; 5591 unsigned int idx, first_key_idx; 5592 5593 /* skip VAR vals */ 5594 first_key_idx = hist_data->n_vals - hist_data->n_vars; 5595 5596 sort_key = &hist_data->sort_keys[i]; 5597 idx = sort_key->field_idx; 5598 5599 if (WARN_ON(idx >= HIST_FIELDS_MAX)) 5600 return -EINVAL; 5601 5602 if (i > 0) 5603 seq_puts(m, ","); 5604 5605 if (idx == HITCOUNT_IDX) 5606 seq_puts(m, "hitcount"); 5607 else { 5608 if (idx >= first_key_idx) 5609 idx += hist_data->n_vars; 5610 hist_field_print(m, hist_data->fields[idx]); 5611 } 5612 5613 if (sort_key->descending) 5614 seq_puts(m, ".descending"); 5615 } 5616 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits)); 5617 if (hist_data->enable_timestamps) 5618 seq_printf(m, ":clock=%s", hist_data->attrs->clock); 5619 5620 print_actions_spec(m, hist_data); 5621 5622 if (data->filter_str) 5623 seq_printf(m, " if %s", data->filter_str); 5624 5625 if (data->paused) 5626 seq_puts(m, " [paused]"); 5627 else 5628 seq_puts(m, " [active]"); 5629 5630 seq_putc(m, '\n'); 5631 5632 return 0; 5633 } 5634 5635 static int event_hist_trigger_init(struct event_trigger_ops *ops, 5636 struct event_trigger_data *data) 5637 { 5638 struct hist_trigger_data *hist_data = data->private_data; 5639 5640 if (!data->ref && hist_data->attrs->name) 5641 save_named_trigger(hist_data->attrs->name, data); 5642 5643 data->ref++; 5644 5645 return 0; 5646 } 5647 5648 static void unregister_field_var_hists(struct hist_trigger_data *hist_data) 5649 { 5650 struct trace_event_file *file; 5651 unsigned int i; 5652 char *cmd; 5653 int ret; 5654 5655 for (i = 0; i < hist_data->n_field_var_hists; i++) { 5656 file = hist_data->field_var_hists[i]->hist_data->event_file; 5657 cmd = hist_data->field_var_hists[i]->cmd; 5658 ret = event_hist_trigger_func(&trigger_hist_cmd, file, 5659 "!hist", "hist", cmd); 5660 } 5661 } 5662 5663 static void event_hist_trigger_free(struct event_trigger_ops *ops, 5664 struct event_trigger_data *data) 5665 { 5666 struct hist_trigger_data *hist_data = data->private_data; 5667 5668 if (WARN_ON_ONCE(data->ref <= 0)) 5669 return; 5670 5671 data->ref--; 5672 if (!data->ref) { 5673 if (data->name) 5674 del_named_trigger(data); 5675 5676 trigger_data_free(data); 5677 5678 remove_hist_vars(hist_data); 5679 5680 unregister_field_var_hists(hist_data); 5681 5682 destroy_hist_data(hist_data); 5683 } 5684 } 5685 5686 static struct event_trigger_ops event_hist_trigger_ops = { 5687 .func = event_hist_trigger, 5688 .print = event_hist_trigger_print, 5689 .init = event_hist_trigger_init, 5690 .free = event_hist_trigger_free, 5691 }; 5692 5693 static int event_hist_trigger_named_init(struct event_trigger_ops *ops, 5694 struct event_trigger_data *data) 5695 { 5696 data->ref++; 5697 5698 save_named_trigger(data->named_data->name, data); 5699 5700 event_hist_trigger_init(ops, data->named_data); 5701 5702 return 0; 5703 } 5704 5705 static void event_hist_trigger_named_free(struct event_trigger_ops *ops, 5706 struct event_trigger_data *data) 5707 { 5708 if (WARN_ON_ONCE(data->ref <= 0)) 5709 return; 5710 5711 event_hist_trigger_free(ops, data->named_data); 5712 5713 data->ref--; 5714 if (!data->ref) { 5715 del_named_trigger(data); 5716 trigger_data_free(data); 5717 } 5718 } 5719 5720 static struct event_trigger_ops event_hist_trigger_named_ops = { 5721 .func = event_hist_trigger, 5722 .print = event_hist_trigger_print, 5723 .init = event_hist_trigger_named_init, 5724 .free = event_hist_trigger_named_free, 5725 }; 5726 5727 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd, 5728 char *param) 5729 { 5730 return &event_hist_trigger_ops; 5731 } 5732 5733 static void hist_clear(struct event_trigger_data *data) 5734 { 5735 struct hist_trigger_data *hist_data = data->private_data; 5736 5737 if (data->name) 5738 pause_named_trigger(data); 5739 5740 tracepoint_synchronize_unregister(); 5741 5742 tracing_map_clear(hist_data->map); 5743 5744 if (data->name) 5745 unpause_named_trigger(data); 5746 } 5747 5748 static bool compatible_field(struct ftrace_event_field *field, 5749 struct ftrace_event_field *test_field) 5750 { 5751 if (field == test_field) 5752 return true; 5753 if (field == NULL || test_field == NULL) 5754 return false; 5755 if (strcmp(field->name, test_field->name) != 0) 5756 return false; 5757 if (strcmp(field->type, test_field->type) != 0) 5758 return false; 5759 if (field->size != test_field->size) 5760 return false; 5761 if (field->is_signed != test_field->is_signed) 5762 return false; 5763 5764 return true; 5765 } 5766 5767 static bool hist_trigger_match(struct event_trigger_data *data, 5768 struct event_trigger_data *data_test, 5769 struct event_trigger_data *named_data, 5770 bool ignore_filter) 5771 { 5772 struct tracing_map_sort_key *sort_key, *sort_key_test; 5773 struct hist_trigger_data *hist_data, *hist_data_test; 5774 struct hist_field *key_field, *key_field_test; 5775 unsigned int i; 5776 5777 if (named_data && (named_data != data_test) && 5778 (named_data != data_test->named_data)) 5779 return false; 5780 5781 if (!named_data && is_named_trigger(data_test)) 5782 return false; 5783 5784 hist_data = data->private_data; 5785 hist_data_test = data_test->private_data; 5786 5787 if (hist_data->n_vals != hist_data_test->n_vals || 5788 hist_data->n_fields != hist_data_test->n_fields || 5789 hist_data->n_sort_keys != hist_data_test->n_sort_keys) 5790 return false; 5791 5792 if (!ignore_filter) { 5793 if ((data->filter_str && !data_test->filter_str) || 5794 (!data->filter_str && data_test->filter_str)) 5795 return false; 5796 } 5797 5798 for_each_hist_field(i, hist_data) { 5799 key_field = hist_data->fields[i]; 5800 key_field_test = hist_data_test->fields[i]; 5801 5802 if (key_field->flags != key_field_test->flags) 5803 return false; 5804 if (!compatible_field(key_field->field, key_field_test->field)) 5805 return false; 5806 if (key_field->offset != key_field_test->offset) 5807 return false; 5808 if (key_field->size != key_field_test->size) 5809 return false; 5810 if (key_field->is_signed != key_field_test->is_signed) 5811 return false; 5812 if (!!key_field->var.name != !!key_field_test->var.name) 5813 return false; 5814 if (key_field->var.name && 5815 strcmp(key_field->var.name, key_field_test->var.name) != 0) 5816 return false; 5817 } 5818 5819 for (i = 0; i < hist_data->n_sort_keys; i++) { 5820 sort_key = &hist_data->sort_keys[i]; 5821 sort_key_test = &hist_data_test->sort_keys[i]; 5822 5823 if (sort_key->field_idx != sort_key_test->field_idx || 5824 sort_key->descending != sort_key_test->descending) 5825 return false; 5826 } 5827 5828 if (!ignore_filter && data->filter_str && 5829 (strcmp(data->filter_str, data_test->filter_str) != 0)) 5830 return false; 5831 5832 if (!actions_match(hist_data, hist_data_test)) 5833 return false; 5834 5835 return true; 5836 } 5837 5838 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops, 5839 struct event_trigger_data *data, 5840 struct trace_event_file *file) 5841 { 5842 struct hist_trigger_data *hist_data = data->private_data; 5843 struct event_trigger_data *test, *named_data = NULL; 5844 struct trace_array *tr = file->tr; 5845 int ret = 0; 5846 5847 if (hist_data->attrs->name) { 5848 named_data = find_named_trigger(hist_data->attrs->name); 5849 if (named_data) { 5850 if (!hist_trigger_match(data, named_data, named_data, 5851 true)) { 5852 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name)); 5853 ret = -EINVAL; 5854 goto out; 5855 } 5856 } 5857 } 5858 5859 if (hist_data->attrs->name && !named_data) 5860 goto new; 5861 5862 list_for_each_entry_rcu(test, &file->triggers, list) { 5863 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5864 if (!hist_trigger_match(data, test, named_data, false)) 5865 continue; 5866 if (hist_data->attrs->pause) 5867 test->paused = true; 5868 else if (hist_data->attrs->cont) 5869 test->paused = false; 5870 else if (hist_data->attrs->clear) 5871 hist_clear(test); 5872 else { 5873 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0); 5874 ret = -EEXIST; 5875 } 5876 goto out; 5877 } 5878 } 5879 new: 5880 if (hist_data->attrs->cont || hist_data->attrs->clear) { 5881 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0); 5882 ret = -ENOENT; 5883 goto out; 5884 } 5885 5886 if (hist_data->attrs->pause) 5887 data->paused = true; 5888 5889 if (named_data) { 5890 data->private_data = named_data->private_data; 5891 set_named_trigger_data(data, named_data); 5892 data->ops = &event_hist_trigger_named_ops; 5893 } 5894 5895 if (data->ops->init) { 5896 ret = data->ops->init(data->ops, data); 5897 if (ret < 0) 5898 goto out; 5899 } 5900 5901 if (hist_data->enable_timestamps) { 5902 char *clock = hist_data->attrs->clock; 5903 5904 ret = tracing_set_clock(file->tr, hist_data->attrs->clock); 5905 if (ret) { 5906 hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock)); 5907 goto out; 5908 } 5909 5910 tracing_set_time_stamp_abs(file->tr, true); 5911 } 5912 5913 if (named_data) 5914 destroy_hist_data(hist_data); 5915 5916 ret++; 5917 out: 5918 return ret; 5919 } 5920 5921 static int hist_trigger_enable(struct event_trigger_data *data, 5922 struct trace_event_file *file) 5923 { 5924 int ret = 0; 5925 5926 list_add_tail_rcu(&data->list, &file->triggers); 5927 5928 update_cond_flag(file); 5929 5930 if (trace_event_trigger_enable_disable(file, 1) < 0) { 5931 list_del_rcu(&data->list); 5932 update_cond_flag(file); 5933 ret--; 5934 } 5935 5936 return ret; 5937 } 5938 5939 static bool have_hist_trigger_match(struct event_trigger_data *data, 5940 struct trace_event_file *file) 5941 { 5942 struct hist_trigger_data *hist_data = data->private_data; 5943 struct event_trigger_data *test, *named_data = NULL; 5944 bool match = false; 5945 5946 if (hist_data->attrs->name) 5947 named_data = find_named_trigger(hist_data->attrs->name); 5948 5949 list_for_each_entry_rcu(test, &file->triggers, list) { 5950 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5951 if (hist_trigger_match(data, test, named_data, false)) { 5952 match = true; 5953 break; 5954 } 5955 } 5956 } 5957 5958 return match; 5959 } 5960 5961 static bool hist_trigger_check_refs(struct event_trigger_data *data, 5962 struct trace_event_file *file) 5963 { 5964 struct hist_trigger_data *hist_data = data->private_data; 5965 struct event_trigger_data *test, *named_data = NULL; 5966 5967 if (hist_data->attrs->name) 5968 named_data = find_named_trigger(hist_data->attrs->name); 5969 5970 list_for_each_entry_rcu(test, &file->triggers, list) { 5971 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5972 if (!hist_trigger_match(data, test, named_data, false)) 5973 continue; 5974 hist_data = test->private_data; 5975 if (check_var_refs(hist_data)) 5976 return true; 5977 break; 5978 } 5979 } 5980 5981 return false; 5982 } 5983 5984 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops, 5985 struct event_trigger_data *data, 5986 struct trace_event_file *file) 5987 { 5988 struct hist_trigger_data *hist_data = data->private_data; 5989 struct event_trigger_data *test, *named_data = NULL; 5990 bool unregistered = false; 5991 5992 if (hist_data->attrs->name) 5993 named_data = find_named_trigger(hist_data->attrs->name); 5994 5995 list_for_each_entry_rcu(test, &file->triggers, list) { 5996 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5997 if (!hist_trigger_match(data, test, named_data, false)) 5998 continue; 5999 unregistered = true; 6000 list_del_rcu(&test->list); 6001 trace_event_trigger_enable_disable(file, 0); 6002 update_cond_flag(file); 6003 break; 6004 } 6005 } 6006 6007 if (unregistered && test->ops->free) 6008 test->ops->free(test->ops, test); 6009 6010 if (hist_data->enable_timestamps) { 6011 if (!hist_data->remove || unregistered) 6012 tracing_set_time_stamp_abs(file->tr, false); 6013 } 6014 } 6015 6016 static bool hist_file_check_refs(struct trace_event_file *file) 6017 { 6018 struct hist_trigger_data *hist_data; 6019 struct event_trigger_data *test; 6020 6021 list_for_each_entry_rcu(test, &file->triggers, list) { 6022 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6023 hist_data = test->private_data; 6024 if (check_var_refs(hist_data)) 6025 return true; 6026 } 6027 } 6028 6029 return false; 6030 } 6031 6032 static void hist_unreg_all(struct trace_event_file *file) 6033 { 6034 struct event_trigger_data *test, *n; 6035 struct hist_trigger_data *hist_data; 6036 struct synth_event *se; 6037 const char *se_name; 6038 6039 lockdep_assert_held(&event_mutex); 6040 6041 if (hist_file_check_refs(file)) 6042 return; 6043 6044 list_for_each_entry_safe(test, n, &file->triggers, list) { 6045 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6046 hist_data = test->private_data; 6047 list_del_rcu(&test->list); 6048 trace_event_trigger_enable_disable(file, 0); 6049 6050 se_name = trace_event_name(file->event_call); 6051 se = find_synth_event(se_name); 6052 if (se) 6053 se->ref--; 6054 6055 update_cond_flag(file); 6056 if (hist_data->enable_timestamps) 6057 tracing_set_time_stamp_abs(file->tr, false); 6058 if (test->ops->free) 6059 test->ops->free(test->ops, test); 6060 } 6061 } 6062 } 6063 6064 static int event_hist_trigger_func(struct event_command *cmd_ops, 6065 struct trace_event_file *file, 6066 char *glob, char *cmd, char *param) 6067 { 6068 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT; 6069 struct event_trigger_data *trigger_data; 6070 struct hist_trigger_attrs *attrs; 6071 struct event_trigger_ops *trigger_ops; 6072 struct hist_trigger_data *hist_data; 6073 struct synth_event *se; 6074 const char *se_name; 6075 bool remove = false; 6076 char *trigger, *p; 6077 int ret = 0; 6078 6079 lockdep_assert_held(&event_mutex); 6080 6081 if (glob && strlen(glob)) { 6082 hist_err_clear(); 6083 last_cmd_set(file, param); 6084 } 6085 6086 if (!param) 6087 return -EINVAL; 6088 6089 if (glob[0] == '!') 6090 remove = true; 6091 6092 /* 6093 * separate the trigger from the filter (k:v [if filter]) 6094 * allowing for whitespace in the trigger 6095 */ 6096 p = trigger = param; 6097 do { 6098 p = strstr(p, "if"); 6099 if (!p) 6100 break; 6101 if (p == param) 6102 return -EINVAL; 6103 if (*(p - 1) != ' ' && *(p - 1) != '\t') { 6104 p++; 6105 continue; 6106 } 6107 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1) 6108 return -EINVAL; 6109 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') { 6110 p++; 6111 continue; 6112 } 6113 break; 6114 } while (p); 6115 6116 if (!p) 6117 param = NULL; 6118 else { 6119 *(p - 1) = '\0'; 6120 param = strstrip(p); 6121 trigger = strstrip(trigger); 6122 } 6123 6124 attrs = parse_hist_trigger_attrs(file->tr, trigger); 6125 if (IS_ERR(attrs)) 6126 return PTR_ERR(attrs); 6127 6128 if (attrs->map_bits) 6129 hist_trigger_bits = attrs->map_bits; 6130 6131 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove); 6132 if (IS_ERR(hist_data)) { 6133 destroy_hist_trigger_attrs(attrs); 6134 return PTR_ERR(hist_data); 6135 } 6136 6137 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger); 6138 6139 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL); 6140 if (!trigger_data) { 6141 ret = -ENOMEM; 6142 goto out_free; 6143 } 6144 6145 trigger_data->count = -1; 6146 trigger_data->ops = trigger_ops; 6147 trigger_data->cmd_ops = cmd_ops; 6148 6149 INIT_LIST_HEAD(&trigger_data->list); 6150 RCU_INIT_POINTER(trigger_data->filter, NULL); 6151 6152 trigger_data->private_data = hist_data; 6153 6154 /* if param is non-empty, it's supposed to be a filter */ 6155 if (param && cmd_ops->set_filter) { 6156 ret = cmd_ops->set_filter(param, trigger_data, file); 6157 if (ret < 0) 6158 goto out_free; 6159 } 6160 6161 if (remove) { 6162 if (!have_hist_trigger_match(trigger_data, file)) 6163 goto out_free; 6164 6165 if (hist_trigger_check_refs(trigger_data, file)) { 6166 ret = -EBUSY; 6167 goto out_free; 6168 } 6169 6170 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file); 6171 se_name = trace_event_name(file->event_call); 6172 se = find_synth_event(se_name); 6173 if (se) 6174 se->ref--; 6175 ret = 0; 6176 goto out_free; 6177 } 6178 6179 ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file); 6180 /* 6181 * The above returns on success the # of triggers registered, 6182 * but if it didn't register any it returns zero. Consider no 6183 * triggers registered a failure too. 6184 */ 6185 if (!ret) { 6186 if (!(attrs->pause || attrs->cont || attrs->clear)) 6187 ret = -ENOENT; 6188 goto out_free; 6189 } else if (ret < 0) 6190 goto out_free; 6191 6192 if (get_named_trigger_data(trigger_data)) 6193 goto enable; 6194 6195 if (has_hist_vars(hist_data)) 6196 save_hist_vars(hist_data); 6197 6198 ret = create_actions(hist_data); 6199 if (ret) 6200 goto out_unreg; 6201 6202 ret = tracing_map_init(hist_data->map); 6203 if (ret) 6204 goto out_unreg; 6205 enable: 6206 ret = hist_trigger_enable(trigger_data, file); 6207 if (ret) 6208 goto out_unreg; 6209 6210 se_name = trace_event_name(file->event_call); 6211 se = find_synth_event(se_name); 6212 if (se) 6213 se->ref++; 6214 /* Just return zero, not the number of registered triggers */ 6215 ret = 0; 6216 out: 6217 if (ret == 0) 6218 hist_err_clear(); 6219 6220 return ret; 6221 out_unreg: 6222 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file); 6223 out_free: 6224 if (cmd_ops->set_filter) 6225 cmd_ops->set_filter(NULL, trigger_data, NULL); 6226 6227 remove_hist_vars(hist_data); 6228 6229 kfree(trigger_data); 6230 6231 destroy_hist_data(hist_data); 6232 goto out; 6233 } 6234 6235 static struct event_command trigger_hist_cmd = { 6236 .name = "hist", 6237 .trigger_type = ETT_EVENT_HIST, 6238 .flags = EVENT_CMD_FL_NEEDS_REC, 6239 .func = event_hist_trigger_func, 6240 .reg = hist_register_trigger, 6241 .unreg = hist_unregister_trigger, 6242 .unreg_all = hist_unreg_all, 6243 .get_trigger_ops = event_hist_get_trigger_ops, 6244 .set_filter = set_trigger_filter, 6245 }; 6246 6247 __init int register_trigger_hist_cmd(void) 6248 { 6249 int ret; 6250 6251 ret = register_event_command(&trigger_hist_cmd); 6252 WARN_ON(ret < 0); 6253 6254 return ret; 6255 } 6256 6257 static void 6258 hist_enable_trigger(struct event_trigger_data *data, void *rec, 6259 struct ring_buffer_event *event) 6260 { 6261 struct enable_trigger_data *enable_data = data->private_data; 6262 struct event_trigger_data *test; 6263 6264 list_for_each_entry_rcu(test, &enable_data->file->triggers, list) { 6265 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6266 if (enable_data->enable) 6267 test->paused = false; 6268 else 6269 test->paused = true; 6270 } 6271 } 6272 } 6273 6274 static void 6275 hist_enable_count_trigger(struct event_trigger_data *data, void *rec, 6276 struct ring_buffer_event *event) 6277 { 6278 if (!data->count) 6279 return; 6280 6281 if (data->count != -1) 6282 (data->count)--; 6283 6284 hist_enable_trigger(data, rec, event); 6285 } 6286 6287 static struct event_trigger_ops hist_enable_trigger_ops = { 6288 .func = hist_enable_trigger, 6289 .print = event_enable_trigger_print, 6290 .init = event_trigger_init, 6291 .free = event_enable_trigger_free, 6292 }; 6293 6294 static struct event_trigger_ops hist_enable_count_trigger_ops = { 6295 .func = hist_enable_count_trigger, 6296 .print = event_enable_trigger_print, 6297 .init = event_trigger_init, 6298 .free = event_enable_trigger_free, 6299 }; 6300 6301 static struct event_trigger_ops hist_disable_trigger_ops = { 6302 .func = hist_enable_trigger, 6303 .print = event_enable_trigger_print, 6304 .init = event_trigger_init, 6305 .free = event_enable_trigger_free, 6306 }; 6307 6308 static struct event_trigger_ops hist_disable_count_trigger_ops = { 6309 .func = hist_enable_count_trigger, 6310 .print = event_enable_trigger_print, 6311 .init = event_trigger_init, 6312 .free = event_enable_trigger_free, 6313 }; 6314 6315 static struct event_trigger_ops * 6316 hist_enable_get_trigger_ops(char *cmd, char *param) 6317 { 6318 struct event_trigger_ops *ops; 6319 bool enable; 6320 6321 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0); 6322 6323 if (enable) 6324 ops = param ? &hist_enable_count_trigger_ops : 6325 &hist_enable_trigger_ops; 6326 else 6327 ops = param ? &hist_disable_count_trigger_ops : 6328 &hist_disable_trigger_ops; 6329 6330 return ops; 6331 } 6332 6333 static void hist_enable_unreg_all(struct trace_event_file *file) 6334 { 6335 struct event_trigger_data *test, *n; 6336 6337 list_for_each_entry_safe(test, n, &file->triggers, list) { 6338 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) { 6339 list_del_rcu(&test->list); 6340 update_cond_flag(file); 6341 trace_event_trigger_enable_disable(file, 0); 6342 if (test->ops->free) 6343 test->ops->free(test->ops, test); 6344 } 6345 } 6346 } 6347 6348 static struct event_command trigger_hist_enable_cmd = { 6349 .name = ENABLE_HIST_STR, 6350 .trigger_type = ETT_HIST_ENABLE, 6351 .func = event_enable_trigger_func, 6352 .reg = event_enable_register_trigger, 6353 .unreg = event_enable_unregister_trigger, 6354 .unreg_all = hist_enable_unreg_all, 6355 .get_trigger_ops = hist_enable_get_trigger_ops, 6356 .set_filter = set_trigger_filter, 6357 }; 6358 6359 static struct event_command trigger_hist_disable_cmd = { 6360 .name = DISABLE_HIST_STR, 6361 .trigger_type = ETT_HIST_ENABLE, 6362 .func = event_enable_trigger_func, 6363 .reg = event_enable_register_trigger, 6364 .unreg = event_enable_unregister_trigger, 6365 .unreg_all = hist_enable_unreg_all, 6366 .get_trigger_ops = hist_enable_get_trigger_ops, 6367 .set_filter = set_trigger_filter, 6368 }; 6369 6370 static __init void unregister_trigger_hist_enable_disable_cmds(void) 6371 { 6372 unregister_event_command(&trigger_hist_enable_cmd); 6373 unregister_event_command(&trigger_hist_disable_cmd); 6374 } 6375 6376 __init int register_trigger_hist_enable_disable_cmds(void) 6377 { 6378 int ret; 6379 6380 ret = register_event_command(&trigger_hist_enable_cmd); 6381 if (WARN_ON(ret < 0)) 6382 return ret; 6383 ret = register_event_command(&trigger_hist_disable_cmd); 6384 if (WARN_ON(ret < 0)) 6385 unregister_trigger_hist_enable_disable_cmds(); 6386 6387 return ret; 6388 } 6389 6390 static __init int trace_events_hist_init(void) 6391 { 6392 struct dentry *entry = NULL; 6393 struct dentry *d_tracer; 6394 int err = 0; 6395 6396 err = dyn_event_register(&synth_event_ops); 6397 if (err) { 6398 pr_warn("Could not register synth_event_ops\n"); 6399 return err; 6400 } 6401 6402 d_tracer = tracing_init_dentry(); 6403 if (IS_ERR(d_tracer)) { 6404 err = PTR_ERR(d_tracer); 6405 goto err; 6406 } 6407 6408 entry = tracefs_create_file("synthetic_events", 0644, d_tracer, 6409 NULL, &synth_events_fops); 6410 if (!entry) { 6411 err = -ENODEV; 6412 goto err; 6413 } 6414 6415 return err; 6416 err: 6417 pr_warn("Could not create tracefs 'synthetic_events' entry\n"); 6418 6419 return err; 6420 } 6421 6422 fs_initcall(trace_events_hist_init); 6423