1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * trace_events_hist - trace event hist triggers 4 * 5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com> 6 */ 7 8 #include <linux/module.h> 9 #include <linux/kallsyms.h> 10 #include <linux/security.h> 11 #include <linux/mutex.h> 12 #include <linux/slab.h> 13 #include <linux/stacktrace.h> 14 #include <linux/rculist.h> 15 #include <linux/tracefs.h> 16 17 /* for gfp flag names */ 18 #include <linux/trace_events.h> 19 #include <trace/events/mmflags.h> 20 21 #include "tracing_map.h" 22 #include "trace_synth.h" 23 24 #define ERRORS \ 25 C(NONE, "No error"), \ 26 C(DUPLICATE_VAR, "Variable already defined"), \ 27 C(VAR_NOT_UNIQUE, "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \ 28 C(TOO_MANY_VARS, "Too many variables defined"), \ 29 C(MALFORMED_ASSIGNMENT, "Malformed assignment"), \ 30 C(NAMED_MISMATCH, "Named hist trigger doesn't match existing named trigger (includes variables)"), \ 31 C(TRIGGER_EEXIST, "Hist trigger already exists"), \ 32 C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \ 33 C(SET_CLOCK_FAIL, "Couldn't set trace_clock"), \ 34 C(BAD_FIELD_MODIFIER, "Invalid field modifier"), \ 35 C(TOO_MANY_SUBEXPR, "Too many subexpressions (3 max)"), \ 36 C(TIMESTAMP_MISMATCH, "Timestamp units in expression don't match"), \ 37 C(TOO_MANY_FIELD_VARS, "Too many field variables defined"), \ 38 C(EVENT_FILE_NOT_FOUND, "Event file not found"), \ 39 C(HIST_NOT_FOUND, "Matching event histogram not found"), \ 40 C(HIST_CREATE_FAIL, "Couldn't create histogram for field"), \ 41 C(SYNTH_VAR_NOT_FOUND, "Couldn't find synthetic variable"), \ 42 C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"), \ 43 C(SYNTH_TYPE_MISMATCH, "Param type doesn't match synthetic event field type"), \ 44 C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \ 45 C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"), \ 46 C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"), \ 47 C(ONX_NOT_VAR, "For onmax(x) or onchange(x), x must be a variable"), \ 48 C(ONX_VAR_NOT_FOUND, "Couldn't find onmax or onchange variable"), \ 49 C(ONX_VAR_CREATE_FAIL, "Couldn't create onmax or onchange variable"), \ 50 C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"), \ 51 C(TOO_MANY_PARAMS, "Too many action params"), \ 52 C(PARAM_NOT_FOUND, "Couldn't find param"), \ 53 C(INVALID_PARAM, "Invalid action param"), \ 54 C(ACTION_NOT_FOUND, "No action found"), \ 55 C(NO_SAVE_PARAMS, "No params found for save()"), \ 56 C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \ 57 C(ACTION_MISMATCH, "Handler doesn't support action"), \ 58 C(NO_CLOSING_PAREN, "No closing paren found"), \ 59 C(SUBSYS_NOT_FOUND, "Missing subsystem"), \ 60 C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"), \ 61 C(INVALID_REF_KEY, "Using variable references in keys not supported"), \ 62 C(VAR_NOT_FOUND, "Couldn't find variable"), \ 63 C(FIELD_NOT_FOUND, "Couldn't find field"), \ 64 C(EMPTY_ASSIGNMENT, "Empty assignment"), \ 65 C(INVALID_SORT_MODIFIER,"Invalid sort modifier"), \ 66 C(EMPTY_SORT_FIELD, "Empty sort field"), \ 67 C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"), \ 68 C(INVALID_SORT_FIELD, "Sort field must be a key or a val"), 69 70 #undef C 71 #define C(a, b) HIST_ERR_##a 72 73 enum { ERRORS }; 74 75 #undef C 76 #define C(a, b) b 77 78 static const char *err_text[] = { ERRORS }; 79 80 struct hist_field; 81 82 typedef u64 (*hist_field_fn_t) (struct hist_field *field, 83 struct tracing_map_elt *elt, 84 struct ring_buffer_event *rbe, 85 void *event); 86 87 #define HIST_FIELD_OPERANDS_MAX 2 88 #define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX) 89 #define HIST_ACTIONS_MAX 8 90 91 enum field_op_id { 92 FIELD_OP_NONE, 93 FIELD_OP_PLUS, 94 FIELD_OP_MINUS, 95 FIELD_OP_UNARY_MINUS, 96 }; 97 98 /* 99 * A hist_var (histogram variable) contains variable information for 100 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF 101 * flag set. A hist_var has a variable name e.g. ts0, and is 102 * associated with a given histogram trigger, as specified by 103 * hist_data. The hist_var idx is the unique index assigned to the 104 * variable by the hist trigger's tracing_map. The idx is what is 105 * used to set a variable's value and, by a variable reference, to 106 * retrieve it. 107 */ 108 struct hist_var { 109 char *name; 110 struct hist_trigger_data *hist_data; 111 unsigned int idx; 112 }; 113 114 struct hist_field { 115 struct ftrace_event_field *field; 116 unsigned long flags; 117 hist_field_fn_t fn; 118 unsigned int ref; 119 unsigned int size; 120 unsigned int offset; 121 unsigned int is_signed; 122 const char *type; 123 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX]; 124 struct hist_trigger_data *hist_data; 125 126 /* 127 * Variable fields contain variable-specific info in var. 128 */ 129 struct hist_var var; 130 enum field_op_id operator; 131 char *system; 132 char *event_name; 133 134 /* 135 * The name field is used for EXPR and VAR_REF fields. VAR 136 * fields contain the variable name in var.name. 137 */ 138 char *name; 139 140 /* 141 * When a histogram trigger is hit, if it has any references 142 * to variables, the values of those variables are collected 143 * into a var_ref_vals array by resolve_var_refs(). The 144 * current value of each variable is read from the tracing_map 145 * using the hist field's hist_var.idx and entered into the 146 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx]. 147 */ 148 unsigned int var_ref_idx; 149 bool read_once; 150 }; 151 152 static u64 hist_field_none(struct hist_field *field, 153 struct tracing_map_elt *elt, 154 struct ring_buffer_event *rbe, 155 void *event) 156 { 157 return 0; 158 } 159 160 static u64 hist_field_counter(struct hist_field *field, 161 struct tracing_map_elt *elt, 162 struct ring_buffer_event *rbe, 163 void *event) 164 { 165 return 1; 166 } 167 168 static u64 hist_field_string(struct hist_field *hist_field, 169 struct tracing_map_elt *elt, 170 struct ring_buffer_event *rbe, 171 void *event) 172 { 173 char *addr = (char *)(event + hist_field->field->offset); 174 175 return (u64)(unsigned long)addr; 176 } 177 178 static u64 hist_field_dynstring(struct hist_field *hist_field, 179 struct tracing_map_elt *elt, 180 struct ring_buffer_event *rbe, 181 void *event) 182 { 183 u32 str_item = *(u32 *)(event + hist_field->field->offset); 184 int str_loc = str_item & 0xffff; 185 char *addr = (char *)(event + str_loc); 186 187 return (u64)(unsigned long)addr; 188 } 189 190 static u64 hist_field_pstring(struct hist_field *hist_field, 191 struct tracing_map_elt *elt, 192 struct ring_buffer_event *rbe, 193 void *event) 194 { 195 char **addr = (char **)(event + hist_field->field->offset); 196 197 return (u64)(unsigned long)*addr; 198 } 199 200 static u64 hist_field_log2(struct hist_field *hist_field, 201 struct tracing_map_elt *elt, 202 struct ring_buffer_event *rbe, 203 void *event) 204 { 205 struct hist_field *operand = hist_field->operands[0]; 206 207 u64 val = operand->fn(operand, elt, rbe, event); 208 209 return (u64) ilog2(roundup_pow_of_two(val)); 210 } 211 212 static u64 hist_field_plus(struct hist_field *hist_field, 213 struct tracing_map_elt *elt, 214 struct ring_buffer_event *rbe, 215 void *event) 216 { 217 struct hist_field *operand1 = hist_field->operands[0]; 218 struct hist_field *operand2 = hist_field->operands[1]; 219 220 u64 val1 = operand1->fn(operand1, elt, rbe, event); 221 u64 val2 = operand2->fn(operand2, elt, rbe, event); 222 223 return val1 + val2; 224 } 225 226 static u64 hist_field_minus(struct hist_field *hist_field, 227 struct tracing_map_elt *elt, 228 struct ring_buffer_event *rbe, 229 void *event) 230 { 231 struct hist_field *operand1 = hist_field->operands[0]; 232 struct hist_field *operand2 = hist_field->operands[1]; 233 234 u64 val1 = operand1->fn(operand1, elt, rbe, event); 235 u64 val2 = operand2->fn(operand2, elt, rbe, event); 236 237 return val1 - val2; 238 } 239 240 static u64 hist_field_unary_minus(struct hist_field *hist_field, 241 struct tracing_map_elt *elt, 242 struct ring_buffer_event *rbe, 243 void *event) 244 { 245 struct hist_field *operand = hist_field->operands[0]; 246 247 s64 sval = (s64)operand->fn(operand, elt, rbe, event); 248 u64 val = (u64)-sval; 249 250 return val; 251 } 252 253 #define DEFINE_HIST_FIELD_FN(type) \ 254 static u64 hist_field_##type(struct hist_field *hist_field, \ 255 struct tracing_map_elt *elt, \ 256 struct ring_buffer_event *rbe, \ 257 void *event) \ 258 { \ 259 type *addr = (type *)(event + hist_field->field->offset); \ 260 \ 261 return (u64)(unsigned long)*addr; \ 262 } 263 264 DEFINE_HIST_FIELD_FN(s64); 265 DEFINE_HIST_FIELD_FN(u64); 266 DEFINE_HIST_FIELD_FN(s32); 267 DEFINE_HIST_FIELD_FN(u32); 268 DEFINE_HIST_FIELD_FN(s16); 269 DEFINE_HIST_FIELD_FN(u16); 270 DEFINE_HIST_FIELD_FN(s8); 271 DEFINE_HIST_FIELD_FN(u8); 272 273 #define for_each_hist_field(i, hist_data) \ 274 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++) 275 276 #define for_each_hist_val_field(i, hist_data) \ 277 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++) 278 279 #define for_each_hist_key_field(i, hist_data) \ 280 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++) 281 282 #define HIST_STACKTRACE_DEPTH 16 283 #define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long)) 284 #define HIST_STACKTRACE_SKIP 5 285 286 #define HITCOUNT_IDX 0 287 #define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE) 288 289 enum hist_field_flags { 290 HIST_FIELD_FL_HITCOUNT = 1 << 0, 291 HIST_FIELD_FL_KEY = 1 << 1, 292 HIST_FIELD_FL_STRING = 1 << 2, 293 HIST_FIELD_FL_HEX = 1 << 3, 294 HIST_FIELD_FL_SYM = 1 << 4, 295 HIST_FIELD_FL_SYM_OFFSET = 1 << 5, 296 HIST_FIELD_FL_EXECNAME = 1 << 6, 297 HIST_FIELD_FL_SYSCALL = 1 << 7, 298 HIST_FIELD_FL_STACKTRACE = 1 << 8, 299 HIST_FIELD_FL_LOG2 = 1 << 9, 300 HIST_FIELD_FL_TIMESTAMP = 1 << 10, 301 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11, 302 HIST_FIELD_FL_VAR = 1 << 12, 303 HIST_FIELD_FL_EXPR = 1 << 13, 304 HIST_FIELD_FL_VAR_REF = 1 << 14, 305 HIST_FIELD_FL_CPU = 1 << 15, 306 HIST_FIELD_FL_ALIAS = 1 << 16, 307 }; 308 309 struct var_defs { 310 unsigned int n_vars; 311 char *name[TRACING_MAP_VARS_MAX]; 312 char *expr[TRACING_MAP_VARS_MAX]; 313 }; 314 315 struct hist_trigger_attrs { 316 char *keys_str; 317 char *vals_str; 318 char *sort_key_str; 319 char *name; 320 char *clock; 321 bool pause; 322 bool cont; 323 bool clear; 324 bool ts_in_usecs; 325 unsigned int map_bits; 326 327 char *assignment_str[TRACING_MAP_VARS_MAX]; 328 unsigned int n_assignments; 329 330 char *action_str[HIST_ACTIONS_MAX]; 331 unsigned int n_actions; 332 333 struct var_defs var_defs; 334 }; 335 336 struct field_var { 337 struct hist_field *var; 338 struct hist_field *val; 339 }; 340 341 struct field_var_hist { 342 struct hist_trigger_data *hist_data; 343 char *cmd; 344 }; 345 346 struct hist_trigger_data { 347 struct hist_field *fields[HIST_FIELDS_MAX]; 348 unsigned int n_vals; 349 unsigned int n_keys; 350 unsigned int n_fields; 351 unsigned int n_vars; 352 unsigned int key_size; 353 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX]; 354 unsigned int n_sort_keys; 355 struct trace_event_file *event_file; 356 struct hist_trigger_attrs *attrs; 357 struct tracing_map *map; 358 bool enable_timestamps; 359 bool remove; 360 struct hist_field *var_refs[TRACING_MAP_VARS_MAX]; 361 unsigned int n_var_refs; 362 363 struct action_data *actions[HIST_ACTIONS_MAX]; 364 unsigned int n_actions; 365 366 struct field_var *field_vars[SYNTH_FIELDS_MAX]; 367 unsigned int n_field_vars; 368 unsigned int n_field_var_str; 369 struct field_var_hist *field_var_hists[SYNTH_FIELDS_MAX]; 370 unsigned int n_field_var_hists; 371 372 struct field_var *save_vars[SYNTH_FIELDS_MAX]; 373 unsigned int n_save_vars; 374 unsigned int n_save_var_str; 375 }; 376 377 struct action_data; 378 379 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data, 380 struct tracing_map_elt *elt, void *rec, 381 struct ring_buffer_event *rbe, void *key, 382 struct action_data *data, u64 *var_ref_vals); 383 384 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val); 385 386 enum handler_id { 387 HANDLER_ONMATCH = 1, 388 HANDLER_ONMAX, 389 HANDLER_ONCHANGE, 390 }; 391 392 enum action_id { 393 ACTION_SAVE = 1, 394 ACTION_TRACE, 395 ACTION_SNAPSHOT, 396 }; 397 398 struct action_data { 399 enum handler_id handler; 400 enum action_id action; 401 char *action_name; 402 action_fn_t fn; 403 404 unsigned int n_params; 405 char *params[SYNTH_FIELDS_MAX]; 406 407 /* 408 * When a histogram trigger is hit, the values of any 409 * references to variables, including variables being passed 410 * as parameters to synthetic events, are collected into a 411 * var_ref_vals array. This var_ref_idx array is an array of 412 * indices into the var_ref_vals array, one for each synthetic 413 * event param, and is passed to the synthetic event 414 * invocation. 415 */ 416 unsigned int var_ref_idx[TRACING_MAP_VARS_MAX]; 417 struct synth_event *synth_event; 418 bool use_trace_keyword; 419 char *synth_event_name; 420 421 union { 422 struct { 423 char *event; 424 char *event_system; 425 } match_data; 426 427 struct { 428 /* 429 * var_str contains the $-unstripped variable 430 * name referenced by var_ref, and used when 431 * printing the action. Because var_ref 432 * creation is deferred to create_actions(), 433 * we need a per-action way to save it until 434 * then, thus var_str. 435 */ 436 char *var_str; 437 438 /* 439 * var_ref refers to the variable being 440 * tracked e.g onmax($var). 441 */ 442 struct hist_field *var_ref; 443 444 /* 445 * track_var contains the 'invisible' tracking 446 * variable created to keep the current 447 * e.g. max value. 448 */ 449 struct hist_field *track_var; 450 451 check_track_val_fn_t check_val; 452 action_fn_t save_data; 453 } track_data; 454 }; 455 }; 456 457 struct track_data { 458 u64 track_val; 459 bool updated; 460 461 unsigned int key_len; 462 void *key; 463 struct tracing_map_elt elt; 464 465 struct action_data *action_data; 466 struct hist_trigger_data *hist_data; 467 }; 468 469 struct hist_elt_data { 470 char *comm; 471 u64 *var_ref_vals; 472 char *field_var_str[SYNTH_FIELDS_MAX]; 473 }; 474 475 struct snapshot_context { 476 struct tracing_map_elt *elt; 477 void *key; 478 }; 479 480 static void track_data_free(struct track_data *track_data) 481 { 482 struct hist_elt_data *elt_data; 483 484 if (!track_data) 485 return; 486 487 kfree(track_data->key); 488 489 elt_data = track_data->elt.private_data; 490 if (elt_data) { 491 kfree(elt_data->comm); 492 kfree(elt_data); 493 } 494 495 kfree(track_data); 496 } 497 498 static struct track_data *track_data_alloc(unsigned int key_len, 499 struct action_data *action_data, 500 struct hist_trigger_data *hist_data) 501 { 502 struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL); 503 struct hist_elt_data *elt_data; 504 505 if (!data) 506 return ERR_PTR(-ENOMEM); 507 508 data->key = kzalloc(key_len, GFP_KERNEL); 509 if (!data->key) { 510 track_data_free(data); 511 return ERR_PTR(-ENOMEM); 512 } 513 514 data->key_len = key_len; 515 data->action_data = action_data; 516 data->hist_data = hist_data; 517 518 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL); 519 if (!elt_data) { 520 track_data_free(data); 521 return ERR_PTR(-ENOMEM); 522 } 523 524 data->elt.private_data = elt_data; 525 526 elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL); 527 if (!elt_data->comm) { 528 track_data_free(data); 529 return ERR_PTR(-ENOMEM); 530 } 531 532 return data; 533 } 534 535 static char last_cmd[MAX_FILTER_STR_VAL]; 536 static char last_cmd_loc[MAX_FILTER_STR_VAL]; 537 538 static int errpos(char *str) 539 { 540 return err_pos(last_cmd, str); 541 } 542 543 static void last_cmd_set(struct trace_event_file *file, char *str) 544 { 545 const char *system = NULL, *name = NULL; 546 struct trace_event_call *call; 547 548 if (!str) 549 return; 550 551 strcpy(last_cmd, "hist:"); 552 strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:")); 553 554 if (file) { 555 call = file->event_call; 556 system = call->class->system; 557 if (system) { 558 name = trace_event_name(call); 559 if (!name) 560 system = NULL; 561 } 562 } 563 564 if (system) 565 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name); 566 } 567 568 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos) 569 { 570 tracing_log_err(tr, last_cmd_loc, last_cmd, err_text, 571 err_type, err_pos); 572 } 573 574 static void hist_err_clear(void) 575 { 576 last_cmd[0] = '\0'; 577 last_cmd_loc[0] = '\0'; 578 } 579 580 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals, 581 unsigned int *var_ref_idx); 582 583 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals, 584 unsigned int *var_ref_idx) 585 { 586 struct tracepoint *tp = event->tp; 587 588 if (unlikely(atomic_read(&tp->key.enabled) > 0)) { 589 struct tracepoint_func *probe_func_ptr; 590 synth_probe_func_t probe_func; 591 void *__data; 592 593 if (!(cpu_online(raw_smp_processor_id()))) 594 return; 595 596 probe_func_ptr = rcu_dereference_sched((tp)->funcs); 597 if (probe_func_ptr) { 598 do { 599 probe_func = probe_func_ptr->func; 600 __data = probe_func_ptr->data; 601 probe_func(__data, var_ref_vals, var_ref_idx); 602 } while ((++probe_func_ptr)->func); 603 } 604 } 605 } 606 607 static void action_trace(struct hist_trigger_data *hist_data, 608 struct tracing_map_elt *elt, void *rec, 609 struct ring_buffer_event *rbe, void *key, 610 struct action_data *data, u64 *var_ref_vals) 611 { 612 struct synth_event *event = data->synth_event; 613 614 trace_synth(event, var_ref_vals, data->var_ref_idx); 615 } 616 617 struct hist_var_data { 618 struct list_head list; 619 struct hist_trigger_data *hist_data; 620 }; 621 622 static u64 hist_field_timestamp(struct hist_field *hist_field, 623 struct tracing_map_elt *elt, 624 struct ring_buffer_event *rbe, 625 void *event) 626 { 627 struct hist_trigger_data *hist_data = hist_field->hist_data; 628 struct trace_array *tr = hist_data->event_file->tr; 629 630 u64 ts = ring_buffer_event_time_stamp(rbe); 631 632 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr)) 633 ts = ns2usecs(ts); 634 635 return ts; 636 } 637 638 static u64 hist_field_cpu(struct hist_field *hist_field, 639 struct tracing_map_elt *elt, 640 struct ring_buffer_event *rbe, 641 void *event) 642 { 643 int cpu = smp_processor_id(); 644 645 return cpu; 646 } 647 648 /** 649 * check_field_for_var_ref - Check if a VAR_REF field references a variable 650 * @hist_field: The VAR_REF field to check 651 * @var_data: The hist trigger that owns the variable 652 * @var_idx: The trigger variable identifier 653 * 654 * Check the given VAR_REF field to see whether or not it references 655 * the given variable associated with the given trigger. 656 * 657 * Return: The VAR_REF field if it does reference the variable, NULL if not 658 */ 659 static struct hist_field * 660 check_field_for_var_ref(struct hist_field *hist_field, 661 struct hist_trigger_data *var_data, 662 unsigned int var_idx) 663 { 664 WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF)); 665 666 if (hist_field && hist_field->var.idx == var_idx && 667 hist_field->var.hist_data == var_data) 668 return hist_field; 669 670 return NULL; 671 } 672 673 /** 674 * find_var_ref - Check if a trigger has a reference to a trigger variable 675 * @hist_data: The hist trigger that might have a reference to the variable 676 * @var_data: The hist trigger that owns the variable 677 * @var_idx: The trigger variable identifier 678 * 679 * Check the list of var_refs[] on the first hist trigger to see 680 * whether any of them are references to the variable on the second 681 * trigger. 682 * 683 * Return: The VAR_REF field referencing the variable if so, NULL if not 684 */ 685 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data, 686 struct hist_trigger_data *var_data, 687 unsigned int var_idx) 688 { 689 struct hist_field *hist_field; 690 unsigned int i; 691 692 for (i = 0; i < hist_data->n_var_refs; i++) { 693 hist_field = hist_data->var_refs[i]; 694 if (check_field_for_var_ref(hist_field, var_data, var_idx)) 695 return hist_field; 696 } 697 698 return NULL; 699 } 700 701 /** 702 * find_any_var_ref - Check if there is a reference to a given trigger variable 703 * @hist_data: The hist trigger 704 * @var_idx: The trigger variable identifier 705 * 706 * Check to see whether the given variable is currently referenced by 707 * any other trigger. 708 * 709 * The trigger the variable is defined on is explicitly excluded - the 710 * assumption being that a self-reference doesn't prevent a trigger 711 * from being removed. 712 * 713 * Return: The VAR_REF field referencing the variable if so, NULL if not 714 */ 715 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data, 716 unsigned int var_idx) 717 { 718 struct trace_array *tr = hist_data->event_file->tr; 719 struct hist_field *found = NULL; 720 struct hist_var_data *var_data; 721 722 list_for_each_entry(var_data, &tr->hist_vars, list) { 723 if (var_data->hist_data == hist_data) 724 continue; 725 found = find_var_ref(var_data->hist_data, hist_data, var_idx); 726 if (found) 727 break; 728 } 729 730 return found; 731 } 732 733 /** 734 * check_var_refs - Check if there is a reference to any of trigger's variables 735 * @hist_data: The hist trigger 736 * 737 * A trigger can define one or more variables. If any one of them is 738 * currently referenced by any other trigger, this function will 739 * determine that. 740 741 * Typically used to determine whether or not a trigger can be removed 742 * - if there are any references to a trigger's variables, it cannot. 743 * 744 * Return: True if there is a reference to any of trigger's variables 745 */ 746 static bool check_var_refs(struct hist_trigger_data *hist_data) 747 { 748 struct hist_field *field; 749 bool found = false; 750 int i; 751 752 for_each_hist_field(i, hist_data) { 753 field = hist_data->fields[i]; 754 if (field && field->flags & HIST_FIELD_FL_VAR) { 755 if (find_any_var_ref(hist_data, field->var.idx)) { 756 found = true; 757 break; 758 } 759 } 760 } 761 762 return found; 763 } 764 765 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data) 766 { 767 struct trace_array *tr = hist_data->event_file->tr; 768 struct hist_var_data *var_data, *found = NULL; 769 770 list_for_each_entry(var_data, &tr->hist_vars, list) { 771 if (var_data->hist_data == hist_data) { 772 found = var_data; 773 break; 774 } 775 } 776 777 return found; 778 } 779 780 static bool field_has_hist_vars(struct hist_field *hist_field, 781 unsigned int level) 782 { 783 int i; 784 785 if (level > 3) 786 return false; 787 788 if (!hist_field) 789 return false; 790 791 if (hist_field->flags & HIST_FIELD_FL_VAR || 792 hist_field->flags & HIST_FIELD_FL_VAR_REF) 793 return true; 794 795 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) { 796 struct hist_field *operand; 797 798 operand = hist_field->operands[i]; 799 if (field_has_hist_vars(operand, level + 1)) 800 return true; 801 } 802 803 return false; 804 } 805 806 static bool has_hist_vars(struct hist_trigger_data *hist_data) 807 { 808 struct hist_field *hist_field; 809 int i; 810 811 for_each_hist_field(i, hist_data) { 812 hist_field = hist_data->fields[i]; 813 if (field_has_hist_vars(hist_field, 0)) 814 return true; 815 } 816 817 return false; 818 } 819 820 static int save_hist_vars(struct hist_trigger_data *hist_data) 821 { 822 struct trace_array *tr = hist_data->event_file->tr; 823 struct hist_var_data *var_data; 824 825 var_data = find_hist_vars(hist_data); 826 if (var_data) 827 return 0; 828 829 if (tracing_check_open_get_tr(tr)) 830 return -ENODEV; 831 832 var_data = kzalloc(sizeof(*var_data), GFP_KERNEL); 833 if (!var_data) { 834 trace_array_put(tr); 835 return -ENOMEM; 836 } 837 838 var_data->hist_data = hist_data; 839 list_add(&var_data->list, &tr->hist_vars); 840 841 return 0; 842 } 843 844 static void remove_hist_vars(struct hist_trigger_data *hist_data) 845 { 846 struct trace_array *tr = hist_data->event_file->tr; 847 struct hist_var_data *var_data; 848 849 var_data = find_hist_vars(hist_data); 850 if (!var_data) 851 return; 852 853 if (WARN_ON(check_var_refs(hist_data))) 854 return; 855 856 list_del(&var_data->list); 857 858 kfree(var_data); 859 860 trace_array_put(tr); 861 } 862 863 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data, 864 const char *var_name) 865 { 866 struct hist_field *hist_field, *found = NULL; 867 int i; 868 869 for_each_hist_field(i, hist_data) { 870 hist_field = hist_data->fields[i]; 871 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR && 872 strcmp(hist_field->var.name, var_name) == 0) { 873 found = hist_field; 874 break; 875 } 876 } 877 878 return found; 879 } 880 881 static struct hist_field *find_var(struct hist_trigger_data *hist_data, 882 struct trace_event_file *file, 883 const char *var_name) 884 { 885 struct hist_trigger_data *test_data; 886 struct event_trigger_data *test; 887 struct hist_field *hist_field; 888 889 lockdep_assert_held(&event_mutex); 890 891 hist_field = find_var_field(hist_data, var_name); 892 if (hist_field) 893 return hist_field; 894 895 list_for_each_entry(test, &file->triggers, list) { 896 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 897 test_data = test->private_data; 898 hist_field = find_var_field(test_data, var_name); 899 if (hist_field) 900 return hist_field; 901 } 902 } 903 904 return NULL; 905 } 906 907 static struct trace_event_file *find_var_file(struct trace_array *tr, 908 char *system, 909 char *event_name, 910 char *var_name) 911 { 912 struct hist_trigger_data *var_hist_data; 913 struct hist_var_data *var_data; 914 struct trace_event_file *file, *found = NULL; 915 916 if (system) 917 return find_event_file(tr, system, event_name); 918 919 list_for_each_entry(var_data, &tr->hist_vars, list) { 920 var_hist_data = var_data->hist_data; 921 file = var_hist_data->event_file; 922 if (file == found) 923 continue; 924 925 if (find_var_field(var_hist_data, var_name)) { 926 if (found) { 927 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name)); 928 return NULL; 929 } 930 931 found = file; 932 } 933 } 934 935 return found; 936 } 937 938 static struct hist_field *find_file_var(struct trace_event_file *file, 939 const char *var_name) 940 { 941 struct hist_trigger_data *test_data; 942 struct event_trigger_data *test; 943 struct hist_field *hist_field; 944 945 lockdep_assert_held(&event_mutex); 946 947 list_for_each_entry(test, &file->triggers, list) { 948 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 949 test_data = test->private_data; 950 hist_field = find_var_field(test_data, var_name); 951 if (hist_field) 952 return hist_field; 953 } 954 } 955 956 return NULL; 957 } 958 959 static struct hist_field * 960 find_match_var(struct hist_trigger_data *hist_data, char *var_name) 961 { 962 struct trace_array *tr = hist_data->event_file->tr; 963 struct hist_field *hist_field, *found = NULL; 964 struct trace_event_file *file; 965 unsigned int i; 966 967 for (i = 0; i < hist_data->n_actions; i++) { 968 struct action_data *data = hist_data->actions[i]; 969 970 if (data->handler == HANDLER_ONMATCH) { 971 char *system = data->match_data.event_system; 972 char *event_name = data->match_data.event; 973 974 file = find_var_file(tr, system, event_name, var_name); 975 if (!file) 976 continue; 977 hist_field = find_file_var(file, var_name); 978 if (hist_field) { 979 if (found) { 980 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, 981 errpos(var_name)); 982 return ERR_PTR(-EINVAL); 983 } 984 985 found = hist_field; 986 } 987 } 988 } 989 return found; 990 } 991 992 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data, 993 char *system, 994 char *event_name, 995 char *var_name) 996 { 997 struct trace_array *tr = hist_data->event_file->tr; 998 struct hist_field *hist_field = NULL; 999 struct trace_event_file *file; 1000 1001 if (!system || !event_name) { 1002 hist_field = find_match_var(hist_data, var_name); 1003 if (IS_ERR(hist_field)) 1004 return NULL; 1005 if (hist_field) 1006 return hist_field; 1007 } 1008 1009 file = find_var_file(tr, system, event_name, var_name); 1010 if (!file) 1011 return NULL; 1012 1013 hist_field = find_file_var(file, var_name); 1014 1015 return hist_field; 1016 } 1017 1018 static u64 hist_field_var_ref(struct hist_field *hist_field, 1019 struct tracing_map_elt *elt, 1020 struct ring_buffer_event *rbe, 1021 void *event) 1022 { 1023 struct hist_elt_data *elt_data; 1024 u64 var_val = 0; 1025 1026 if (WARN_ON_ONCE(!elt)) 1027 return var_val; 1028 1029 elt_data = elt->private_data; 1030 var_val = elt_data->var_ref_vals[hist_field->var_ref_idx]; 1031 1032 return var_val; 1033 } 1034 1035 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key, 1036 u64 *var_ref_vals, bool self) 1037 { 1038 struct hist_trigger_data *var_data; 1039 struct tracing_map_elt *var_elt; 1040 struct hist_field *hist_field; 1041 unsigned int i, var_idx; 1042 bool resolved = true; 1043 u64 var_val = 0; 1044 1045 for (i = 0; i < hist_data->n_var_refs; i++) { 1046 hist_field = hist_data->var_refs[i]; 1047 var_idx = hist_field->var.idx; 1048 var_data = hist_field->var.hist_data; 1049 1050 if (var_data == NULL) { 1051 resolved = false; 1052 break; 1053 } 1054 1055 if ((self && var_data != hist_data) || 1056 (!self && var_data == hist_data)) 1057 continue; 1058 1059 var_elt = tracing_map_lookup(var_data->map, key); 1060 if (!var_elt) { 1061 resolved = false; 1062 break; 1063 } 1064 1065 if (!tracing_map_var_set(var_elt, var_idx)) { 1066 resolved = false; 1067 break; 1068 } 1069 1070 if (self || !hist_field->read_once) 1071 var_val = tracing_map_read_var(var_elt, var_idx); 1072 else 1073 var_val = tracing_map_read_var_once(var_elt, var_idx); 1074 1075 var_ref_vals[i] = var_val; 1076 } 1077 1078 return resolved; 1079 } 1080 1081 static const char *hist_field_name(struct hist_field *field, 1082 unsigned int level) 1083 { 1084 const char *field_name = ""; 1085 1086 if (level > 1) 1087 return field_name; 1088 1089 if (field->field) 1090 field_name = field->field->name; 1091 else if (field->flags & HIST_FIELD_FL_LOG2 || 1092 field->flags & HIST_FIELD_FL_ALIAS) 1093 field_name = hist_field_name(field->operands[0], ++level); 1094 else if (field->flags & HIST_FIELD_FL_CPU) 1095 field_name = "cpu"; 1096 else if (field->flags & HIST_FIELD_FL_EXPR || 1097 field->flags & HIST_FIELD_FL_VAR_REF) { 1098 if (field->system) { 1099 static char full_name[MAX_FILTER_STR_VAL]; 1100 1101 strcat(full_name, field->system); 1102 strcat(full_name, "."); 1103 strcat(full_name, field->event_name); 1104 strcat(full_name, "."); 1105 strcat(full_name, field->name); 1106 field_name = full_name; 1107 } else 1108 field_name = field->name; 1109 } else if (field->flags & HIST_FIELD_FL_TIMESTAMP) 1110 field_name = "common_timestamp"; 1111 1112 if (field_name == NULL) 1113 field_name = ""; 1114 1115 return field_name; 1116 } 1117 1118 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed) 1119 { 1120 hist_field_fn_t fn = NULL; 1121 1122 switch (field_size) { 1123 case 8: 1124 if (field_is_signed) 1125 fn = hist_field_s64; 1126 else 1127 fn = hist_field_u64; 1128 break; 1129 case 4: 1130 if (field_is_signed) 1131 fn = hist_field_s32; 1132 else 1133 fn = hist_field_u32; 1134 break; 1135 case 2: 1136 if (field_is_signed) 1137 fn = hist_field_s16; 1138 else 1139 fn = hist_field_u16; 1140 break; 1141 case 1: 1142 if (field_is_signed) 1143 fn = hist_field_s8; 1144 else 1145 fn = hist_field_u8; 1146 break; 1147 } 1148 1149 return fn; 1150 } 1151 1152 static int parse_map_size(char *str) 1153 { 1154 unsigned long size, map_bits; 1155 int ret; 1156 1157 ret = kstrtoul(str, 0, &size); 1158 if (ret) 1159 goto out; 1160 1161 map_bits = ilog2(roundup_pow_of_two(size)); 1162 if (map_bits < TRACING_MAP_BITS_MIN || 1163 map_bits > TRACING_MAP_BITS_MAX) 1164 ret = -EINVAL; 1165 else 1166 ret = map_bits; 1167 out: 1168 return ret; 1169 } 1170 1171 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs) 1172 { 1173 unsigned int i; 1174 1175 if (!attrs) 1176 return; 1177 1178 for (i = 0; i < attrs->n_assignments; i++) 1179 kfree(attrs->assignment_str[i]); 1180 1181 for (i = 0; i < attrs->n_actions; i++) 1182 kfree(attrs->action_str[i]); 1183 1184 kfree(attrs->name); 1185 kfree(attrs->sort_key_str); 1186 kfree(attrs->keys_str); 1187 kfree(attrs->vals_str); 1188 kfree(attrs->clock); 1189 kfree(attrs); 1190 } 1191 1192 static int parse_action(char *str, struct hist_trigger_attrs *attrs) 1193 { 1194 int ret = -EINVAL; 1195 1196 if (attrs->n_actions >= HIST_ACTIONS_MAX) 1197 return ret; 1198 1199 if ((str_has_prefix(str, "onmatch(")) || 1200 (str_has_prefix(str, "onmax(")) || 1201 (str_has_prefix(str, "onchange("))) { 1202 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL); 1203 if (!attrs->action_str[attrs->n_actions]) { 1204 ret = -ENOMEM; 1205 return ret; 1206 } 1207 attrs->n_actions++; 1208 ret = 0; 1209 } 1210 return ret; 1211 } 1212 1213 static int parse_assignment(struct trace_array *tr, 1214 char *str, struct hist_trigger_attrs *attrs) 1215 { 1216 int len, ret = 0; 1217 1218 if ((len = str_has_prefix(str, "key=")) || 1219 (len = str_has_prefix(str, "keys="))) { 1220 attrs->keys_str = kstrdup(str + len, GFP_KERNEL); 1221 if (!attrs->keys_str) { 1222 ret = -ENOMEM; 1223 goto out; 1224 } 1225 } else if ((len = str_has_prefix(str, "val=")) || 1226 (len = str_has_prefix(str, "vals=")) || 1227 (len = str_has_prefix(str, "values="))) { 1228 attrs->vals_str = kstrdup(str + len, GFP_KERNEL); 1229 if (!attrs->vals_str) { 1230 ret = -ENOMEM; 1231 goto out; 1232 } 1233 } else if ((len = str_has_prefix(str, "sort="))) { 1234 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL); 1235 if (!attrs->sort_key_str) { 1236 ret = -ENOMEM; 1237 goto out; 1238 } 1239 } else if (str_has_prefix(str, "name=")) { 1240 attrs->name = kstrdup(str, GFP_KERNEL); 1241 if (!attrs->name) { 1242 ret = -ENOMEM; 1243 goto out; 1244 } 1245 } else if ((len = str_has_prefix(str, "clock="))) { 1246 str += len; 1247 1248 str = strstrip(str); 1249 attrs->clock = kstrdup(str, GFP_KERNEL); 1250 if (!attrs->clock) { 1251 ret = -ENOMEM; 1252 goto out; 1253 } 1254 } else if ((len = str_has_prefix(str, "size="))) { 1255 int map_bits = parse_map_size(str + len); 1256 1257 if (map_bits < 0) { 1258 ret = map_bits; 1259 goto out; 1260 } 1261 attrs->map_bits = map_bits; 1262 } else { 1263 char *assignment; 1264 1265 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) { 1266 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str)); 1267 ret = -EINVAL; 1268 goto out; 1269 } 1270 1271 assignment = kstrdup(str, GFP_KERNEL); 1272 if (!assignment) { 1273 ret = -ENOMEM; 1274 goto out; 1275 } 1276 1277 attrs->assignment_str[attrs->n_assignments++] = assignment; 1278 } 1279 out: 1280 return ret; 1281 } 1282 1283 static struct hist_trigger_attrs * 1284 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str) 1285 { 1286 struct hist_trigger_attrs *attrs; 1287 int ret = 0; 1288 1289 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL); 1290 if (!attrs) 1291 return ERR_PTR(-ENOMEM); 1292 1293 while (trigger_str) { 1294 char *str = strsep(&trigger_str, ":"); 1295 char *rhs; 1296 1297 rhs = strchr(str, '='); 1298 if (rhs) { 1299 if (!strlen(++rhs)) { 1300 ret = -EINVAL; 1301 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str)); 1302 goto free; 1303 } 1304 ret = parse_assignment(tr, str, attrs); 1305 if (ret) 1306 goto free; 1307 } else if (strcmp(str, "pause") == 0) 1308 attrs->pause = true; 1309 else if ((strcmp(str, "cont") == 0) || 1310 (strcmp(str, "continue") == 0)) 1311 attrs->cont = true; 1312 else if (strcmp(str, "clear") == 0) 1313 attrs->clear = true; 1314 else { 1315 ret = parse_action(str, attrs); 1316 if (ret) 1317 goto free; 1318 } 1319 } 1320 1321 if (!attrs->keys_str) { 1322 ret = -EINVAL; 1323 goto free; 1324 } 1325 1326 if (!attrs->clock) { 1327 attrs->clock = kstrdup("global", GFP_KERNEL); 1328 if (!attrs->clock) { 1329 ret = -ENOMEM; 1330 goto free; 1331 } 1332 } 1333 1334 return attrs; 1335 free: 1336 destroy_hist_trigger_attrs(attrs); 1337 1338 return ERR_PTR(ret); 1339 } 1340 1341 static inline void save_comm(char *comm, struct task_struct *task) 1342 { 1343 if (!task->pid) { 1344 strcpy(comm, "<idle>"); 1345 return; 1346 } 1347 1348 if (WARN_ON_ONCE(task->pid < 0)) { 1349 strcpy(comm, "<XXX>"); 1350 return; 1351 } 1352 1353 strncpy(comm, task->comm, TASK_COMM_LEN); 1354 } 1355 1356 static void hist_elt_data_free(struct hist_elt_data *elt_data) 1357 { 1358 unsigned int i; 1359 1360 for (i = 0; i < SYNTH_FIELDS_MAX; i++) 1361 kfree(elt_data->field_var_str[i]); 1362 1363 kfree(elt_data->comm); 1364 kfree(elt_data); 1365 } 1366 1367 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt) 1368 { 1369 struct hist_elt_data *elt_data = elt->private_data; 1370 1371 hist_elt_data_free(elt_data); 1372 } 1373 1374 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt) 1375 { 1376 struct hist_trigger_data *hist_data = elt->map->private_data; 1377 unsigned int size = TASK_COMM_LEN; 1378 struct hist_elt_data *elt_data; 1379 struct hist_field *key_field; 1380 unsigned int i, n_str; 1381 1382 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL); 1383 if (!elt_data) 1384 return -ENOMEM; 1385 1386 for_each_hist_key_field(i, hist_data) { 1387 key_field = hist_data->fields[i]; 1388 1389 if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 1390 elt_data->comm = kzalloc(size, GFP_KERNEL); 1391 if (!elt_data->comm) { 1392 kfree(elt_data); 1393 return -ENOMEM; 1394 } 1395 break; 1396 } 1397 } 1398 1399 n_str = hist_data->n_field_var_str + hist_data->n_save_var_str; 1400 1401 size = STR_VAR_LEN_MAX; 1402 1403 for (i = 0; i < n_str; i++) { 1404 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL); 1405 if (!elt_data->field_var_str[i]) { 1406 hist_elt_data_free(elt_data); 1407 return -ENOMEM; 1408 } 1409 } 1410 1411 elt->private_data = elt_data; 1412 1413 return 0; 1414 } 1415 1416 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt) 1417 { 1418 struct hist_elt_data *elt_data = elt->private_data; 1419 1420 if (elt_data->comm) 1421 save_comm(elt_data->comm, current); 1422 } 1423 1424 static const struct tracing_map_ops hist_trigger_elt_data_ops = { 1425 .elt_alloc = hist_trigger_elt_data_alloc, 1426 .elt_free = hist_trigger_elt_data_free, 1427 .elt_init = hist_trigger_elt_data_init, 1428 }; 1429 1430 static const char *get_hist_field_flags(struct hist_field *hist_field) 1431 { 1432 const char *flags_str = NULL; 1433 1434 if (hist_field->flags & HIST_FIELD_FL_HEX) 1435 flags_str = "hex"; 1436 else if (hist_field->flags & HIST_FIELD_FL_SYM) 1437 flags_str = "sym"; 1438 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET) 1439 flags_str = "sym-offset"; 1440 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME) 1441 flags_str = "execname"; 1442 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL) 1443 flags_str = "syscall"; 1444 else if (hist_field->flags & HIST_FIELD_FL_LOG2) 1445 flags_str = "log2"; 1446 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS) 1447 flags_str = "usecs"; 1448 1449 return flags_str; 1450 } 1451 1452 static void expr_field_str(struct hist_field *field, char *expr) 1453 { 1454 if (field->flags & HIST_FIELD_FL_VAR_REF) 1455 strcat(expr, "$"); 1456 1457 strcat(expr, hist_field_name(field, 0)); 1458 1459 if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) { 1460 const char *flags_str = get_hist_field_flags(field); 1461 1462 if (flags_str) { 1463 strcat(expr, "."); 1464 strcat(expr, flags_str); 1465 } 1466 } 1467 } 1468 1469 static char *expr_str(struct hist_field *field, unsigned int level) 1470 { 1471 char *expr; 1472 1473 if (level > 1) 1474 return NULL; 1475 1476 expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 1477 if (!expr) 1478 return NULL; 1479 1480 if (!field->operands[0]) { 1481 expr_field_str(field, expr); 1482 return expr; 1483 } 1484 1485 if (field->operator == FIELD_OP_UNARY_MINUS) { 1486 char *subexpr; 1487 1488 strcat(expr, "-("); 1489 subexpr = expr_str(field->operands[0], ++level); 1490 if (!subexpr) { 1491 kfree(expr); 1492 return NULL; 1493 } 1494 strcat(expr, subexpr); 1495 strcat(expr, ")"); 1496 1497 kfree(subexpr); 1498 1499 return expr; 1500 } 1501 1502 expr_field_str(field->operands[0], expr); 1503 1504 switch (field->operator) { 1505 case FIELD_OP_MINUS: 1506 strcat(expr, "-"); 1507 break; 1508 case FIELD_OP_PLUS: 1509 strcat(expr, "+"); 1510 break; 1511 default: 1512 kfree(expr); 1513 return NULL; 1514 } 1515 1516 expr_field_str(field->operands[1], expr); 1517 1518 return expr; 1519 } 1520 1521 static int contains_operator(char *str) 1522 { 1523 enum field_op_id field_op = FIELD_OP_NONE; 1524 char *op; 1525 1526 op = strpbrk(str, "+-"); 1527 if (!op) 1528 return FIELD_OP_NONE; 1529 1530 switch (*op) { 1531 case '-': 1532 if (*str == '-') 1533 field_op = FIELD_OP_UNARY_MINUS; 1534 else 1535 field_op = FIELD_OP_MINUS; 1536 break; 1537 case '+': 1538 field_op = FIELD_OP_PLUS; 1539 break; 1540 default: 1541 break; 1542 } 1543 1544 return field_op; 1545 } 1546 1547 static void get_hist_field(struct hist_field *hist_field) 1548 { 1549 hist_field->ref++; 1550 } 1551 1552 static void __destroy_hist_field(struct hist_field *hist_field) 1553 { 1554 if (--hist_field->ref > 1) 1555 return; 1556 1557 kfree(hist_field->var.name); 1558 kfree(hist_field->name); 1559 kfree(hist_field->type); 1560 1561 kfree(hist_field->system); 1562 kfree(hist_field->event_name); 1563 1564 kfree(hist_field); 1565 } 1566 1567 static void destroy_hist_field(struct hist_field *hist_field, 1568 unsigned int level) 1569 { 1570 unsigned int i; 1571 1572 if (level > 3) 1573 return; 1574 1575 if (!hist_field) 1576 return; 1577 1578 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) 1579 return; /* var refs will be destroyed separately */ 1580 1581 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) 1582 destroy_hist_field(hist_field->operands[i], level + 1); 1583 1584 __destroy_hist_field(hist_field); 1585 } 1586 1587 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, 1588 struct ftrace_event_field *field, 1589 unsigned long flags, 1590 char *var_name) 1591 { 1592 struct hist_field *hist_field; 1593 1594 if (field && is_function_field(field)) 1595 return NULL; 1596 1597 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL); 1598 if (!hist_field) 1599 return NULL; 1600 1601 hist_field->ref = 1; 1602 1603 hist_field->hist_data = hist_data; 1604 1605 if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS) 1606 goto out; /* caller will populate */ 1607 1608 if (flags & HIST_FIELD_FL_VAR_REF) { 1609 hist_field->fn = hist_field_var_ref; 1610 goto out; 1611 } 1612 1613 if (flags & HIST_FIELD_FL_HITCOUNT) { 1614 hist_field->fn = hist_field_counter; 1615 hist_field->size = sizeof(u64); 1616 hist_field->type = kstrdup("u64", GFP_KERNEL); 1617 if (!hist_field->type) 1618 goto free; 1619 goto out; 1620 } 1621 1622 if (flags & HIST_FIELD_FL_STACKTRACE) { 1623 hist_field->fn = hist_field_none; 1624 goto out; 1625 } 1626 1627 if (flags & HIST_FIELD_FL_LOG2) { 1628 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2; 1629 hist_field->fn = hist_field_log2; 1630 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL); 1631 hist_field->size = hist_field->operands[0]->size; 1632 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL); 1633 if (!hist_field->type) 1634 goto free; 1635 goto out; 1636 } 1637 1638 if (flags & HIST_FIELD_FL_TIMESTAMP) { 1639 hist_field->fn = hist_field_timestamp; 1640 hist_field->size = sizeof(u64); 1641 hist_field->type = kstrdup("u64", GFP_KERNEL); 1642 if (!hist_field->type) 1643 goto free; 1644 goto out; 1645 } 1646 1647 if (flags & HIST_FIELD_FL_CPU) { 1648 hist_field->fn = hist_field_cpu; 1649 hist_field->size = sizeof(int); 1650 hist_field->type = kstrdup("unsigned int", GFP_KERNEL); 1651 if (!hist_field->type) 1652 goto free; 1653 goto out; 1654 } 1655 1656 if (WARN_ON_ONCE(!field)) 1657 goto out; 1658 1659 if (is_string_field(field)) { 1660 flags |= HIST_FIELD_FL_STRING; 1661 1662 hist_field->size = MAX_FILTER_STR_VAL; 1663 hist_field->type = kstrdup(field->type, GFP_KERNEL); 1664 if (!hist_field->type) 1665 goto free; 1666 1667 if (field->filter_type == FILTER_STATIC_STRING) 1668 hist_field->fn = hist_field_string; 1669 else if (field->filter_type == FILTER_DYN_STRING) 1670 hist_field->fn = hist_field_dynstring; 1671 else 1672 hist_field->fn = hist_field_pstring; 1673 } else { 1674 hist_field->size = field->size; 1675 hist_field->is_signed = field->is_signed; 1676 hist_field->type = kstrdup(field->type, GFP_KERNEL); 1677 if (!hist_field->type) 1678 goto free; 1679 1680 hist_field->fn = select_value_fn(field->size, 1681 field->is_signed); 1682 if (!hist_field->fn) { 1683 destroy_hist_field(hist_field, 0); 1684 return NULL; 1685 } 1686 } 1687 out: 1688 hist_field->field = field; 1689 hist_field->flags = flags; 1690 1691 if (var_name) { 1692 hist_field->var.name = kstrdup(var_name, GFP_KERNEL); 1693 if (!hist_field->var.name) 1694 goto free; 1695 } 1696 1697 return hist_field; 1698 free: 1699 destroy_hist_field(hist_field, 0); 1700 return NULL; 1701 } 1702 1703 static void destroy_hist_fields(struct hist_trigger_data *hist_data) 1704 { 1705 unsigned int i; 1706 1707 for (i = 0; i < HIST_FIELDS_MAX; i++) { 1708 if (hist_data->fields[i]) { 1709 destroy_hist_field(hist_data->fields[i], 0); 1710 hist_data->fields[i] = NULL; 1711 } 1712 } 1713 1714 for (i = 0; i < hist_data->n_var_refs; i++) { 1715 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF)); 1716 __destroy_hist_field(hist_data->var_refs[i]); 1717 hist_data->var_refs[i] = NULL; 1718 } 1719 } 1720 1721 static int init_var_ref(struct hist_field *ref_field, 1722 struct hist_field *var_field, 1723 char *system, char *event_name) 1724 { 1725 int err = 0; 1726 1727 ref_field->var.idx = var_field->var.idx; 1728 ref_field->var.hist_data = var_field->hist_data; 1729 ref_field->size = var_field->size; 1730 ref_field->is_signed = var_field->is_signed; 1731 ref_field->flags |= var_field->flags & 1732 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 1733 1734 if (system) { 1735 ref_field->system = kstrdup(system, GFP_KERNEL); 1736 if (!ref_field->system) 1737 return -ENOMEM; 1738 } 1739 1740 if (event_name) { 1741 ref_field->event_name = kstrdup(event_name, GFP_KERNEL); 1742 if (!ref_field->event_name) { 1743 err = -ENOMEM; 1744 goto free; 1745 } 1746 } 1747 1748 if (var_field->var.name) { 1749 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL); 1750 if (!ref_field->name) { 1751 err = -ENOMEM; 1752 goto free; 1753 } 1754 } else if (var_field->name) { 1755 ref_field->name = kstrdup(var_field->name, GFP_KERNEL); 1756 if (!ref_field->name) { 1757 err = -ENOMEM; 1758 goto free; 1759 } 1760 } 1761 1762 ref_field->type = kstrdup(var_field->type, GFP_KERNEL); 1763 if (!ref_field->type) { 1764 err = -ENOMEM; 1765 goto free; 1766 } 1767 out: 1768 return err; 1769 free: 1770 kfree(ref_field->system); 1771 kfree(ref_field->event_name); 1772 kfree(ref_field->name); 1773 1774 goto out; 1775 } 1776 1777 static int find_var_ref_idx(struct hist_trigger_data *hist_data, 1778 struct hist_field *var_field) 1779 { 1780 struct hist_field *ref_field; 1781 int i; 1782 1783 for (i = 0; i < hist_data->n_var_refs; i++) { 1784 ref_field = hist_data->var_refs[i]; 1785 if (ref_field->var.idx == var_field->var.idx && 1786 ref_field->var.hist_data == var_field->hist_data) 1787 return i; 1788 } 1789 1790 return -ENOENT; 1791 } 1792 1793 /** 1794 * create_var_ref - Create a variable reference and attach it to trigger 1795 * @hist_data: The trigger that will be referencing the variable 1796 * @var_field: The VAR field to create a reference to 1797 * @system: The optional system string 1798 * @event_name: The optional event_name string 1799 * 1800 * Given a variable hist_field, create a VAR_REF hist_field that 1801 * represents a reference to it. 1802 * 1803 * This function also adds the reference to the trigger that 1804 * now references the variable. 1805 * 1806 * Return: The VAR_REF field if successful, NULL if not 1807 */ 1808 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data, 1809 struct hist_field *var_field, 1810 char *system, char *event_name) 1811 { 1812 unsigned long flags = HIST_FIELD_FL_VAR_REF; 1813 struct hist_field *ref_field; 1814 int i; 1815 1816 /* Check if the variable already exists */ 1817 for (i = 0; i < hist_data->n_var_refs; i++) { 1818 ref_field = hist_data->var_refs[i]; 1819 if (ref_field->var.idx == var_field->var.idx && 1820 ref_field->var.hist_data == var_field->hist_data) { 1821 get_hist_field(ref_field); 1822 return ref_field; 1823 } 1824 } 1825 1826 ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL); 1827 if (ref_field) { 1828 if (init_var_ref(ref_field, var_field, system, event_name)) { 1829 destroy_hist_field(ref_field, 0); 1830 return NULL; 1831 } 1832 1833 hist_data->var_refs[hist_data->n_var_refs] = ref_field; 1834 ref_field->var_ref_idx = hist_data->n_var_refs++; 1835 } 1836 1837 return ref_field; 1838 } 1839 1840 static bool is_var_ref(char *var_name) 1841 { 1842 if (!var_name || strlen(var_name) < 2 || var_name[0] != '$') 1843 return false; 1844 1845 return true; 1846 } 1847 1848 static char *field_name_from_var(struct hist_trigger_data *hist_data, 1849 char *var_name) 1850 { 1851 char *name, *field; 1852 unsigned int i; 1853 1854 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 1855 name = hist_data->attrs->var_defs.name[i]; 1856 1857 if (strcmp(var_name, name) == 0) { 1858 field = hist_data->attrs->var_defs.expr[i]; 1859 if (contains_operator(field) || is_var_ref(field)) 1860 continue; 1861 return field; 1862 } 1863 } 1864 1865 return NULL; 1866 } 1867 1868 static char *local_field_var_ref(struct hist_trigger_data *hist_data, 1869 char *system, char *event_name, 1870 char *var_name) 1871 { 1872 struct trace_event_call *call; 1873 1874 if (system && event_name) { 1875 call = hist_data->event_file->event_call; 1876 1877 if (strcmp(system, call->class->system) != 0) 1878 return NULL; 1879 1880 if (strcmp(event_name, trace_event_name(call)) != 0) 1881 return NULL; 1882 } 1883 1884 if (!!system != !!event_name) 1885 return NULL; 1886 1887 if (!is_var_ref(var_name)) 1888 return NULL; 1889 1890 var_name++; 1891 1892 return field_name_from_var(hist_data, var_name); 1893 } 1894 1895 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data, 1896 char *system, char *event_name, 1897 char *var_name) 1898 { 1899 struct hist_field *var_field = NULL, *ref_field = NULL; 1900 struct trace_array *tr = hist_data->event_file->tr; 1901 1902 if (!is_var_ref(var_name)) 1903 return NULL; 1904 1905 var_name++; 1906 1907 var_field = find_event_var(hist_data, system, event_name, var_name); 1908 if (var_field) 1909 ref_field = create_var_ref(hist_data, var_field, 1910 system, event_name); 1911 1912 if (!ref_field) 1913 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name)); 1914 1915 return ref_field; 1916 } 1917 1918 static struct ftrace_event_field * 1919 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, 1920 char *field_str, unsigned long *flags) 1921 { 1922 struct ftrace_event_field *field = NULL; 1923 char *field_name, *modifier, *str; 1924 struct trace_array *tr = file->tr; 1925 1926 modifier = str = kstrdup(field_str, GFP_KERNEL); 1927 if (!modifier) 1928 return ERR_PTR(-ENOMEM); 1929 1930 field_name = strsep(&modifier, "."); 1931 if (modifier) { 1932 if (strcmp(modifier, "hex") == 0) 1933 *flags |= HIST_FIELD_FL_HEX; 1934 else if (strcmp(modifier, "sym") == 0) 1935 *flags |= HIST_FIELD_FL_SYM; 1936 else if (strcmp(modifier, "sym-offset") == 0) 1937 *flags |= HIST_FIELD_FL_SYM_OFFSET; 1938 else if ((strcmp(modifier, "execname") == 0) && 1939 (strcmp(field_name, "common_pid") == 0)) 1940 *flags |= HIST_FIELD_FL_EXECNAME; 1941 else if (strcmp(modifier, "syscall") == 0) 1942 *flags |= HIST_FIELD_FL_SYSCALL; 1943 else if (strcmp(modifier, "log2") == 0) 1944 *flags |= HIST_FIELD_FL_LOG2; 1945 else if (strcmp(modifier, "usecs") == 0) 1946 *flags |= HIST_FIELD_FL_TIMESTAMP_USECS; 1947 else { 1948 hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier)); 1949 field = ERR_PTR(-EINVAL); 1950 goto out; 1951 } 1952 } 1953 1954 if (strcmp(field_name, "common_timestamp") == 0) { 1955 *flags |= HIST_FIELD_FL_TIMESTAMP; 1956 hist_data->enable_timestamps = true; 1957 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS) 1958 hist_data->attrs->ts_in_usecs = true; 1959 } else if (strcmp(field_name, "cpu") == 0) 1960 *flags |= HIST_FIELD_FL_CPU; 1961 else { 1962 field = trace_find_event_field(file->event_call, field_name); 1963 if (!field || !field->size) { 1964 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name)); 1965 field = ERR_PTR(-EINVAL); 1966 goto out; 1967 } 1968 } 1969 out: 1970 kfree(str); 1971 1972 return field; 1973 } 1974 1975 static struct hist_field *create_alias(struct hist_trigger_data *hist_data, 1976 struct hist_field *var_ref, 1977 char *var_name) 1978 { 1979 struct hist_field *alias = NULL; 1980 unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR; 1981 1982 alias = create_hist_field(hist_data, NULL, flags, var_name); 1983 if (!alias) 1984 return NULL; 1985 1986 alias->fn = var_ref->fn; 1987 alias->operands[0] = var_ref; 1988 1989 if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) { 1990 destroy_hist_field(alias, 0); 1991 return NULL; 1992 } 1993 1994 alias->var_ref_idx = var_ref->var_ref_idx; 1995 1996 return alias; 1997 } 1998 1999 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data, 2000 struct trace_event_file *file, char *str, 2001 unsigned long *flags, char *var_name) 2002 { 2003 char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str; 2004 struct ftrace_event_field *field = NULL; 2005 struct hist_field *hist_field = NULL; 2006 int ret = 0; 2007 2008 s = strchr(str, '.'); 2009 if (s) { 2010 s = strchr(++s, '.'); 2011 if (s) { 2012 ref_system = strsep(&str, "."); 2013 if (!str) { 2014 ret = -EINVAL; 2015 goto out; 2016 } 2017 ref_event = strsep(&str, "."); 2018 if (!str) { 2019 ret = -EINVAL; 2020 goto out; 2021 } 2022 ref_var = str; 2023 } 2024 } 2025 2026 s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var); 2027 if (!s) { 2028 hist_field = parse_var_ref(hist_data, ref_system, 2029 ref_event, ref_var); 2030 if (hist_field) { 2031 if (var_name) { 2032 hist_field = create_alias(hist_data, hist_field, var_name); 2033 if (!hist_field) { 2034 ret = -ENOMEM; 2035 goto out; 2036 } 2037 } 2038 return hist_field; 2039 } 2040 } else 2041 str = s; 2042 2043 field = parse_field(hist_data, file, str, flags); 2044 if (IS_ERR(field)) { 2045 ret = PTR_ERR(field); 2046 goto out; 2047 } 2048 2049 hist_field = create_hist_field(hist_data, field, *flags, var_name); 2050 if (!hist_field) { 2051 ret = -ENOMEM; 2052 goto out; 2053 } 2054 2055 return hist_field; 2056 out: 2057 return ERR_PTR(ret); 2058 } 2059 2060 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2061 struct trace_event_file *file, 2062 char *str, unsigned long flags, 2063 char *var_name, unsigned int level); 2064 2065 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data, 2066 struct trace_event_file *file, 2067 char *str, unsigned long flags, 2068 char *var_name, unsigned int level) 2069 { 2070 struct hist_field *operand1, *expr = NULL; 2071 unsigned long operand_flags; 2072 int ret = 0; 2073 char *s; 2074 2075 /* we support only -(xxx) i.e. explicit parens required */ 2076 2077 if (level > 3) { 2078 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str)); 2079 ret = -EINVAL; 2080 goto free; 2081 } 2082 2083 str++; /* skip leading '-' */ 2084 2085 s = strchr(str, '('); 2086 if (s) 2087 str++; 2088 else { 2089 ret = -EINVAL; 2090 goto free; 2091 } 2092 2093 s = strrchr(str, ')'); 2094 if (s) 2095 *s = '\0'; 2096 else { 2097 ret = -EINVAL; /* no closing ')' */ 2098 goto free; 2099 } 2100 2101 flags |= HIST_FIELD_FL_EXPR; 2102 expr = create_hist_field(hist_data, NULL, flags, var_name); 2103 if (!expr) { 2104 ret = -ENOMEM; 2105 goto free; 2106 } 2107 2108 operand_flags = 0; 2109 operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level); 2110 if (IS_ERR(operand1)) { 2111 ret = PTR_ERR(operand1); 2112 goto free; 2113 } 2114 2115 expr->flags |= operand1->flags & 2116 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2117 expr->fn = hist_field_unary_minus; 2118 expr->operands[0] = operand1; 2119 expr->operator = FIELD_OP_UNARY_MINUS; 2120 expr->name = expr_str(expr, 0); 2121 expr->type = kstrdup(operand1->type, GFP_KERNEL); 2122 if (!expr->type) { 2123 ret = -ENOMEM; 2124 goto free; 2125 } 2126 2127 return expr; 2128 free: 2129 destroy_hist_field(expr, 0); 2130 return ERR_PTR(ret); 2131 } 2132 2133 static int check_expr_operands(struct trace_array *tr, 2134 struct hist_field *operand1, 2135 struct hist_field *operand2) 2136 { 2137 unsigned long operand1_flags = operand1->flags; 2138 unsigned long operand2_flags = operand2->flags; 2139 2140 if ((operand1_flags & HIST_FIELD_FL_VAR_REF) || 2141 (operand1_flags & HIST_FIELD_FL_ALIAS)) { 2142 struct hist_field *var; 2143 2144 var = find_var_field(operand1->var.hist_data, operand1->name); 2145 if (!var) 2146 return -EINVAL; 2147 operand1_flags = var->flags; 2148 } 2149 2150 if ((operand2_flags & HIST_FIELD_FL_VAR_REF) || 2151 (operand2_flags & HIST_FIELD_FL_ALIAS)) { 2152 struct hist_field *var; 2153 2154 var = find_var_field(operand2->var.hist_data, operand2->name); 2155 if (!var) 2156 return -EINVAL; 2157 operand2_flags = var->flags; 2158 } 2159 2160 if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) != 2161 (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) { 2162 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0); 2163 return -EINVAL; 2164 } 2165 2166 return 0; 2167 } 2168 2169 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2170 struct trace_event_file *file, 2171 char *str, unsigned long flags, 2172 char *var_name, unsigned int level) 2173 { 2174 struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL; 2175 unsigned long operand_flags; 2176 int field_op, ret = -EINVAL; 2177 char *sep, *operand1_str; 2178 2179 if (level > 3) { 2180 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str)); 2181 return ERR_PTR(-EINVAL); 2182 } 2183 2184 field_op = contains_operator(str); 2185 2186 if (field_op == FIELD_OP_NONE) 2187 return parse_atom(hist_data, file, str, &flags, var_name); 2188 2189 if (field_op == FIELD_OP_UNARY_MINUS) 2190 return parse_unary(hist_data, file, str, flags, var_name, ++level); 2191 2192 switch (field_op) { 2193 case FIELD_OP_MINUS: 2194 sep = "-"; 2195 break; 2196 case FIELD_OP_PLUS: 2197 sep = "+"; 2198 break; 2199 default: 2200 goto free; 2201 } 2202 2203 operand1_str = strsep(&str, sep); 2204 if (!operand1_str || !str) 2205 goto free; 2206 2207 operand_flags = 0; 2208 operand1 = parse_atom(hist_data, file, operand1_str, 2209 &operand_flags, NULL); 2210 if (IS_ERR(operand1)) { 2211 ret = PTR_ERR(operand1); 2212 operand1 = NULL; 2213 goto free; 2214 } 2215 2216 /* rest of string could be another expression e.g. b+c in a+b+c */ 2217 operand_flags = 0; 2218 operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level); 2219 if (IS_ERR(operand2)) { 2220 ret = PTR_ERR(operand2); 2221 operand2 = NULL; 2222 goto free; 2223 } 2224 2225 ret = check_expr_operands(file->tr, operand1, operand2); 2226 if (ret) 2227 goto free; 2228 2229 flags |= HIST_FIELD_FL_EXPR; 2230 2231 flags |= operand1->flags & 2232 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2233 2234 expr = create_hist_field(hist_data, NULL, flags, var_name); 2235 if (!expr) { 2236 ret = -ENOMEM; 2237 goto free; 2238 } 2239 2240 operand1->read_once = true; 2241 operand2->read_once = true; 2242 2243 expr->operands[0] = operand1; 2244 expr->operands[1] = operand2; 2245 expr->operator = field_op; 2246 expr->name = expr_str(expr, 0); 2247 expr->type = kstrdup(operand1->type, GFP_KERNEL); 2248 if (!expr->type) { 2249 ret = -ENOMEM; 2250 goto free; 2251 } 2252 2253 switch (field_op) { 2254 case FIELD_OP_MINUS: 2255 expr->fn = hist_field_minus; 2256 break; 2257 case FIELD_OP_PLUS: 2258 expr->fn = hist_field_plus; 2259 break; 2260 default: 2261 ret = -EINVAL; 2262 goto free; 2263 } 2264 2265 return expr; 2266 free: 2267 destroy_hist_field(operand1, 0); 2268 destroy_hist_field(operand2, 0); 2269 destroy_hist_field(expr, 0); 2270 2271 return ERR_PTR(ret); 2272 } 2273 2274 static char *find_trigger_filter(struct hist_trigger_data *hist_data, 2275 struct trace_event_file *file) 2276 { 2277 struct event_trigger_data *test; 2278 2279 lockdep_assert_held(&event_mutex); 2280 2281 list_for_each_entry(test, &file->triggers, list) { 2282 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2283 if (test->private_data == hist_data) 2284 return test->filter_str; 2285 } 2286 } 2287 2288 return NULL; 2289 } 2290 2291 static struct event_command trigger_hist_cmd; 2292 static int event_hist_trigger_func(struct event_command *cmd_ops, 2293 struct trace_event_file *file, 2294 char *glob, char *cmd, char *param); 2295 2296 static bool compatible_keys(struct hist_trigger_data *target_hist_data, 2297 struct hist_trigger_data *hist_data, 2298 unsigned int n_keys) 2299 { 2300 struct hist_field *target_hist_field, *hist_field; 2301 unsigned int n, i, j; 2302 2303 if (hist_data->n_fields - hist_data->n_vals != n_keys) 2304 return false; 2305 2306 i = hist_data->n_vals; 2307 j = target_hist_data->n_vals; 2308 2309 for (n = 0; n < n_keys; n++) { 2310 hist_field = hist_data->fields[i + n]; 2311 target_hist_field = target_hist_data->fields[j + n]; 2312 2313 if (strcmp(hist_field->type, target_hist_field->type) != 0) 2314 return false; 2315 if (hist_field->size != target_hist_field->size) 2316 return false; 2317 if (hist_field->is_signed != target_hist_field->is_signed) 2318 return false; 2319 } 2320 2321 return true; 2322 } 2323 2324 static struct hist_trigger_data * 2325 find_compatible_hist(struct hist_trigger_data *target_hist_data, 2326 struct trace_event_file *file) 2327 { 2328 struct hist_trigger_data *hist_data; 2329 struct event_trigger_data *test; 2330 unsigned int n_keys; 2331 2332 lockdep_assert_held(&event_mutex); 2333 2334 n_keys = target_hist_data->n_fields - target_hist_data->n_vals; 2335 2336 list_for_each_entry(test, &file->triggers, list) { 2337 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2338 hist_data = test->private_data; 2339 2340 if (compatible_keys(target_hist_data, hist_data, n_keys)) 2341 return hist_data; 2342 } 2343 } 2344 2345 return NULL; 2346 } 2347 2348 static struct trace_event_file *event_file(struct trace_array *tr, 2349 char *system, char *event_name) 2350 { 2351 struct trace_event_file *file; 2352 2353 file = __find_event_file(tr, system, event_name); 2354 if (!file) 2355 return ERR_PTR(-EINVAL); 2356 2357 return file; 2358 } 2359 2360 static struct hist_field * 2361 find_synthetic_field_var(struct hist_trigger_data *target_hist_data, 2362 char *system, char *event_name, char *field_name) 2363 { 2364 struct hist_field *event_var; 2365 char *synthetic_name; 2366 2367 synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2368 if (!synthetic_name) 2369 return ERR_PTR(-ENOMEM); 2370 2371 strcpy(synthetic_name, "synthetic_"); 2372 strcat(synthetic_name, field_name); 2373 2374 event_var = find_event_var(target_hist_data, system, event_name, synthetic_name); 2375 2376 kfree(synthetic_name); 2377 2378 return event_var; 2379 } 2380 2381 /** 2382 * create_field_var_hist - Automatically create a histogram and var for a field 2383 * @target_hist_data: The target hist trigger 2384 * @subsys_name: Optional subsystem name 2385 * @event_name: Optional event name 2386 * @field_name: The name of the field (and the resulting variable) 2387 * 2388 * Hist trigger actions fetch data from variables, not directly from 2389 * events. However, for convenience, users are allowed to directly 2390 * specify an event field in an action, which will be automatically 2391 * converted into a variable on their behalf. 2392 2393 * If a user specifies a field on an event that isn't the event the 2394 * histogram currently being defined (the target event histogram), the 2395 * only way that can be accomplished is if a new hist trigger is 2396 * created and the field variable defined on that. 2397 * 2398 * This function creates a new histogram compatible with the target 2399 * event (meaning a histogram with the same key as the target 2400 * histogram), and creates a variable for the specified field, but 2401 * with 'synthetic_' prepended to the variable name in order to avoid 2402 * collision with normal field variables. 2403 * 2404 * Return: The variable created for the field. 2405 */ 2406 static struct hist_field * 2407 create_field_var_hist(struct hist_trigger_data *target_hist_data, 2408 char *subsys_name, char *event_name, char *field_name) 2409 { 2410 struct trace_array *tr = target_hist_data->event_file->tr; 2411 struct hist_field *event_var = ERR_PTR(-EINVAL); 2412 struct hist_trigger_data *hist_data; 2413 unsigned int i, n, first = true; 2414 struct field_var_hist *var_hist; 2415 struct trace_event_file *file; 2416 struct hist_field *key_field; 2417 char *saved_filter; 2418 char *cmd; 2419 int ret; 2420 2421 if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) { 2422 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name)); 2423 return ERR_PTR(-EINVAL); 2424 } 2425 2426 file = event_file(tr, subsys_name, event_name); 2427 2428 if (IS_ERR(file)) { 2429 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name)); 2430 ret = PTR_ERR(file); 2431 return ERR_PTR(ret); 2432 } 2433 2434 /* 2435 * Look for a histogram compatible with target. We'll use the 2436 * found histogram specification to create a new matching 2437 * histogram with our variable on it. target_hist_data is not 2438 * yet a registered histogram so we can't use that. 2439 */ 2440 hist_data = find_compatible_hist(target_hist_data, file); 2441 if (!hist_data) { 2442 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name)); 2443 return ERR_PTR(-EINVAL); 2444 } 2445 2446 /* See if a synthetic field variable has already been created */ 2447 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 2448 event_name, field_name); 2449 if (!IS_ERR_OR_NULL(event_var)) 2450 return event_var; 2451 2452 var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL); 2453 if (!var_hist) 2454 return ERR_PTR(-ENOMEM); 2455 2456 cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2457 if (!cmd) { 2458 kfree(var_hist); 2459 return ERR_PTR(-ENOMEM); 2460 } 2461 2462 /* Use the same keys as the compatible histogram */ 2463 strcat(cmd, "keys="); 2464 2465 for_each_hist_key_field(i, hist_data) { 2466 key_field = hist_data->fields[i]; 2467 if (!first) 2468 strcat(cmd, ","); 2469 strcat(cmd, key_field->field->name); 2470 first = false; 2471 } 2472 2473 /* Create the synthetic field variable specification */ 2474 strcat(cmd, ":synthetic_"); 2475 strcat(cmd, field_name); 2476 strcat(cmd, "="); 2477 strcat(cmd, field_name); 2478 2479 /* Use the same filter as the compatible histogram */ 2480 saved_filter = find_trigger_filter(hist_data, file); 2481 if (saved_filter) { 2482 strcat(cmd, " if "); 2483 strcat(cmd, saved_filter); 2484 } 2485 2486 var_hist->cmd = kstrdup(cmd, GFP_KERNEL); 2487 if (!var_hist->cmd) { 2488 kfree(cmd); 2489 kfree(var_hist); 2490 return ERR_PTR(-ENOMEM); 2491 } 2492 2493 /* Save the compatible histogram information */ 2494 var_hist->hist_data = hist_data; 2495 2496 /* Create the new histogram with our variable */ 2497 ret = event_hist_trigger_func(&trigger_hist_cmd, file, 2498 "", "hist", cmd); 2499 if (ret) { 2500 kfree(cmd); 2501 kfree(var_hist->cmd); 2502 kfree(var_hist); 2503 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name)); 2504 return ERR_PTR(ret); 2505 } 2506 2507 kfree(cmd); 2508 2509 /* If we can't find the variable, something went wrong */ 2510 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 2511 event_name, field_name); 2512 if (IS_ERR_OR_NULL(event_var)) { 2513 kfree(var_hist->cmd); 2514 kfree(var_hist); 2515 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name)); 2516 return ERR_PTR(-EINVAL); 2517 } 2518 2519 n = target_hist_data->n_field_var_hists; 2520 target_hist_data->field_var_hists[n] = var_hist; 2521 target_hist_data->n_field_var_hists++; 2522 2523 return event_var; 2524 } 2525 2526 static struct hist_field * 2527 find_target_event_var(struct hist_trigger_data *hist_data, 2528 char *subsys_name, char *event_name, char *var_name) 2529 { 2530 struct trace_event_file *file = hist_data->event_file; 2531 struct hist_field *hist_field = NULL; 2532 2533 if (subsys_name) { 2534 struct trace_event_call *call; 2535 2536 if (!event_name) 2537 return NULL; 2538 2539 call = file->event_call; 2540 2541 if (strcmp(subsys_name, call->class->system) != 0) 2542 return NULL; 2543 2544 if (strcmp(event_name, trace_event_name(call)) != 0) 2545 return NULL; 2546 } 2547 2548 hist_field = find_var_field(hist_data, var_name); 2549 2550 return hist_field; 2551 } 2552 2553 static inline void __update_field_vars(struct tracing_map_elt *elt, 2554 struct ring_buffer_event *rbe, 2555 void *rec, 2556 struct field_var **field_vars, 2557 unsigned int n_field_vars, 2558 unsigned int field_var_str_start) 2559 { 2560 struct hist_elt_data *elt_data = elt->private_data; 2561 unsigned int i, j, var_idx; 2562 u64 var_val; 2563 2564 for (i = 0, j = field_var_str_start; i < n_field_vars; i++) { 2565 struct field_var *field_var = field_vars[i]; 2566 struct hist_field *var = field_var->var; 2567 struct hist_field *val = field_var->val; 2568 2569 var_val = val->fn(val, elt, rbe, rec); 2570 var_idx = var->var.idx; 2571 2572 if (val->flags & HIST_FIELD_FL_STRING) { 2573 char *str = elt_data->field_var_str[j++]; 2574 char *val_str = (char *)(uintptr_t)var_val; 2575 2576 strscpy(str, val_str, STR_VAR_LEN_MAX); 2577 var_val = (u64)(uintptr_t)str; 2578 } 2579 tracing_map_set_var(elt, var_idx, var_val); 2580 } 2581 } 2582 2583 static void update_field_vars(struct hist_trigger_data *hist_data, 2584 struct tracing_map_elt *elt, 2585 struct ring_buffer_event *rbe, 2586 void *rec) 2587 { 2588 __update_field_vars(elt, rbe, rec, hist_data->field_vars, 2589 hist_data->n_field_vars, 0); 2590 } 2591 2592 static void save_track_data_vars(struct hist_trigger_data *hist_data, 2593 struct tracing_map_elt *elt, void *rec, 2594 struct ring_buffer_event *rbe, void *key, 2595 struct action_data *data, u64 *var_ref_vals) 2596 { 2597 __update_field_vars(elt, rbe, rec, hist_data->save_vars, 2598 hist_data->n_save_vars, hist_data->n_field_var_str); 2599 } 2600 2601 static struct hist_field *create_var(struct hist_trigger_data *hist_data, 2602 struct trace_event_file *file, 2603 char *name, int size, const char *type) 2604 { 2605 struct hist_field *var; 2606 int idx; 2607 2608 if (find_var(hist_data, file, name) && !hist_data->remove) { 2609 var = ERR_PTR(-EINVAL); 2610 goto out; 2611 } 2612 2613 var = kzalloc(sizeof(struct hist_field), GFP_KERNEL); 2614 if (!var) { 2615 var = ERR_PTR(-ENOMEM); 2616 goto out; 2617 } 2618 2619 idx = tracing_map_add_var(hist_data->map); 2620 if (idx < 0) { 2621 kfree(var); 2622 var = ERR_PTR(-EINVAL); 2623 goto out; 2624 } 2625 2626 var->ref = 1; 2627 var->flags = HIST_FIELD_FL_VAR; 2628 var->var.idx = idx; 2629 var->var.hist_data = var->hist_data = hist_data; 2630 var->size = size; 2631 var->var.name = kstrdup(name, GFP_KERNEL); 2632 var->type = kstrdup(type, GFP_KERNEL); 2633 if (!var->var.name || !var->type) { 2634 kfree(var->var.name); 2635 kfree(var->type); 2636 kfree(var); 2637 var = ERR_PTR(-ENOMEM); 2638 } 2639 out: 2640 return var; 2641 } 2642 2643 static struct field_var *create_field_var(struct hist_trigger_data *hist_data, 2644 struct trace_event_file *file, 2645 char *field_name) 2646 { 2647 struct hist_field *val = NULL, *var = NULL; 2648 unsigned long flags = HIST_FIELD_FL_VAR; 2649 struct trace_array *tr = file->tr; 2650 struct field_var *field_var; 2651 int ret = 0; 2652 2653 if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) { 2654 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name)); 2655 ret = -EINVAL; 2656 goto err; 2657 } 2658 2659 val = parse_atom(hist_data, file, field_name, &flags, NULL); 2660 if (IS_ERR(val)) { 2661 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name)); 2662 ret = PTR_ERR(val); 2663 goto err; 2664 } 2665 2666 var = create_var(hist_data, file, field_name, val->size, val->type); 2667 if (IS_ERR(var)) { 2668 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name)); 2669 kfree(val); 2670 ret = PTR_ERR(var); 2671 goto err; 2672 } 2673 2674 field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL); 2675 if (!field_var) { 2676 kfree(val); 2677 kfree(var); 2678 ret = -ENOMEM; 2679 goto err; 2680 } 2681 2682 field_var->var = var; 2683 field_var->val = val; 2684 out: 2685 return field_var; 2686 err: 2687 field_var = ERR_PTR(ret); 2688 goto out; 2689 } 2690 2691 /** 2692 * create_target_field_var - Automatically create a variable for a field 2693 * @target_hist_data: The target hist trigger 2694 * @subsys_name: Optional subsystem name 2695 * @event_name: Optional event name 2696 * @var_name: The name of the field (and the resulting variable) 2697 * 2698 * Hist trigger actions fetch data from variables, not directly from 2699 * events. However, for convenience, users are allowed to directly 2700 * specify an event field in an action, which will be automatically 2701 * converted into a variable on their behalf. 2702 2703 * This function creates a field variable with the name var_name on 2704 * the hist trigger currently being defined on the target event. If 2705 * subsys_name and event_name are specified, this function simply 2706 * verifies that they do in fact match the target event subsystem and 2707 * event name. 2708 * 2709 * Return: The variable created for the field. 2710 */ 2711 static struct field_var * 2712 create_target_field_var(struct hist_trigger_data *target_hist_data, 2713 char *subsys_name, char *event_name, char *var_name) 2714 { 2715 struct trace_event_file *file = target_hist_data->event_file; 2716 2717 if (subsys_name) { 2718 struct trace_event_call *call; 2719 2720 if (!event_name) 2721 return NULL; 2722 2723 call = file->event_call; 2724 2725 if (strcmp(subsys_name, call->class->system) != 0) 2726 return NULL; 2727 2728 if (strcmp(event_name, trace_event_name(call)) != 0) 2729 return NULL; 2730 } 2731 2732 return create_field_var(target_hist_data, file, var_name); 2733 } 2734 2735 static bool check_track_val_max(u64 track_val, u64 var_val) 2736 { 2737 if (var_val <= track_val) 2738 return false; 2739 2740 return true; 2741 } 2742 2743 static bool check_track_val_changed(u64 track_val, u64 var_val) 2744 { 2745 if (var_val == track_val) 2746 return false; 2747 2748 return true; 2749 } 2750 2751 static u64 get_track_val(struct hist_trigger_data *hist_data, 2752 struct tracing_map_elt *elt, 2753 struct action_data *data) 2754 { 2755 unsigned int track_var_idx = data->track_data.track_var->var.idx; 2756 u64 track_val; 2757 2758 track_val = tracing_map_read_var(elt, track_var_idx); 2759 2760 return track_val; 2761 } 2762 2763 static void save_track_val(struct hist_trigger_data *hist_data, 2764 struct tracing_map_elt *elt, 2765 struct action_data *data, u64 var_val) 2766 { 2767 unsigned int track_var_idx = data->track_data.track_var->var.idx; 2768 2769 tracing_map_set_var(elt, track_var_idx, var_val); 2770 } 2771 2772 static void save_track_data(struct hist_trigger_data *hist_data, 2773 struct tracing_map_elt *elt, void *rec, 2774 struct ring_buffer_event *rbe, void *key, 2775 struct action_data *data, u64 *var_ref_vals) 2776 { 2777 if (data->track_data.save_data) 2778 data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals); 2779 } 2780 2781 static bool check_track_val(struct tracing_map_elt *elt, 2782 struct action_data *data, 2783 u64 var_val) 2784 { 2785 struct hist_trigger_data *hist_data; 2786 u64 track_val; 2787 2788 hist_data = data->track_data.track_var->hist_data; 2789 track_val = get_track_val(hist_data, elt, data); 2790 2791 return data->track_data.check_val(track_val, var_val); 2792 } 2793 2794 #ifdef CONFIG_TRACER_SNAPSHOT 2795 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data) 2796 { 2797 /* called with tr->max_lock held */ 2798 struct track_data *track_data = tr->cond_snapshot->cond_data; 2799 struct hist_elt_data *elt_data, *track_elt_data; 2800 struct snapshot_context *context = cond_data; 2801 struct action_data *action; 2802 u64 track_val; 2803 2804 if (!track_data) 2805 return false; 2806 2807 action = track_data->action_data; 2808 2809 track_val = get_track_val(track_data->hist_data, context->elt, 2810 track_data->action_data); 2811 2812 if (!action->track_data.check_val(track_data->track_val, track_val)) 2813 return false; 2814 2815 track_data->track_val = track_val; 2816 memcpy(track_data->key, context->key, track_data->key_len); 2817 2818 elt_data = context->elt->private_data; 2819 track_elt_data = track_data->elt.private_data; 2820 if (elt_data->comm) 2821 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN); 2822 2823 track_data->updated = true; 2824 2825 return true; 2826 } 2827 2828 static void save_track_data_snapshot(struct hist_trigger_data *hist_data, 2829 struct tracing_map_elt *elt, void *rec, 2830 struct ring_buffer_event *rbe, void *key, 2831 struct action_data *data, 2832 u64 *var_ref_vals) 2833 { 2834 struct trace_event_file *file = hist_data->event_file; 2835 struct snapshot_context context; 2836 2837 context.elt = elt; 2838 context.key = key; 2839 2840 tracing_snapshot_cond(file->tr, &context); 2841 } 2842 2843 static void hist_trigger_print_key(struct seq_file *m, 2844 struct hist_trigger_data *hist_data, 2845 void *key, 2846 struct tracing_map_elt *elt); 2847 2848 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data) 2849 { 2850 unsigned int i; 2851 2852 if (!hist_data->n_actions) 2853 return NULL; 2854 2855 for (i = 0; i < hist_data->n_actions; i++) { 2856 struct action_data *data = hist_data->actions[i]; 2857 2858 if (data->action == ACTION_SNAPSHOT) 2859 return data; 2860 } 2861 2862 return NULL; 2863 } 2864 2865 static void track_data_snapshot_print(struct seq_file *m, 2866 struct hist_trigger_data *hist_data) 2867 { 2868 struct trace_event_file *file = hist_data->event_file; 2869 struct track_data *track_data; 2870 struct action_data *action; 2871 2872 track_data = tracing_cond_snapshot_data(file->tr); 2873 if (!track_data) 2874 return; 2875 2876 if (!track_data->updated) 2877 return; 2878 2879 action = snapshot_action(hist_data); 2880 if (!action) 2881 return; 2882 2883 seq_puts(m, "\nSnapshot taken (see tracing/snapshot). Details:\n"); 2884 seq_printf(m, "\ttriggering value { %s(%s) }: %10llu", 2885 action->handler == HANDLER_ONMAX ? "onmax" : "onchange", 2886 action->track_data.var_str, track_data->track_val); 2887 2888 seq_puts(m, "\ttriggered by event with key: "); 2889 hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt); 2890 seq_putc(m, '\n'); 2891 } 2892 #else 2893 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data) 2894 { 2895 return false; 2896 } 2897 static void save_track_data_snapshot(struct hist_trigger_data *hist_data, 2898 struct tracing_map_elt *elt, void *rec, 2899 struct ring_buffer_event *rbe, void *key, 2900 struct action_data *data, 2901 u64 *var_ref_vals) {} 2902 static void track_data_snapshot_print(struct seq_file *m, 2903 struct hist_trigger_data *hist_data) {} 2904 #endif /* CONFIG_TRACER_SNAPSHOT */ 2905 2906 static void track_data_print(struct seq_file *m, 2907 struct hist_trigger_data *hist_data, 2908 struct tracing_map_elt *elt, 2909 struct action_data *data) 2910 { 2911 u64 track_val = get_track_val(hist_data, elt, data); 2912 unsigned int i, save_var_idx; 2913 2914 if (data->handler == HANDLER_ONMAX) 2915 seq_printf(m, "\n\tmax: %10llu", track_val); 2916 else if (data->handler == HANDLER_ONCHANGE) 2917 seq_printf(m, "\n\tchanged: %10llu", track_val); 2918 2919 if (data->action == ACTION_SNAPSHOT) 2920 return; 2921 2922 for (i = 0; i < hist_data->n_save_vars; i++) { 2923 struct hist_field *save_val = hist_data->save_vars[i]->val; 2924 struct hist_field *save_var = hist_data->save_vars[i]->var; 2925 u64 val; 2926 2927 save_var_idx = save_var->var.idx; 2928 2929 val = tracing_map_read_var(elt, save_var_idx); 2930 2931 if (save_val->flags & HIST_FIELD_FL_STRING) { 2932 seq_printf(m, " %s: %-32s", save_var->var.name, 2933 (char *)(uintptr_t)(val)); 2934 } else 2935 seq_printf(m, " %s: %10llu", save_var->var.name, val); 2936 } 2937 } 2938 2939 static void ontrack_action(struct hist_trigger_data *hist_data, 2940 struct tracing_map_elt *elt, void *rec, 2941 struct ring_buffer_event *rbe, void *key, 2942 struct action_data *data, u64 *var_ref_vals) 2943 { 2944 u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx]; 2945 2946 if (check_track_val(elt, data, var_val)) { 2947 save_track_val(hist_data, elt, data, var_val); 2948 save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals); 2949 } 2950 } 2951 2952 static void action_data_destroy(struct action_data *data) 2953 { 2954 unsigned int i; 2955 2956 lockdep_assert_held(&event_mutex); 2957 2958 kfree(data->action_name); 2959 2960 for (i = 0; i < data->n_params; i++) 2961 kfree(data->params[i]); 2962 2963 if (data->synth_event) 2964 data->synth_event->ref--; 2965 2966 kfree(data->synth_event_name); 2967 2968 kfree(data); 2969 } 2970 2971 static void track_data_destroy(struct hist_trigger_data *hist_data, 2972 struct action_data *data) 2973 { 2974 struct trace_event_file *file = hist_data->event_file; 2975 2976 destroy_hist_field(data->track_data.track_var, 0); 2977 2978 if (data->action == ACTION_SNAPSHOT) { 2979 struct track_data *track_data; 2980 2981 track_data = tracing_cond_snapshot_data(file->tr); 2982 if (track_data && track_data->hist_data == hist_data) { 2983 tracing_snapshot_cond_disable(file->tr); 2984 track_data_free(track_data); 2985 } 2986 } 2987 2988 kfree(data->track_data.var_str); 2989 2990 action_data_destroy(data); 2991 } 2992 2993 static int action_create(struct hist_trigger_data *hist_data, 2994 struct action_data *data); 2995 2996 static int track_data_create(struct hist_trigger_data *hist_data, 2997 struct action_data *data) 2998 { 2999 struct hist_field *var_field, *ref_field, *track_var = NULL; 3000 struct trace_event_file *file = hist_data->event_file; 3001 struct trace_array *tr = file->tr; 3002 char *track_data_var_str; 3003 int ret = 0; 3004 3005 track_data_var_str = data->track_data.var_str; 3006 if (track_data_var_str[0] != '$') { 3007 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str)); 3008 return -EINVAL; 3009 } 3010 track_data_var_str++; 3011 3012 var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str); 3013 if (!var_field) { 3014 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str)); 3015 return -EINVAL; 3016 } 3017 3018 ref_field = create_var_ref(hist_data, var_field, NULL, NULL); 3019 if (!ref_field) 3020 return -ENOMEM; 3021 3022 data->track_data.var_ref = ref_field; 3023 3024 if (data->handler == HANDLER_ONMAX) 3025 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64"); 3026 if (IS_ERR(track_var)) { 3027 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0); 3028 ret = PTR_ERR(track_var); 3029 goto out; 3030 } 3031 3032 if (data->handler == HANDLER_ONCHANGE) 3033 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64"); 3034 if (IS_ERR(track_var)) { 3035 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0); 3036 ret = PTR_ERR(track_var); 3037 goto out; 3038 } 3039 data->track_data.track_var = track_var; 3040 3041 ret = action_create(hist_data, data); 3042 out: 3043 return ret; 3044 } 3045 3046 static int parse_action_params(struct trace_array *tr, char *params, 3047 struct action_data *data) 3048 { 3049 char *param, *saved_param; 3050 bool first_param = true; 3051 int ret = 0; 3052 3053 while (params) { 3054 if (data->n_params >= SYNTH_FIELDS_MAX) { 3055 hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0); 3056 goto out; 3057 } 3058 3059 param = strsep(¶ms, ","); 3060 if (!param) { 3061 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0); 3062 ret = -EINVAL; 3063 goto out; 3064 } 3065 3066 param = strstrip(param); 3067 if (strlen(param) < 2) { 3068 hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param)); 3069 ret = -EINVAL; 3070 goto out; 3071 } 3072 3073 saved_param = kstrdup(param, GFP_KERNEL); 3074 if (!saved_param) { 3075 ret = -ENOMEM; 3076 goto out; 3077 } 3078 3079 if (first_param && data->use_trace_keyword) { 3080 data->synth_event_name = saved_param; 3081 first_param = false; 3082 continue; 3083 } 3084 first_param = false; 3085 3086 data->params[data->n_params++] = saved_param; 3087 } 3088 out: 3089 return ret; 3090 } 3091 3092 static int action_parse(struct trace_array *tr, char *str, struct action_data *data, 3093 enum handler_id handler) 3094 { 3095 char *action_name; 3096 int ret = 0; 3097 3098 strsep(&str, "."); 3099 if (!str) { 3100 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0); 3101 ret = -EINVAL; 3102 goto out; 3103 } 3104 3105 action_name = strsep(&str, "("); 3106 if (!action_name || !str) { 3107 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0); 3108 ret = -EINVAL; 3109 goto out; 3110 } 3111 3112 if (str_has_prefix(action_name, "save")) { 3113 char *params = strsep(&str, ")"); 3114 3115 if (!params) { 3116 hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0); 3117 ret = -EINVAL; 3118 goto out; 3119 } 3120 3121 ret = parse_action_params(tr, params, data); 3122 if (ret) 3123 goto out; 3124 3125 if (handler == HANDLER_ONMAX) 3126 data->track_data.check_val = check_track_val_max; 3127 else if (handler == HANDLER_ONCHANGE) 3128 data->track_data.check_val = check_track_val_changed; 3129 else { 3130 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name)); 3131 ret = -EINVAL; 3132 goto out; 3133 } 3134 3135 data->track_data.save_data = save_track_data_vars; 3136 data->fn = ontrack_action; 3137 data->action = ACTION_SAVE; 3138 } else if (str_has_prefix(action_name, "snapshot")) { 3139 char *params = strsep(&str, ")"); 3140 3141 if (!str) { 3142 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params)); 3143 ret = -EINVAL; 3144 goto out; 3145 } 3146 3147 if (handler == HANDLER_ONMAX) 3148 data->track_data.check_val = check_track_val_max; 3149 else if (handler == HANDLER_ONCHANGE) 3150 data->track_data.check_val = check_track_val_changed; 3151 else { 3152 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name)); 3153 ret = -EINVAL; 3154 goto out; 3155 } 3156 3157 data->track_data.save_data = save_track_data_snapshot; 3158 data->fn = ontrack_action; 3159 data->action = ACTION_SNAPSHOT; 3160 } else { 3161 char *params = strsep(&str, ")"); 3162 3163 if (str_has_prefix(action_name, "trace")) 3164 data->use_trace_keyword = true; 3165 3166 if (params) { 3167 ret = parse_action_params(tr, params, data); 3168 if (ret) 3169 goto out; 3170 } 3171 3172 if (handler == HANDLER_ONMAX) 3173 data->track_data.check_val = check_track_val_max; 3174 else if (handler == HANDLER_ONCHANGE) 3175 data->track_data.check_val = check_track_val_changed; 3176 3177 if (handler != HANDLER_ONMATCH) { 3178 data->track_data.save_data = action_trace; 3179 data->fn = ontrack_action; 3180 } else 3181 data->fn = action_trace; 3182 3183 data->action = ACTION_TRACE; 3184 } 3185 3186 data->action_name = kstrdup(action_name, GFP_KERNEL); 3187 if (!data->action_name) { 3188 ret = -ENOMEM; 3189 goto out; 3190 } 3191 3192 data->handler = handler; 3193 out: 3194 return ret; 3195 } 3196 3197 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data, 3198 char *str, enum handler_id handler) 3199 { 3200 struct action_data *data; 3201 int ret = -EINVAL; 3202 char *var_str; 3203 3204 data = kzalloc(sizeof(*data), GFP_KERNEL); 3205 if (!data) 3206 return ERR_PTR(-ENOMEM); 3207 3208 var_str = strsep(&str, ")"); 3209 if (!var_str || !str) { 3210 ret = -EINVAL; 3211 goto free; 3212 } 3213 3214 data->track_data.var_str = kstrdup(var_str, GFP_KERNEL); 3215 if (!data->track_data.var_str) { 3216 ret = -ENOMEM; 3217 goto free; 3218 } 3219 3220 ret = action_parse(hist_data->event_file->tr, str, data, handler); 3221 if (ret) 3222 goto free; 3223 out: 3224 return data; 3225 free: 3226 track_data_destroy(hist_data, data); 3227 data = ERR_PTR(ret); 3228 goto out; 3229 } 3230 3231 static void onmatch_destroy(struct action_data *data) 3232 { 3233 kfree(data->match_data.event); 3234 kfree(data->match_data.event_system); 3235 3236 action_data_destroy(data); 3237 } 3238 3239 static void destroy_field_var(struct field_var *field_var) 3240 { 3241 if (!field_var) 3242 return; 3243 3244 destroy_hist_field(field_var->var, 0); 3245 destroy_hist_field(field_var->val, 0); 3246 3247 kfree(field_var); 3248 } 3249 3250 static void destroy_field_vars(struct hist_trigger_data *hist_data) 3251 { 3252 unsigned int i; 3253 3254 for (i = 0; i < hist_data->n_field_vars; i++) 3255 destroy_field_var(hist_data->field_vars[i]); 3256 3257 for (i = 0; i < hist_data->n_save_vars; i++) 3258 destroy_field_var(hist_data->save_vars[i]); 3259 } 3260 3261 static void save_field_var(struct hist_trigger_data *hist_data, 3262 struct field_var *field_var) 3263 { 3264 hist_data->field_vars[hist_data->n_field_vars++] = field_var; 3265 3266 if (field_var->val->flags & HIST_FIELD_FL_STRING) 3267 hist_data->n_field_var_str++; 3268 } 3269 3270 3271 static int check_synth_field(struct synth_event *event, 3272 struct hist_field *hist_field, 3273 unsigned int field_pos) 3274 { 3275 struct synth_field *field; 3276 3277 if (field_pos >= event->n_fields) 3278 return -EINVAL; 3279 3280 field = event->fields[field_pos]; 3281 3282 if (strcmp(field->type, hist_field->type) != 0) { 3283 if (field->size != hist_field->size || 3284 field->is_signed != hist_field->is_signed) 3285 return -EINVAL; 3286 } 3287 3288 return 0; 3289 } 3290 3291 static struct hist_field * 3292 trace_action_find_var(struct hist_trigger_data *hist_data, 3293 struct action_data *data, 3294 char *system, char *event, char *var) 3295 { 3296 struct trace_array *tr = hist_data->event_file->tr; 3297 struct hist_field *hist_field; 3298 3299 var++; /* skip '$' */ 3300 3301 hist_field = find_target_event_var(hist_data, system, event, var); 3302 if (!hist_field) { 3303 if (!system && data->handler == HANDLER_ONMATCH) { 3304 system = data->match_data.event_system; 3305 event = data->match_data.event; 3306 } 3307 3308 hist_field = find_event_var(hist_data, system, event, var); 3309 } 3310 3311 if (!hist_field) 3312 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var)); 3313 3314 return hist_field; 3315 } 3316 3317 static struct hist_field * 3318 trace_action_create_field_var(struct hist_trigger_data *hist_data, 3319 struct action_data *data, char *system, 3320 char *event, char *var) 3321 { 3322 struct hist_field *hist_field = NULL; 3323 struct field_var *field_var; 3324 3325 /* 3326 * First try to create a field var on the target event (the 3327 * currently being defined). This will create a variable for 3328 * unqualified fields on the target event, or if qualified, 3329 * target fields that have qualified names matching the target. 3330 */ 3331 field_var = create_target_field_var(hist_data, system, event, var); 3332 3333 if (field_var && !IS_ERR(field_var)) { 3334 save_field_var(hist_data, field_var); 3335 hist_field = field_var->var; 3336 } else { 3337 field_var = NULL; 3338 /* 3339 * If no explicit system.event is specfied, default to 3340 * looking for fields on the onmatch(system.event.xxx) 3341 * event. 3342 */ 3343 if (!system && data->handler == HANDLER_ONMATCH) { 3344 system = data->match_data.event_system; 3345 event = data->match_data.event; 3346 } 3347 3348 /* 3349 * At this point, we're looking at a field on another 3350 * event. Because we can't modify a hist trigger on 3351 * another event to add a variable for a field, we need 3352 * to create a new trigger on that event and create the 3353 * variable at the same time. 3354 */ 3355 hist_field = create_field_var_hist(hist_data, system, event, var); 3356 if (IS_ERR(hist_field)) 3357 goto free; 3358 } 3359 out: 3360 return hist_field; 3361 free: 3362 destroy_field_var(field_var); 3363 hist_field = NULL; 3364 goto out; 3365 } 3366 3367 static int trace_action_create(struct hist_trigger_data *hist_data, 3368 struct action_data *data) 3369 { 3370 struct trace_array *tr = hist_data->event_file->tr; 3371 char *event_name, *param, *system = NULL; 3372 struct hist_field *hist_field, *var_ref; 3373 unsigned int i; 3374 unsigned int field_pos = 0; 3375 struct synth_event *event; 3376 char *synth_event_name; 3377 int var_ref_idx, ret = 0; 3378 3379 lockdep_assert_held(&event_mutex); 3380 3381 if (data->use_trace_keyword) 3382 synth_event_name = data->synth_event_name; 3383 else 3384 synth_event_name = data->action_name; 3385 3386 event = find_synth_event(synth_event_name); 3387 if (!event) { 3388 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name)); 3389 return -EINVAL; 3390 } 3391 3392 event->ref++; 3393 3394 for (i = 0; i < data->n_params; i++) { 3395 char *p; 3396 3397 p = param = kstrdup(data->params[i], GFP_KERNEL); 3398 if (!param) { 3399 ret = -ENOMEM; 3400 goto err; 3401 } 3402 3403 system = strsep(¶m, "."); 3404 if (!param) { 3405 param = (char *)system; 3406 system = event_name = NULL; 3407 } else { 3408 event_name = strsep(¶m, "."); 3409 if (!param) { 3410 kfree(p); 3411 ret = -EINVAL; 3412 goto err; 3413 } 3414 } 3415 3416 if (param[0] == '$') 3417 hist_field = trace_action_find_var(hist_data, data, 3418 system, event_name, 3419 param); 3420 else 3421 hist_field = trace_action_create_field_var(hist_data, 3422 data, 3423 system, 3424 event_name, 3425 param); 3426 3427 if (!hist_field) { 3428 kfree(p); 3429 ret = -EINVAL; 3430 goto err; 3431 } 3432 3433 if (check_synth_field(event, hist_field, field_pos) == 0) { 3434 var_ref = create_var_ref(hist_data, hist_field, 3435 system, event_name); 3436 if (!var_ref) { 3437 kfree(p); 3438 ret = -ENOMEM; 3439 goto err; 3440 } 3441 3442 var_ref_idx = find_var_ref_idx(hist_data, var_ref); 3443 if (WARN_ON(var_ref_idx < 0)) { 3444 ret = var_ref_idx; 3445 goto err; 3446 } 3447 3448 data->var_ref_idx[i] = var_ref_idx; 3449 3450 field_pos++; 3451 kfree(p); 3452 continue; 3453 } 3454 3455 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param)); 3456 kfree(p); 3457 ret = -EINVAL; 3458 goto err; 3459 } 3460 3461 if (field_pos != event->n_fields) { 3462 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name)); 3463 ret = -EINVAL; 3464 goto err; 3465 } 3466 3467 data->synth_event = event; 3468 out: 3469 return ret; 3470 err: 3471 event->ref--; 3472 3473 goto out; 3474 } 3475 3476 static int action_create(struct hist_trigger_data *hist_data, 3477 struct action_data *data) 3478 { 3479 struct trace_event_file *file = hist_data->event_file; 3480 struct trace_array *tr = file->tr; 3481 struct track_data *track_data; 3482 struct field_var *field_var; 3483 unsigned int i; 3484 char *param; 3485 int ret = 0; 3486 3487 if (data->action == ACTION_TRACE) 3488 return trace_action_create(hist_data, data); 3489 3490 if (data->action == ACTION_SNAPSHOT) { 3491 track_data = track_data_alloc(hist_data->key_size, data, hist_data); 3492 if (IS_ERR(track_data)) { 3493 ret = PTR_ERR(track_data); 3494 goto out; 3495 } 3496 3497 ret = tracing_snapshot_cond_enable(file->tr, track_data, 3498 cond_snapshot_update); 3499 if (ret) 3500 track_data_free(track_data); 3501 3502 goto out; 3503 } 3504 3505 if (data->action == ACTION_SAVE) { 3506 if (hist_data->n_save_vars) { 3507 ret = -EEXIST; 3508 hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0); 3509 goto out; 3510 } 3511 3512 for (i = 0; i < data->n_params; i++) { 3513 param = kstrdup(data->params[i], GFP_KERNEL); 3514 if (!param) { 3515 ret = -ENOMEM; 3516 goto out; 3517 } 3518 3519 field_var = create_target_field_var(hist_data, NULL, NULL, param); 3520 if (IS_ERR(field_var)) { 3521 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL, 3522 errpos(param)); 3523 ret = PTR_ERR(field_var); 3524 kfree(param); 3525 goto out; 3526 } 3527 3528 hist_data->save_vars[hist_data->n_save_vars++] = field_var; 3529 if (field_var->val->flags & HIST_FIELD_FL_STRING) 3530 hist_data->n_save_var_str++; 3531 kfree(param); 3532 } 3533 } 3534 out: 3535 return ret; 3536 } 3537 3538 static int onmatch_create(struct hist_trigger_data *hist_data, 3539 struct action_data *data) 3540 { 3541 return action_create(hist_data, data); 3542 } 3543 3544 static struct action_data *onmatch_parse(struct trace_array *tr, char *str) 3545 { 3546 char *match_event, *match_event_system; 3547 struct action_data *data; 3548 int ret = -EINVAL; 3549 3550 data = kzalloc(sizeof(*data), GFP_KERNEL); 3551 if (!data) 3552 return ERR_PTR(-ENOMEM); 3553 3554 match_event = strsep(&str, ")"); 3555 if (!match_event || !str) { 3556 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event)); 3557 goto free; 3558 } 3559 3560 match_event_system = strsep(&match_event, "."); 3561 if (!match_event) { 3562 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system)); 3563 goto free; 3564 } 3565 3566 if (IS_ERR(event_file(tr, match_event_system, match_event))) { 3567 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event)); 3568 goto free; 3569 } 3570 3571 data->match_data.event = kstrdup(match_event, GFP_KERNEL); 3572 if (!data->match_data.event) { 3573 ret = -ENOMEM; 3574 goto free; 3575 } 3576 3577 data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL); 3578 if (!data->match_data.event_system) { 3579 ret = -ENOMEM; 3580 goto free; 3581 } 3582 3583 ret = action_parse(tr, str, data, HANDLER_ONMATCH); 3584 if (ret) 3585 goto free; 3586 out: 3587 return data; 3588 free: 3589 onmatch_destroy(data); 3590 data = ERR_PTR(ret); 3591 goto out; 3592 } 3593 3594 static int create_hitcount_val(struct hist_trigger_data *hist_data) 3595 { 3596 hist_data->fields[HITCOUNT_IDX] = 3597 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL); 3598 if (!hist_data->fields[HITCOUNT_IDX]) 3599 return -ENOMEM; 3600 3601 hist_data->n_vals++; 3602 hist_data->n_fields++; 3603 3604 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX)) 3605 return -EINVAL; 3606 3607 return 0; 3608 } 3609 3610 static int __create_val_field(struct hist_trigger_data *hist_data, 3611 unsigned int val_idx, 3612 struct trace_event_file *file, 3613 char *var_name, char *field_str, 3614 unsigned long flags) 3615 { 3616 struct hist_field *hist_field; 3617 int ret = 0; 3618 3619 hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0); 3620 if (IS_ERR(hist_field)) { 3621 ret = PTR_ERR(hist_field); 3622 goto out; 3623 } 3624 3625 hist_data->fields[val_idx] = hist_field; 3626 3627 ++hist_data->n_vals; 3628 ++hist_data->n_fields; 3629 3630 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 3631 ret = -EINVAL; 3632 out: 3633 return ret; 3634 } 3635 3636 static int create_val_field(struct hist_trigger_data *hist_data, 3637 unsigned int val_idx, 3638 struct trace_event_file *file, 3639 char *field_str) 3640 { 3641 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX)) 3642 return -EINVAL; 3643 3644 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0); 3645 } 3646 3647 static int create_var_field(struct hist_trigger_data *hist_data, 3648 unsigned int val_idx, 3649 struct trace_event_file *file, 3650 char *var_name, char *expr_str) 3651 { 3652 struct trace_array *tr = hist_data->event_file->tr; 3653 unsigned long flags = 0; 3654 3655 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 3656 return -EINVAL; 3657 3658 if (find_var(hist_data, file, var_name) && !hist_data->remove) { 3659 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name)); 3660 return -EINVAL; 3661 } 3662 3663 flags |= HIST_FIELD_FL_VAR; 3664 hist_data->n_vars++; 3665 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX)) 3666 return -EINVAL; 3667 3668 return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags); 3669 } 3670 3671 static int create_val_fields(struct hist_trigger_data *hist_data, 3672 struct trace_event_file *file) 3673 { 3674 char *fields_str, *field_str; 3675 unsigned int i, j = 1; 3676 int ret; 3677 3678 ret = create_hitcount_val(hist_data); 3679 if (ret) 3680 goto out; 3681 3682 fields_str = hist_data->attrs->vals_str; 3683 if (!fields_str) 3684 goto out; 3685 3686 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX && 3687 j < TRACING_MAP_VALS_MAX; i++) { 3688 field_str = strsep(&fields_str, ","); 3689 if (!field_str) 3690 break; 3691 3692 if (strcmp(field_str, "hitcount") == 0) 3693 continue; 3694 3695 ret = create_val_field(hist_data, j++, file, field_str); 3696 if (ret) 3697 goto out; 3698 } 3699 3700 if (fields_str && (strcmp(fields_str, "hitcount") != 0)) 3701 ret = -EINVAL; 3702 out: 3703 return ret; 3704 } 3705 3706 static int create_key_field(struct hist_trigger_data *hist_data, 3707 unsigned int key_idx, 3708 unsigned int key_offset, 3709 struct trace_event_file *file, 3710 char *field_str) 3711 { 3712 struct trace_array *tr = hist_data->event_file->tr; 3713 struct hist_field *hist_field = NULL; 3714 unsigned long flags = 0; 3715 unsigned int key_size; 3716 int ret = 0; 3717 3718 if (WARN_ON(key_idx >= HIST_FIELDS_MAX)) 3719 return -EINVAL; 3720 3721 flags |= HIST_FIELD_FL_KEY; 3722 3723 if (strcmp(field_str, "stacktrace") == 0) { 3724 flags |= HIST_FIELD_FL_STACKTRACE; 3725 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH; 3726 hist_field = create_hist_field(hist_data, NULL, flags, NULL); 3727 } else { 3728 hist_field = parse_expr(hist_data, file, field_str, flags, 3729 NULL, 0); 3730 if (IS_ERR(hist_field)) { 3731 ret = PTR_ERR(hist_field); 3732 goto out; 3733 } 3734 3735 if (field_has_hist_vars(hist_field, 0)) { 3736 hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str)); 3737 destroy_hist_field(hist_field, 0); 3738 ret = -EINVAL; 3739 goto out; 3740 } 3741 3742 key_size = hist_field->size; 3743 } 3744 3745 hist_data->fields[key_idx] = hist_field; 3746 3747 key_size = ALIGN(key_size, sizeof(u64)); 3748 hist_data->fields[key_idx]->size = key_size; 3749 hist_data->fields[key_idx]->offset = key_offset; 3750 3751 hist_data->key_size += key_size; 3752 3753 if (hist_data->key_size > HIST_KEY_SIZE_MAX) { 3754 ret = -EINVAL; 3755 goto out; 3756 } 3757 3758 hist_data->n_keys++; 3759 hist_data->n_fields++; 3760 3761 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX)) 3762 return -EINVAL; 3763 3764 ret = key_size; 3765 out: 3766 return ret; 3767 } 3768 3769 static int create_key_fields(struct hist_trigger_data *hist_data, 3770 struct trace_event_file *file) 3771 { 3772 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals; 3773 char *fields_str, *field_str; 3774 int ret = -EINVAL; 3775 3776 fields_str = hist_data->attrs->keys_str; 3777 if (!fields_str) 3778 goto out; 3779 3780 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) { 3781 field_str = strsep(&fields_str, ","); 3782 if (!field_str) 3783 break; 3784 ret = create_key_field(hist_data, i, key_offset, 3785 file, field_str); 3786 if (ret < 0) 3787 goto out; 3788 key_offset += ret; 3789 } 3790 if (fields_str) { 3791 ret = -EINVAL; 3792 goto out; 3793 } 3794 ret = 0; 3795 out: 3796 return ret; 3797 } 3798 3799 static int create_var_fields(struct hist_trigger_data *hist_data, 3800 struct trace_event_file *file) 3801 { 3802 unsigned int i, j = hist_data->n_vals; 3803 int ret = 0; 3804 3805 unsigned int n_vars = hist_data->attrs->var_defs.n_vars; 3806 3807 for (i = 0; i < n_vars; i++) { 3808 char *var_name = hist_data->attrs->var_defs.name[i]; 3809 char *expr = hist_data->attrs->var_defs.expr[i]; 3810 3811 ret = create_var_field(hist_data, j++, file, var_name, expr); 3812 if (ret) 3813 goto out; 3814 } 3815 out: 3816 return ret; 3817 } 3818 3819 static void free_var_defs(struct hist_trigger_data *hist_data) 3820 { 3821 unsigned int i; 3822 3823 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 3824 kfree(hist_data->attrs->var_defs.name[i]); 3825 kfree(hist_data->attrs->var_defs.expr[i]); 3826 } 3827 3828 hist_data->attrs->var_defs.n_vars = 0; 3829 } 3830 3831 static int parse_var_defs(struct hist_trigger_data *hist_data) 3832 { 3833 struct trace_array *tr = hist_data->event_file->tr; 3834 char *s, *str, *var_name, *field_str; 3835 unsigned int i, j, n_vars = 0; 3836 int ret = 0; 3837 3838 for (i = 0; i < hist_data->attrs->n_assignments; i++) { 3839 str = hist_data->attrs->assignment_str[i]; 3840 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) { 3841 field_str = strsep(&str, ","); 3842 if (!field_str) 3843 break; 3844 3845 var_name = strsep(&field_str, "="); 3846 if (!var_name || !field_str) { 3847 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT, 3848 errpos(var_name)); 3849 ret = -EINVAL; 3850 goto free; 3851 } 3852 3853 if (n_vars == TRACING_MAP_VARS_MAX) { 3854 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name)); 3855 ret = -EINVAL; 3856 goto free; 3857 } 3858 3859 s = kstrdup(var_name, GFP_KERNEL); 3860 if (!s) { 3861 ret = -ENOMEM; 3862 goto free; 3863 } 3864 hist_data->attrs->var_defs.name[n_vars] = s; 3865 3866 s = kstrdup(field_str, GFP_KERNEL); 3867 if (!s) { 3868 ret = -ENOMEM; 3869 goto free; 3870 } 3871 hist_data->attrs->var_defs.expr[n_vars++] = s; 3872 3873 hist_data->attrs->var_defs.n_vars = n_vars; 3874 } 3875 } 3876 3877 return ret; 3878 free: 3879 free_var_defs(hist_data); 3880 3881 return ret; 3882 } 3883 3884 static int create_hist_fields(struct hist_trigger_data *hist_data, 3885 struct trace_event_file *file) 3886 { 3887 int ret; 3888 3889 ret = parse_var_defs(hist_data); 3890 if (ret) 3891 goto out; 3892 3893 ret = create_val_fields(hist_data, file); 3894 if (ret) 3895 goto out; 3896 3897 ret = create_var_fields(hist_data, file); 3898 if (ret) 3899 goto out; 3900 3901 ret = create_key_fields(hist_data, file); 3902 if (ret) 3903 goto out; 3904 out: 3905 free_var_defs(hist_data); 3906 3907 return ret; 3908 } 3909 3910 static int is_descending(struct trace_array *tr, const char *str) 3911 { 3912 if (!str) 3913 return 0; 3914 3915 if (strcmp(str, "descending") == 0) 3916 return 1; 3917 3918 if (strcmp(str, "ascending") == 0) 3919 return 0; 3920 3921 hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str)); 3922 3923 return -EINVAL; 3924 } 3925 3926 static int create_sort_keys(struct hist_trigger_data *hist_data) 3927 { 3928 struct trace_array *tr = hist_data->event_file->tr; 3929 char *fields_str = hist_data->attrs->sort_key_str; 3930 struct tracing_map_sort_key *sort_key; 3931 int descending, ret = 0; 3932 unsigned int i, j, k; 3933 3934 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */ 3935 3936 if (!fields_str) 3937 goto out; 3938 3939 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) { 3940 struct hist_field *hist_field; 3941 char *field_str, *field_name; 3942 const char *test_name; 3943 3944 sort_key = &hist_data->sort_keys[i]; 3945 3946 field_str = strsep(&fields_str, ","); 3947 if (!field_str) 3948 break; 3949 3950 if (!*field_str) { 3951 ret = -EINVAL; 3952 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort=")); 3953 break; 3954 } 3955 3956 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) { 3957 hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort=")); 3958 ret = -EINVAL; 3959 break; 3960 } 3961 3962 field_name = strsep(&field_str, "."); 3963 if (!field_name || !*field_name) { 3964 ret = -EINVAL; 3965 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort=")); 3966 break; 3967 } 3968 3969 if (strcmp(field_name, "hitcount") == 0) { 3970 descending = is_descending(tr, field_str); 3971 if (descending < 0) { 3972 ret = descending; 3973 break; 3974 } 3975 sort_key->descending = descending; 3976 continue; 3977 } 3978 3979 for (j = 1, k = 1; j < hist_data->n_fields; j++) { 3980 unsigned int idx; 3981 3982 hist_field = hist_data->fields[j]; 3983 if (hist_field->flags & HIST_FIELD_FL_VAR) 3984 continue; 3985 3986 idx = k++; 3987 3988 test_name = hist_field_name(hist_field, 0); 3989 3990 if (strcmp(field_name, test_name) == 0) { 3991 sort_key->field_idx = idx; 3992 descending = is_descending(tr, field_str); 3993 if (descending < 0) { 3994 ret = descending; 3995 goto out; 3996 } 3997 sort_key->descending = descending; 3998 break; 3999 } 4000 } 4001 if (j == hist_data->n_fields) { 4002 ret = -EINVAL; 4003 hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name)); 4004 break; 4005 } 4006 } 4007 4008 hist_data->n_sort_keys = i; 4009 out: 4010 return ret; 4011 } 4012 4013 static void destroy_actions(struct hist_trigger_data *hist_data) 4014 { 4015 unsigned int i; 4016 4017 for (i = 0; i < hist_data->n_actions; i++) { 4018 struct action_data *data = hist_data->actions[i]; 4019 4020 if (data->handler == HANDLER_ONMATCH) 4021 onmatch_destroy(data); 4022 else if (data->handler == HANDLER_ONMAX || 4023 data->handler == HANDLER_ONCHANGE) 4024 track_data_destroy(hist_data, data); 4025 else 4026 kfree(data); 4027 } 4028 } 4029 4030 static int parse_actions(struct hist_trigger_data *hist_data) 4031 { 4032 struct trace_array *tr = hist_data->event_file->tr; 4033 struct action_data *data; 4034 unsigned int i; 4035 int ret = 0; 4036 char *str; 4037 int len; 4038 4039 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4040 str = hist_data->attrs->action_str[i]; 4041 4042 if ((len = str_has_prefix(str, "onmatch("))) { 4043 char *action_str = str + len; 4044 4045 data = onmatch_parse(tr, action_str); 4046 if (IS_ERR(data)) { 4047 ret = PTR_ERR(data); 4048 break; 4049 } 4050 } else if ((len = str_has_prefix(str, "onmax("))) { 4051 char *action_str = str + len; 4052 4053 data = track_data_parse(hist_data, action_str, 4054 HANDLER_ONMAX); 4055 if (IS_ERR(data)) { 4056 ret = PTR_ERR(data); 4057 break; 4058 } 4059 } else if ((len = str_has_prefix(str, "onchange("))) { 4060 char *action_str = str + len; 4061 4062 data = track_data_parse(hist_data, action_str, 4063 HANDLER_ONCHANGE); 4064 if (IS_ERR(data)) { 4065 ret = PTR_ERR(data); 4066 break; 4067 } 4068 } else { 4069 ret = -EINVAL; 4070 break; 4071 } 4072 4073 hist_data->actions[hist_data->n_actions++] = data; 4074 } 4075 4076 return ret; 4077 } 4078 4079 static int create_actions(struct hist_trigger_data *hist_data) 4080 { 4081 struct action_data *data; 4082 unsigned int i; 4083 int ret = 0; 4084 4085 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4086 data = hist_data->actions[i]; 4087 4088 if (data->handler == HANDLER_ONMATCH) { 4089 ret = onmatch_create(hist_data, data); 4090 if (ret) 4091 break; 4092 } else if (data->handler == HANDLER_ONMAX || 4093 data->handler == HANDLER_ONCHANGE) { 4094 ret = track_data_create(hist_data, data); 4095 if (ret) 4096 break; 4097 } else { 4098 ret = -EINVAL; 4099 break; 4100 } 4101 } 4102 4103 return ret; 4104 } 4105 4106 static void print_actions(struct seq_file *m, 4107 struct hist_trigger_data *hist_data, 4108 struct tracing_map_elt *elt) 4109 { 4110 unsigned int i; 4111 4112 for (i = 0; i < hist_data->n_actions; i++) { 4113 struct action_data *data = hist_data->actions[i]; 4114 4115 if (data->action == ACTION_SNAPSHOT) 4116 continue; 4117 4118 if (data->handler == HANDLER_ONMAX || 4119 data->handler == HANDLER_ONCHANGE) 4120 track_data_print(m, hist_data, elt, data); 4121 } 4122 } 4123 4124 static void print_action_spec(struct seq_file *m, 4125 struct hist_trigger_data *hist_data, 4126 struct action_data *data) 4127 { 4128 unsigned int i; 4129 4130 if (data->action == ACTION_SAVE) { 4131 for (i = 0; i < hist_data->n_save_vars; i++) { 4132 seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name); 4133 if (i < hist_data->n_save_vars - 1) 4134 seq_puts(m, ","); 4135 } 4136 } else if (data->action == ACTION_TRACE) { 4137 if (data->use_trace_keyword) 4138 seq_printf(m, "%s", data->synth_event_name); 4139 for (i = 0; i < data->n_params; i++) { 4140 if (i || data->use_trace_keyword) 4141 seq_puts(m, ","); 4142 seq_printf(m, "%s", data->params[i]); 4143 } 4144 } 4145 } 4146 4147 static void print_track_data_spec(struct seq_file *m, 4148 struct hist_trigger_data *hist_data, 4149 struct action_data *data) 4150 { 4151 if (data->handler == HANDLER_ONMAX) 4152 seq_puts(m, ":onmax("); 4153 else if (data->handler == HANDLER_ONCHANGE) 4154 seq_puts(m, ":onchange("); 4155 seq_printf(m, "%s", data->track_data.var_str); 4156 seq_printf(m, ").%s(", data->action_name); 4157 4158 print_action_spec(m, hist_data, data); 4159 4160 seq_puts(m, ")"); 4161 } 4162 4163 static void print_onmatch_spec(struct seq_file *m, 4164 struct hist_trigger_data *hist_data, 4165 struct action_data *data) 4166 { 4167 seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system, 4168 data->match_data.event); 4169 4170 seq_printf(m, "%s(", data->action_name); 4171 4172 print_action_spec(m, hist_data, data); 4173 4174 seq_puts(m, ")"); 4175 } 4176 4177 static bool actions_match(struct hist_trigger_data *hist_data, 4178 struct hist_trigger_data *hist_data_test) 4179 { 4180 unsigned int i, j; 4181 4182 if (hist_data->n_actions != hist_data_test->n_actions) 4183 return false; 4184 4185 for (i = 0; i < hist_data->n_actions; i++) { 4186 struct action_data *data = hist_data->actions[i]; 4187 struct action_data *data_test = hist_data_test->actions[i]; 4188 char *action_name, *action_name_test; 4189 4190 if (data->handler != data_test->handler) 4191 return false; 4192 if (data->action != data_test->action) 4193 return false; 4194 4195 if (data->n_params != data_test->n_params) 4196 return false; 4197 4198 for (j = 0; j < data->n_params; j++) { 4199 if (strcmp(data->params[j], data_test->params[j]) != 0) 4200 return false; 4201 } 4202 4203 if (data->use_trace_keyword) 4204 action_name = data->synth_event_name; 4205 else 4206 action_name = data->action_name; 4207 4208 if (data_test->use_trace_keyword) 4209 action_name_test = data_test->synth_event_name; 4210 else 4211 action_name_test = data_test->action_name; 4212 4213 if (strcmp(action_name, action_name_test) != 0) 4214 return false; 4215 4216 if (data->handler == HANDLER_ONMATCH) { 4217 if (strcmp(data->match_data.event_system, 4218 data_test->match_data.event_system) != 0) 4219 return false; 4220 if (strcmp(data->match_data.event, 4221 data_test->match_data.event) != 0) 4222 return false; 4223 } else if (data->handler == HANDLER_ONMAX || 4224 data->handler == HANDLER_ONCHANGE) { 4225 if (strcmp(data->track_data.var_str, 4226 data_test->track_data.var_str) != 0) 4227 return false; 4228 } 4229 } 4230 4231 return true; 4232 } 4233 4234 4235 static void print_actions_spec(struct seq_file *m, 4236 struct hist_trigger_data *hist_data) 4237 { 4238 unsigned int i; 4239 4240 for (i = 0; i < hist_data->n_actions; i++) { 4241 struct action_data *data = hist_data->actions[i]; 4242 4243 if (data->handler == HANDLER_ONMATCH) 4244 print_onmatch_spec(m, hist_data, data); 4245 else if (data->handler == HANDLER_ONMAX || 4246 data->handler == HANDLER_ONCHANGE) 4247 print_track_data_spec(m, hist_data, data); 4248 } 4249 } 4250 4251 static void destroy_field_var_hists(struct hist_trigger_data *hist_data) 4252 { 4253 unsigned int i; 4254 4255 for (i = 0; i < hist_data->n_field_var_hists; i++) { 4256 kfree(hist_data->field_var_hists[i]->cmd); 4257 kfree(hist_data->field_var_hists[i]); 4258 } 4259 } 4260 4261 static void destroy_hist_data(struct hist_trigger_data *hist_data) 4262 { 4263 if (!hist_data) 4264 return; 4265 4266 destroy_hist_trigger_attrs(hist_data->attrs); 4267 destroy_hist_fields(hist_data); 4268 tracing_map_destroy(hist_data->map); 4269 4270 destroy_actions(hist_data); 4271 destroy_field_vars(hist_data); 4272 destroy_field_var_hists(hist_data); 4273 4274 kfree(hist_data); 4275 } 4276 4277 static int create_tracing_map_fields(struct hist_trigger_data *hist_data) 4278 { 4279 struct tracing_map *map = hist_data->map; 4280 struct ftrace_event_field *field; 4281 struct hist_field *hist_field; 4282 int i, idx = 0; 4283 4284 for_each_hist_field(i, hist_data) { 4285 hist_field = hist_data->fields[i]; 4286 if (hist_field->flags & HIST_FIELD_FL_KEY) { 4287 tracing_map_cmp_fn_t cmp_fn; 4288 4289 field = hist_field->field; 4290 4291 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE) 4292 cmp_fn = tracing_map_cmp_none; 4293 else if (!field) 4294 cmp_fn = tracing_map_cmp_num(hist_field->size, 4295 hist_field->is_signed); 4296 else if (is_string_field(field)) 4297 cmp_fn = tracing_map_cmp_string; 4298 else 4299 cmp_fn = tracing_map_cmp_num(field->size, 4300 field->is_signed); 4301 idx = tracing_map_add_key_field(map, 4302 hist_field->offset, 4303 cmp_fn); 4304 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR)) 4305 idx = tracing_map_add_sum_field(map); 4306 4307 if (idx < 0) 4308 return idx; 4309 4310 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4311 idx = tracing_map_add_var(map); 4312 if (idx < 0) 4313 return idx; 4314 hist_field->var.idx = idx; 4315 hist_field->var.hist_data = hist_data; 4316 } 4317 } 4318 4319 return 0; 4320 } 4321 4322 static struct hist_trigger_data * 4323 create_hist_data(unsigned int map_bits, 4324 struct hist_trigger_attrs *attrs, 4325 struct trace_event_file *file, 4326 bool remove) 4327 { 4328 const struct tracing_map_ops *map_ops = NULL; 4329 struct hist_trigger_data *hist_data; 4330 int ret = 0; 4331 4332 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL); 4333 if (!hist_data) 4334 return ERR_PTR(-ENOMEM); 4335 4336 hist_data->attrs = attrs; 4337 hist_data->remove = remove; 4338 hist_data->event_file = file; 4339 4340 ret = parse_actions(hist_data); 4341 if (ret) 4342 goto free; 4343 4344 ret = create_hist_fields(hist_data, file); 4345 if (ret) 4346 goto free; 4347 4348 ret = create_sort_keys(hist_data); 4349 if (ret) 4350 goto free; 4351 4352 map_ops = &hist_trigger_elt_data_ops; 4353 4354 hist_data->map = tracing_map_create(map_bits, hist_data->key_size, 4355 map_ops, hist_data); 4356 if (IS_ERR(hist_data->map)) { 4357 ret = PTR_ERR(hist_data->map); 4358 hist_data->map = NULL; 4359 goto free; 4360 } 4361 4362 ret = create_tracing_map_fields(hist_data); 4363 if (ret) 4364 goto free; 4365 out: 4366 return hist_data; 4367 free: 4368 hist_data->attrs = NULL; 4369 4370 destroy_hist_data(hist_data); 4371 4372 hist_data = ERR_PTR(ret); 4373 4374 goto out; 4375 } 4376 4377 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, 4378 struct tracing_map_elt *elt, void *rec, 4379 struct ring_buffer_event *rbe, 4380 u64 *var_ref_vals) 4381 { 4382 struct hist_elt_data *elt_data; 4383 struct hist_field *hist_field; 4384 unsigned int i, var_idx; 4385 u64 hist_val; 4386 4387 elt_data = elt->private_data; 4388 elt_data->var_ref_vals = var_ref_vals; 4389 4390 for_each_hist_val_field(i, hist_data) { 4391 hist_field = hist_data->fields[i]; 4392 hist_val = hist_field->fn(hist_field, elt, rbe, rec); 4393 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4394 var_idx = hist_field->var.idx; 4395 tracing_map_set_var(elt, var_idx, hist_val); 4396 continue; 4397 } 4398 tracing_map_update_sum(elt, i, hist_val); 4399 } 4400 4401 for_each_hist_key_field(i, hist_data) { 4402 hist_field = hist_data->fields[i]; 4403 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4404 hist_val = hist_field->fn(hist_field, elt, rbe, rec); 4405 var_idx = hist_field->var.idx; 4406 tracing_map_set_var(elt, var_idx, hist_val); 4407 } 4408 } 4409 4410 update_field_vars(hist_data, elt, rbe, rec); 4411 } 4412 4413 static inline void add_to_key(char *compound_key, void *key, 4414 struct hist_field *key_field, void *rec) 4415 { 4416 size_t size = key_field->size; 4417 4418 if (key_field->flags & HIST_FIELD_FL_STRING) { 4419 struct ftrace_event_field *field; 4420 4421 field = key_field->field; 4422 if (field->filter_type == FILTER_DYN_STRING) 4423 size = *(u32 *)(rec + field->offset) >> 16; 4424 else if (field->filter_type == FILTER_PTR_STRING) 4425 size = strlen(key); 4426 else if (field->filter_type == FILTER_STATIC_STRING) 4427 size = field->size; 4428 4429 /* ensure NULL-termination */ 4430 if (size > key_field->size - 1) 4431 size = key_field->size - 1; 4432 4433 strncpy(compound_key + key_field->offset, (char *)key, size); 4434 } else 4435 memcpy(compound_key + key_field->offset, key, size); 4436 } 4437 4438 static void 4439 hist_trigger_actions(struct hist_trigger_data *hist_data, 4440 struct tracing_map_elt *elt, void *rec, 4441 struct ring_buffer_event *rbe, void *key, 4442 u64 *var_ref_vals) 4443 { 4444 struct action_data *data; 4445 unsigned int i; 4446 4447 for (i = 0; i < hist_data->n_actions; i++) { 4448 data = hist_data->actions[i]; 4449 data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals); 4450 } 4451 } 4452 4453 static void event_hist_trigger(struct event_trigger_data *data, void *rec, 4454 struct ring_buffer_event *rbe) 4455 { 4456 struct hist_trigger_data *hist_data = data->private_data; 4457 bool use_compound_key = (hist_data->n_keys > 1); 4458 unsigned long entries[HIST_STACKTRACE_DEPTH]; 4459 u64 var_ref_vals[TRACING_MAP_VARS_MAX]; 4460 char compound_key[HIST_KEY_SIZE_MAX]; 4461 struct tracing_map_elt *elt = NULL; 4462 struct hist_field *key_field; 4463 u64 field_contents; 4464 void *key = NULL; 4465 unsigned int i; 4466 4467 memset(compound_key, 0, hist_data->key_size); 4468 4469 for_each_hist_key_field(i, hist_data) { 4470 key_field = hist_data->fields[i]; 4471 4472 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 4473 memset(entries, 0, HIST_STACKTRACE_SIZE); 4474 stack_trace_save(entries, HIST_STACKTRACE_DEPTH, 4475 HIST_STACKTRACE_SKIP); 4476 key = entries; 4477 } else { 4478 field_contents = key_field->fn(key_field, elt, rbe, rec); 4479 if (key_field->flags & HIST_FIELD_FL_STRING) { 4480 key = (void *)(unsigned long)field_contents; 4481 use_compound_key = true; 4482 } else 4483 key = (void *)&field_contents; 4484 } 4485 4486 if (use_compound_key) 4487 add_to_key(compound_key, key, key_field, rec); 4488 } 4489 4490 if (use_compound_key) 4491 key = compound_key; 4492 4493 if (hist_data->n_var_refs && 4494 !resolve_var_refs(hist_data, key, var_ref_vals, false)) 4495 return; 4496 4497 elt = tracing_map_insert(hist_data->map, key); 4498 if (!elt) 4499 return; 4500 4501 hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals); 4502 4503 if (resolve_var_refs(hist_data, key, var_ref_vals, true)) 4504 hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals); 4505 } 4506 4507 static void hist_trigger_stacktrace_print(struct seq_file *m, 4508 unsigned long *stacktrace_entries, 4509 unsigned int max_entries) 4510 { 4511 char str[KSYM_SYMBOL_LEN]; 4512 unsigned int spaces = 8; 4513 unsigned int i; 4514 4515 for (i = 0; i < max_entries; i++) { 4516 if (!stacktrace_entries[i]) 4517 return; 4518 4519 seq_printf(m, "%*c", 1 + spaces, ' '); 4520 sprint_symbol(str, stacktrace_entries[i]); 4521 seq_printf(m, "%s\n", str); 4522 } 4523 } 4524 4525 static void hist_trigger_print_key(struct seq_file *m, 4526 struct hist_trigger_data *hist_data, 4527 void *key, 4528 struct tracing_map_elt *elt) 4529 { 4530 struct hist_field *key_field; 4531 char str[KSYM_SYMBOL_LEN]; 4532 bool multiline = false; 4533 const char *field_name; 4534 unsigned int i; 4535 u64 uval; 4536 4537 seq_puts(m, "{ "); 4538 4539 for_each_hist_key_field(i, hist_data) { 4540 key_field = hist_data->fields[i]; 4541 4542 if (i > hist_data->n_vals) 4543 seq_puts(m, ", "); 4544 4545 field_name = hist_field_name(key_field, 0); 4546 4547 if (key_field->flags & HIST_FIELD_FL_HEX) { 4548 uval = *(u64 *)(key + key_field->offset); 4549 seq_printf(m, "%s: %llx", field_name, uval); 4550 } else if (key_field->flags & HIST_FIELD_FL_SYM) { 4551 uval = *(u64 *)(key + key_field->offset); 4552 sprint_symbol_no_offset(str, uval); 4553 seq_printf(m, "%s: [%llx] %-45s", field_name, 4554 uval, str); 4555 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) { 4556 uval = *(u64 *)(key + key_field->offset); 4557 sprint_symbol(str, uval); 4558 seq_printf(m, "%s: [%llx] %-55s", field_name, 4559 uval, str); 4560 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 4561 struct hist_elt_data *elt_data = elt->private_data; 4562 char *comm; 4563 4564 if (WARN_ON_ONCE(!elt_data)) 4565 return; 4566 4567 comm = elt_data->comm; 4568 4569 uval = *(u64 *)(key + key_field->offset); 4570 seq_printf(m, "%s: %-16s[%10llu]", field_name, 4571 comm, uval); 4572 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) { 4573 const char *syscall_name; 4574 4575 uval = *(u64 *)(key + key_field->offset); 4576 syscall_name = get_syscall_name(uval); 4577 if (!syscall_name) 4578 syscall_name = "unknown_syscall"; 4579 4580 seq_printf(m, "%s: %-30s[%3llu]", field_name, 4581 syscall_name, uval); 4582 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 4583 seq_puts(m, "stacktrace:\n"); 4584 hist_trigger_stacktrace_print(m, 4585 key + key_field->offset, 4586 HIST_STACKTRACE_DEPTH); 4587 multiline = true; 4588 } else if (key_field->flags & HIST_FIELD_FL_LOG2) { 4589 seq_printf(m, "%s: ~ 2^%-2llu", field_name, 4590 *(u64 *)(key + key_field->offset)); 4591 } else if (key_field->flags & HIST_FIELD_FL_STRING) { 4592 seq_printf(m, "%s: %-50s", field_name, 4593 (char *)(key + key_field->offset)); 4594 } else { 4595 uval = *(u64 *)(key + key_field->offset); 4596 seq_printf(m, "%s: %10llu", field_name, uval); 4597 } 4598 } 4599 4600 if (!multiline) 4601 seq_puts(m, " "); 4602 4603 seq_puts(m, "}"); 4604 } 4605 4606 static void hist_trigger_entry_print(struct seq_file *m, 4607 struct hist_trigger_data *hist_data, 4608 void *key, 4609 struct tracing_map_elt *elt) 4610 { 4611 const char *field_name; 4612 unsigned int i; 4613 4614 hist_trigger_print_key(m, hist_data, key, elt); 4615 4616 seq_printf(m, " hitcount: %10llu", 4617 tracing_map_read_sum(elt, HITCOUNT_IDX)); 4618 4619 for (i = 1; i < hist_data->n_vals; i++) { 4620 field_name = hist_field_name(hist_data->fields[i], 0); 4621 4622 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR || 4623 hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR) 4624 continue; 4625 4626 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) { 4627 seq_printf(m, " %s: %10llx", field_name, 4628 tracing_map_read_sum(elt, i)); 4629 } else { 4630 seq_printf(m, " %s: %10llu", field_name, 4631 tracing_map_read_sum(elt, i)); 4632 } 4633 } 4634 4635 print_actions(m, hist_data, elt); 4636 4637 seq_puts(m, "\n"); 4638 } 4639 4640 static int print_entries(struct seq_file *m, 4641 struct hist_trigger_data *hist_data) 4642 { 4643 struct tracing_map_sort_entry **sort_entries = NULL; 4644 struct tracing_map *map = hist_data->map; 4645 int i, n_entries; 4646 4647 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys, 4648 hist_data->n_sort_keys, 4649 &sort_entries); 4650 if (n_entries < 0) 4651 return n_entries; 4652 4653 for (i = 0; i < n_entries; i++) 4654 hist_trigger_entry_print(m, hist_data, 4655 sort_entries[i]->key, 4656 sort_entries[i]->elt); 4657 4658 tracing_map_destroy_sort_entries(sort_entries, n_entries); 4659 4660 return n_entries; 4661 } 4662 4663 static void hist_trigger_show(struct seq_file *m, 4664 struct event_trigger_data *data, int n) 4665 { 4666 struct hist_trigger_data *hist_data; 4667 int n_entries; 4668 4669 if (n > 0) 4670 seq_puts(m, "\n\n"); 4671 4672 seq_puts(m, "# event histogram\n#\n# trigger info: "); 4673 data->ops->print(m, data->ops, data); 4674 seq_puts(m, "#\n\n"); 4675 4676 hist_data = data->private_data; 4677 n_entries = print_entries(m, hist_data); 4678 if (n_entries < 0) 4679 n_entries = 0; 4680 4681 track_data_snapshot_print(m, hist_data); 4682 4683 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n", 4684 (u64)atomic64_read(&hist_data->map->hits), 4685 n_entries, (u64)atomic64_read(&hist_data->map->drops)); 4686 } 4687 4688 static int hist_show(struct seq_file *m, void *v) 4689 { 4690 struct event_trigger_data *data; 4691 struct trace_event_file *event_file; 4692 int n = 0, ret = 0; 4693 4694 mutex_lock(&event_mutex); 4695 4696 event_file = event_file_data(m->private); 4697 if (unlikely(!event_file)) { 4698 ret = -ENODEV; 4699 goto out_unlock; 4700 } 4701 4702 list_for_each_entry(data, &event_file->triggers, list) { 4703 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 4704 hist_trigger_show(m, data, n++); 4705 } 4706 4707 out_unlock: 4708 mutex_unlock(&event_mutex); 4709 4710 return ret; 4711 } 4712 4713 static int event_hist_open(struct inode *inode, struct file *file) 4714 { 4715 int ret; 4716 4717 ret = security_locked_down(LOCKDOWN_TRACEFS); 4718 if (ret) 4719 return ret; 4720 4721 return single_open(file, hist_show, file); 4722 } 4723 4724 const struct file_operations event_hist_fops = { 4725 .open = event_hist_open, 4726 .read = seq_read, 4727 .llseek = seq_lseek, 4728 .release = single_release, 4729 }; 4730 4731 #ifdef CONFIG_HIST_TRIGGERS_DEBUG 4732 static void hist_field_debug_show_flags(struct seq_file *m, 4733 unsigned long flags) 4734 { 4735 seq_puts(m, " flags:\n"); 4736 4737 if (flags & HIST_FIELD_FL_KEY) 4738 seq_puts(m, " HIST_FIELD_FL_KEY\n"); 4739 else if (flags & HIST_FIELD_FL_HITCOUNT) 4740 seq_puts(m, " VAL: HIST_FIELD_FL_HITCOUNT\n"); 4741 else if (flags & HIST_FIELD_FL_VAR) 4742 seq_puts(m, " HIST_FIELD_FL_VAR\n"); 4743 else if (flags & HIST_FIELD_FL_VAR_REF) 4744 seq_puts(m, " HIST_FIELD_FL_VAR_REF\n"); 4745 else 4746 seq_puts(m, " VAL: normal u64 value\n"); 4747 4748 if (flags & HIST_FIELD_FL_ALIAS) 4749 seq_puts(m, " HIST_FIELD_FL_ALIAS\n"); 4750 } 4751 4752 static int hist_field_debug_show(struct seq_file *m, 4753 struct hist_field *field, unsigned long flags) 4754 { 4755 if ((field->flags & flags) != flags) { 4756 seq_printf(m, "ERROR: bad flags - %lx\n", flags); 4757 return -EINVAL; 4758 } 4759 4760 hist_field_debug_show_flags(m, field->flags); 4761 if (field->field) 4762 seq_printf(m, " ftrace_event_field name: %s\n", 4763 field->field->name); 4764 4765 if (field->flags & HIST_FIELD_FL_VAR) { 4766 seq_printf(m, " var.name: %s\n", field->var.name); 4767 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 4768 field->var.idx); 4769 } 4770 4771 if (field->flags & HIST_FIELD_FL_ALIAS) 4772 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n", 4773 field->var_ref_idx); 4774 4775 if (field->flags & HIST_FIELD_FL_VAR_REF) { 4776 seq_printf(m, " name: %s\n", field->name); 4777 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 4778 field->var.idx); 4779 seq_printf(m, " var.hist_data: %p\n", field->var.hist_data); 4780 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n", 4781 field->var_ref_idx); 4782 if (field->system) 4783 seq_printf(m, " system: %s\n", field->system); 4784 if (field->event_name) 4785 seq_printf(m, " event_name: %s\n", field->event_name); 4786 } 4787 4788 seq_printf(m, " type: %s\n", field->type); 4789 seq_printf(m, " size: %u\n", field->size); 4790 seq_printf(m, " is_signed: %u\n", field->is_signed); 4791 4792 return 0; 4793 } 4794 4795 static int field_var_debug_show(struct seq_file *m, 4796 struct field_var *field_var, unsigned int i, 4797 bool save_vars) 4798 { 4799 const char *vars_name = save_vars ? "save_vars" : "field_vars"; 4800 struct hist_field *field; 4801 int ret = 0; 4802 4803 seq_printf(m, "\n hist_data->%s[%d]:\n", vars_name, i); 4804 4805 field = field_var->var; 4806 4807 seq_printf(m, "\n %s[%d].var:\n", vars_name, i); 4808 4809 hist_field_debug_show_flags(m, field->flags); 4810 seq_printf(m, " var.name: %s\n", field->var.name); 4811 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 4812 field->var.idx); 4813 4814 field = field_var->val; 4815 4816 seq_printf(m, "\n %s[%d].val:\n", vars_name, i); 4817 if (field->field) 4818 seq_printf(m, " ftrace_event_field name: %s\n", 4819 field->field->name); 4820 else { 4821 ret = -EINVAL; 4822 goto out; 4823 } 4824 4825 seq_printf(m, " type: %s\n", field->type); 4826 seq_printf(m, " size: %u\n", field->size); 4827 seq_printf(m, " is_signed: %u\n", field->is_signed); 4828 out: 4829 return ret; 4830 } 4831 4832 static int hist_action_debug_show(struct seq_file *m, 4833 struct action_data *data, int i) 4834 { 4835 int ret = 0; 4836 4837 if (data->handler == HANDLER_ONMAX || 4838 data->handler == HANDLER_ONCHANGE) { 4839 seq_printf(m, "\n hist_data->actions[%d].track_data.var_ref:\n", i); 4840 ret = hist_field_debug_show(m, data->track_data.var_ref, 4841 HIST_FIELD_FL_VAR_REF); 4842 if (ret) 4843 goto out; 4844 4845 seq_printf(m, "\n hist_data->actions[%d].track_data.track_var:\n", i); 4846 ret = hist_field_debug_show(m, data->track_data.track_var, 4847 HIST_FIELD_FL_VAR); 4848 if (ret) 4849 goto out; 4850 } 4851 4852 if (data->handler == HANDLER_ONMATCH) { 4853 seq_printf(m, "\n hist_data->actions[%d].match_data.event_system: %s\n", 4854 i, data->match_data.event_system); 4855 seq_printf(m, " hist_data->actions[%d].match_data.event: %s\n", 4856 i, data->match_data.event); 4857 } 4858 out: 4859 return ret; 4860 } 4861 4862 static int hist_actions_debug_show(struct seq_file *m, 4863 struct hist_trigger_data *hist_data) 4864 { 4865 int i, ret = 0; 4866 4867 if (hist_data->n_actions) 4868 seq_puts(m, "\n action tracking variables (for onmax()/onchange()/onmatch()):\n"); 4869 4870 for (i = 0; i < hist_data->n_actions; i++) { 4871 struct action_data *action = hist_data->actions[i]; 4872 4873 ret = hist_action_debug_show(m, action, i); 4874 if (ret) 4875 goto out; 4876 } 4877 4878 if (hist_data->n_save_vars) 4879 seq_puts(m, "\n save action variables (save() params):\n"); 4880 4881 for (i = 0; i < hist_data->n_save_vars; i++) { 4882 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true); 4883 if (ret) 4884 goto out; 4885 } 4886 out: 4887 return ret; 4888 } 4889 4890 static void hist_trigger_debug_show(struct seq_file *m, 4891 struct event_trigger_data *data, int n) 4892 { 4893 struct hist_trigger_data *hist_data; 4894 int i, ret; 4895 4896 if (n > 0) 4897 seq_puts(m, "\n\n"); 4898 4899 seq_puts(m, "# event histogram\n#\n# trigger info: "); 4900 data->ops->print(m, data->ops, data); 4901 seq_puts(m, "#\n\n"); 4902 4903 hist_data = data->private_data; 4904 4905 seq_printf(m, "hist_data: %p\n\n", hist_data); 4906 seq_printf(m, " n_vals: %u\n", hist_data->n_vals); 4907 seq_printf(m, " n_keys: %u\n", hist_data->n_keys); 4908 seq_printf(m, " n_fields: %u\n", hist_data->n_fields); 4909 4910 seq_puts(m, "\n val fields:\n\n"); 4911 4912 seq_puts(m, " hist_data->fields[0]:\n"); 4913 ret = hist_field_debug_show(m, hist_data->fields[0], 4914 HIST_FIELD_FL_HITCOUNT); 4915 if (ret) 4916 return; 4917 4918 for (i = 1; i < hist_data->n_vals; i++) { 4919 seq_printf(m, "\n hist_data->fields[%d]:\n", i); 4920 ret = hist_field_debug_show(m, hist_data->fields[i], 0); 4921 if (ret) 4922 return; 4923 } 4924 4925 seq_puts(m, "\n key fields:\n"); 4926 4927 for (i = hist_data->n_vals; i < hist_data->n_fields; i++) { 4928 seq_printf(m, "\n hist_data->fields[%d]:\n", i); 4929 ret = hist_field_debug_show(m, hist_data->fields[i], 4930 HIST_FIELD_FL_KEY); 4931 if (ret) 4932 return; 4933 } 4934 4935 if (hist_data->n_var_refs) 4936 seq_puts(m, "\n variable reference fields:\n"); 4937 4938 for (i = 0; i < hist_data->n_var_refs; i++) { 4939 seq_printf(m, "\n hist_data->var_refs[%d]:\n", i); 4940 ret = hist_field_debug_show(m, hist_data->var_refs[i], 4941 HIST_FIELD_FL_VAR_REF); 4942 if (ret) 4943 return; 4944 } 4945 4946 if (hist_data->n_field_vars) 4947 seq_puts(m, "\n field variables:\n"); 4948 4949 for (i = 0; i < hist_data->n_field_vars; i++) { 4950 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false); 4951 if (ret) 4952 return; 4953 } 4954 4955 ret = hist_actions_debug_show(m, hist_data); 4956 if (ret) 4957 return; 4958 } 4959 4960 static int hist_debug_show(struct seq_file *m, void *v) 4961 { 4962 struct event_trigger_data *data; 4963 struct trace_event_file *event_file; 4964 int n = 0, ret = 0; 4965 4966 mutex_lock(&event_mutex); 4967 4968 event_file = event_file_data(m->private); 4969 if (unlikely(!event_file)) { 4970 ret = -ENODEV; 4971 goto out_unlock; 4972 } 4973 4974 list_for_each_entry(data, &event_file->triggers, list) { 4975 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 4976 hist_trigger_debug_show(m, data, n++); 4977 } 4978 4979 out_unlock: 4980 mutex_unlock(&event_mutex); 4981 4982 return ret; 4983 } 4984 4985 static int event_hist_debug_open(struct inode *inode, struct file *file) 4986 { 4987 int ret; 4988 4989 ret = security_locked_down(LOCKDOWN_TRACEFS); 4990 if (ret) 4991 return ret; 4992 4993 return single_open(file, hist_debug_show, file); 4994 } 4995 4996 const struct file_operations event_hist_debug_fops = { 4997 .open = event_hist_debug_open, 4998 .read = seq_read, 4999 .llseek = seq_lseek, 5000 .release = single_release, 5001 }; 5002 #endif 5003 5004 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field) 5005 { 5006 const char *field_name = hist_field_name(hist_field, 0); 5007 5008 if (hist_field->var.name) 5009 seq_printf(m, "%s=", hist_field->var.name); 5010 5011 if (hist_field->flags & HIST_FIELD_FL_CPU) 5012 seq_puts(m, "cpu"); 5013 else if (field_name) { 5014 if (hist_field->flags & HIST_FIELD_FL_VAR_REF || 5015 hist_field->flags & HIST_FIELD_FL_ALIAS) 5016 seq_putc(m, '$'); 5017 seq_printf(m, "%s", field_name); 5018 } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP) 5019 seq_puts(m, "common_timestamp"); 5020 5021 if (hist_field->flags) { 5022 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) && 5023 !(hist_field->flags & HIST_FIELD_FL_EXPR)) { 5024 const char *flags = get_hist_field_flags(hist_field); 5025 5026 if (flags) 5027 seq_printf(m, ".%s", flags); 5028 } 5029 } 5030 } 5031 5032 static int event_hist_trigger_print(struct seq_file *m, 5033 struct event_trigger_ops *ops, 5034 struct event_trigger_data *data) 5035 { 5036 struct hist_trigger_data *hist_data = data->private_data; 5037 struct hist_field *field; 5038 bool have_var = false; 5039 unsigned int i; 5040 5041 seq_puts(m, "hist:"); 5042 5043 if (data->name) 5044 seq_printf(m, "%s:", data->name); 5045 5046 seq_puts(m, "keys="); 5047 5048 for_each_hist_key_field(i, hist_data) { 5049 field = hist_data->fields[i]; 5050 5051 if (i > hist_data->n_vals) 5052 seq_puts(m, ","); 5053 5054 if (field->flags & HIST_FIELD_FL_STACKTRACE) 5055 seq_puts(m, "stacktrace"); 5056 else 5057 hist_field_print(m, field); 5058 } 5059 5060 seq_puts(m, ":vals="); 5061 5062 for_each_hist_val_field(i, hist_data) { 5063 field = hist_data->fields[i]; 5064 if (field->flags & HIST_FIELD_FL_VAR) { 5065 have_var = true; 5066 continue; 5067 } 5068 5069 if (i == HITCOUNT_IDX) 5070 seq_puts(m, "hitcount"); 5071 else { 5072 seq_puts(m, ","); 5073 hist_field_print(m, field); 5074 } 5075 } 5076 5077 if (have_var) { 5078 unsigned int n = 0; 5079 5080 seq_puts(m, ":"); 5081 5082 for_each_hist_val_field(i, hist_data) { 5083 field = hist_data->fields[i]; 5084 5085 if (field->flags & HIST_FIELD_FL_VAR) { 5086 if (n++) 5087 seq_puts(m, ","); 5088 hist_field_print(m, field); 5089 } 5090 } 5091 } 5092 5093 seq_puts(m, ":sort="); 5094 5095 for (i = 0; i < hist_data->n_sort_keys; i++) { 5096 struct tracing_map_sort_key *sort_key; 5097 unsigned int idx, first_key_idx; 5098 5099 /* skip VAR vals */ 5100 first_key_idx = hist_data->n_vals - hist_data->n_vars; 5101 5102 sort_key = &hist_data->sort_keys[i]; 5103 idx = sort_key->field_idx; 5104 5105 if (WARN_ON(idx >= HIST_FIELDS_MAX)) 5106 return -EINVAL; 5107 5108 if (i > 0) 5109 seq_puts(m, ","); 5110 5111 if (idx == HITCOUNT_IDX) 5112 seq_puts(m, "hitcount"); 5113 else { 5114 if (idx >= first_key_idx) 5115 idx += hist_data->n_vars; 5116 hist_field_print(m, hist_data->fields[idx]); 5117 } 5118 5119 if (sort_key->descending) 5120 seq_puts(m, ".descending"); 5121 } 5122 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits)); 5123 if (hist_data->enable_timestamps) 5124 seq_printf(m, ":clock=%s", hist_data->attrs->clock); 5125 5126 print_actions_spec(m, hist_data); 5127 5128 if (data->filter_str) 5129 seq_printf(m, " if %s", data->filter_str); 5130 5131 if (data->paused) 5132 seq_puts(m, " [paused]"); 5133 else 5134 seq_puts(m, " [active]"); 5135 5136 seq_putc(m, '\n'); 5137 5138 return 0; 5139 } 5140 5141 static int event_hist_trigger_init(struct event_trigger_ops *ops, 5142 struct event_trigger_data *data) 5143 { 5144 struct hist_trigger_data *hist_data = data->private_data; 5145 5146 if (!data->ref && hist_data->attrs->name) 5147 save_named_trigger(hist_data->attrs->name, data); 5148 5149 data->ref++; 5150 5151 return 0; 5152 } 5153 5154 static void unregister_field_var_hists(struct hist_trigger_data *hist_data) 5155 { 5156 struct trace_event_file *file; 5157 unsigned int i; 5158 char *cmd; 5159 int ret; 5160 5161 for (i = 0; i < hist_data->n_field_var_hists; i++) { 5162 file = hist_data->field_var_hists[i]->hist_data->event_file; 5163 cmd = hist_data->field_var_hists[i]->cmd; 5164 ret = event_hist_trigger_func(&trigger_hist_cmd, file, 5165 "!hist", "hist", cmd); 5166 } 5167 } 5168 5169 static void event_hist_trigger_free(struct event_trigger_ops *ops, 5170 struct event_trigger_data *data) 5171 { 5172 struct hist_trigger_data *hist_data = data->private_data; 5173 5174 if (WARN_ON_ONCE(data->ref <= 0)) 5175 return; 5176 5177 data->ref--; 5178 if (!data->ref) { 5179 if (data->name) 5180 del_named_trigger(data); 5181 5182 trigger_data_free(data); 5183 5184 remove_hist_vars(hist_data); 5185 5186 unregister_field_var_hists(hist_data); 5187 5188 destroy_hist_data(hist_data); 5189 } 5190 } 5191 5192 static struct event_trigger_ops event_hist_trigger_ops = { 5193 .func = event_hist_trigger, 5194 .print = event_hist_trigger_print, 5195 .init = event_hist_trigger_init, 5196 .free = event_hist_trigger_free, 5197 }; 5198 5199 static int event_hist_trigger_named_init(struct event_trigger_ops *ops, 5200 struct event_trigger_data *data) 5201 { 5202 data->ref++; 5203 5204 save_named_trigger(data->named_data->name, data); 5205 5206 event_hist_trigger_init(ops, data->named_data); 5207 5208 return 0; 5209 } 5210 5211 static void event_hist_trigger_named_free(struct event_trigger_ops *ops, 5212 struct event_trigger_data *data) 5213 { 5214 if (WARN_ON_ONCE(data->ref <= 0)) 5215 return; 5216 5217 event_hist_trigger_free(ops, data->named_data); 5218 5219 data->ref--; 5220 if (!data->ref) { 5221 del_named_trigger(data); 5222 trigger_data_free(data); 5223 } 5224 } 5225 5226 static struct event_trigger_ops event_hist_trigger_named_ops = { 5227 .func = event_hist_trigger, 5228 .print = event_hist_trigger_print, 5229 .init = event_hist_trigger_named_init, 5230 .free = event_hist_trigger_named_free, 5231 }; 5232 5233 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd, 5234 char *param) 5235 { 5236 return &event_hist_trigger_ops; 5237 } 5238 5239 static void hist_clear(struct event_trigger_data *data) 5240 { 5241 struct hist_trigger_data *hist_data = data->private_data; 5242 5243 if (data->name) 5244 pause_named_trigger(data); 5245 5246 tracepoint_synchronize_unregister(); 5247 5248 tracing_map_clear(hist_data->map); 5249 5250 if (data->name) 5251 unpause_named_trigger(data); 5252 } 5253 5254 static bool compatible_field(struct ftrace_event_field *field, 5255 struct ftrace_event_field *test_field) 5256 { 5257 if (field == test_field) 5258 return true; 5259 if (field == NULL || test_field == NULL) 5260 return false; 5261 if (strcmp(field->name, test_field->name) != 0) 5262 return false; 5263 if (strcmp(field->type, test_field->type) != 0) 5264 return false; 5265 if (field->size != test_field->size) 5266 return false; 5267 if (field->is_signed != test_field->is_signed) 5268 return false; 5269 5270 return true; 5271 } 5272 5273 static bool hist_trigger_match(struct event_trigger_data *data, 5274 struct event_trigger_data *data_test, 5275 struct event_trigger_data *named_data, 5276 bool ignore_filter) 5277 { 5278 struct tracing_map_sort_key *sort_key, *sort_key_test; 5279 struct hist_trigger_data *hist_data, *hist_data_test; 5280 struct hist_field *key_field, *key_field_test; 5281 unsigned int i; 5282 5283 if (named_data && (named_data != data_test) && 5284 (named_data != data_test->named_data)) 5285 return false; 5286 5287 if (!named_data && is_named_trigger(data_test)) 5288 return false; 5289 5290 hist_data = data->private_data; 5291 hist_data_test = data_test->private_data; 5292 5293 if (hist_data->n_vals != hist_data_test->n_vals || 5294 hist_data->n_fields != hist_data_test->n_fields || 5295 hist_data->n_sort_keys != hist_data_test->n_sort_keys) 5296 return false; 5297 5298 if (!ignore_filter) { 5299 if ((data->filter_str && !data_test->filter_str) || 5300 (!data->filter_str && data_test->filter_str)) 5301 return false; 5302 } 5303 5304 for_each_hist_field(i, hist_data) { 5305 key_field = hist_data->fields[i]; 5306 key_field_test = hist_data_test->fields[i]; 5307 5308 if (key_field->flags != key_field_test->flags) 5309 return false; 5310 if (!compatible_field(key_field->field, key_field_test->field)) 5311 return false; 5312 if (key_field->offset != key_field_test->offset) 5313 return false; 5314 if (key_field->size != key_field_test->size) 5315 return false; 5316 if (key_field->is_signed != key_field_test->is_signed) 5317 return false; 5318 if (!!key_field->var.name != !!key_field_test->var.name) 5319 return false; 5320 if (key_field->var.name && 5321 strcmp(key_field->var.name, key_field_test->var.name) != 0) 5322 return false; 5323 } 5324 5325 for (i = 0; i < hist_data->n_sort_keys; i++) { 5326 sort_key = &hist_data->sort_keys[i]; 5327 sort_key_test = &hist_data_test->sort_keys[i]; 5328 5329 if (sort_key->field_idx != sort_key_test->field_idx || 5330 sort_key->descending != sort_key_test->descending) 5331 return false; 5332 } 5333 5334 if (!ignore_filter && data->filter_str && 5335 (strcmp(data->filter_str, data_test->filter_str) != 0)) 5336 return false; 5337 5338 if (!actions_match(hist_data, hist_data_test)) 5339 return false; 5340 5341 return true; 5342 } 5343 5344 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops, 5345 struct event_trigger_data *data, 5346 struct trace_event_file *file) 5347 { 5348 struct hist_trigger_data *hist_data = data->private_data; 5349 struct event_trigger_data *test, *named_data = NULL; 5350 struct trace_array *tr = file->tr; 5351 int ret = 0; 5352 5353 if (hist_data->attrs->name) { 5354 named_data = find_named_trigger(hist_data->attrs->name); 5355 if (named_data) { 5356 if (!hist_trigger_match(data, named_data, named_data, 5357 true)) { 5358 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name)); 5359 ret = -EINVAL; 5360 goto out; 5361 } 5362 } 5363 } 5364 5365 if (hist_data->attrs->name && !named_data) 5366 goto new; 5367 5368 lockdep_assert_held(&event_mutex); 5369 5370 list_for_each_entry(test, &file->triggers, list) { 5371 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5372 if (!hist_trigger_match(data, test, named_data, false)) 5373 continue; 5374 if (hist_data->attrs->pause) 5375 test->paused = true; 5376 else if (hist_data->attrs->cont) 5377 test->paused = false; 5378 else if (hist_data->attrs->clear) 5379 hist_clear(test); 5380 else { 5381 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0); 5382 ret = -EEXIST; 5383 } 5384 goto out; 5385 } 5386 } 5387 new: 5388 if (hist_data->attrs->cont || hist_data->attrs->clear) { 5389 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0); 5390 ret = -ENOENT; 5391 goto out; 5392 } 5393 5394 if (hist_data->attrs->pause) 5395 data->paused = true; 5396 5397 if (named_data) { 5398 data->private_data = named_data->private_data; 5399 set_named_trigger_data(data, named_data); 5400 data->ops = &event_hist_trigger_named_ops; 5401 } 5402 5403 if (data->ops->init) { 5404 ret = data->ops->init(data->ops, data); 5405 if (ret < 0) 5406 goto out; 5407 } 5408 5409 if (hist_data->enable_timestamps) { 5410 char *clock = hist_data->attrs->clock; 5411 5412 ret = tracing_set_clock(file->tr, hist_data->attrs->clock); 5413 if (ret) { 5414 hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock)); 5415 goto out; 5416 } 5417 5418 tracing_set_time_stamp_abs(file->tr, true); 5419 } 5420 5421 if (named_data) 5422 destroy_hist_data(hist_data); 5423 5424 ret++; 5425 out: 5426 return ret; 5427 } 5428 5429 static int hist_trigger_enable(struct event_trigger_data *data, 5430 struct trace_event_file *file) 5431 { 5432 int ret = 0; 5433 5434 list_add_tail_rcu(&data->list, &file->triggers); 5435 5436 update_cond_flag(file); 5437 5438 if (trace_event_trigger_enable_disable(file, 1) < 0) { 5439 list_del_rcu(&data->list); 5440 update_cond_flag(file); 5441 ret--; 5442 } 5443 5444 return ret; 5445 } 5446 5447 static bool have_hist_trigger_match(struct event_trigger_data *data, 5448 struct trace_event_file *file) 5449 { 5450 struct hist_trigger_data *hist_data = data->private_data; 5451 struct event_trigger_data *test, *named_data = NULL; 5452 bool match = false; 5453 5454 lockdep_assert_held(&event_mutex); 5455 5456 if (hist_data->attrs->name) 5457 named_data = find_named_trigger(hist_data->attrs->name); 5458 5459 list_for_each_entry(test, &file->triggers, list) { 5460 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5461 if (hist_trigger_match(data, test, named_data, false)) { 5462 match = true; 5463 break; 5464 } 5465 } 5466 } 5467 5468 return match; 5469 } 5470 5471 static bool hist_trigger_check_refs(struct event_trigger_data *data, 5472 struct trace_event_file *file) 5473 { 5474 struct hist_trigger_data *hist_data = data->private_data; 5475 struct event_trigger_data *test, *named_data = NULL; 5476 5477 lockdep_assert_held(&event_mutex); 5478 5479 if (hist_data->attrs->name) 5480 named_data = find_named_trigger(hist_data->attrs->name); 5481 5482 list_for_each_entry(test, &file->triggers, list) { 5483 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5484 if (!hist_trigger_match(data, test, named_data, false)) 5485 continue; 5486 hist_data = test->private_data; 5487 if (check_var_refs(hist_data)) 5488 return true; 5489 break; 5490 } 5491 } 5492 5493 return false; 5494 } 5495 5496 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops, 5497 struct event_trigger_data *data, 5498 struct trace_event_file *file) 5499 { 5500 struct hist_trigger_data *hist_data = data->private_data; 5501 struct event_trigger_data *test, *named_data = NULL; 5502 bool unregistered = false; 5503 5504 lockdep_assert_held(&event_mutex); 5505 5506 if (hist_data->attrs->name) 5507 named_data = find_named_trigger(hist_data->attrs->name); 5508 5509 list_for_each_entry(test, &file->triggers, list) { 5510 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5511 if (!hist_trigger_match(data, test, named_data, false)) 5512 continue; 5513 unregistered = true; 5514 list_del_rcu(&test->list); 5515 trace_event_trigger_enable_disable(file, 0); 5516 update_cond_flag(file); 5517 break; 5518 } 5519 } 5520 5521 if (unregistered && test->ops->free) 5522 test->ops->free(test->ops, test); 5523 5524 if (hist_data->enable_timestamps) { 5525 if (!hist_data->remove || unregistered) 5526 tracing_set_time_stamp_abs(file->tr, false); 5527 } 5528 } 5529 5530 static bool hist_file_check_refs(struct trace_event_file *file) 5531 { 5532 struct hist_trigger_data *hist_data; 5533 struct event_trigger_data *test; 5534 5535 lockdep_assert_held(&event_mutex); 5536 5537 list_for_each_entry(test, &file->triggers, list) { 5538 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5539 hist_data = test->private_data; 5540 if (check_var_refs(hist_data)) 5541 return true; 5542 } 5543 } 5544 5545 return false; 5546 } 5547 5548 static void hist_unreg_all(struct trace_event_file *file) 5549 { 5550 struct event_trigger_data *test, *n; 5551 struct hist_trigger_data *hist_data; 5552 struct synth_event *se; 5553 const char *se_name; 5554 5555 lockdep_assert_held(&event_mutex); 5556 5557 if (hist_file_check_refs(file)) 5558 return; 5559 5560 list_for_each_entry_safe(test, n, &file->triggers, list) { 5561 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5562 hist_data = test->private_data; 5563 list_del_rcu(&test->list); 5564 trace_event_trigger_enable_disable(file, 0); 5565 5566 se_name = trace_event_name(file->event_call); 5567 se = find_synth_event(se_name); 5568 if (se) 5569 se->ref--; 5570 5571 update_cond_flag(file); 5572 if (hist_data->enable_timestamps) 5573 tracing_set_time_stamp_abs(file->tr, false); 5574 if (test->ops->free) 5575 test->ops->free(test->ops, test); 5576 } 5577 } 5578 } 5579 5580 static int event_hist_trigger_func(struct event_command *cmd_ops, 5581 struct trace_event_file *file, 5582 char *glob, char *cmd, char *param) 5583 { 5584 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT; 5585 struct event_trigger_data *trigger_data; 5586 struct hist_trigger_attrs *attrs; 5587 struct event_trigger_ops *trigger_ops; 5588 struct hist_trigger_data *hist_data; 5589 struct synth_event *se; 5590 const char *se_name; 5591 bool remove = false; 5592 char *trigger, *p; 5593 int ret = 0; 5594 5595 lockdep_assert_held(&event_mutex); 5596 5597 if (glob && strlen(glob)) { 5598 hist_err_clear(); 5599 last_cmd_set(file, param); 5600 } 5601 5602 if (!param) 5603 return -EINVAL; 5604 5605 if (glob[0] == '!') 5606 remove = true; 5607 5608 /* 5609 * separate the trigger from the filter (k:v [if filter]) 5610 * allowing for whitespace in the trigger 5611 */ 5612 p = trigger = param; 5613 do { 5614 p = strstr(p, "if"); 5615 if (!p) 5616 break; 5617 if (p == param) 5618 return -EINVAL; 5619 if (*(p - 1) != ' ' && *(p - 1) != '\t') { 5620 p++; 5621 continue; 5622 } 5623 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1) 5624 return -EINVAL; 5625 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') { 5626 p++; 5627 continue; 5628 } 5629 break; 5630 } while (p); 5631 5632 if (!p) 5633 param = NULL; 5634 else { 5635 *(p - 1) = '\0'; 5636 param = strstrip(p); 5637 trigger = strstrip(trigger); 5638 } 5639 5640 attrs = parse_hist_trigger_attrs(file->tr, trigger); 5641 if (IS_ERR(attrs)) 5642 return PTR_ERR(attrs); 5643 5644 if (attrs->map_bits) 5645 hist_trigger_bits = attrs->map_bits; 5646 5647 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove); 5648 if (IS_ERR(hist_data)) { 5649 destroy_hist_trigger_attrs(attrs); 5650 return PTR_ERR(hist_data); 5651 } 5652 5653 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger); 5654 5655 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL); 5656 if (!trigger_data) { 5657 ret = -ENOMEM; 5658 goto out_free; 5659 } 5660 5661 trigger_data->count = -1; 5662 trigger_data->ops = trigger_ops; 5663 trigger_data->cmd_ops = cmd_ops; 5664 5665 INIT_LIST_HEAD(&trigger_data->list); 5666 RCU_INIT_POINTER(trigger_data->filter, NULL); 5667 5668 trigger_data->private_data = hist_data; 5669 5670 /* if param is non-empty, it's supposed to be a filter */ 5671 if (param && cmd_ops->set_filter) { 5672 ret = cmd_ops->set_filter(param, trigger_data, file); 5673 if (ret < 0) 5674 goto out_free; 5675 } 5676 5677 if (remove) { 5678 if (!have_hist_trigger_match(trigger_data, file)) 5679 goto out_free; 5680 5681 if (hist_trigger_check_refs(trigger_data, file)) { 5682 ret = -EBUSY; 5683 goto out_free; 5684 } 5685 5686 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file); 5687 se_name = trace_event_name(file->event_call); 5688 se = find_synth_event(se_name); 5689 if (se) 5690 se->ref--; 5691 ret = 0; 5692 goto out_free; 5693 } 5694 5695 ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file); 5696 /* 5697 * The above returns on success the # of triggers registered, 5698 * but if it didn't register any it returns zero. Consider no 5699 * triggers registered a failure too. 5700 */ 5701 if (!ret) { 5702 if (!(attrs->pause || attrs->cont || attrs->clear)) 5703 ret = -ENOENT; 5704 goto out_free; 5705 } else if (ret < 0) 5706 goto out_free; 5707 5708 if (get_named_trigger_data(trigger_data)) 5709 goto enable; 5710 5711 if (has_hist_vars(hist_data)) 5712 save_hist_vars(hist_data); 5713 5714 ret = create_actions(hist_data); 5715 if (ret) 5716 goto out_unreg; 5717 5718 ret = tracing_map_init(hist_data->map); 5719 if (ret) 5720 goto out_unreg; 5721 enable: 5722 ret = hist_trigger_enable(trigger_data, file); 5723 if (ret) 5724 goto out_unreg; 5725 5726 se_name = trace_event_name(file->event_call); 5727 se = find_synth_event(se_name); 5728 if (se) 5729 se->ref++; 5730 /* Just return zero, not the number of registered triggers */ 5731 ret = 0; 5732 out: 5733 if (ret == 0) 5734 hist_err_clear(); 5735 5736 return ret; 5737 out_unreg: 5738 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file); 5739 out_free: 5740 if (cmd_ops->set_filter) 5741 cmd_ops->set_filter(NULL, trigger_data, NULL); 5742 5743 remove_hist_vars(hist_data); 5744 5745 kfree(trigger_data); 5746 5747 destroy_hist_data(hist_data); 5748 goto out; 5749 } 5750 5751 static struct event_command trigger_hist_cmd = { 5752 .name = "hist", 5753 .trigger_type = ETT_EVENT_HIST, 5754 .flags = EVENT_CMD_FL_NEEDS_REC, 5755 .func = event_hist_trigger_func, 5756 .reg = hist_register_trigger, 5757 .unreg = hist_unregister_trigger, 5758 .unreg_all = hist_unreg_all, 5759 .get_trigger_ops = event_hist_get_trigger_ops, 5760 .set_filter = set_trigger_filter, 5761 }; 5762 5763 __init int register_trigger_hist_cmd(void) 5764 { 5765 int ret; 5766 5767 ret = register_event_command(&trigger_hist_cmd); 5768 WARN_ON(ret < 0); 5769 5770 return ret; 5771 } 5772 5773 static void 5774 hist_enable_trigger(struct event_trigger_data *data, void *rec, 5775 struct ring_buffer_event *event) 5776 { 5777 struct enable_trigger_data *enable_data = data->private_data; 5778 struct event_trigger_data *test; 5779 5780 list_for_each_entry_rcu(test, &enable_data->file->triggers, list, 5781 lockdep_is_held(&event_mutex)) { 5782 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5783 if (enable_data->enable) 5784 test->paused = false; 5785 else 5786 test->paused = true; 5787 } 5788 } 5789 } 5790 5791 static void 5792 hist_enable_count_trigger(struct event_trigger_data *data, void *rec, 5793 struct ring_buffer_event *event) 5794 { 5795 if (!data->count) 5796 return; 5797 5798 if (data->count != -1) 5799 (data->count)--; 5800 5801 hist_enable_trigger(data, rec, event); 5802 } 5803 5804 static struct event_trigger_ops hist_enable_trigger_ops = { 5805 .func = hist_enable_trigger, 5806 .print = event_enable_trigger_print, 5807 .init = event_trigger_init, 5808 .free = event_enable_trigger_free, 5809 }; 5810 5811 static struct event_trigger_ops hist_enable_count_trigger_ops = { 5812 .func = hist_enable_count_trigger, 5813 .print = event_enable_trigger_print, 5814 .init = event_trigger_init, 5815 .free = event_enable_trigger_free, 5816 }; 5817 5818 static struct event_trigger_ops hist_disable_trigger_ops = { 5819 .func = hist_enable_trigger, 5820 .print = event_enable_trigger_print, 5821 .init = event_trigger_init, 5822 .free = event_enable_trigger_free, 5823 }; 5824 5825 static struct event_trigger_ops hist_disable_count_trigger_ops = { 5826 .func = hist_enable_count_trigger, 5827 .print = event_enable_trigger_print, 5828 .init = event_trigger_init, 5829 .free = event_enable_trigger_free, 5830 }; 5831 5832 static struct event_trigger_ops * 5833 hist_enable_get_trigger_ops(char *cmd, char *param) 5834 { 5835 struct event_trigger_ops *ops; 5836 bool enable; 5837 5838 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0); 5839 5840 if (enable) 5841 ops = param ? &hist_enable_count_trigger_ops : 5842 &hist_enable_trigger_ops; 5843 else 5844 ops = param ? &hist_disable_count_trigger_ops : 5845 &hist_disable_trigger_ops; 5846 5847 return ops; 5848 } 5849 5850 static void hist_enable_unreg_all(struct trace_event_file *file) 5851 { 5852 struct event_trigger_data *test, *n; 5853 5854 list_for_each_entry_safe(test, n, &file->triggers, list) { 5855 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) { 5856 list_del_rcu(&test->list); 5857 update_cond_flag(file); 5858 trace_event_trigger_enable_disable(file, 0); 5859 if (test->ops->free) 5860 test->ops->free(test->ops, test); 5861 } 5862 } 5863 } 5864 5865 static struct event_command trigger_hist_enable_cmd = { 5866 .name = ENABLE_HIST_STR, 5867 .trigger_type = ETT_HIST_ENABLE, 5868 .func = event_enable_trigger_func, 5869 .reg = event_enable_register_trigger, 5870 .unreg = event_enable_unregister_trigger, 5871 .unreg_all = hist_enable_unreg_all, 5872 .get_trigger_ops = hist_enable_get_trigger_ops, 5873 .set_filter = set_trigger_filter, 5874 }; 5875 5876 static struct event_command trigger_hist_disable_cmd = { 5877 .name = DISABLE_HIST_STR, 5878 .trigger_type = ETT_HIST_ENABLE, 5879 .func = event_enable_trigger_func, 5880 .reg = event_enable_register_trigger, 5881 .unreg = event_enable_unregister_trigger, 5882 .unreg_all = hist_enable_unreg_all, 5883 .get_trigger_ops = hist_enable_get_trigger_ops, 5884 .set_filter = set_trigger_filter, 5885 }; 5886 5887 static __init void unregister_trigger_hist_enable_disable_cmds(void) 5888 { 5889 unregister_event_command(&trigger_hist_enable_cmd); 5890 unregister_event_command(&trigger_hist_disable_cmd); 5891 } 5892 5893 __init int register_trigger_hist_enable_disable_cmds(void) 5894 { 5895 int ret; 5896 5897 ret = register_event_command(&trigger_hist_enable_cmd); 5898 if (WARN_ON(ret < 0)) 5899 return ret; 5900 ret = register_event_command(&trigger_hist_disable_cmd); 5901 if (WARN_ON(ret < 0)) 5902 unregister_trigger_hist_enable_disable_cmds(); 5903 5904 return ret; 5905 } 5906