1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * trace_events_hist - trace event hist triggers 4 * 5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com> 6 */ 7 8 #include <linux/module.h> 9 #include <linux/kallsyms.h> 10 #include <linux/security.h> 11 #include <linux/mutex.h> 12 #include <linux/slab.h> 13 #include <linux/stacktrace.h> 14 #include <linux/rculist.h> 15 #include <linux/tracefs.h> 16 17 /* for gfp flag names */ 18 #include <linux/trace_events.h> 19 #include <trace/events/mmflags.h> 20 21 #include "tracing_map.h" 22 #include "trace_synth.h" 23 24 #define ERRORS \ 25 C(NONE, "No error"), \ 26 C(DUPLICATE_VAR, "Variable already defined"), \ 27 C(VAR_NOT_UNIQUE, "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \ 28 C(TOO_MANY_VARS, "Too many variables defined"), \ 29 C(MALFORMED_ASSIGNMENT, "Malformed assignment"), \ 30 C(NAMED_MISMATCH, "Named hist trigger doesn't match existing named trigger (includes variables)"), \ 31 C(TRIGGER_EEXIST, "Hist trigger already exists"), \ 32 C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \ 33 C(SET_CLOCK_FAIL, "Couldn't set trace_clock"), \ 34 C(BAD_FIELD_MODIFIER, "Invalid field modifier"), \ 35 C(TOO_MANY_SUBEXPR, "Too many subexpressions (3 max)"), \ 36 C(TIMESTAMP_MISMATCH, "Timestamp units in expression don't match"), \ 37 C(TOO_MANY_FIELD_VARS, "Too many field variables defined"), \ 38 C(EVENT_FILE_NOT_FOUND, "Event file not found"), \ 39 C(HIST_NOT_FOUND, "Matching event histogram not found"), \ 40 C(HIST_CREATE_FAIL, "Couldn't create histogram for field"), \ 41 C(SYNTH_VAR_NOT_FOUND, "Couldn't find synthetic variable"), \ 42 C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"), \ 43 C(SYNTH_TYPE_MISMATCH, "Param type doesn't match synthetic event field type"), \ 44 C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \ 45 C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"), \ 46 C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"), \ 47 C(ONX_NOT_VAR, "For onmax(x) or onchange(x), x must be a variable"), \ 48 C(ONX_VAR_NOT_FOUND, "Couldn't find onmax or onchange variable"), \ 49 C(ONX_VAR_CREATE_FAIL, "Couldn't create onmax or onchange variable"), \ 50 C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"), \ 51 C(TOO_MANY_PARAMS, "Too many action params"), \ 52 C(PARAM_NOT_FOUND, "Couldn't find param"), \ 53 C(INVALID_PARAM, "Invalid action param"), \ 54 C(ACTION_NOT_FOUND, "No action found"), \ 55 C(NO_SAVE_PARAMS, "No params found for save()"), \ 56 C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \ 57 C(ACTION_MISMATCH, "Handler doesn't support action"), \ 58 C(NO_CLOSING_PAREN, "No closing paren found"), \ 59 C(SUBSYS_NOT_FOUND, "Missing subsystem"), \ 60 C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"), \ 61 C(INVALID_REF_KEY, "Using variable references in keys not supported"), \ 62 C(VAR_NOT_FOUND, "Couldn't find variable"), \ 63 C(FIELD_NOT_FOUND, "Couldn't find field"), \ 64 C(EMPTY_ASSIGNMENT, "Empty assignment"), \ 65 C(INVALID_SORT_MODIFIER,"Invalid sort modifier"), \ 66 C(EMPTY_SORT_FIELD, "Empty sort field"), \ 67 C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"), \ 68 C(INVALID_SORT_FIELD, "Sort field must be a key or a val"), 69 70 #undef C 71 #define C(a, b) HIST_ERR_##a 72 73 enum { ERRORS }; 74 75 #undef C 76 #define C(a, b) b 77 78 static const char *err_text[] = { ERRORS }; 79 80 struct hist_field; 81 82 typedef u64 (*hist_field_fn_t) (struct hist_field *field, 83 struct tracing_map_elt *elt, 84 struct ring_buffer_event *rbe, 85 void *event); 86 87 #define HIST_FIELD_OPERANDS_MAX 2 88 #define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX) 89 #define HIST_ACTIONS_MAX 8 90 91 enum field_op_id { 92 FIELD_OP_NONE, 93 FIELD_OP_PLUS, 94 FIELD_OP_MINUS, 95 FIELD_OP_UNARY_MINUS, 96 }; 97 98 /* 99 * A hist_var (histogram variable) contains variable information for 100 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF 101 * flag set. A hist_var has a variable name e.g. ts0, and is 102 * associated with a given histogram trigger, as specified by 103 * hist_data. The hist_var idx is the unique index assigned to the 104 * variable by the hist trigger's tracing_map. The idx is what is 105 * used to set a variable's value and, by a variable reference, to 106 * retrieve it. 107 */ 108 struct hist_var { 109 char *name; 110 struct hist_trigger_data *hist_data; 111 unsigned int idx; 112 }; 113 114 struct hist_field { 115 struct ftrace_event_field *field; 116 unsigned long flags; 117 hist_field_fn_t fn; 118 unsigned int ref; 119 unsigned int size; 120 unsigned int offset; 121 unsigned int is_signed; 122 const char *type; 123 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX]; 124 struct hist_trigger_data *hist_data; 125 126 /* 127 * Variable fields contain variable-specific info in var. 128 */ 129 struct hist_var var; 130 enum field_op_id operator; 131 char *system; 132 char *event_name; 133 134 /* 135 * The name field is used for EXPR and VAR_REF fields. VAR 136 * fields contain the variable name in var.name. 137 */ 138 char *name; 139 140 /* 141 * When a histogram trigger is hit, if it has any references 142 * to variables, the values of those variables are collected 143 * into a var_ref_vals array by resolve_var_refs(). The 144 * current value of each variable is read from the tracing_map 145 * using the hist field's hist_var.idx and entered into the 146 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx]. 147 */ 148 unsigned int var_ref_idx; 149 bool read_once; 150 }; 151 152 static u64 hist_field_none(struct hist_field *field, 153 struct tracing_map_elt *elt, 154 struct ring_buffer_event *rbe, 155 void *event) 156 { 157 return 0; 158 } 159 160 static u64 hist_field_counter(struct hist_field *field, 161 struct tracing_map_elt *elt, 162 struct ring_buffer_event *rbe, 163 void *event) 164 { 165 return 1; 166 } 167 168 static u64 hist_field_string(struct hist_field *hist_field, 169 struct tracing_map_elt *elt, 170 struct ring_buffer_event *rbe, 171 void *event) 172 { 173 char *addr = (char *)(event + hist_field->field->offset); 174 175 return (u64)(unsigned long)addr; 176 } 177 178 static u64 hist_field_dynstring(struct hist_field *hist_field, 179 struct tracing_map_elt *elt, 180 struct ring_buffer_event *rbe, 181 void *event) 182 { 183 u32 str_item = *(u32 *)(event + hist_field->field->offset); 184 int str_loc = str_item & 0xffff; 185 char *addr = (char *)(event + str_loc); 186 187 return (u64)(unsigned long)addr; 188 } 189 190 static u64 hist_field_pstring(struct hist_field *hist_field, 191 struct tracing_map_elt *elt, 192 struct ring_buffer_event *rbe, 193 void *event) 194 { 195 char **addr = (char **)(event + hist_field->field->offset); 196 197 return (u64)(unsigned long)*addr; 198 } 199 200 static u64 hist_field_log2(struct hist_field *hist_field, 201 struct tracing_map_elt *elt, 202 struct ring_buffer_event *rbe, 203 void *event) 204 { 205 struct hist_field *operand = hist_field->operands[0]; 206 207 u64 val = operand->fn(operand, elt, rbe, event); 208 209 return (u64) ilog2(roundup_pow_of_two(val)); 210 } 211 212 static u64 hist_field_plus(struct hist_field *hist_field, 213 struct tracing_map_elt *elt, 214 struct ring_buffer_event *rbe, 215 void *event) 216 { 217 struct hist_field *operand1 = hist_field->operands[0]; 218 struct hist_field *operand2 = hist_field->operands[1]; 219 220 u64 val1 = operand1->fn(operand1, elt, rbe, event); 221 u64 val2 = operand2->fn(operand2, elt, rbe, event); 222 223 return val1 + val2; 224 } 225 226 static u64 hist_field_minus(struct hist_field *hist_field, 227 struct tracing_map_elt *elt, 228 struct ring_buffer_event *rbe, 229 void *event) 230 { 231 struct hist_field *operand1 = hist_field->operands[0]; 232 struct hist_field *operand2 = hist_field->operands[1]; 233 234 u64 val1 = operand1->fn(operand1, elt, rbe, event); 235 u64 val2 = operand2->fn(operand2, elt, rbe, event); 236 237 return val1 - val2; 238 } 239 240 static u64 hist_field_unary_minus(struct hist_field *hist_field, 241 struct tracing_map_elt *elt, 242 struct ring_buffer_event *rbe, 243 void *event) 244 { 245 struct hist_field *operand = hist_field->operands[0]; 246 247 s64 sval = (s64)operand->fn(operand, elt, rbe, event); 248 u64 val = (u64)-sval; 249 250 return val; 251 } 252 253 #define DEFINE_HIST_FIELD_FN(type) \ 254 static u64 hist_field_##type(struct hist_field *hist_field, \ 255 struct tracing_map_elt *elt, \ 256 struct ring_buffer_event *rbe, \ 257 void *event) \ 258 { \ 259 type *addr = (type *)(event + hist_field->field->offset); \ 260 \ 261 return (u64)(unsigned long)*addr; \ 262 } 263 264 DEFINE_HIST_FIELD_FN(s64); 265 DEFINE_HIST_FIELD_FN(u64); 266 DEFINE_HIST_FIELD_FN(s32); 267 DEFINE_HIST_FIELD_FN(u32); 268 DEFINE_HIST_FIELD_FN(s16); 269 DEFINE_HIST_FIELD_FN(u16); 270 DEFINE_HIST_FIELD_FN(s8); 271 DEFINE_HIST_FIELD_FN(u8); 272 273 #define for_each_hist_field(i, hist_data) \ 274 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++) 275 276 #define for_each_hist_val_field(i, hist_data) \ 277 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++) 278 279 #define for_each_hist_key_field(i, hist_data) \ 280 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++) 281 282 #define HIST_STACKTRACE_DEPTH 16 283 #define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long)) 284 #define HIST_STACKTRACE_SKIP 5 285 286 #define HITCOUNT_IDX 0 287 #define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE) 288 289 enum hist_field_flags { 290 HIST_FIELD_FL_HITCOUNT = 1 << 0, 291 HIST_FIELD_FL_KEY = 1 << 1, 292 HIST_FIELD_FL_STRING = 1 << 2, 293 HIST_FIELD_FL_HEX = 1 << 3, 294 HIST_FIELD_FL_SYM = 1 << 4, 295 HIST_FIELD_FL_SYM_OFFSET = 1 << 5, 296 HIST_FIELD_FL_EXECNAME = 1 << 6, 297 HIST_FIELD_FL_SYSCALL = 1 << 7, 298 HIST_FIELD_FL_STACKTRACE = 1 << 8, 299 HIST_FIELD_FL_LOG2 = 1 << 9, 300 HIST_FIELD_FL_TIMESTAMP = 1 << 10, 301 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11, 302 HIST_FIELD_FL_VAR = 1 << 12, 303 HIST_FIELD_FL_EXPR = 1 << 13, 304 HIST_FIELD_FL_VAR_REF = 1 << 14, 305 HIST_FIELD_FL_CPU = 1 << 15, 306 HIST_FIELD_FL_ALIAS = 1 << 16, 307 }; 308 309 struct var_defs { 310 unsigned int n_vars; 311 char *name[TRACING_MAP_VARS_MAX]; 312 char *expr[TRACING_MAP_VARS_MAX]; 313 }; 314 315 struct hist_trigger_attrs { 316 char *keys_str; 317 char *vals_str; 318 char *sort_key_str; 319 char *name; 320 char *clock; 321 bool pause; 322 bool cont; 323 bool clear; 324 bool ts_in_usecs; 325 unsigned int map_bits; 326 327 char *assignment_str[TRACING_MAP_VARS_MAX]; 328 unsigned int n_assignments; 329 330 char *action_str[HIST_ACTIONS_MAX]; 331 unsigned int n_actions; 332 333 struct var_defs var_defs; 334 }; 335 336 struct field_var { 337 struct hist_field *var; 338 struct hist_field *val; 339 }; 340 341 struct field_var_hist { 342 struct hist_trigger_data *hist_data; 343 char *cmd; 344 }; 345 346 struct hist_trigger_data { 347 struct hist_field *fields[HIST_FIELDS_MAX]; 348 unsigned int n_vals; 349 unsigned int n_keys; 350 unsigned int n_fields; 351 unsigned int n_vars; 352 unsigned int key_size; 353 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX]; 354 unsigned int n_sort_keys; 355 struct trace_event_file *event_file; 356 struct hist_trigger_attrs *attrs; 357 struct tracing_map *map; 358 bool enable_timestamps; 359 bool remove; 360 struct hist_field *var_refs[TRACING_MAP_VARS_MAX]; 361 unsigned int n_var_refs; 362 363 struct action_data *actions[HIST_ACTIONS_MAX]; 364 unsigned int n_actions; 365 366 struct field_var *field_vars[SYNTH_FIELDS_MAX]; 367 unsigned int n_field_vars; 368 unsigned int n_field_var_str; 369 struct field_var_hist *field_var_hists[SYNTH_FIELDS_MAX]; 370 unsigned int n_field_var_hists; 371 372 struct field_var *save_vars[SYNTH_FIELDS_MAX]; 373 unsigned int n_save_vars; 374 unsigned int n_save_var_str; 375 }; 376 377 struct action_data; 378 379 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data, 380 struct tracing_map_elt *elt, void *rec, 381 struct ring_buffer_event *rbe, void *key, 382 struct action_data *data, u64 *var_ref_vals); 383 384 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val); 385 386 enum handler_id { 387 HANDLER_ONMATCH = 1, 388 HANDLER_ONMAX, 389 HANDLER_ONCHANGE, 390 }; 391 392 enum action_id { 393 ACTION_SAVE = 1, 394 ACTION_TRACE, 395 ACTION_SNAPSHOT, 396 }; 397 398 struct action_data { 399 enum handler_id handler; 400 enum action_id action; 401 char *action_name; 402 action_fn_t fn; 403 404 unsigned int n_params; 405 char *params[SYNTH_FIELDS_MAX]; 406 407 /* 408 * When a histogram trigger is hit, the values of any 409 * references to variables, including variables being passed 410 * as parameters to synthetic events, are collected into a 411 * var_ref_vals array. This var_ref_idx array is an array of 412 * indices into the var_ref_vals array, one for each synthetic 413 * event param, and is passed to the synthetic event 414 * invocation. 415 */ 416 unsigned int var_ref_idx[TRACING_MAP_VARS_MAX]; 417 struct synth_event *synth_event; 418 bool use_trace_keyword; 419 char *synth_event_name; 420 421 union { 422 struct { 423 char *event; 424 char *event_system; 425 } match_data; 426 427 struct { 428 /* 429 * var_str contains the $-unstripped variable 430 * name referenced by var_ref, and used when 431 * printing the action. Because var_ref 432 * creation is deferred to create_actions(), 433 * we need a per-action way to save it until 434 * then, thus var_str. 435 */ 436 char *var_str; 437 438 /* 439 * var_ref refers to the variable being 440 * tracked e.g onmax($var). 441 */ 442 struct hist_field *var_ref; 443 444 /* 445 * track_var contains the 'invisible' tracking 446 * variable created to keep the current 447 * e.g. max value. 448 */ 449 struct hist_field *track_var; 450 451 check_track_val_fn_t check_val; 452 action_fn_t save_data; 453 } track_data; 454 }; 455 }; 456 457 struct track_data { 458 u64 track_val; 459 bool updated; 460 461 unsigned int key_len; 462 void *key; 463 struct tracing_map_elt elt; 464 465 struct action_data *action_data; 466 struct hist_trigger_data *hist_data; 467 }; 468 469 struct hist_elt_data { 470 char *comm; 471 u64 *var_ref_vals; 472 char *field_var_str[SYNTH_FIELDS_MAX]; 473 }; 474 475 struct snapshot_context { 476 struct tracing_map_elt *elt; 477 void *key; 478 }; 479 480 static void track_data_free(struct track_data *track_data) 481 { 482 struct hist_elt_data *elt_data; 483 484 if (!track_data) 485 return; 486 487 kfree(track_data->key); 488 489 elt_data = track_data->elt.private_data; 490 if (elt_data) { 491 kfree(elt_data->comm); 492 kfree(elt_data); 493 } 494 495 kfree(track_data); 496 } 497 498 static struct track_data *track_data_alloc(unsigned int key_len, 499 struct action_data *action_data, 500 struct hist_trigger_data *hist_data) 501 { 502 struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL); 503 struct hist_elt_data *elt_data; 504 505 if (!data) 506 return ERR_PTR(-ENOMEM); 507 508 data->key = kzalloc(key_len, GFP_KERNEL); 509 if (!data->key) { 510 track_data_free(data); 511 return ERR_PTR(-ENOMEM); 512 } 513 514 data->key_len = key_len; 515 data->action_data = action_data; 516 data->hist_data = hist_data; 517 518 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL); 519 if (!elt_data) { 520 track_data_free(data); 521 return ERR_PTR(-ENOMEM); 522 } 523 524 data->elt.private_data = elt_data; 525 526 elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL); 527 if (!elt_data->comm) { 528 track_data_free(data); 529 return ERR_PTR(-ENOMEM); 530 } 531 532 return data; 533 } 534 535 static char last_cmd[MAX_FILTER_STR_VAL]; 536 static char last_cmd_loc[MAX_FILTER_STR_VAL]; 537 538 static int errpos(char *str) 539 { 540 return err_pos(last_cmd, str); 541 } 542 543 static void last_cmd_set(struct trace_event_file *file, char *str) 544 { 545 const char *system = NULL, *name = NULL; 546 struct trace_event_call *call; 547 548 if (!str) 549 return; 550 551 strcpy(last_cmd, "hist:"); 552 strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:")); 553 554 if (file) { 555 call = file->event_call; 556 system = call->class->system; 557 if (system) { 558 name = trace_event_name(call); 559 if (!name) 560 system = NULL; 561 } 562 } 563 564 if (system) 565 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name); 566 } 567 568 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos) 569 { 570 tracing_log_err(tr, last_cmd_loc, last_cmd, err_text, 571 err_type, err_pos); 572 } 573 574 static void hist_err_clear(void) 575 { 576 last_cmd[0] = '\0'; 577 last_cmd_loc[0] = '\0'; 578 } 579 580 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals, 581 unsigned int *var_ref_idx); 582 583 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals, 584 unsigned int *var_ref_idx) 585 { 586 struct tracepoint *tp = event->tp; 587 588 if (unlikely(atomic_read(&tp->key.enabled) > 0)) { 589 struct tracepoint_func *probe_func_ptr; 590 synth_probe_func_t probe_func; 591 void *__data; 592 593 if (!(cpu_online(raw_smp_processor_id()))) 594 return; 595 596 probe_func_ptr = rcu_dereference_sched((tp)->funcs); 597 if (probe_func_ptr) { 598 do { 599 probe_func = probe_func_ptr->func; 600 __data = probe_func_ptr->data; 601 probe_func(__data, var_ref_vals, var_ref_idx); 602 } while ((++probe_func_ptr)->func); 603 } 604 } 605 } 606 607 static void action_trace(struct hist_trigger_data *hist_data, 608 struct tracing_map_elt *elt, void *rec, 609 struct ring_buffer_event *rbe, void *key, 610 struct action_data *data, u64 *var_ref_vals) 611 { 612 struct synth_event *event = data->synth_event; 613 614 trace_synth(event, var_ref_vals, data->var_ref_idx); 615 } 616 617 struct hist_var_data { 618 struct list_head list; 619 struct hist_trigger_data *hist_data; 620 }; 621 622 static u64 hist_field_timestamp(struct hist_field *hist_field, 623 struct tracing_map_elt *elt, 624 struct ring_buffer_event *rbe, 625 void *event) 626 { 627 struct hist_trigger_data *hist_data = hist_field->hist_data; 628 struct trace_array *tr = hist_data->event_file->tr; 629 630 u64 ts = ring_buffer_event_time_stamp(rbe); 631 632 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr)) 633 ts = ns2usecs(ts); 634 635 return ts; 636 } 637 638 static u64 hist_field_cpu(struct hist_field *hist_field, 639 struct tracing_map_elt *elt, 640 struct ring_buffer_event *rbe, 641 void *event) 642 { 643 int cpu = smp_processor_id(); 644 645 return cpu; 646 } 647 648 /** 649 * check_field_for_var_ref - Check if a VAR_REF field references a variable 650 * @hist_field: The VAR_REF field to check 651 * @var_data: The hist trigger that owns the variable 652 * @var_idx: The trigger variable identifier 653 * 654 * Check the given VAR_REF field to see whether or not it references 655 * the given variable associated with the given trigger. 656 * 657 * Return: The VAR_REF field if it does reference the variable, NULL if not 658 */ 659 static struct hist_field * 660 check_field_for_var_ref(struct hist_field *hist_field, 661 struct hist_trigger_data *var_data, 662 unsigned int var_idx) 663 { 664 WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF)); 665 666 if (hist_field && hist_field->var.idx == var_idx && 667 hist_field->var.hist_data == var_data) 668 return hist_field; 669 670 return NULL; 671 } 672 673 /** 674 * find_var_ref - Check if a trigger has a reference to a trigger variable 675 * @hist_data: The hist trigger that might have a reference to the variable 676 * @var_data: The hist trigger that owns the variable 677 * @var_idx: The trigger variable identifier 678 * 679 * Check the list of var_refs[] on the first hist trigger to see 680 * whether any of them are references to the variable on the second 681 * trigger. 682 * 683 * Return: The VAR_REF field referencing the variable if so, NULL if not 684 */ 685 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data, 686 struct hist_trigger_data *var_data, 687 unsigned int var_idx) 688 { 689 struct hist_field *hist_field; 690 unsigned int i; 691 692 for (i = 0; i < hist_data->n_var_refs; i++) { 693 hist_field = hist_data->var_refs[i]; 694 if (check_field_for_var_ref(hist_field, var_data, var_idx)) 695 return hist_field; 696 } 697 698 return NULL; 699 } 700 701 /** 702 * find_any_var_ref - Check if there is a reference to a given trigger variable 703 * @hist_data: The hist trigger 704 * @var_idx: The trigger variable identifier 705 * 706 * Check to see whether the given variable is currently referenced by 707 * any other trigger. 708 * 709 * The trigger the variable is defined on is explicitly excluded - the 710 * assumption being that a self-reference doesn't prevent a trigger 711 * from being removed. 712 * 713 * Return: The VAR_REF field referencing the variable if so, NULL if not 714 */ 715 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data, 716 unsigned int var_idx) 717 { 718 struct trace_array *tr = hist_data->event_file->tr; 719 struct hist_field *found = NULL; 720 struct hist_var_data *var_data; 721 722 list_for_each_entry(var_data, &tr->hist_vars, list) { 723 if (var_data->hist_data == hist_data) 724 continue; 725 found = find_var_ref(var_data->hist_data, hist_data, var_idx); 726 if (found) 727 break; 728 } 729 730 return found; 731 } 732 733 /** 734 * check_var_refs - Check if there is a reference to any of trigger's variables 735 * @hist_data: The hist trigger 736 * 737 * A trigger can define one or more variables. If any one of them is 738 * currently referenced by any other trigger, this function will 739 * determine that. 740 741 * Typically used to determine whether or not a trigger can be removed 742 * - if there are any references to a trigger's variables, it cannot. 743 * 744 * Return: True if there is a reference to any of trigger's variables 745 */ 746 static bool check_var_refs(struct hist_trigger_data *hist_data) 747 { 748 struct hist_field *field; 749 bool found = false; 750 int i; 751 752 for_each_hist_field(i, hist_data) { 753 field = hist_data->fields[i]; 754 if (field && field->flags & HIST_FIELD_FL_VAR) { 755 if (find_any_var_ref(hist_data, field->var.idx)) { 756 found = true; 757 break; 758 } 759 } 760 } 761 762 return found; 763 } 764 765 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data) 766 { 767 struct trace_array *tr = hist_data->event_file->tr; 768 struct hist_var_data *var_data, *found = NULL; 769 770 list_for_each_entry(var_data, &tr->hist_vars, list) { 771 if (var_data->hist_data == hist_data) { 772 found = var_data; 773 break; 774 } 775 } 776 777 return found; 778 } 779 780 static bool field_has_hist_vars(struct hist_field *hist_field, 781 unsigned int level) 782 { 783 int i; 784 785 if (level > 3) 786 return false; 787 788 if (!hist_field) 789 return false; 790 791 if (hist_field->flags & HIST_FIELD_FL_VAR || 792 hist_field->flags & HIST_FIELD_FL_VAR_REF) 793 return true; 794 795 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) { 796 struct hist_field *operand; 797 798 operand = hist_field->operands[i]; 799 if (field_has_hist_vars(operand, level + 1)) 800 return true; 801 } 802 803 return false; 804 } 805 806 static bool has_hist_vars(struct hist_trigger_data *hist_data) 807 { 808 struct hist_field *hist_field; 809 int i; 810 811 for_each_hist_field(i, hist_data) { 812 hist_field = hist_data->fields[i]; 813 if (field_has_hist_vars(hist_field, 0)) 814 return true; 815 } 816 817 return false; 818 } 819 820 static int save_hist_vars(struct hist_trigger_data *hist_data) 821 { 822 struct trace_array *tr = hist_data->event_file->tr; 823 struct hist_var_data *var_data; 824 825 var_data = find_hist_vars(hist_data); 826 if (var_data) 827 return 0; 828 829 if (tracing_check_open_get_tr(tr)) 830 return -ENODEV; 831 832 var_data = kzalloc(sizeof(*var_data), GFP_KERNEL); 833 if (!var_data) { 834 trace_array_put(tr); 835 return -ENOMEM; 836 } 837 838 var_data->hist_data = hist_data; 839 list_add(&var_data->list, &tr->hist_vars); 840 841 return 0; 842 } 843 844 static void remove_hist_vars(struct hist_trigger_data *hist_data) 845 { 846 struct trace_array *tr = hist_data->event_file->tr; 847 struct hist_var_data *var_data; 848 849 var_data = find_hist_vars(hist_data); 850 if (!var_data) 851 return; 852 853 if (WARN_ON(check_var_refs(hist_data))) 854 return; 855 856 list_del(&var_data->list); 857 858 kfree(var_data); 859 860 trace_array_put(tr); 861 } 862 863 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data, 864 const char *var_name) 865 { 866 struct hist_field *hist_field, *found = NULL; 867 int i; 868 869 for_each_hist_field(i, hist_data) { 870 hist_field = hist_data->fields[i]; 871 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR && 872 strcmp(hist_field->var.name, var_name) == 0) { 873 found = hist_field; 874 break; 875 } 876 } 877 878 return found; 879 } 880 881 static struct hist_field *find_var(struct hist_trigger_data *hist_data, 882 struct trace_event_file *file, 883 const char *var_name) 884 { 885 struct hist_trigger_data *test_data; 886 struct event_trigger_data *test; 887 struct hist_field *hist_field; 888 889 lockdep_assert_held(&event_mutex); 890 891 hist_field = find_var_field(hist_data, var_name); 892 if (hist_field) 893 return hist_field; 894 895 list_for_each_entry(test, &file->triggers, list) { 896 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 897 test_data = test->private_data; 898 hist_field = find_var_field(test_data, var_name); 899 if (hist_field) 900 return hist_field; 901 } 902 } 903 904 return NULL; 905 } 906 907 static struct trace_event_file *find_var_file(struct trace_array *tr, 908 char *system, 909 char *event_name, 910 char *var_name) 911 { 912 struct hist_trigger_data *var_hist_data; 913 struct hist_var_data *var_data; 914 struct trace_event_file *file, *found = NULL; 915 916 if (system) 917 return find_event_file(tr, system, event_name); 918 919 list_for_each_entry(var_data, &tr->hist_vars, list) { 920 var_hist_data = var_data->hist_data; 921 file = var_hist_data->event_file; 922 if (file == found) 923 continue; 924 925 if (find_var_field(var_hist_data, var_name)) { 926 if (found) { 927 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name)); 928 return NULL; 929 } 930 931 found = file; 932 } 933 } 934 935 return found; 936 } 937 938 static struct hist_field *find_file_var(struct trace_event_file *file, 939 const char *var_name) 940 { 941 struct hist_trigger_data *test_data; 942 struct event_trigger_data *test; 943 struct hist_field *hist_field; 944 945 lockdep_assert_held(&event_mutex); 946 947 list_for_each_entry(test, &file->triggers, list) { 948 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 949 test_data = test->private_data; 950 hist_field = find_var_field(test_data, var_name); 951 if (hist_field) 952 return hist_field; 953 } 954 } 955 956 return NULL; 957 } 958 959 static struct hist_field * 960 find_match_var(struct hist_trigger_data *hist_data, char *var_name) 961 { 962 struct trace_array *tr = hist_data->event_file->tr; 963 struct hist_field *hist_field, *found = NULL; 964 struct trace_event_file *file; 965 unsigned int i; 966 967 for (i = 0; i < hist_data->n_actions; i++) { 968 struct action_data *data = hist_data->actions[i]; 969 970 if (data->handler == HANDLER_ONMATCH) { 971 char *system = data->match_data.event_system; 972 char *event_name = data->match_data.event; 973 974 file = find_var_file(tr, system, event_name, var_name); 975 if (!file) 976 continue; 977 hist_field = find_file_var(file, var_name); 978 if (hist_field) { 979 if (found) { 980 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, 981 errpos(var_name)); 982 return ERR_PTR(-EINVAL); 983 } 984 985 found = hist_field; 986 } 987 } 988 } 989 return found; 990 } 991 992 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data, 993 char *system, 994 char *event_name, 995 char *var_name) 996 { 997 struct trace_array *tr = hist_data->event_file->tr; 998 struct hist_field *hist_field = NULL; 999 struct trace_event_file *file; 1000 1001 if (!system || !event_name) { 1002 hist_field = find_match_var(hist_data, var_name); 1003 if (IS_ERR(hist_field)) 1004 return NULL; 1005 if (hist_field) 1006 return hist_field; 1007 } 1008 1009 file = find_var_file(tr, system, event_name, var_name); 1010 if (!file) 1011 return NULL; 1012 1013 hist_field = find_file_var(file, var_name); 1014 1015 return hist_field; 1016 } 1017 1018 static u64 hist_field_var_ref(struct hist_field *hist_field, 1019 struct tracing_map_elt *elt, 1020 struct ring_buffer_event *rbe, 1021 void *event) 1022 { 1023 struct hist_elt_data *elt_data; 1024 u64 var_val = 0; 1025 1026 if (WARN_ON_ONCE(!elt)) 1027 return var_val; 1028 1029 elt_data = elt->private_data; 1030 var_val = elt_data->var_ref_vals[hist_field->var_ref_idx]; 1031 1032 return var_val; 1033 } 1034 1035 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key, 1036 u64 *var_ref_vals, bool self) 1037 { 1038 struct hist_trigger_data *var_data; 1039 struct tracing_map_elt *var_elt; 1040 struct hist_field *hist_field; 1041 unsigned int i, var_idx; 1042 bool resolved = true; 1043 u64 var_val = 0; 1044 1045 for (i = 0; i < hist_data->n_var_refs; i++) { 1046 hist_field = hist_data->var_refs[i]; 1047 var_idx = hist_field->var.idx; 1048 var_data = hist_field->var.hist_data; 1049 1050 if (var_data == NULL) { 1051 resolved = false; 1052 break; 1053 } 1054 1055 if ((self && var_data != hist_data) || 1056 (!self && var_data == hist_data)) 1057 continue; 1058 1059 var_elt = tracing_map_lookup(var_data->map, key); 1060 if (!var_elt) { 1061 resolved = false; 1062 break; 1063 } 1064 1065 if (!tracing_map_var_set(var_elt, var_idx)) { 1066 resolved = false; 1067 break; 1068 } 1069 1070 if (self || !hist_field->read_once) 1071 var_val = tracing_map_read_var(var_elt, var_idx); 1072 else 1073 var_val = tracing_map_read_var_once(var_elt, var_idx); 1074 1075 var_ref_vals[i] = var_val; 1076 } 1077 1078 return resolved; 1079 } 1080 1081 static const char *hist_field_name(struct hist_field *field, 1082 unsigned int level) 1083 { 1084 const char *field_name = ""; 1085 1086 if (level > 1) 1087 return field_name; 1088 1089 if (field->field) 1090 field_name = field->field->name; 1091 else if (field->flags & HIST_FIELD_FL_LOG2 || 1092 field->flags & HIST_FIELD_FL_ALIAS) 1093 field_name = hist_field_name(field->operands[0], ++level); 1094 else if (field->flags & HIST_FIELD_FL_CPU) 1095 field_name = "cpu"; 1096 else if (field->flags & HIST_FIELD_FL_EXPR || 1097 field->flags & HIST_FIELD_FL_VAR_REF) { 1098 if (field->system) { 1099 static char full_name[MAX_FILTER_STR_VAL]; 1100 1101 strcat(full_name, field->system); 1102 strcat(full_name, "."); 1103 strcat(full_name, field->event_name); 1104 strcat(full_name, "."); 1105 strcat(full_name, field->name); 1106 field_name = full_name; 1107 } else 1108 field_name = field->name; 1109 } else if (field->flags & HIST_FIELD_FL_TIMESTAMP) 1110 field_name = "common_timestamp"; 1111 1112 if (field_name == NULL) 1113 field_name = ""; 1114 1115 return field_name; 1116 } 1117 1118 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed) 1119 { 1120 hist_field_fn_t fn = NULL; 1121 1122 switch (field_size) { 1123 case 8: 1124 if (field_is_signed) 1125 fn = hist_field_s64; 1126 else 1127 fn = hist_field_u64; 1128 break; 1129 case 4: 1130 if (field_is_signed) 1131 fn = hist_field_s32; 1132 else 1133 fn = hist_field_u32; 1134 break; 1135 case 2: 1136 if (field_is_signed) 1137 fn = hist_field_s16; 1138 else 1139 fn = hist_field_u16; 1140 break; 1141 case 1: 1142 if (field_is_signed) 1143 fn = hist_field_s8; 1144 else 1145 fn = hist_field_u8; 1146 break; 1147 } 1148 1149 return fn; 1150 } 1151 1152 static int parse_map_size(char *str) 1153 { 1154 unsigned long size, map_bits; 1155 int ret; 1156 1157 ret = kstrtoul(str, 0, &size); 1158 if (ret) 1159 goto out; 1160 1161 map_bits = ilog2(roundup_pow_of_two(size)); 1162 if (map_bits < TRACING_MAP_BITS_MIN || 1163 map_bits > TRACING_MAP_BITS_MAX) 1164 ret = -EINVAL; 1165 else 1166 ret = map_bits; 1167 out: 1168 return ret; 1169 } 1170 1171 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs) 1172 { 1173 unsigned int i; 1174 1175 if (!attrs) 1176 return; 1177 1178 for (i = 0; i < attrs->n_assignments; i++) 1179 kfree(attrs->assignment_str[i]); 1180 1181 for (i = 0; i < attrs->n_actions; i++) 1182 kfree(attrs->action_str[i]); 1183 1184 kfree(attrs->name); 1185 kfree(attrs->sort_key_str); 1186 kfree(attrs->keys_str); 1187 kfree(attrs->vals_str); 1188 kfree(attrs->clock); 1189 kfree(attrs); 1190 } 1191 1192 static int parse_action(char *str, struct hist_trigger_attrs *attrs) 1193 { 1194 int ret = -EINVAL; 1195 1196 if (attrs->n_actions >= HIST_ACTIONS_MAX) 1197 return ret; 1198 1199 if ((str_has_prefix(str, "onmatch(")) || 1200 (str_has_prefix(str, "onmax(")) || 1201 (str_has_prefix(str, "onchange("))) { 1202 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL); 1203 if (!attrs->action_str[attrs->n_actions]) { 1204 ret = -ENOMEM; 1205 return ret; 1206 } 1207 attrs->n_actions++; 1208 ret = 0; 1209 } 1210 return ret; 1211 } 1212 1213 static int parse_assignment(struct trace_array *tr, 1214 char *str, struct hist_trigger_attrs *attrs) 1215 { 1216 int len, ret = 0; 1217 1218 if ((len = str_has_prefix(str, "key=")) || 1219 (len = str_has_prefix(str, "keys="))) { 1220 attrs->keys_str = kstrdup(str + len, GFP_KERNEL); 1221 if (!attrs->keys_str) { 1222 ret = -ENOMEM; 1223 goto out; 1224 } 1225 } else if ((len = str_has_prefix(str, "val=")) || 1226 (len = str_has_prefix(str, "vals=")) || 1227 (len = str_has_prefix(str, "values="))) { 1228 attrs->vals_str = kstrdup(str + len, GFP_KERNEL); 1229 if (!attrs->vals_str) { 1230 ret = -ENOMEM; 1231 goto out; 1232 } 1233 } else if ((len = str_has_prefix(str, "sort="))) { 1234 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL); 1235 if (!attrs->sort_key_str) { 1236 ret = -ENOMEM; 1237 goto out; 1238 } 1239 } else if (str_has_prefix(str, "name=")) { 1240 attrs->name = kstrdup(str, GFP_KERNEL); 1241 if (!attrs->name) { 1242 ret = -ENOMEM; 1243 goto out; 1244 } 1245 } else if ((len = str_has_prefix(str, "clock="))) { 1246 str += len; 1247 1248 str = strstrip(str); 1249 attrs->clock = kstrdup(str, GFP_KERNEL); 1250 if (!attrs->clock) { 1251 ret = -ENOMEM; 1252 goto out; 1253 } 1254 } else if ((len = str_has_prefix(str, "size="))) { 1255 int map_bits = parse_map_size(str + len); 1256 1257 if (map_bits < 0) { 1258 ret = map_bits; 1259 goto out; 1260 } 1261 attrs->map_bits = map_bits; 1262 } else { 1263 char *assignment; 1264 1265 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) { 1266 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str)); 1267 ret = -EINVAL; 1268 goto out; 1269 } 1270 1271 assignment = kstrdup(str, GFP_KERNEL); 1272 if (!assignment) { 1273 ret = -ENOMEM; 1274 goto out; 1275 } 1276 1277 attrs->assignment_str[attrs->n_assignments++] = assignment; 1278 } 1279 out: 1280 return ret; 1281 } 1282 1283 static struct hist_trigger_attrs * 1284 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str) 1285 { 1286 struct hist_trigger_attrs *attrs; 1287 int ret = 0; 1288 1289 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL); 1290 if (!attrs) 1291 return ERR_PTR(-ENOMEM); 1292 1293 while (trigger_str) { 1294 char *str = strsep(&trigger_str, ":"); 1295 char *rhs; 1296 1297 rhs = strchr(str, '='); 1298 if (rhs) { 1299 if (!strlen(++rhs)) { 1300 ret = -EINVAL; 1301 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str)); 1302 goto free; 1303 } 1304 ret = parse_assignment(tr, str, attrs); 1305 if (ret) 1306 goto free; 1307 } else if (strcmp(str, "pause") == 0) 1308 attrs->pause = true; 1309 else if ((strcmp(str, "cont") == 0) || 1310 (strcmp(str, "continue") == 0)) 1311 attrs->cont = true; 1312 else if (strcmp(str, "clear") == 0) 1313 attrs->clear = true; 1314 else { 1315 ret = parse_action(str, attrs); 1316 if (ret) 1317 goto free; 1318 } 1319 } 1320 1321 if (!attrs->keys_str) { 1322 ret = -EINVAL; 1323 goto free; 1324 } 1325 1326 if (!attrs->clock) { 1327 attrs->clock = kstrdup("global", GFP_KERNEL); 1328 if (!attrs->clock) { 1329 ret = -ENOMEM; 1330 goto free; 1331 } 1332 } 1333 1334 return attrs; 1335 free: 1336 destroy_hist_trigger_attrs(attrs); 1337 1338 return ERR_PTR(ret); 1339 } 1340 1341 static inline void save_comm(char *comm, struct task_struct *task) 1342 { 1343 if (!task->pid) { 1344 strcpy(comm, "<idle>"); 1345 return; 1346 } 1347 1348 if (WARN_ON_ONCE(task->pid < 0)) { 1349 strcpy(comm, "<XXX>"); 1350 return; 1351 } 1352 1353 strncpy(comm, task->comm, TASK_COMM_LEN); 1354 } 1355 1356 static void hist_elt_data_free(struct hist_elt_data *elt_data) 1357 { 1358 unsigned int i; 1359 1360 for (i = 0; i < SYNTH_FIELDS_MAX; i++) 1361 kfree(elt_data->field_var_str[i]); 1362 1363 kfree(elt_data->comm); 1364 kfree(elt_data); 1365 } 1366 1367 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt) 1368 { 1369 struct hist_elt_data *elt_data = elt->private_data; 1370 1371 hist_elt_data_free(elt_data); 1372 } 1373 1374 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt) 1375 { 1376 struct hist_trigger_data *hist_data = elt->map->private_data; 1377 unsigned int size = TASK_COMM_LEN; 1378 struct hist_elt_data *elt_data; 1379 struct hist_field *key_field; 1380 unsigned int i, n_str; 1381 1382 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL); 1383 if (!elt_data) 1384 return -ENOMEM; 1385 1386 for_each_hist_key_field(i, hist_data) { 1387 key_field = hist_data->fields[i]; 1388 1389 if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 1390 elt_data->comm = kzalloc(size, GFP_KERNEL); 1391 if (!elt_data->comm) { 1392 kfree(elt_data); 1393 return -ENOMEM; 1394 } 1395 break; 1396 } 1397 } 1398 1399 n_str = hist_data->n_field_var_str + hist_data->n_save_var_str; 1400 1401 size = STR_VAR_LEN_MAX; 1402 1403 for (i = 0; i < n_str; i++) { 1404 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL); 1405 if (!elt_data->field_var_str[i]) { 1406 hist_elt_data_free(elt_data); 1407 return -ENOMEM; 1408 } 1409 } 1410 1411 elt->private_data = elt_data; 1412 1413 return 0; 1414 } 1415 1416 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt) 1417 { 1418 struct hist_elt_data *elt_data = elt->private_data; 1419 1420 if (elt_data->comm) 1421 save_comm(elt_data->comm, current); 1422 } 1423 1424 static const struct tracing_map_ops hist_trigger_elt_data_ops = { 1425 .elt_alloc = hist_trigger_elt_data_alloc, 1426 .elt_free = hist_trigger_elt_data_free, 1427 .elt_init = hist_trigger_elt_data_init, 1428 }; 1429 1430 static const char *get_hist_field_flags(struct hist_field *hist_field) 1431 { 1432 const char *flags_str = NULL; 1433 1434 if (hist_field->flags & HIST_FIELD_FL_HEX) 1435 flags_str = "hex"; 1436 else if (hist_field->flags & HIST_FIELD_FL_SYM) 1437 flags_str = "sym"; 1438 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET) 1439 flags_str = "sym-offset"; 1440 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME) 1441 flags_str = "execname"; 1442 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL) 1443 flags_str = "syscall"; 1444 else if (hist_field->flags & HIST_FIELD_FL_LOG2) 1445 flags_str = "log2"; 1446 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS) 1447 flags_str = "usecs"; 1448 1449 return flags_str; 1450 } 1451 1452 static void expr_field_str(struct hist_field *field, char *expr) 1453 { 1454 if (field->flags & HIST_FIELD_FL_VAR_REF) 1455 strcat(expr, "$"); 1456 1457 strcat(expr, hist_field_name(field, 0)); 1458 1459 if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) { 1460 const char *flags_str = get_hist_field_flags(field); 1461 1462 if (flags_str) { 1463 strcat(expr, "."); 1464 strcat(expr, flags_str); 1465 } 1466 } 1467 } 1468 1469 static char *expr_str(struct hist_field *field, unsigned int level) 1470 { 1471 char *expr; 1472 1473 if (level > 1) 1474 return NULL; 1475 1476 expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 1477 if (!expr) 1478 return NULL; 1479 1480 if (!field->operands[0]) { 1481 expr_field_str(field, expr); 1482 return expr; 1483 } 1484 1485 if (field->operator == FIELD_OP_UNARY_MINUS) { 1486 char *subexpr; 1487 1488 strcat(expr, "-("); 1489 subexpr = expr_str(field->operands[0], ++level); 1490 if (!subexpr) { 1491 kfree(expr); 1492 return NULL; 1493 } 1494 strcat(expr, subexpr); 1495 strcat(expr, ")"); 1496 1497 kfree(subexpr); 1498 1499 return expr; 1500 } 1501 1502 expr_field_str(field->operands[0], expr); 1503 1504 switch (field->operator) { 1505 case FIELD_OP_MINUS: 1506 strcat(expr, "-"); 1507 break; 1508 case FIELD_OP_PLUS: 1509 strcat(expr, "+"); 1510 break; 1511 default: 1512 kfree(expr); 1513 return NULL; 1514 } 1515 1516 expr_field_str(field->operands[1], expr); 1517 1518 return expr; 1519 } 1520 1521 static int contains_operator(char *str) 1522 { 1523 enum field_op_id field_op = FIELD_OP_NONE; 1524 char *op; 1525 1526 op = strpbrk(str, "+-"); 1527 if (!op) 1528 return FIELD_OP_NONE; 1529 1530 switch (*op) { 1531 case '-': 1532 if (*str == '-') 1533 field_op = FIELD_OP_UNARY_MINUS; 1534 else 1535 field_op = FIELD_OP_MINUS; 1536 break; 1537 case '+': 1538 field_op = FIELD_OP_PLUS; 1539 break; 1540 default: 1541 break; 1542 } 1543 1544 return field_op; 1545 } 1546 1547 static void get_hist_field(struct hist_field *hist_field) 1548 { 1549 hist_field->ref++; 1550 } 1551 1552 static void __destroy_hist_field(struct hist_field *hist_field) 1553 { 1554 if (--hist_field->ref > 1) 1555 return; 1556 1557 kfree(hist_field->var.name); 1558 kfree(hist_field->name); 1559 kfree(hist_field->type); 1560 1561 kfree(hist_field->system); 1562 kfree(hist_field->event_name); 1563 1564 kfree(hist_field); 1565 } 1566 1567 static void destroy_hist_field(struct hist_field *hist_field, 1568 unsigned int level) 1569 { 1570 unsigned int i; 1571 1572 if (level > 3) 1573 return; 1574 1575 if (!hist_field) 1576 return; 1577 1578 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) 1579 return; /* var refs will be destroyed separately */ 1580 1581 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) 1582 destroy_hist_field(hist_field->operands[i], level + 1); 1583 1584 __destroy_hist_field(hist_field); 1585 } 1586 1587 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, 1588 struct ftrace_event_field *field, 1589 unsigned long flags, 1590 char *var_name) 1591 { 1592 struct hist_field *hist_field; 1593 1594 if (field && is_function_field(field)) 1595 return NULL; 1596 1597 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL); 1598 if (!hist_field) 1599 return NULL; 1600 1601 hist_field->ref = 1; 1602 1603 hist_field->hist_data = hist_data; 1604 1605 if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS) 1606 goto out; /* caller will populate */ 1607 1608 if (flags & HIST_FIELD_FL_VAR_REF) { 1609 hist_field->fn = hist_field_var_ref; 1610 goto out; 1611 } 1612 1613 if (flags & HIST_FIELD_FL_HITCOUNT) { 1614 hist_field->fn = hist_field_counter; 1615 hist_field->size = sizeof(u64); 1616 hist_field->type = kstrdup("u64", GFP_KERNEL); 1617 if (!hist_field->type) 1618 goto free; 1619 goto out; 1620 } 1621 1622 if (flags & HIST_FIELD_FL_STACKTRACE) { 1623 hist_field->fn = hist_field_none; 1624 goto out; 1625 } 1626 1627 if (flags & HIST_FIELD_FL_LOG2) { 1628 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2; 1629 hist_field->fn = hist_field_log2; 1630 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL); 1631 hist_field->size = hist_field->operands[0]->size; 1632 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL); 1633 if (!hist_field->type) 1634 goto free; 1635 goto out; 1636 } 1637 1638 if (flags & HIST_FIELD_FL_TIMESTAMP) { 1639 hist_field->fn = hist_field_timestamp; 1640 hist_field->size = sizeof(u64); 1641 hist_field->type = kstrdup("u64", GFP_KERNEL); 1642 if (!hist_field->type) 1643 goto free; 1644 goto out; 1645 } 1646 1647 if (flags & HIST_FIELD_FL_CPU) { 1648 hist_field->fn = hist_field_cpu; 1649 hist_field->size = sizeof(int); 1650 hist_field->type = kstrdup("unsigned int", GFP_KERNEL); 1651 if (!hist_field->type) 1652 goto free; 1653 goto out; 1654 } 1655 1656 if (WARN_ON_ONCE(!field)) 1657 goto out; 1658 1659 if (is_string_field(field)) { 1660 flags |= HIST_FIELD_FL_STRING; 1661 1662 hist_field->size = MAX_FILTER_STR_VAL; 1663 hist_field->type = kstrdup(field->type, GFP_KERNEL); 1664 if (!hist_field->type) 1665 goto free; 1666 1667 if (field->filter_type == FILTER_STATIC_STRING) 1668 hist_field->fn = hist_field_string; 1669 else if (field->filter_type == FILTER_DYN_STRING) 1670 hist_field->fn = hist_field_dynstring; 1671 else 1672 hist_field->fn = hist_field_pstring; 1673 } else { 1674 hist_field->size = field->size; 1675 hist_field->is_signed = field->is_signed; 1676 hist_field->type = kstrdup(field->type, GFP_KERNEL); 1677 if (!hist_field->type) 1678 goto free; 1679 1680 hist_field->fn = select_value_fn(field->size, 1681 field->is_signed); 1682 if (!hist_field->fn) { 1683 destroy_hist_field(hist_field, 0); 1684 return NULL; 1685 } 1686 } 1687 out: 1688 hist_field->field = field; 1689 hist_field->flags = flags; 1690 1691 if (var_name) { 1692 hist_field->var.name = kstrdup(var_name, GFP_KERNEL); 1693 if (!hist_field->var.name) 1694 goto free; 1695 } 1696 1697 return hist_field; 1698 free: 1699 destroy_hist_field(hist_field, 0); 1700 return NULL; 1701 } 1702 1703 static void destroy_hist_fields(struct hist_trigger_data *hist_data) 1704 { 1705 unsigned int i; 1706 1707 for (i = 0; i < HIST_FIELDS_MAX; i++) { 1708 if (hist_data->fields[i]) { 1709 destroy_hist_field(hist_data->fields[i], 0); 1710 hist_data->fields[i] = NULL; 1711 } 1712 } 1713 1714 for (i = 0; i < hist_data->n_var_refs; i++) { 1715 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF)); 1716 __destroy_hist_field(hist_data->var_refs[i]); 1717 hist_data->var_refs[i] = NULL; 1718 } 1719 } 1720 1721 static int init_var_ref(struct hist_field *ref_field, 1722 struct hist_field *var_field, 1723 char *system, char *event_name) 1724 { 1725 int err = 0; 1726 1727 ref_field->var.idx = var_field->var.idx; 1728 ref_field->var.hist_data = var_field->hist_data; 1729 ref_field->size = var_field->size; 1730 ref_field->is_signed = var_field->is_signed; 1731 ref_field->flags |= var_field->flags & 1732 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 1733 1734 if (system) { 1735 ref_field->system = kstrdup(system, GFP_KERNEL); 1736 if (!ref_field->system) 1737 return -ENOMEM; 1738 } 1739 1740 if (event_name) { 1741 ref_field->event_name = kstrdup(event_name, GFP_KERNEL); 1742 if (!ref_field->event_name) { 1743 err = -ENOMEM; 1744 goto free; 1745 } 1746 } 1747 1748 if (var_field->var.name) { 1749 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL); 1750 if (!ref_field->name) { 1751 err = -ENOMEM; 1752 goto free; 1753 } 1754 } else if (var_field->name) { 1755 ref_field->name = kstrdup(var_field->name, GFP_KERNEL); 1756 if (!ref_field->name) { 1757 err = -ENOMEM; 1758 goto free; 1759 } 1760 } 1761 1762 ref_field->type = kstrdup(var_field->type, GFP_KERNEL); 1763 if (!ref_field->type) { 1764 err = -ENOMEM; 1765 goto free; 1766 } 1767 out: 1768 return err; 1769 free: 1770 kfree(ref_field->system); 1771 kfree(ref_field->event_name); 1772 kfree(ref_field->name); 1773 1774 goto out; 1775 } 1776 1777 static int find_var_ref_idx(struct hist_trigger_data *hist_data, 1778 struct hist_field *var_field) 1779 { 1780 struct hist_field *ref_field; 1781 int i; 1782 1783 for (i = 0; i < hist_data->n_var_refs; i++) { 1784 ref_field = hist_data->var_refs[i]; 1785 if (ref_field->var.idx == var_field->var.idx && 1786 ref_field->var.hist_data == var_field->hist_data) 1787 return i; 1788 } 1789 1790 return -ENOENT; 1791 } 1792 1793 /** 1794 * create_var_ref - Create a variable reference and attach it to trigger 1795 * @hist_data: The trigger that will be referencing the variable 1796 * @var_field: The VAR field to create a reference to 1797 * @system: The optional system string 1798 * @event_name: The optional event_name string 1799 * 1800 * Given a variable hist_field, create a VAR_REF hist_field that 1801 * represents a reference to it. 1802 * 1803 * This function also adds the reference to the trigger that 1804 * now references the variable. 1805 * 1806 * Return: The VAR_REF field if successful, NULL if not 1807 */ 1808 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data, 1809 struct hist_field *var_field, 1810 char *system, char *event_name) 1811 { 1812 unsigned long flags = HIST_FIELD_FL_VAR_REF; 1813 struct hist_field *ref_field; 1814 int i; 1815 1816 /* Check if the variable already exists */ 1817 for (i = 0; i < hist_data->n_var_refs; i++) { 1818 ref_field = hist_data->var_refs[i]; 1819 if (ref_field->var.idx == var_field->var.idx && 1820 ref_field->var.hist_data == var_field->hist_data) { 1821 get_hist_field(ref_field); 1822 return ref_field; 1823 } 1824 } 1825 1826 ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL); 1827 if (ref_field) { 1828 if (init_var_ref(ref_field, var_field, system, event_name)) { 1829 destroy_hist_field(ref_field, 0); 1830 return NULL; 1831 } 1832 1833 hist_data->var_refs[hist_data->n_var_refs] = ref_field; 1834 ref_field->var_ref_idx = hist_data->n_var_refs++; 1835 } 1836 1837 return ref_field; 1838 } 1839 1840 static bool is_var_ref(char *var_name) 1841 { 1842 if (!var_name || strlen(var_name) < 2 || var_name[0] != '$') 1843 return false; 1844 1845 return true; 1846 } 1847 1848 static char *field_name_from_var(struct hist_trigger_data *hist_data, 1849 char *var_name) 1850 { 1851 char *name, *field; 1852 unsigned int i; 1853 1854 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 1855 name = hist_data->attrs->var_defs.name[i]; 1856 1857 if (strcmp(var_name, name) == 0) { 1858 field = hist_data->attrs->var_defs.expr[i]; 1859 if (contains_operator(field) || is_var_ref(field)) 1860 continue; 1861 return field; 1862 } 1863 } 1864 1865 return NULL; 1866 } 1867 1868 static char *local_field_var_ref(struct hist_trigger_data *hist_data, 1869 char *system, char *event_name, 1870 char *var_name) 1871 { 1872 struct trace_event_call *call; 1873 1874 if (system && event_name) { 1875 call = hist_data->event_file->event_call; 1876 1877 if (strcmp(system, call->class->system) != 0) 1878 return NULL; 1879 1880 if (strcmp(event_name, trace_event_name(call)) != 0) 1881 return NULL; 1882 } 1883 1884 if (!!system != !!event_name) 1885 return NULL; 1886 1887 if (!is_var_ref(var_name)) 1888 return NULL; 1889 1890 var_name++; 1891 1892 return field_name_from_var(hist_data, var_name); 1893 } 1894 1895 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data, 1896 char *system, char *event_name, 1897 char *var_name) 1898 { 1899 struct hist_field *var_field = NULL, *ref_field = NULL; 1900 struct trace_array *tr = hist_data->event_file->tr; 1901 1902 if (!is_var_ref(var_name)) 1903 return NULL; 1904 1905 var_name++; 1906 1907 var_field = find_event_var(hist_data, system, event_name, var_name); 1908 if (var_field) 1909 ref_field = create_var_ref(hist_data, var_field, 1910 system, event_name); 1911 1912 if (!ref_field) 1913 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name)); 1914 1915 return ref_field; 1916 } 1917 1918 static struct ftrace_event_field * 1919 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, 1920 char *field_str, unsigned long *flags) 1921 { 1922 struct ftrace_event_field *field = NULL; 1923 char *field_name, *modifier, *str; 1924 struct trace_array *tr = file->tr; 1925 1926 modifier = str = kstrdup(field_str, GFP_KERNEL); 1927 if (!modifier) 1928 return ERR_PTR(-ENOMEM); 1929 1930 field_name = strsep(&modifier, "."); 1931 if (modifier) { 1932 if (strcmp(modifier, "hex") == 0) 1933 *flags |= HIST_FIELD_FL_HEX; 1934 else if (strcmp(modifier, "sym") == 0) 1935 *flags |= HIST_FIELD_FL_SYM; 1936 else if (strcmp(modifier, "sym-offset") == 0) 1937 *flags |= HIST_FIELD_FL_SYM_OFFSET; 1938 else if ((strcmp(modifier, "execname") == 0) && 1939 (strcmp(field_name, "common_pid") == 0)) 1940 *flags |= HIST_FIELD_FL_EXECNAME; 1941 else if (strcmp(modifier, "syscall") == 0) 1942 *flags |= HIST_FIELD_FL_SYSCALL; 1943 else if (strcmp(modifier, "log2") == 0) 1944 *flags |= HIST_FIELD_FL_LOG2; 1945 else if (strcmp(modifier, "usecs") == 0) 1946 *flags |= HIST_FIELD_FL_TIMESTAMP_USECS; 1947 else { 1948 hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier)); 1949 field = ERR_PTR(-EINVAL); 1950 goto out; 1951 } 1952 } 1953 1954 if (strcmp(field_name, "common_timestamp") == 0) { 1955 *flags |= HIST_FIELD_FL_TIMESTAMP; 1956 hist_data->enable_timestamps = true; 1957 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS) 1958 hist_data->attrs->ts_in_usecs = true; 1959 } else if (strcmp(field_name, "cpu") == 0) 1960 *flags |= HIST_FIELD_FL_CPU; 1961 else { 1962 field = trace_find_event_field(file->event_call, field_name); 1963 if (!field || !field->size) { 1964 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name)); 1965 field = ERR_PTR(-EINVAL); 1966 goto out; 1967 } 1968 } 1969 out: 1970 kfree(str); 1971 1972 return field; 1973 } 1974 1975 static struct hist_field *create_alias(struct hist_trigger_data *hist_data, 1976 struct hist_field *var_ref, 1977 char *var_name) 1978 { 1979 struct hist_field *alias = NULL; 1980 unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR; 1981 1982 alias = create_hist_field(hist_data, NULL, flags, var_name); 1983 if (!alias) 1984 return NULL; 1985 1986 alias->fn = var_ref->fn; 1987 alias->operands[0] = var_ref; 1988 1989 if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) { 1990 destroy_hist_field(alias, 0); 1991 return NULL; 1992 } 1993 1994 alias->var_ref_idx = var_ref->var_ref_idx; 1995 1996 return alias; 1997 } 1998 1999 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data, 2000 struct trace_event_file *file, char *str, 2001 unsigned long *flags, char *var_name) 2002 { 2003 char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str; 2004 struct ftrace_event_field *field = NULL; 2005 struct hist_field *hist_field = NULL; 2006 int ret = 0; 2007 2008 s = strchr(str, '.'); 2009 if (s) { 2010 s = strchr(++s, '.'); 2011 if (s) { 2012 ref_system = strsep(&str, "."); 2013 if (!str) { 2014 ret = -EINVAL; 2015 goto out; 2016 } 2017 ref_event = strsep(&str, "."); 2018 if (!str) { 2019 ret = -EINVAL; 2020 goto out; 2021 } 2022 ref_var = str; 2023 } 2024 } 2025 2026 s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var); 2027 if (!s) { 2028 hist_field = parse_var_ref(hist_data, ref_system, 2029 ref_event, ref_var); 2030 if (hist_field) { 2031 if (var_name) { 2032 hist_field = create_alias(hist_data, hist_field, var_name); 2033 if (!hist_field) { 2034 ret = -ENOMEM; 2035 goto out; 2036 } 2037 } 2038 return hist_field; 2039 } 2040 } else 2041 str = s; 2042 2043 field = parse_field(hist_data, file, str, flags); 2044 if (IS_ERR(field)) { 2045 ret = PTR_ERR(field); 2046 goto out; 2047 } 2048 2049 hist_field = create_hist_field(hist_data, field, *flags, var_name); 2050 if (!hist_field) { 2051 ret = -ENOMEM; 2052 goto out; 2053 } 2054 2055 return hist_field; 2056 out: 2057 return ERR_PTR(ret); 2058 } 2059 2060 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2061 struct trace_event_file *file, 2062 char *str, unsigned long flags, 2063 char *var_name, unsigned int level); 2064 2065 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data, 2066 struct trace_event_file *file, 2067 char *str, unsigned long flags, 2068 char *var_name, unsigned int level) 2069 { 2070 struct hist_field *operand1, *expr = NULL; 2071 unsigned long operand_flags; 2072 int ret = 0; 2073 char *s; 2074 2075 /* we support only -(xxx) i.e. explicit parens required */ 2076 2077 if (level > 3) { 2078 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str)); 2079 ret = -EINVAL; 2080 goto free; 2081 } 2082 2083 str++; /* skip leading '-' */ 2084 2085 s = strchr(str, '('); 2086 if (s) 2087 str++; 2088 else { 2089 ret = -EINVAL; 2090 goto free; 2091 } 2092 2093 s = strrchr(str, ')'); 2094 if (s) 2095 *s = '\0'; 2096 else { 2097 ret = -EINVAL; /* no closing ')' */ 2098 goto free; 2099 } 2100 2101 flags |= HIST_FIELD_FL_EXPR; 2102 expr = create_hist_field(hist_data, NULL, flags, var_name); 2103 if (!expr) { 2104 ret = -ENOMEM; 2105 goto free; 2106 } 2107 2108 operand_flags = 0; 2109 operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level); 2110 if (IS_ERR(operand1)) { 2111 ret = PTR_ERR(operand1); 2112 goto free; 2113 } 2114 2115 expr->flags |= operand1->flags & 2116 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2117 expr->fn = hist_field_unary_minus; 2118 expr->operands[0] = operand1; 2119 expr->operator = FIELD_OP_UNARY_MINUS; 2120 expr->name = expr_str(expr, 0); 2121 expr->type = kstrdup(operand1->type, GFP_KERNEL); 2122 if (!expr->type) { 2123 ret = -ENOMEM; 2124 goto free; 2125 } 2126 2127 return expr; 2128 free: 2129 destroy_hist_field(expr, 0); 2130 return ERR_PTR(ret); 2131 } 2132 2133 static int check_expr_operands(struct trace_array *tr, 2134 struct hist_field *operand1, 2135 struct hist_field *operand2) 2136 { 2137 unsigned long operand1_flags = operand1->flags; 2138 unsigned long operand2_flags = operand2->flags; 2139 2140 if ((operand1_flags & HIST_FIELD_FL_VAR_REF) || 2141 (operand1_flags & HIST_FIELD_FL_ALIAS)) { 2142 struct hist_field *var; 2143 2144 var = find_var_field(operand1->var.hist_data, operand1->name); 2145 if (!var) 2146 return -EINVAL; 2147 operand1_flags = var->flags; 2148 } 2149 2150 if ((operand2_flags & HIST_FIELD_FL_VAR_REF) || 2151 (operand2_flags & HIST_FIELD_FL_ALIAS)) { 2152 struct hist_field *var; 2153 2154 var = find_var_field(operand2->var.hist_data, operand2->name); 2155 if (!var) 2156 return -EINVAL; 2157 operand2_flags = var->flags; 2158 } 2159 2160 if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) != 2161 (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) { 2162 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0); 2163 return -EINVAL; 2164 } 2165 2166 return 0; 2167 } 2168 2169 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2170 struct trace_event_file *file, 2171 char *str, unsigned long flags, 2172 char *var_name, unsigned int level) 2173 { 2174 struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL; 2175 unsigned long operand_flags; 2176 int field_op, ret = -EINVAL; 2177 char *sep, *operand1_str; 2178 2179 if (level > 3) { 2180 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str)); 2181 return ERR_PTR(-EINVAL); 2182 } 2183 2184 field_op = contains_operator(str); 2185 2186 if (field_op == FIELD_OP_NONE) 2187 return parse_atom(hist_data, file, str, &flags, var_name); 2188 2189 if (field_op == FIELD_OP_UNARY_MINUS) 2190 return parse_unary(hist_data, file, str, flags, var_name, ++level); 2191 2192 switch (field_op) { 2193 case FIELD_OP_MINUS: 2194 sep = "-"; 2195 break; 2196 case FIELD_OP_PLUS: 2197 sep = "+"; 2198 break; 2199 default: 2200 goto free; 2201 } 2202 2203 operand1_str = strsep(&str, sep); 2204 if (!operand1_str || !str) 2205 goto free; 2206 2207 operand_flags = 0; 2208 operand1 = parse_atom(hist_data, file, operand1_str, 2209 &operand_flags, NULL); 2210 if (IS_ERR(operand1)) { 2211 ret = PTR_ERR(operand1); 2212 operand1 = NULL; 2213 goto free; 2214 } 2215 2216 /* rest of string could be another expression e.g. b+c in a+b+c */ 2217 operand_flags = 0; 2218 operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level); 2219 if (IS_ERR(operand2)) { 2220 ret = PTR_ERR(operand2); 2221 operand2 = NULL; 2222 goto free; 2223 } 2224 2225 ret = check_expr_operands(file->tr, operand1, operand2); 2226 if (ret) 2227 goto free; 2228 2229 flags |= HIST_FIELD_FL_EXPR; 2230 2231 flags |= operand1->flags & 2232 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2233 2234 expr = create_hist_field(hist_data, NULL, flags, var_name); 2235 if (!expr) { 2236 ret = -ENOMEM; 2237 goto free; 2238 } 2239 2240 operand1->read_once = true; 2241 operand2->read_once = true; 2242 2243 expr->operands[0] = operand1; 2244 expr->operands[1] = operand2; 2245 expr->operator = field_op; 2246 expr->name = expr_str(expr, 0); 2247 expr->type = kstrdup(operand1->type, GFP_KERNEL); 2248 if (!expr->type) { 2249 ret = -ENOMEM; 2250 goto free; 2251 } 2252 2253 switch (field_op) { 2254 case FIELD_OP_MINUS: 2255 expr->fn = hist_field_minus; 2256 break; 2257 case FIELD_OP_PLUS: 2258 expr->fn = hist_field_plus; 2259 break; 2260 default: 2261 ret = -EINVAL; 2262 goto free; 2263 } 2264 2265 return expr; 2266 free: 2267 destroy_hist_field(operand1, 0); 2268 destroy_hist_field(operand2, 0); 2269 destroy_hist_field(expr, 0); 2270 2271 return ERR_PTR(ret); 2272 } 2273 2274 static char *find_trigger_filter(struct hist_trigger_data *hist_data, 2275 struct trace_event_file *file) 2276 { 2277 struct event_trigger_data *test; 2278 2279 lockdep_assert_held(&event_mutex); 2280 2281 list_for_each_entry(test, &file->triggers, list) { 2282 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2283 if (test->private_data == hist_data) 2284 return test->filter_str; 2285 } 2286 } 2287 2288 return NULL; 2289 } 2290 2291 static struct event_command trigger_hist_cmd; 2292 static int event_hist_trigger_func(struct event_command *cmd_ops, 2293 struct trace_event_file *file, 2294 char *glob, char *cmd, char *param); 2295 2296 static bool compatible_keys(struct hist_trigger_data *target_hist_data, 2297 struct hist_trigger_data *hist_data, 2298 unsigned int n_keys) 2299 { 2300 struct hist_field *target_hist_field, *hist_field; 2301 unsigned int n, i, j; 2302 2303 if (hist_data->n_fields - hist_data->n_vals != n_keys) 2304 return false; 2305 2306 i = hist_data->n_vals; 2307 j = target_hist_data->n_vals; 2308 2309 for (n = 0; n < n_keys; n++) { 2310 hist_field = hist_data->fields[i + n]; 2311 target_hist_field = target_hist_data->fields[j + n]; 2312 2313 if (strcmp(hist_field->type, target_hist_field->type) != 0) 2314 return false; 2315 if (hist_field->size != target_hist_field->size) 2316 return false; 2317 if (hist_field->is_signed != target_hist_field->is_signed) 2318 return false; 2319 } 2320 2321 return true; 2322 } 2323 2324 static struct hist_trigger_data * 2325 find_compatible_hist(struct hist_trigger_data *target_hist_data, 2326 struct trace_event_file *file) 2327 { 2328 struct hist_trigger_data *hist_data; 2329 struct event_trigger_data *test; 2330 unsigned int n_keys; 2331 2332 lockdep_assert_held(&event_mutex); 2333 2334 n_keys = target_hist_data->n_fields - target_hist_data->n_vals; 2335 2336 list_for_each_entry(test, &file->triggers, list) { 2337 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2338 hist_data = test->private_data; 2339 2340 if (compatible_keys(target_hist_data, hist_data, n_keys)) 2341 return hist_data; 2342 } 2343 } 2344 2345 return NULL; 2346 } 2347 2348 static struct trace_event_file *event_file(struct trace_array *tr, 2349 char *system, char *event_name) 2350 { 2351 struct trace_event_file *file; 2352 2353 file = __find_event_file(tr, system, event_name); 2354 if (!file) 2355 return ERR_PTR(-EINVAL); 2356 2357 return file; 2358 } 2359 2360 static struct hist_field * 2361 find_synthetic_field_var(struct hist_trigger_data *target_hist_data, 2362 char *system, char *event_name, char *field_name) 2363 { 2364 struct hist_field *event_var; 2365 char *synthetic_name; 2366 2367 synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2368 if (!synthetic_name) 2369 return ERR_PTR(-ENOMEM); 2370 2371 strcpy(synthetic_name, "synthetic_"); 2372 strcat(synthetic_name, field_name); 2373 2374 event_var = find_event_var(target_hist_data, system, event_name, synthetic_name); 2375 2376 kfree(synthetic_name); 2377 2378 return event_var; 2379 } 2380 2381 /** 2382 * create_field_var_hist - Automatically create a histogram and var for a field 2383 * @target_hist_data: The target hist trigger 2384 * @subsys_name: Optional subsystem name 2385 * @event_name: Optional event name 2386 * @field_name: The name of the field (and the resulting variable) 2387 * 2388 * Hist trigger actions fetch data from variables, not directly from 2389 * events. However, for convenience, users are allowed to directly 2390 * specify an event field in an action, which will be automatically 2391 * converted into a variable on their behalf. 2392 2393 * If a user specifies a field on an event that isn't the event the 2394 * histogram currently being defined (the target event histogram), the 2395 * only way that can be accomplished is if a new hist trigger is 2396 * created and the field variable defined on that. 2397 * 2398 * This function creates a new histogram compatible with the target 2399 * event (meaning a histogram with the same key as the target 2400 * histogram), and creates a variable for the specified field, but 2401 * with 'synthetic_' prepended to the variable name in order to avoid 2402 * collision with normal field variables. 2403 * 2404 * Return: The variable created for the field. 2405 */ 2406 static struct hist_field * 2407 create_field_var_hist(struct hist_trigger_data *target_hist_data, 2408 char *subsys_name, char *event_name, char *field_name) 2409 { 2410 struct trace_array *tr = target_hist_data->event_file->tr; 2411 struct hist_field *event_var = ERR_PTR(-EINVAL); 2412 struct hist_trigger_data *hist_data; 2413 unsigned int i, n, first = true; 2414 struct field_var_hist *var_hist; 2415 struct trace_event_file *file; 2416 struct hist_field *key_field; 2417 char *saved_filter; 2418 char *cmd; 2419 int ret; 2420 2421 if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) { 2422 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name)); 2423 return ERR_PTR(-EINVAL); 2424 } 2425 2426 file = event_file(tr, subsys_name, event_name); 2427 2428 if (IS_ERR(file)) { 2429 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name)); 2430 ret = PTR_ERR(file); 2431 return ERR_PTR(ret); 2432 } 2433 2434 /* 2435 * Look for a histogram compatible with target. We'll use the 2436 * found histogram specification to create a new matching 2437 * histogram with our variable on it. target_hist_data is not 2438 * yet a registered histogram so we can't use that. 2439 */ 2440 hist_data = find_compatible_hist(target_hist_data, file); 2441 if (!hist_data) { 2442 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name)); 2443 return ERR_PTR(-EINVAL); 2444 } 2445 2446 /* See if a synthetic field variable has already been created */ 2447 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 2448 event_name, field_name); 2449 if (!IS_ERR_OR_NULL(event_var)) 2450 return event_var; 2451 2452 var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL); 2453 if (!var_hist) 2454 return ERR_PTR(-ENOMEM); 2455 2456 cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2457 if (!cmd) { 2458 kfree(var_hist); 2459 return ERR_PTR(-ENOMEM); 2460 } 2461 2462 /* Use the same keys as the compatible histogram */ 2463 strcat(cmd, "keys="); 2464 2465 for_each_hist_key_field(i, hist_data) { 2466 key_field = hist_data->fields[i]; 2467 if (!first) 2468 strcat(cmd, ","); 2469 strcat(cmd, key_field->field->name); 2470 first = false; 2471 } 2472 2473 /* Create the synthetic field variable specification */ 2474 strcat(cmd, ":synthetic_"); 2475 strcat(cmd, field_name); 2476 strcat(cmd, "="); 2477 strcat(cmd, field_name); 2478 2479 /* Use the same filter as the compatible histogram */ 2480 saved_filter = find_trigger_filter(hist_data, file); 2481 if (saved_filter) { 2482 strcat(cmd, " if "); 2483 strcat(cmd, saved_filter); 2484 } 2485 2486 var_hist->cmd = kstrdup(cmd, GFP_KERNEL); 2487 if (!var_hist->cmd) { 2488 kfree(cmd); 2489 kfree(var_hist); 2490 return ERR_PTR(-ENOMEM); 2491 } 2492 2493 /* Save the compatible histogram information */ 2494 var_hist->hist_data = hist_data; 2495 2496 /* Create the new histogram with our variable */ 2497 ret = event_hist_trigger_func(&trigger_hist_cmd, file, 2498 "", "hist", cmd); 2499 if (ret) { 2500 kfree(cmd); 2501 kfree(var_hist->cmd); 2502 kfree(var_hist); 2503 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name)); 2504 return ERR_PTR(ret); 2505 } 2506 2507 kfree(cmd); 2508 2509 /* If we can't find the variable, something went wrong */ 2510 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 2511 event_name, field_name); 2512 if (IS_ERR_OR_NULL(event_var)) { 2513 kfree(var_hist->cmd); 2514 kfree(var_hist); 2515 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name)); 2516 return ERR_PTR(-EINVAL); 2517 } 2518 2519 n = target_hist_data->n_field_var_hists; 2520 target_hist_data->field_var_hists[n] = var_hist; 2521 target_hist_data->n_field_var_hists++; 2522 2523 return event_var; 2524 } 2525 2526 static struct hist_field * 2527 find_target_event_var(struct hist_trigger_data *hist_data, 2528 char *subsys_name, char *event_name, char *var_name) 2529 { 2530 struct trace_event_file *file = hist_data->event_file; 2531 struct hist_field *hist_field = NULL; 2532 2533 if (subsys_name) { 2534 struct trace_event_call *call; 2535 2536 if (!event_name) 2537 return NULL; 2538 2539 call = file->event_call; 2540 2541 if (strcmp(subsys_name, call->class->system) != 0) 2542 return NULL; 2543 2544 if (strcmp(event_name, trace_event_name(call)) != 0) 2545 return NULL; 2546 } 2547 2548 hist_field = find_var_field(hist_data, var_name); 2549 2550 return hist_field; 2551 } 2552 2553 static inline void __update_field_vars(struct tracing_map_elt *elt, 2554 struct ring_buffer_event *rbe, 2555 void *rec, 2556 struct field_var **field_vars, 2557 unsigned int n_field_vars, 2558 unsigned int field_var_str_start) 2559 { 2560 struct hist_elt_data *elt_data = elt->private_data; 2561 unsigned int i, j, var_idx; 2562 u64 var_val; 2563 2564 for (i = 0, j = field_var_str_start; i < n_field_vars; i++) { 2565 struct field_var *field_var = field_vars[i]; 2566 struct hist_field *var = field_var->var; 2567 struct hist_field *val = field_var->val; 2568 2569 var_val = val->fn(val, elt, rbe, rec); 2570 var_idx = var->var.idx; 2571 2572 if (val->flags & HIST_FIELD_FL_STRING) { 2573 char *str = elt_data->field_var_str[j++]; 2574 char *val_str = (char *)(uintptr_t)var_val; 2575 2576 strscpy(str, val_str, STR_VAR_LEN_MAX); 2577 var_val = (u64)(uintptr_t)str; 2578 } 2579 tracing_map_set_var(elt, var_idx, var_val); 2580 } 2581 } 2582 2583 static void update_field_vars(struct hist_trigger_data *hist_data, 2584 struct tracing_map_elt *elt, 2585 struct ring_buffer_event *rbe, 2586 void *rec) 2587 { 2588 __update_field_vars(elt, rbe, rec, hist_data->field_vars, 2589 hist_data->n_field_vars, 0); 2590 } 2591 2592 static void save_track_data_vars(struct hist_trigger_data *hist_data, 2593 struct tracing_map_elt *elt, void *rec, 2594 struct ring_buffer_event *rbe, void *key, 2595 struct action_data *data, u64 *var_ref_vals) 2596 { 2597 __update_field_vars(elt, rbe, rec, hist_data->save_vars, 2598 hist_data->n_save_vars, hist_data->n_field_var_str); 2599 } 2600 2601 static struct hist_field *create_var(struct hist_trigger_data *hist_data, 2602 struct trace_event_file *file, 2603 char *name, int size, const char *type) 2604 { 2605 struct hist_field *var; 2606 int idx; 2607 2608 if (find_var(hist_data, file, name) && !hist_data->remove) { 2609 var = ERR_PTR(-EINVAL); 2610 goto out; 2611 } 2612 2613 var = kzalloc(sizeof(struct hist_field), GFP_KERNEL); 2614 if (!var) { 2615 var = ERR_PTR(-ENOMEM); 2616 goto out; 2617 } 2618 2619 idx = tracing_map_add_var(hist_data->map); 2620 if (idx < 0) { 2621 kfree(var); 2622 var = ERR_PTR(-EINVAL); 2623 goto out; 2624 } 2625 2626 var->ref = 1; 2627 var->flags = HIST_FIELD_FL_VAR; 2628 var->var.idx = idx; 2629 var->var.hist_data = var->hist_data = hist_data; 2630 var->size = size; 2631 var->var.name = kstrdup(name, GFP_KERNEL); 2632 var->type = kstrdup(type, GFP_KERNEL); 2633 if (!var->var.name || !var->type) { 2634 kfree(var->var.name); 2635 kfree(var->type); 2636 kfree(var); 2637 var = ERR_PTR(-ENOMEM); 2638 } 2639 out: 2640 return var; 2641 } 2642 2643 static struct field_var *create_field_var(struct hist_trigger_data *hist_data, 2644 struct trace_event_file *file, 2645 char *field_name) 2646 { 2647 struct hist_field *val = NULL, *var = NULL; 2648 unsigned long flags = HIST_FIELD_FL_VAR; 2649 struct trace_array *tr = file->tr; 2650 struct field_var *field_var; 2651 int ret = 0; 2652 2653 if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) { 2654 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name)); 2655 ret = -EINVAL; 2656 goto err; 2657 } 2658 2659 val = parse_atom(hist_data, file, field_name, &flags, NULL); 2660 if (IS_ERR(val)) { 2661 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name)); 2662 ret = PTR_ERR(val); 2663 goto err; 2664 } 2665 2666 var = create_var(hist_data, file, field_name, val->size, val->type); 2667 if (IS_ERR(var)) { 2668 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name)); 2669 kfree(val); 2670 ret = PTR_ERR(var); 2671 goto err; 2672 } 2673 2674 field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL); 2675 if (!field_var) { 2676 kfree(val); 2677 kfree(var); 2678 ret = -ENOMEM; 2679 goto err; 2680 } 2681 2682 field_var->var = var; 2683 field_var->val = val; 2684 out: 2685 return field_var; 2686 err: 2687 field_var = ERR_PTR(ret); 2688 goto out; 2689 } 2690 2691 /** 2692 * create_target_field_var - Automatically create a variable for a field 2693 * @target_hist_data: The target hist trigger 2694 * @subsys_name: Optional subsystem name 2695 * @event_name: Optional event name 2696 * @var_name: The name of the field (and the resulting variable) 2697 * 2698 * Hist trigger actions fetch data from variables, not directly from 2699 * events. However, for convenience, users are allowed to directly 2700 * specify an event field in an action, which will be automatically 2701 * converted into a variable on their behalf. 2702 2703 * This function creates a field variable with the name var_name on 2704 * the hist trigger currently being defined on the target event. If 2705 * subsys_name and event_name are specified, this function simply 2706 * verifies that they do in fact match the target event subsystem and 2707 * event name. 2708 * 2709 * Return: The variable created for the field. 2710 */ 2711 static struct field_var * 2712 create_target_field_var(struct hist_trigger_data *target_hist_data, 2713 char *subsys_name, char *event_name, char *var_name) 2714 { 2715 struct trace_event_file *file = target_hist_data->event_file; 2716 2717 if (subsys_name) { 2718 struct trace_event_call *call; 2719 2720 if (!event_name) 2721 return NULL; 2722 2723 call = file->event_call; 2724 2725 if (strcmp(subsys_name, call->class->system) != 0) 2726 return NULL; 2727 2728 if (strcmp(event_name, trace_event_name(call)) != 0) 2729 return NULL; 2730 } 2731 2732 return create_field_var(target_hist_data, file, var_name); 2733 } 2734 2735 static bool check_track_val_max(u64 track_val, u64 var_val) 2736 { 2737 if (var_val <= track_val) 2738 return false; 2739 2740 return true; 2741 } 2742 2743 static bool check_track_val_changed(u64 track_val, u64 var_val) 2744 { 2745 if (var_val == track_val) 2746 return false; 2747 2748 return true; 2749 } 2750 2751 static u64 get_track_val(struct hist_trigger_data *hist_data, 2752 struct tracing_map_elt *elt, 2753 struct action_data *data) 2754 { 2755 unsigned int track_var_idx = data->track_data.track_var->var.idx; 2756 u64 track_val; 2757 2758 track_val = tracing_map_read_var(elt, track_var_idx); 2759 2760 return track_val; 2761 } 2762 2763 static void save_track_val(struct hist_trigger_data *hist_data, 2764 struct tracing_map_elt *elt, 2765 struct action_data *data, u64 var_val) 2766 { 2767 unsigned int track_var_idx = data->track_data.track_var->var.idx; 2768 2769 tracing_map_set_var(elt, track_var_idx, var_val); 2770 } 2771 2772 static void save_track_data(struct hist_trigger_data *hist_data, 2773 struct tracing_map_elt *elt, void *rec, 2774 struct ring_buffer_event *rbe, void *key, 2775 struct action_data *data, u64 *var_ref_vals) 2776 { 2777 if (data->track_data.save_data) 2778 data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals); 2779 } 2780 2781 static bool check_track_val(struct tracing_map_elt *elt, 2782 struct action_data *data, 2783 u64 var_val) 2784 { 2785 struct hist_trigger_data *hist_data; 2786 u64 track_val; 2787 2788 hist_data = data->track_data.track_var->hist_data; 2789 track_val = get_track_val(hist_data, elt, data); 2790 2791 return data->track_data.check_val(track_val, var_val); 2792 } 2793 2794 #ifdef CONFIG_TRACER_SNAPSHOT 2795 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data) 2796 { 2797 /* called with tr->max_lock held */ 2798 struct track_data *track_data = tr->cond_snapshot->cond_data; 2799 struct hist_elt_data *elt_data, *track_elt_data; 2800 struct snapshot_context *context = cond_data; 2801 struct action_data *action; 2802 u64 track_val; 2803 2804 if (!track_data) 2805 return false; 2806 2807 action = track_data->action_data; 2808 2809 track_val = get_track_val(track_data->hist_data, context->elt, 2810 track_data->action_data); 2811 2812 if (!action->track_data.check_val(track_data->track_val, track_val)) 2813 return false; 2814 2815 track_data->track_val = track_val; 2816 memcpy(track_data->key, context->key, track_data->key_len); 2817 2818 elt_data = context->elt->private_data; 2819 track_elt_data = track_data->elt.private_data; 2820 if (elt_data->comm) 2821 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN); 2822 2823 track_data->updated = true; 2824 2825 return true; 2826 } 2827 2828 static void save_track_data_snapshot(struct hist_trigger_data *hist_data, 2829 struct tracing_map_elt *elt, void *rec, 2830 struct ring_buffer_event *rbe, void *key, 2831 struct action_data *data, 2832 u64 *var_ref_vals) 2833 { 2834 struct trace_event_file *file = hist_data->event_file; 2835 struct snapshot_context context; 2836 2837 context.elt = elt; 2838 context.key = key; 2839 2840 tracing_snapshot_cond(file->tr, &context); 2841 } 2842 2843 static void hist_trigger_print_key(struct seq_file *m, 2844 struct hist_trigger_data *hist_data, 2845 void *key, 2846 struct tracing_map_elt *elt); 2847 2848 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data) 2849 { 2850 unsigned int i; 2851 2852 if (!hist_data->n_actions) 2853 return NULL; 2854 2855 for (i = 0; i < hist_data->n_actions; i++) { 2856 struct action_data *data = hist_data->actions[i]; 2857 2858 if (data->action == ACTION_SNAPSHOT) 2859 return data; 2860 } 2861 2862 return NULL; 2863 } 2864 2865 static void track_data_snapshot_print(struct seq_file *m, 2866 struct hist_trigger_data *hist_data) 2867 { 2868 struct trace_event_file *file = hist_data->event_file; 2869 struct track_data *track_data; 2870 struct action_data *action; 2871 2872 track_data = tracing_cond_snapshot_data(file->tr); 2873 if (!track_data) 2874 return; 2875 2876 if (!track_data->updated) 2877 return; 2878 2879 action = snapshot_action(hist_data); 2880 if (!action) 2881 return; 2882 2883 seq_puts(m, "\nSnapshot taken (see tracing/snapshot). Details:\n"); 2884 seq_printf(m, "\ttriggering value { %s(%s) }: %10llu", 2885 action->handler == HANDLER_ONMAX ? "onmax" : "onchange", 2886 action->track_data.var_str, track_data->track_val); 2887 2888 seq_puts(m, "\ttriggered by event with key: "); 2889 hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt); 2890 seq_putc(m, '\n'); 2891 } 2892 #else 2893 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data) 2894 { 2895 return false; 2896 } 2897 static void save_track_data_snapshot(struct hist_trigger_data *hist_data, 2898 struct tracing_map_elt *elt, void *rec, 2899 struct ring_buffer_event *rbe, void *key, 2900 struct action_data *data, 2901 u64 *var_ref_vals) {} 2902 static void track_data_snapshot_print(struct seq_file *m, 2903 struct hist_trigger_data *hist_data) {} 2904 #endif /* CONFIG_TRACER_SNAPSHOT */ 2905 2906 static void track_data_print(struct seq_file *m, 2907 struct hist_trigger_data *hist_data, 2908 struct tracing_map_elt *elt, 2909 struct action_data *data) 2910 { 2911 u64 track_val = get_track_val(hist_data, elt, data); 2912 unsigned int i, save_var_idx; 2913 2914 if (data->handler == HANDLER_ONMAX) 2915 seq_printf(m, "\n\tmax: %10llu", track_val); 2916 else if (data->handler == HANDLER_ONCHANGE) 2917 seq_printf(m, "\n\tchanged: %10llu", track_val); 2918 2919 if (data->action == ACTION_SNAPSHOT) 2920 return; 2921 2922 for (i = 0; i < hist_data->n_save_vars; i++) { 2923 struct hist_field *save_val = hist_data->save_vars[i]->val; 2924 struct hist_field *save_var = hist_data->save_vars[i]->var; 2925 u64 val; 2926 2927 save_var_idx = save_var->var.idx; 2928 2929 val = tracing_map_read_var(elt, save_var_idx); 2930 2931 if (save_val->flags & HIST_FIELD_FL_STRING) { 2932 seq_printf(m, " %s: %-32s", save_var->var.name, 2933 (char *)(uintptr_t)(val)); 2934 } else 2935 seq_printf(m, " %s: %10llu", save_var->var.name, val); 2936 } 2937 } 2938 2939 static void ontrack_action(struct hist_trigger_data *hist_data, 2940 struct tracing_map_elt *elt, void *rec, 2941 struct ring_buffer_event *rbe, void *key, 2942 struct action_data *data, u64 *var_ref_vals) 2943 { 2944 u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx]; 2945 2946 if (check_track_val(elt, data, var_val)) { 2947 save_track_val(hist_data, elt, data, var_val); 2948 save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals); 2949 } 2950 } 2951 2952 static void action_data_destroy(struct action_data *data) 2953 { 2954 unsigned int i; 2955 2956 lockdep_assert_held(&event_mutex); 2957 2958 kfree(data->action_name); 2959 2960 for (i = 0; i < data->n_params; i++) 2961 kfree(data->params[i]); 2962 2963 if (data->synth_event) 2964 data->synth_event->ref--; 2965 2966 kfree(data->synth_event_name); 2967 2968 kfree(data); 2969 } 2970 2971 static void track_data_destroy(struct hist_trigger_data *hist_data, 2972 struct action_data *data) 2973 { 2974 struct trace_event_file *file = hist_data->event_file; 2975 2976 destroy_hist_field(data->track_data.track_var, 0); 2977 2978 if (data->action == ACTION_SNAPSHOT) { 2979 struct track_data *track_data; 2980 2981 track_data = tracing_cond_snapshot_data(file->tr); 2982 if (track_data && track_data->hist_data == hist_data) { 2983 tracing_snapshot_cond_disable(file->tr); 2984 track_data_free(track_data); 2985 } 2986 } 2987 2988 kfree(data->track_data.var_str); 2989 2990 action_data_destroy(data); 2991 } 2992 2993 static int action_create(struct hist_trigger_data *hist_data, 2994 struct action_data *data); 2995 2996 static int track_data_create(struct hist_trigger_data *hist_data, 2997 struct action_data *data) 2998 { 2999 struct hist_field *var_field, *ref_field, *track_var = NULL; 3000 struct trace_event_file *file = hist_data->event_file; 3001 struct trace_array *tr = file->tr; 3002 char *track_data_var_str; 3003 int ret = 0; 3004 3005 track_data_var_str = data->track_data.var_str; 3006 if (track_data_var_str[0] != '$') { 3007 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str)); 3008 return -EINVAL; 3009 } 3010 track_data_var_str++; 3011 3012 var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str); 3013 if (!var_field) { 3014 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str)); 3015 return -EINVAL; 3016 } 3017 3018 ref_field = create_var_ref(hist_data, var_field, NULL, NULL); 3019 if (!ref_field) 3020 return -ENOMEM; 3021 3022 data->track_data.var_ref = ref_field; 3023 3024 if (data->handler == HANDLER_ONMAX) 3025 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64"); 3026 if (IS_ERR(track_var)) { 3027 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0); 3028 ret = PTR_ERR(track_var); 3029 goto out; 3030 } 3031 3032 if (data->handler == HANDLER_ONCHANGE) 3033 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64"); 3034 if (IS_ERR(track_var)) { 3035 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0); 3036 ret = PTR_ERR(track_var); 3037 goto out; 3038 } 3039 data->track_data.track_var = track_var; 3040 3041 ret = action_create(hist_data, data); 3042 out: 3043 return ret; 3044 } 3045 3046 static int parse_action_params(struct trace_array *tr, char *params, 3047 struct action_data *data) 3048 { 3049 char *param, *saved_param; 3050 bool first_param = true; 3051 int ret = 0; 3052 3053 while (params) { 3054 if (data->n_params >= SYNTH_FIELDS_MAX) { 3055 hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0); 3056 goto out; 3057 } 3058 3059 param = strsep(¶ms, ","); 3060 if (!param) { 3061 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0); 3062 ret = -EINVAL; 3063 goto out; 3064 } 3065 3066 param = strstrip(param); 3067 if (strlen(param) < 2) { 3068 hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param)); 3069 ret = -EINVAL; 3070 goto out; 3071 } 3072 3073 saved_param = kstrdup(param, GFP_KERNEL); 3074 if (!saved_param) { 3075 ret = -ENOMEM; 3076 goto out; 3077 } 3078 3079 if (first_param && data->use_trace_keyword) { 3080 data->synth_event_name = saved_param; 3081 first_param = false; 3082 continue; 3083 } 3084 first_param = false; 3085 3086 data->params[data->n_params++] = saved_param; 3087 } 3088 out: 3089 return ret; 3090 } 3091 3092 static int action_parse(struct trace_array *tr, char *str, struct action_data *data, 3093 enum handler_id handler) 3094 { 3095 char *action_name; 3096 int ret = 0; 3097 3098 strsep(&str, "."); 3099 if (!str) { 3100 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0); 3101 ret = -EINVAL; 3102 goto out; 3103 } 3104 3105 action_name = strsep(&str, "("); 3106 if (!action_name || !str) { 3107 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0); 3108 ret = -EINVAL; 3109 goto out; 3110 } 3111 3112 if (str_has_prefix(action_name, "save")) { 3113 char *params = strsep(&str, ")"); 3114 3115 if (!params) { 3116 hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0); 3117 ret = -EINVAL; 3118 goto out; 3119 } 3120 3121 ret = parse_action_params(tr, params, data); 3122 if (ret) 3123 goto out; 3124 3125 if (handler == HANDLER_ONMAX) 3126 data->track_data.check_val = check_track_val_max; 3127 else if (handler == HANDLER_ONCHANGE) 3128 data->track_data.check_val = check_track_val_changed; 3129 else { 3130 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name)); 3131 ret = -EINVAL; 3132 goto out; 3133 } 3134 3135 data->track_data.save_data = save_track_data_vars; 3136 data->fn = ontrack_action; 3137 data->action = ACTION_SAVE; 3138 } else if (str_has_prefix(action_name, "snapshot")) { 3139 char *params = strsep(&str, ")"); 3140 3141 if (!str) { 3142 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params)); 3143 ret = -EINVAL; 3144 goto out; 3145 } 3146 3147 if (handler == HANDLER_ONMAX) 3148 data->track_data.check_val = check_track_val_max; 3149 else if (handler == HANDLER_ONCHANGE) 3150 data->track_data.check_val = check_track_val_changed; 3151 else { 3152 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name)); 3153 ret = -EINVAL; 3154 goto out; 3155 } 3156 3157 data->track_data.save_data = save_track_data_snapshot; 3158 data->fn = ontrack_action; 3159 data->action = ACTION_SNAPSHOT; 3160 } else { 3161 char *params = strsep(&str, ")"); 3162 3163 if (str_has_prefix(action_name, "trace")) 3164 data->use_trace_keyword = true; 3165 3166 if (params) { 3167 ret = parse_action_params(tr, params, data); 3168 if (ret) 3169 goto out; 3170 } 3171 3172 if (handler == HANDLER_ONMAX) 3173 data->track_data.check_val = check_track_val_max; 3174 else if (handler == HANDLER_ONCHANGE) 3175 data->track_data.check_val = check_track_val_changed; 3176 3177 if (handler != HANDLER_ONMATCH) { 3178 data->track_data.save_data = action_trace; 3179 data->fn = ontrack_action; 3180 } else 3181 data->fn = action_trace; 3182 3183 data->action = ACTION_TRACE; 3184 } 3185 3186 data->action_name = kstrdup(action_name, GFP_KERNEL); 3187 if (!data->action_name) { 3188 ret = -ENOMEM; 3189 goto out; 3190 } 3191 3192 data->handler = handler; 3193 out: 3194 return ret; 3195 } 3196 3197 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data, 3198 char *str, enum handler_id handler) 3199 { 3200 struct action_data *data; 3201 int ret = -EINVAL; 3202 char *var_str; 3203 3204 data = kzalloc(sizeof(*data), GFP_KERNEL); 3205 if (!data) 3206 return ERR_PTR(-ENOMEM); 3207 3208 var_str = strsep(&str, ")"); 3209 if (!var_str || !str) { 3210 ret = -EINVAL; 3211 goto free; 3212 } 3213 3214 data->track_data.var_str = kstrdup(var_str, GFP_KERNEL); 3215 if (!data->track_data.var_str) { 3216 ret = -ENOMEM; 3217 goto free; 3218 } 3219 3220 ret = action_parse(hist_data->event_file->tr, str, data, handler); 3221 if (ret) 3222 goto free; 3223 out: 3224 return data; 3225 free: 3226 track_data_destroy(hist_data, data); 3227 data = ERR_PTR(ret); 3228 goto out; 3229 } 3230 3231 static void onmatch_destroy(struct action_data *data) 3232 { 3233 kfree(data->match_data.event); 3234 kfree(data->match_data.event_system); 3235 3236 action_data_destroy(data); 3237 } 3238 3239 static void destroy_field_var(struct field_var *field_var) 3240 { 3241 if (!field_var) 3242 return; 3243 3244 destroy_hist_field(field_var->var, 0); 3245 destroy_hist_field(field_var->val, 0); 3246 3247 kfree(field_var); 3248 } 3249 3250 static void destroy_field_vars(struct hist_trigger_data *hist_data) 3251 { 3252 unsigned int i; 3253 3254 for (i = 0; i < hist_data->n_field_vars; i++) 3255 destroy_field_var(hist_data->field_vars[i]); 3256 3257 for (i = 0; i < hist_data->n_save_vars; i++) 3258 destroy_field_var(hist_data->save_vars[i]); 3259 } 3260 3261 static void save_field_var(struct hist_trigger_data *hist_data, 3262 struct field_var *field_var) 3263 { 3264 hist_data->field_vars[hist_data->n_field_vars++] = field_var; 3265 3266 if (field_var->val->flags & HIST_FIELD_FL_STRING) 3267 hist_data->n_field_var_str++; 3268 } 3269 3270 3271 static int check_synth_field(struct synth_event *event, 3272 struct hist_field *hist_field, 3273 unsigned int field_pos) 3274 { 3275 struct synth_field *field; 3276 3277 if (field_pos >= event->n_fields) 3278 return -EINVAL; 3279 3280 field = event->fields[field_pos]; 3281 3282 if (strcmp(field->type, hist_field->type) != 0) { 3283 if (field->size != hist_field->size || 3284 field->is_signed != hist_field->is_signed) 3285 return -EINVAL; 3286 } 3287 3288 return 0; 3289 } 3290 3291 static struct hist_field * 3292 trace_action_find_var(struct hist_trigger_data *hist_data, 3293 struct action_data *data, 3294 char *system, char *event, char *var) 3295 { 3296 struct trace_array *tr = hist_data->event_file->tr; 3297 struct hist_field *hist_field; 3298 3299 var++; /* skip '$' */ 3300 3301 hist_field = find_target_event_var(hist_data, system, event, var); 3302 if (!hist_field) { 3303 if (!system && data->handler == HANDLER_ONMATCH) { 3304 system = data->match_data.event_system; 3305 event = data->match_data.event; 3306 } 3307 3308 hist_field = find_event_var(hist_data, system, event, var); 3309 } 3310 3311 if (!hist_field) 3312 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var)); 3313 3314 return hist_field; 3315 } 3316 3317 static struct hist_field * 3318 trace_action_create_field_var(struct hist_trigger_data *hist_data, 3319 struct action_data *data, char *system, 3320 char *event, char *var) 3321 { 3322 struct hist_field *hist_field = NULL; 3323 struct field_var *field_var; 3324 3325 /* 3326 * First try to create a field var on the target event (the 3327 * currently being defined). This will create a variable for 3328 * unqualified fields on the target event, or if qualified, 3329 * target fields that have qualified names matching the target. 3330 */ 3331 field_var = create_target_field_var(hist_data, system, event, var); 3332 3333 if (field_var && !IS_ERR(field_var)) { 3334 save_field_var(hist_data, field_var); 3335 hist_field = field_var->var; 3336 } else { 3337 field_var = NULL; 3338 /* 3339 * If no explicit system.event is specfied, default to 3340 * looking for fields on the onmatch(system.event.xxx) 3341 * event. 3342 */ 3343 if (!system && data->handler == HANDLER_ONMATCH) { 3344 system = data->match_data.event_system; 3345 event = data->match_data.event; 3346 } 3347 3348 /* 3349 * At this point, we're looking at a field on another 3350 * event. Because we can't modify a hist trigger on 3351 * another event to add a variable for a field, we need 3352 * to create a new trigger on that event and create the 3353 * variable at the same time. 3354 */ 3355 hist_field = create_field_var_hist(hist_data, system, event, var); 3356 if (IS_ERR(hist_field)) 3357 goto free; 3358 } 3359 out: 3360 return hist_field; 3361 free: 3362 destroy_field_var(field_var); 3363 hist_field = NULL; 3364 goto out; 3365 } 3366 3367 static int trace_action_create(struct hist_trigger_data *hist_data, 3368 struct action_data *data) 3369 { 3370 struct trace_array *tr = hist_data->event_file->tr; 3371 char *event_name, *param, *system = NULL; 3372 struct hist_field *hist_field, *var_ref; 3373 unsigned int i; 3374 unsigned int field_pos = 0; 3375 struct synth_event *event; 3376 char *synth_event_name; 3377 int var_ref_idx, ret = 0; 3378 3379 lockdep_assert_held(&event_mutex); 3380 3381 if (data->use_trace_keyword) 3382 synth_event_name = data->synth_event_name; 3383 else 3384 synth_event_name = data->action_name; 3385 3386 event = find_synth_event(synth_event_name); 3387 if (!event) { 3388 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name)); 3389 return -EINVAL; 3390 } 3391 3392 event->ref++; 3393 3394 for (i = 0; i < data->n_params; i++) { 3395 char *p; 3396 3397 p = param = kstrdup(data->params[i], GFP_KERNEL); 3398 if (!param) { 3399 ret = -ENOMEM; 3400 goto err; 3401 } 3402 3403 system = strsep(¶m, "."); 3404 if (!param) { 3405 param = (char *)system; 3406 system = event_name = NULL; 3407 } else { 3408 event_name = strsep(¶m, "."); 3409 if (!param) { 3410 kfree(p); 3411 ret = -EINVAL; 3412 goto err; 3413 } 3414 } 3415 3416 if (param[0] == '$') 3417 hist_field = trace_action_find_var(hist_data, data, 3418 system, event_name, 3419 param); 3420 else 3421 hist_field = trace_action_create_field_var(hist_data, 3422 data, 3423 system, 3424 event_name, 3425 param); 3426 3427 if (!hist_field) { 3428 kfree(p); 3429 ret = -EINVAL; 3430 goto err; 3431 } 3432 3433 if (check_synth_field(event, hist_field, field_pos) == 0) { 3434 var_ref = create_var_ref(hist_data, hist_field, 3435 system, event_name); 3436 if (!var_ref) { 3437 kfree(p); 3438 ret = -ENOMEM; 3439 goto err; 3440 } 3441 3442 var_ref_idx = find_var_ref_idx(hist_data, var_ref); 3443 if (WARN_ON(var_ref_idx < 0)) { 3444 ret = var_ref_idx; 3445 goto err; 3446 } 3447 3448 data->var_ref_idx[i] = var_ref_idx; 3449 3450 field_pos++; 3451 kfree(p); 3452 continue; 3453 } 3454 3455 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param)); 3456 kfree(p); 3457 ret = -EINVAL; 3458 goto err; 3459 } 3460 3461 if (field_pos != event->n_fields) { 3462 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name)); 3463 ret = -EINVAL; 3464 goto err; 3465 } 3466 3467 data->synth_event = event; 3468 out: 3469 return ret; 3470 err: 3471 event->ref--; 3472 3473 goto out; 3474 } 3475 3476 static int action_create(struct hist_trigger_data *hist_data, 3477 struct action_data *data) 3478 { 3479 struct trace_event_file *file = hist_data->event_file; 3480 struct trace_array *tr = file->tr; 3481 struct track_data *track_data; 3482 struct field_var *field_var; 3483 unsigned int i; 3484 char *param; 3485 int ret = 0; 3486 3487 if (data->action == ACTION_TRACE) 3488 return trace_action_create(hist_data, data); 3489 3490 if (data->action == ACTION_SNAPSHOT) { 3491 track_data = track_data_alloc(hist_data->key_size, data, hist_data); 3492 if (IS_ERR(track_data)) { 3493 ret = PTR_ERR(track_data); 3494 goto out; 3495 } 3496 3497 ret = tracing_snapshot_cond_enable(file->tr, track_data, 3498 cond_snapshot_update); 3499 if (ret) 3500 track_data_free(track_data); 3501 3502 goto out; 3503 } 3504 3505 if (data->action == ACTION_SAVE) { 3506 if (hist_data->n_save_vars) { 3507 ret = -EEXIST; 3508 hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0); 3509 goto out; 3510 } 3511 3512 for (i = 0; i < data->n_params; i++) { 3513 param = kstrdup(data->params[i], GFP_KERNEL); 3514 if (!param) { 3515 ret = -ENOMEM; 3516 goto out; 3517 } 3518 3519 field_var = create_target_field_var(hist_data, NULL, NULL, param); 3520 if (IS_ERR(field_var)) { 3521 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL, 3522 errpos(param)); 3523 ret = PTR_ERR(field_var); 3524 kfree(param); 3525 goto out; 3526 } 3527 3528 hist_data->save_vars[hist_data->n_save_vars++] = field_var; 3529 if (field_var->val->flags & HIST_FIELD_FL_STRING) 3530 hist_data->n_save_var_str++; 3531 kfree(param); 3532 } 3533 } 3534 out: 3535 return ret; 3536 } 3537 3538 static int onmatch_create(struct hist_trigger_data *hist_data, 3539 struct action_data *data) 3540 { 3541 return action_create(hist_data, data); 3542 } 3543 3544 static struct action_data *onmatch_parse(struct trace_array *tr, char *str) 3545 { 3546 char *match_event, *match_event_system; 3547 struct action_data *data; 3548 int ret = -EINVAL; 3549 3550 data = kzalloc(sizeof(*data), GFP_KERNEL); 3551 if (!data) 3552 return ERR_PTR(-ENOMEM); 3553 3554 match_event = strsep(&str, ")"); 3555 if (!match_event || !str) { 3556 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event)); 3557 goto free; 3558 } 3559 3560 match_event_system = strsep(&match_event, "."); 3561 if (!match_event) { 3562 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system)); 3563 goto free; 3564 } 3565 3566 if (IS_ERR(event_file(tr, match_event_system, match_event))) { 3567 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event)); 3568 goto free; 3569 } 3570 3571 data->match_data.event = kstrdup(match_event, GFP_KERNEL); 3572 if (!data->match_data.event) { 3573 ret = -ENOMEM; 3574 goto free; 3575 } 3576 3577 data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL); 3578 if (!data->match_data.event_system) { 3579 ret = -ENOMEM; 3580 goto free; 3581 } 3582 3583 ret = action_parse(tr, str, data, HANDLER_ONMATCH); 3584 if (ret) 3585 goto free; 3586 out: 3587 return data; 3588 free: 3589 onmatch_destroy(data); 3590 data = ERR_PTR(ret); 3591 goto out; 3592 } 3593 3594 static int create_hitcount_val(struct hist_trigger_data *hist_data) 3595 { 3596 hist_data->fields[HITCOUNT_IDX] = 3597 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL); 3598 if (!hist_data->fields[HITCOUNT_IDX]) 3599 return -ENOMEM; 3600 3601 hist_data->n_vals++; 3602 hist_data->n_fields++; 3603 3604 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX)) 3605 return -EINVAL; 3606 3607 return 0; 3608 } 3609 3610 static int __create_val_field(struct hist_trigger_data *hist_data, 3611 unsigned int val_idx, 3612 struct trace_event_file *file, 3613 char *var_name, char *field_str, 3614 unsigned long flags) 3615 { 3616 struct hist_field *hist_field; 3617 int ret = 0; 3618 3619 hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0); 3620 if (IS_ERR(hist_field)) { 3621 ret = PTR_ERR(hist_field); 3622 goto out; 3623 } 3624 3625 hist_data->fields[val_idx] = hist_field; 3626 3627 ++hist_data->n_vals; 3628 ++hist_data->n_fields; 3629 3630 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 3631 ret = -EINVAL; 3632 out: 3633 return ret; 3634 } 3635 3636 static int create_val_field(struct hist_trigger_data *hist_data, 3637 unsigned int val_idx, 3638 struct trace_event_file *file, 3639 char *field_str) 3640 { 3641 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX)) 3642 return -EINVAL; 3643 3644 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0); 3645 } 3646 3647 static int create_var_field(struct hist_trigger_data *hist_data, 3648 unsigned int val_idx, 3649 struct trace_event_file *file, 3650 char *var_name, char *expr_str) 3651 { 3652 struct trace_array *tr = hist_data->event_file->tr; 3653 unsigned long flags = 0; 3654 3655 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 3656 return -EINVAL; 3657 3658 if (find_var(hist_data, file, var_name) && !hist_data->remove) { 3659 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name)); 3660 return -EINVAL; 3661 } 3662 3663 flags |= HIST_FIELD_FL_VAR; 3664 hist_data->n_vars++; 3665 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX)) 3666 return -EINVAL; 3667 3668 return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags); 3669 } 3670 3671 static int create_val_fields(struct hist_trigger_data *hist_data, 3672 struct trace_event_file *file) 3673 { 3674 char *fields_str, *field_str; 3675 unsigned int i, j = 1; 3676 int ret; 3677 3678 ret = create_hitcount_val(hist_data); 3679 if (ret) 3680 goto out; 3681 3682 fields_str = hist_data->attrs->vals_str; 3683 if (!fields_str) 3684 goto out; 3685 3686 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX && 3687 j < TRACING_MAP_VALS_MAX; i++) { 3688 field_str = strsep(&fields_str, ","); 3689 if (!field_str) 3690 break; 3691 3692 if (strcmp(field_str, "hitcount") == 0) 3693 continue; 3694 3695 ret = create_val_field(hist_data, j++, file, field_str); 3696 if (ret) 3697 goto out; 3698 } 3699 3700 if (fields_str && (strcmp(fields_str, "hitcount") != 0)) 3701 ret = -EINVAL; 3702 out: 3703 return ret; 3704 } 3705 3706 static int create_key_field(struct hist_trigger_data *hist_data, 3707 unsigned int key_idx, 3708 unsigned int key_offset, 3709 struct trace_event_file *file, 3710 char *field_str) 3711 { 3712 struct trace_array *tr = hist_data->event_file->tr; 3713 struct hist_field *hist_field = NULL; 3714 unsigned long flags = 0; 3715 unsigned int key_size; 3716 int ret = 0; 3717 3718 if (WARN_ON(key_idx >= HIST_FIELDS_MAX)) 3719 return -EINVAL; 3720 3721 flags |= HIST_FIELD_FL_KEY; 3722 3723 if (strcmp(field_str, "stacktrace") == 0) { 3724 flags |= HIST_FIELD_FL_STACKTRACE; 3725 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH; 3726 hist_field = create_hist_field(hist_data, NULL, flags, NULL); 3727 } else { 3728 hist_field = parse_expr(hist_data, file, field_str, flags, 3729 NULL, 0); 3730 if (IS_ERR(hist_field)) { 3731 ret = PTR_ERR(hist_field); 3732 goto out; 3733 } 3734 3735 if (field_has_hist_vars(hist_field, 0)) { 3736 hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str)); 3737 destroy_hist_field(hist_field, 0); 3738 ret = -EINVAL; 3739 goto out; 3740 } 3741 3742 key_size = hist_field->size; 3743 } 3744 3745 hist_data->fields[key_idx] = hist_field; 3746 3747 key_size = ALIGN(key_size, sizeof(u64)); 3748 hist_data->fields[key_idx]->size = key_size; 3749 hist_data->fields[key_idx]->offset = key_offset; 3750 3751 hist_data->key_size += key_size; 3752 3753 if (hist_data->key_size > HIST_KEY_SIZE_MAX) { 3754 ret = -EINVAL; 3755 goto out; 3756 } 3757 3758 hist_data->n_keys++; 3759 hist_data->n_fields++; 3760 3761 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX)) 3762 return -EINVAL; 3763 3764 ret = key_size; 3765 out: 3766 return ret; 3767 } 3768 3769 static int create_key_fields(struct hist_trigger_data *hist_data, 3770 struct trace_event_file *file) 3771 { 3772 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals; 3773 char *fields_str, *field_str; 3774 int ret = -EINVAL; 3775 3776 fields_str = hist_data->attrs->keys_str; 3777 if (!fields_str) 3778 goto out; 3779 3780 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) { 3781 field_str = strsep(&fields_str, ","); 3782 if (!field_str) 3783 break; 3784 ret = create_key_field(hist_data, i, key_offset, 3785 file, field_str); 3786 if (ret < 0) 3787 goto out; 3788 key_offset += ret; 3789 } 3790 if (fields_str) { 3791 ret = -EINVAL; 3792 goto out; 3793 } 3794 ret = 0; 3795 out: 3796 return ret; 3797 } 3798 3799 static int create_var_fields(struct hist_trigger_data *hist_data, 3800 struct trace_event_file *file) 3801 { 3802 unsigned int i, j = hist_data->n_vals; 3803 int ret = 0; 3804 3805 unsigned int n_vars = hist_data->attrs->var_defs.n_vars; 3806 3807 for (i = 0; i < n_vars; i++) { 3808 char *var_name = hist_data->attrs->var_defs.name[i]; 3809 char *expr = hist_data->attrs->var_defs.expr[i]; 3810 3811 ret = create_var_field(hist_data, j++, file, var_name, expr); 3812 if (ret) 3813 goto out; 3814 } 3815 out: 3816 return ret; 3817 } 3818 3819 static void free_var_defs(struct hist_trigger_data *hist_data) 3820 { 3821 unsigned int i; 3822 3823 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 3824 kfree(hist_data->attrs->var_defs.name[i]); 3825 kfree(hist_data->attrs->var_defs.expr[i]); 3826 } 3827 3828 hist_data->attrs->var_defs.n_vars = 0; 3829 } 3830 3831 static int parse_var_defs(struct hist_trigger_data *hist_data) 3832 { 3833 struct trace_array *tr = hist_data->event_file->tr; 3834 char *s, *str, *var_name, *field_str; 3835 unsigned int i, j, n_vars = 0; 3836 int ret = 0; 3837 3838 for (i = 0; i < hist_data->attrs->n_assignments; i++) { 3839 str = hist_data->attrs->assignment_str[i]; 3840 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) { 3841 field_str = strsep(&str, ","); 3842 if (!field_str) 3843 break; 3844 3845 var_name = strsep(&field_str, "="); 3846 if (!var_name || !field_str) { 3847 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT, 3848 errpos(var_name)); 3849 ret = -EINVAL; 3850 goto free; 3851 } 3852 3853 if (n_vars == TRACING_MAP_VARS_MAX) { 3854 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name)); 3855 ret = -EINVAL; 3856 goto free; 3857 } 3858 3859 s = kstrdup(var_name, GFP_KERNEL); 3860 if (!s) { 3861 ret = -ENOMEM; 3862 goto free; 3863 } 3864 hist_data->attrs->var_defs.name[n_vars] = s; 3865 3866 s = kstrdup(field_str, GFP_KERNEL); 3867 if (!s) { 3868 kfree(hist_data->attrs->var_defs.name[n_vars]); 3869 ret = -ENOMEM; 3870 goto free; 3871 } 3872 hist_data->attrs->var_defs.expr[n_vars++] = s; 3873 3874 hist_data->attrs->var_defs.n_vars = n_vars; 3875 } 3876 } 3877 3878 return ret; 3879 free: 3880 free_var_defs(hist_data); 3881 3882 return ret; 3883 } 3884 3885 static int create_hist_fields(struct hist_trigger_data *hist_data, 3886 struct trace_event_file *file) 3887 { 3888 int ret; 3889 3890 ret = parse_var_defs(hist_data); 3891 if (ret) 3892 goto out; 3893 3894 ret = create_val_fields(hist_data, file); 3895 if (ret) 3896 goto out; 3897 3898 ret = create_var_fields(hist_data, file); 3899 if (ret) 3900 goto out; 3901 3902 ret = create_key_fields(hist_data, file); 3903 if (ret) 3904 goto out; 3905 out: 3906 free_var_defs(hist_data); 3907 3908 return ret; 3909 } 3910 3911 static int is_descending(struct trace_array *tr, const char *str) 3912 { 3913 if (!str) 3914 return 0; 3915 3916 if (strcmp(str, "descending") == 0) 3917 return 1; 3918 3919 if (strcmp(str, "ascending") == 0) 3920 return 0; 3921 3922 hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str)); 3923 3924 return -EINVAL; 3925 } 3926 3927 static int create_sort_keys(struct hist_trigger_data *hist_data) 3928 { 3929 struct trace_array *tr = hist_data->event_file->tr; 3930 char *fields_str = hist_data->attrs->sort_key_str; 3931 struct tracing_map_sort_key *sort_key; 3932 int descending, ret = 0; 3933 unsigned int i, j, k; 3934 3935 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */ 3936 3937 if (!fields_str) 3938 goto out; 3939 3940 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) { 3941 struct hist_field *hist_field; 3942 char *field_str, *field_name; 3943 const char *test_name; 3944 3945 sort_key = &hist_data->sort_keys[i]; 3946 3947 field_str = strsep(&fields_str, ","); 3948 if (!field_str) 3949 break; 3950 3951 if (!*field_str) { 3952 ret = -EINVAL; 3953 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort=")); 3954 break; 3955 } 3956 3957 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) { 3958 hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort=")); 3959 ret = -EINVAL; 3960 break; 3961 } 3962 3963 field_name = strsep(&field_str, "."); 3964 if (!field_name || !*field_name) { 3965 ret = -EINVAL; 3966 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort=")); 3967 break; 3968 } 3969 3970 if (strcmp(field_name, "hitcount") == 0) { 3971 descending = is_descending(tr, field_str); 3972 if (descending < 0) { 3973 ret = descending; 3974 break; 3975 } 3976 sort_key->descending = descending; 3977 continue; 3978 } 3979 3980 for (j = 1, k = 1; j < hist_data->n_fields; j++) { 3981 unsigned int idx; 3982 3983 hist_field = hist_data->fields[j]; 3984 if (hist_field->flags & HIST_FIELD_FL_VAR) 3985 continue; 3986 3987 idx = k++; 3988 3989 test_name = hist_field_name(hist_field, 0); 3990 3991 if (strcmp(field_name, test_name) == 0) { 3992 sort_key->field_idx = idx; 3993 descending = is_descending(tr, field_str); 3994 if (descending < 0) { 3995 ret = descending; 3996 goto out; 3997 } 3998 sort_key->descending = descending; 3999 break; 4000 } 4001 } 4002 if (j == hist_data->n_fields) { 4003 ret = -EINVAL; 4004 hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name)); 4005 break; 4006 } 4007 } 4008 4009 hist_data->n_sort_keys = i; 4010 out: 4011 return ret; 4012 } 4013 4014 static void destroy_actions(struct hist_trigger_data *hist_data) 4015 { 4016 unsigned int i; 4017 4018 for (i = 0; i < hist_data->n_actions; i++) { 4019 struct action_data *data = hist_data->actions[i]; 4020 4021 if (data->handler == HANDLER_ONMATCH) 4022 onmatch_destroy(data); 4023 else if (data->handler == HANDLER_ONMAX || 4024 data->handler == HANDLER_ONCHANGE) 4025 track_data_destroy(hist_data, data); 4026 else 4027 kfree(data); 4028 } 4029 } 4030 4031 static int parse_actions(struct hist_trigger_data *hist_data) 4032 { 4033 struct trace_array *tr = hist_data->event_file->tr; 4034 struct action_data *data; 4035 unsigned int i; 4036 int ret = 0; 4037 char *str; 4038 int len; 4039 4040 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4041 str = hist_data->attrs->action_str[i]; 4042 4043 if ((len = str_has_prefix(str, "onmatch("))) { 4044 char *action_str = str + len; 4045 4046 data = onmatch_parse(tr, action_str); 4047 if (IS_ERR(data)) { 4048 ret = PTR_ERR(data); 4049 break; 4050 } 4051 } else if ((len = str_has_prefix(str, "onmax("))) { 4052 char *action_str = str + len; 4053 4054 data = track_data_parse(hist_data, action_str, 4055 HANDLER_ONMAX); 4056 if (IS_ERR(data)) { 4057 ret = PTR_ERR(data); 4058 break; 4059 } 4060 } else if ((len = str_has_prefix(str, "onchange("))) { 4061 char *action_str = str + len; 4062 4063 data = track_data_parse(hist_data, action_str, 4064 HANDLER_ONCHANGE); 4065 if (IS_ERR(data)) { 4066 ret = PTR_ERR(data); 4067 break; 4068 } 4069 } else { 4070 ret = -EINVAL; 4071 break; 4072 } 4073 4074 hist_data->actions[hist_data->n_actions++] = data; 4075 } 4076 4077 return ret; 4078 } 4079 4080 static int create_actions(struct hist_trigger_data *hist_data) 4081 { 4082 struct action_data *data; 4083 unsigned int i; 4084 int ret = 0; 4085 4086 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4087 data = hist_data->actions[i]; 4088 4089 if (data->handler == HANDLER_ONMATCH) { 4090 ret = onmatch_create(hist_data, data); 4091 if (ret) 4092 break; 4093 } else if (data->handler == HANDLER_ONMAX || 4094 data->handler == HANDLER_ONCHANGE) { 4095 ret = track_data_create(hist_data, data); 4096 if (ret) 4097 break; 4098 } else { 4099 ret = -EINVAL; 4100 break; 4101 } 4102 } 4103 4104 return ret; 4105 } 4106 4107 static void print_actions(struct seq_file *m, 4108 struct hist_trigger_data *hist_data, 4109 struct tracing_map_elt *elt) 4110 { 4111 unsigned int i; 4112 4113 for (i = 0; i < hist_data->n_actions; i++) { 4114 struct action_data *data = hist_data->actions[i]; 4115 4116 if (data->action == ACTION_SNAPSHOT) 4117 continue; 4118 4119 if (data->handler == HANDLER_ONMAX || 4120 data->handler == HANDLER_ONCHANGE) 4121 track_data_print(m, hist_data, elt, data); 4122 } 4123 } 4124 4125 static void print_action_spec(struct seq_file *m, 4126 struct hist_trigger_data *hist_data, 4127 struct action_data *data) 4128 { 4129 unsigned int i; 4130 4131 if (data->action == ACTION_SAVE) { 4132 for (i = 0; i < hist_data->n_save_vars; i++) { 4133 seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name); 4134 if (i < hist_data->n_save_vars - 1) 4135 seq_puts(m, ","); 4136 } 4137 } else if (data->action == ACTION_TRACE) { 4138 if (data->use_trace_keyword) 4139 seq_printf(m, "%s", data->synth_event_name); 4140 for (i = 0; i < data->n_params; i++) { 4141 if (i || data->use_trace_keyword) 4142 seq_puts(m, ","); 4143 seq_printf(m, "%s", data->params[i]); 4144 } 4145 } 4146 } 4147 4148 static void print_track_data_spec(struct seq_file *m, 4149 struct hist_trigger_data *hist_data, 4150 struct action_data *data) 4151 { 4152 if (data->handler == HANDLER_ONMAX) 4153 seq_puts(m, ":onmax("); 4154 else if (data->handler == HANDLER_ONCHANGE) 4155 seq_puts(m, ":onchange("); 4156 seq_printf(m, "%s", data->track_data.var_str); 4157 seq_printf(m, ").%s(", data->action_name); 4158 4159 print_action_spec(m, hist_data, data); 4160 4161 seq_puts(m, ")"); 4162 } 4163 4164 static void print_onmatch_spec(struct seq_file *m, 4165 struct hist_trigger_data *hist_data, 4166 struct action_data *data) 4167 { 4168 seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system, 4169 data->match_data.event); 4170 4171 seq_printf(m, "%s(", data->action_name); 4172 4173 print_action_spec(m, hist_data, data); 4174 4175 seq_puts(m, ")"); 4176 } 4177 4178 static bool actions_match(struct hist_trigger_data *hist_data, 4179 struct hist_trigger_data *hist_data_test) 4180 { 4181 unsigned int i, j; 4182 4183 if (hist_data->n_actions != hist_data_test->n_actions) 4184 return false; 4185 4186 for (i = 0; i < hist_data->n_actions; i++) { 4187 struct action_data *data = hist_data->actions[i]; 4188 struct action_data *data_test = hist_data_test->actions[i]; 4189 char *action_name, *action_name_test; 4190 4191 if (data->handler != data_test->handler) 4192 return false; 4193 if (data->action != data_test->action) 4194 return false; 4195 4196 if (data->n_params != data_test->n_params) 4197 return false; 4198 4199 for (j = 0; j < data->n_params; j++) { 4200 if (strcmp(data->params[j], data_test->params[j]) != 0) 4201 return false; 4202 } 4203 4204 if (data->use_trace_keyword) 4205 action_name = data->synth_event_name; 4206 else 4207 action_name = data->action_name; 4208 4209 if (data_test->use_trace_keyword) 4210 action_name_test = data_test->synth_event_name; 4211 else 4212 action_name_test = data_test->action_name; 4213 4214 if (strcmp(action_name, action_name_test) != 0) 4215 return false; 4216 4217 if (data->handler == HANDLER_ONMATCH) { 4218 if (strcmp(data->match_data.event_system, 4219 data_test->match_data.event_system) != 0) 4220 return false; 4221 if (strcmp(data->match_data.event, 4222 data_test->match_data.event) != 0) 4223 return false; 4224 } else if (data->handler == HANDLER_ONMAX || 4225 data->handler == HANDLER_ONCHANGE) { 4226 if (strcmp(data->track_data.var_str, 4227 data_test->track_data.var_str) != 0) 4228 return false; 4229 } 4230 } 4231 4232 return true; 4233 } 4234 4235 4236 static void print_actions_spec(struct seq_file *m, 4237 struct hist_trigger_data *hist_data) 4238 { 4239 unsigned int i; 4240 4241 for (i = 0; i < hist_data->n_actions; i++) { 4242 struct action_data *data = hist_data->actions[i]; 4243 4244 if (data->handler == HANDLER_ONMATCH) 4245 print_onmatch_spec(m, hist_data, data); 4246 else if (data->handler == HANDLER_ONMAX || 4247 data->handler == HANDLER_ONCHANGE) 4248 print_track_data_spec(m, hist_data, data); 4249 } 4250 } 4251 4252 static void destroy_field_var_hists(struct hist_trigger_data *hist_data) 4253 { 4254 unsigned int i; 4255 4256 for (i = 0; i < hist_data->n_field_var_hists; i++) { 4257 kfree(hist_data->field_var_hists[i]->cmd); 4258 kfree(hist_data->field_var_hists[i]); 4259 } 4260 } 4261 4262 static void destroy_hist_data(struct hist_trigger_data *hist_data) 4263 { 4264 if (!hist_data) 4265 return; 4266 4267 destroy_hist_trigger_attrs(hist_data->attrs); 4268 destroy_hist_fields(hist_data); 4269 tracing_map_destroy(hist_data->map); 4270 4271 destroy_actions(hist_data); 4272 destroy_field_vars(hist_data); 4273 destroy_field_var_hists(hist_data); 4274 4275 kfree(hist_data); 4276 } 4277 4278 static int create_tracing_map_fields(struct hist_trigger_data *hist_data) 4279 { 4280 struct tracing_map *map = hist_data->map; 4281 struct ftrace_event_field *field; 4282 struct hist_field *hist_field; 4283 int i, idx = 0; 4284 4285 for_each_hist_field(i, hist_data) { 4286 hist_field = hist_data->fields[i]; 4287 if (hist_field->flags & HIST_FIELD_FL_KEY) { 4288 tracing_map_cmp_fn_t cmp_fn; 4289 4290 field = hist_field->field; 4291 4292 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE) 4293 cmp_fn = tracing_map_cmp_none; 4294 else if (!field) 4295 cmp_fn = tracing_map_cmp_num(hist_field->size, 4296 hist_field->is_signed); 4297 else if (is_string_field(field)) 4298 cmp_fn = tracing_map_cmp_string; 4299 else 4300 cmp_fn = tracing_map_cmp_num(field->size, 4301 field->is_signed); 4302 idx = tracing_map_add_key_field(map, 4303 hist_field->offset, 4304 cmp_fn); 4305 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR)) 4306 idx = tracing_map_add_sum_field(map); 4307 4308 if (idx < 0) 4309 return idx; 4310 4311 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4312 idx = tracing_map_add_var(map); 4313 if (idx < 0) 4314 return idx; 4315 hist_field->var.idx = idx; 4316 hist_field->var.hist_data = hist_data; 4317 } 4318 } 4319 4320 return 0; 4321 } 4322 4323 static struct hist_trigger_data * 4324 create_hist_data(unsigned int map_bits, 4325 struct hist_trigger_attrs *attrs, 4326 struct trace_event_file *file, 4327 bool remove) 4328 { 4329 const struct tracing_map_ops *map_ops = NULL; 4330 struct hist_trigger_data *hist_data; 4331 int ret = 0; 4332 4333 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL); 4334 if (!hist_data) 4335 return ERR_PTR(-ENOMEM); 4336 4337 hist_data->attrs = attrs; 4338 hist_data->remove = remove; 4339 hist_data->event_file = file; 4340 4341 ret = parse_actions(hist_data); 4342 if (ret) 4343 goto free; 4344 4345 ret = create_hist_fields(hist_data, file); 4346 if (ret) 4347 goto free; 4348 4349 ret = create_sort_keys(hist_data); 4350 if (ret) 4351 goto free; 4352 4353 map_ops = &hist_trigger_elt_data_ops; 4354 4355 hist_data->map = tracing_map_create(map_bits, hist_data->key_size, 4356 map_ops, hist_data); 4357 if (IS_ERR(hist_data->map)) { 4358 ret = PTR_ERR(hist_data->map); 4359 hist_data->map = NULL; 4360 goto free; 4361 } 4362 4363 ret = create_tracing_map_fields(hist_data); 4364 if (ret) 4365 goto free; 4366 out: 4367 return hist_data; 4368 free: 4369 hist_data->attrs = NULL; 4370 4371 destroy_hist_data(hist_data); 4372 4373 hist_data = ERR_PTR(ret); 4374 4375 goto out; 4376 } 4377 4378 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, 4379 struct tracing_map_elt *elt, void *rec, 4380 struct ring_buffer_event *rbe, 4381 u64 *var_ref_vals) 4382 { 4383 struct hist_elt_data *elt_data; 4384 struct hist_field *hist_field; 4385 unsigned int i, var_idx; 4386 u64 hist_val; 4387 4388 elt_data = elt->private_data; 4389 elt_data->var_ref_vals = var_ref_vals; 4390 4391 for_each_hist_val_field(i, hist_data) { 4392 hist_field = hist_data->fields[i]; 4393 hist_val = hist_field->fn(hist_field, elt, rbe, rec); 4394 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4395 var_idx = hist_field->var.idx; 4396 tracing_map_set_var(elt, var_idx, hist_val); 4397 continue; 4398 } 4399 tracing_map_update_sum(elt, i, hist_val); 4400 } 4401 4402 for_each_hist_key_field(i, hist_data) { 4403 hist_field = hist_data->fields[i]; 4404 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4405 hist_val = hist_field->fn(hist_field, elt, rbe, rec); 4406 var_idx = hist_field->var.idx; 4407 tracing_map_set_var(elt, var_idx, hist_val); 4408 } 4409 } 4410 4411 update_field_vars(hist_data, elt, rbe, rec); 4412 } 4413 4414 static inline void add_to_key(char *compound_key, void *key, 4415 struct hist_field *key_field, void *rec) 4416 { 4417 size_t size = key_field->size; 4418 4419 if (key_field->flags & HIST_FIELD_FL_STRING) { 4420 struct ftrace_event_field *field; 4421 4422 field = key_field->field; 4423 if (field->filter_type == FILTER_DYN_STRING) 4424 size = *(u32 *)(rec + field->offset) >> 16; 4425 else if (field->filter_type == FILTER_PTR_STRING) 4426 size = strlen(key); 4427 else if (field->filter_type == FILTER_STATIC_STRING) 4428 size = field->size; 4429 4430 /* ensure NULL-termination */ 4431 if (size > key_field->size - 1) 4432 size = key_field->size - 1; 4433 4434 strncpy(compound_key + key_field->offset, (char *)key, size); 4435 } else 4436 memcpy(compound_key + key_field->offset, key, size); 4437 } 4438 4439 static void 4440 hist_trigger_actions(struct hist_trigger_data *hist_data, 4441 struct tracing_map_elt *elt, void *rec, 4442 struct ring_buffer_event *rbe, void *key, 4443 u64 *var_ref_vals) 4444 { 4445 struct action_data *data; 4446 unsigned int i; 4447 4448 for (i = 0; i < hist_data->n_actions; i++) { 4449 data = hist_data->actions[i]; 4450 data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals); 4451 } 4452 } 4453 4454 static void event_hist_trigger(struct event_trigger_data *data, void *rec, 4455 struct ring_buffer_event *rbe) 4456 { 4457 struct hist_trigger_data *hist_data = data->private_data; 4458 bool use_compound_key = (hist_data->n_keys > 1); 4459 unsigned long entries[HIST_STACKTRACE_DEPTH]; 4460 u64 var_ref_vals[TRACING_MAP_VARS_MAX]; 4461 char compound_key[HIST_KEY_SIZE_MAX]; 4462 struct tracing_map_elt *elt = NULL; 4463 struct hist_field *key_field; 4464 u64 field_contents; 4465 void *key = NULL; 4466 unsigned int i; 4467 4468 memset(compound_key, 0, hist_data->key_size); 4469 4470 for_each_hist_key_field(i, hist_data) { 4471 key_field = hist_data->fields[i]; 4472 4473 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 4474 memset(entries, 0, HIST_STACKTRACE_SIZE); 4475 stack_trace_save(entries, HIST_STACKTRACE_DEPTH, 4476 HIST_STACKTRACE_SKIP); 4477 key = entries; 4478 } else { 4479 field_contents = key_field->fn(key_field, elt, rbe, rec); 4480 if (key_field->flags & HIST_FIELD_FL_STRING) { 4481 key = (void *)(unsigned long)field_contents; 4482 use_compound_key = true; 4483 } else 4484 key = (void *)&field_contents; 4485 } 4486 4487 if (use_compound_key) 4488 add_to_key(compound_key, key, key_field, rec); 4489 } 4490 4491 if (use_compound_key) 4492 key = compound_key; 4493 4494 if (hist_data->n_var_refs && 4495 !resolve_var_refs(hist_data, key, var_ref_vals, false)) 4496 return; 4497 4498 elt = tracing_map_insert(hist_data->map, key); 4499 if (!elt) 4500 return; 4501 4502 hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals); 4503 4504 if (resolve_var_refs(hist_data, key, var_ref_vals, true)) 4505 hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals); 4506 } 4507 4508 static void hist_trigger_stacktrace_print(struct seq_file *m, 4509 unsigned long *stacktrace_entries, 4510 unsigned int max_entries) 4511 { 4512 char str[KSYM_SYMBOL_LEN]; 4513 unsigned int spaces = 8; 4514 unsigned int i; 4515 4516 for (i = 0; i < max_entries; i++) { 4517 if (!stacktrace_entries[i]) 4518 return; 4519 4520 seq_printf(m, "%*c", 1 + spaces, ' '); 4521 sprint_symbol(str, stacktrace_entries[i]); 4522 seq_printf(m, "%s\n", str); 4523 } 4524 } 4525 4526 static void hist_trigger_print_key(struct seq_file *m, 4527 struct hist_trigger_data *hist_data, 4528 void *key, 4529 struct tracing_map_elt *elt) 4530 { 4531 struct hist_field *key_field; 4532 char str[KSYM_SYMBOL_LEN]; 4533 bool multiline = false; 4534 const char *field_name; 4535 unsigned int i; 4536 u64 uval; 4537 4538 seq_puts(m, "{ "); 4539 4540 for_each_hist_key_field(i, hist_data) { 4541 key_field = hist_data->fields[i]; 4542 4543 if (i > hist_data->n_vals) 4544 seq_puts(m, ", "); 4545 4546 field_name = hist_field_name(key_field, 0); 4547 4548 if (key_field->flags & HIST_FIELD_FL_HEX) { 4549 uval = *(u64 *)(key + key_field->offset); 4550 seq_printf(m, "%s: %llx", field_name, uval); 4551 } else if (key_field->flags & HIST_FIELD_FL_SYM) { 4552 uval = *(u64 *)(key + key_field->offset); 4553 sprint_symbol_no_offset(str, uval); 4554 seq_printf(m, "%s: [%llx] %-45s", field_name, 4555 uval, str); 4556 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) { 4557 uval = *(u64 *)(key + key_field->offset); 4558 sprint_symbol(str, uval); 4559 seq_printf(m, "%s: [%llx] %-55s", field_name, 4560 uval, str); 4561 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 4562 struct hist_elt_data *elt_data = elt->private_data; 4563 char *comm; 4564 4565 if (WARN_ON_ONCE(!elt_data)) 4566 return; 4567 4568 comm = elt_data->comm; 4569 4570 uval = *(u64 *)(key + key_field->offset); 4571 seq_printf(m, "%s: %-16s[%10llu]", field_name, 4572 comm, uval); 4573 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) { 4574 const char *syscall_name; 4575 4576 uval = *(u64 *)(key + key_field->offset); 4577 syscall_name = get_syscall_name(uval); 4578 if (!syscall_name) 4579 syscall_name = "unknown_syscall"; 4580 4581 seq_printf(m, "%s: %-30s[%3llu]", field_name, 4582 syscall_name, uval); 4583 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 4584 seq_puts(m, "stacktrace:\n"); 4585 hist_trigger_stacktrace_print(m, 4586 key + key_field->offset, 4587 HIST_STACKTRACE_DEPTH); 4588 multiline = true; 4589 } else if (key_field->flags & HIST_FIELD_FL_LOG2) { 4590 seq_printf(m, "%s: ~ 2^%-2llu", field_name, 4591 *(u64 *)(key + key_field->offset)); 4592 } else if (key_field->flags & HIST_FIELD_FL_STRING) { 4593 seq_printf(m, "%s: %-50s", field_name, 4594 (char *)(key + key_field->offset)); 4595 } else { 4596 uval = *(u64 *)(key + key_field->offset); 4597 seq_printf(m, "%s: %10llu", field_name, uval); 4598 } 4599 } 4600 4601 if (!multiline) 4602 seq_puts(m, " "); 4603 4604 seq_puts(m, "}"); 4605 } 4606 4607 static void hist_trigger_entry_print(struct seq_file *m, 4608 struct hist_trigger_data *hist_data, 4609 void *key, 4610 struct tracing_map_elt *elt) 4611 { 4612 const char *field_name; 4613 unsigned int i; 4614 4615 hist_trigger_print_key(m, hist_data, key, elt); 4616 4617 seq_printf(m, " hitcount: %10llu", 4618 tracing_map_read_sum(elt, HITCOUNT_IDX)); 4619 4620 for (i = 1; i < hist_data->n_vals; i++) { 4621 field_name = hist_field_name(hist_data->fields[i], 0); 4622 4623 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR || 4624 hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR) 4625 continue; 4626 4627 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) { 4628 seq_printf(m, " %s: %10llx", field_name, 4629 tracing_map_read_sum(elt, i)); 4630 } else { 4631 seq_printf(m, " %s: %10llu", field_name, 4632 tracing_map_read_sum(elt, i)); 4633 } 4634 } 4635 4636 print_actions(m, hist_data, elt); 4637 4638 seq_puts(m, "\n"); 4639 } 4640 4641 static int print_entries(struct seq_file *m, 4642 struct hist_trigger_data *hist_data) 4643 { 4644 struct tracing_map_sort_entry **sort_entries = NULL; 4645 struct tracing_map *map = hist_data->map; 4646 int i, n_entries; 4647 4648 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys, 4649 hist_data->n_sort_keys, 4650 &sort_entries); 4651 if (n_entries < 0) 4652 return n_entries; 4653 4654 for (i = 0; i < n_entries; i++) 4655 hist_trigger_entry_print(m, hist_data, 4656 sort_entries[i]->key, 4657 sort_entries[i]->elt); 4658 4659 tracing_map_destroy_sort_entries(sort_entries, n_entries); 4660 4661 return n_entries; 4662 } 4663 4664 static void hist_trigger_show(struct seq_file *m, 4665 struct event_trigger_data *data, int n) 4666 { 4667 struct hist_trigger_data *hist_data; 4668 int n_entries; 4669 4670 if (n > 0) 4671 seq_puts(m, "\n\n"); 4672 4673 seq_puts(m, "# event histogram\n#\n# trigger info: "); 4674 data->ops->print(m, data->ops, data); 4675 seq_puts(m, "#\n\n"); 4676 4677 hist_data = data->private_data; 4678 n_entries = print_entries(m, hist_data); 4679 if (n_entries < 0) 4680 n_entries = 0; 4681 4682 track_data_snapshot_print(m, hist_data); 4683 4684 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n", 4685 (u64)atomic64_read(&hist_data->map->hits), 4686 n_entries, (u64)atomic64_read(&hist_data->map->drops)); 4687 } 4688 4689 static int hist_show(struct seq_file *m, void *v) 4690 { 4691 struct event_trigger_data *data; 4692 struct trace_event_file *event_file; 4693 int n = 0, ret = 0; 4694 4695 mutex_lock(&event_mutex); 4696 4697 event_file = event_file_data(m->private); 4698 if (unlikely(!event_file)) { 4699 ret = -ENODEV; 4700 goto out_unlock; 4701 } 4702 4703 list_for_each_entry(data, &event_file->triggers, list) { 4704 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 4705 hist_trigger_show(m, data, n++); 4706 } 4707 4708 out_unlock: 4709 mutex_unlock(&event_mutex); 4710 4711 return ret; 4712 } 4713 4714 static int event_hist_open(struct inode *inode, struct file *file) 4715 { 4716 int ret; 4717 4718 ret = security_locked_down(LOCKDOWN_TRACEFS); 4719 if (ret) 4720 return ret; 4721 4722 return single_open(file, hist_show, file); 4723 } 4724 4725 const struct file_operations event_hist_fops = { 4726 .open = event_hist_open, 4727 .read = seq_read, 4728 .llseek = seq_lseek, 4729 .release = single_release, 4730 }; 4731 4732 #ifdef CONFIG_HIST_TRIGGERS_DEBUG 4733 static void hist_field_debug_show_flags(struct seq_file *m, 4734 unsigned long flags) 4735 { 4736 seq_puts(m, " flags:\n"); 4737 4738 if (flags & HIST_FIELD_FL_KEY) 4739 seq_puts(m, " HIST_FIELD_FL_KEY\n"); 4740 else if (flags & HIST_FIELD_FL_HITCOUNT) 4741 seq_puts(m, " VAL: HIST_FIELD_FL_HITCOUNT\n"); 4742 else if (flags & HIST_FIELD_FL_VAR) 4743 seq_puts(m, " HIST_FIELD_FL_VAR\n"); 4744 else if (flags & HIST_FIELD_FL_VAR_REF) 4745 seq_puts(m, " HIST_FIELD_FL_VAR_REF\n"); 4746 else 4747 seq_puts(m, " VAL: normal u64 value\n"); 4748 4749 if (flags & HIST_FIELD_FL_ALIAS) 4750 seq_puts(m, " HIST_FIELD_FL_ALIAS\n"); 4751 } 4752 4753 static int hist_field_debug_show(struct seq_file *m, 4754 struct hist_field *field, unsigned long flags) 4755 { 4756 if ((field->flags & flags) != flags) { 4757 seq_printf(m, "ERROR: bad flags - %lx\n", flags); 4758 return -EINVAL; 4759 } 4760 4761 hist_field_debug_show_flags(m, field->flags); 4762 if (field->field) 4763 seq_printf(m, " ftrace_event_field name: %s\n", 4764 field->field->name); 4765 4766 if (field->flags & HIST_FIELD_FL_VAR) { 4767 seq_printf(m, " var.name: %s\n", field->var.name); 4768 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 4769 field->var.idx); 4770 } 4771 4772 if (field->flags & HIST_FIELD_FL_ALIAS) 4773 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n", 4774 field->var_ref_idx); 4775 4776 if (field->flags & HIST_FIELD_FL_VAR_REF) { 4777 seq_printf(m, " name: %s\n", field->name); 4778 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 4779 field->var.idx); 4780 seq_printf(m, " var.hist_data: %p\n", field->var.hist_data); 4781 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n", 4782 field->var_ref_idx); 4783 if (field->system) 4784 seq_printf(m, " system: %s\n", field->system); 4785 if (field->event_name) 4786 seq_printf(m, " event_name: %s\n", field->event_name); 4787 } 4788 4789 seq_printf(m, " type: %s\n", field->type); 4790 seq_printf(m, " size: %u\n", field->size); 4791 seq_printf(m, " is_signed: %u\n", field->is_signed); 4792 4793 return 0; 4794 } 4795 4796 static int field_var_debug_show(struct seq_file *m, 4797 struct field_var *field_var, unsigned int i, 4798 bool save_vars) 4799 { 4800 const char *vars_name = save_vars ? "save_vars" : "field_vars"; 4801 struct hist_field *field; 4802 int ret = 0; 4803 4804 seq_printf(m, "\n hist_data->%s[%d]:\n", vars_name, i); 4805 4806 field = field_var->var; 4807 4808 seq_printf(m, "\n %s[%d].var:\n", vars_name, i); 4809 4810 hist_field_debug_show_flags(m, field->flags); 4811 seq_printf(m, " var.name: %s\n", field->var.name); 4812 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 4813 field->var.idx); 4814 4815 field = field_var->val; 4816 4817 seq_printf(m, "\n %s[%d].val:\n", vars_name, i); 4818 if (field->field) 4819 seq_printf(m, " ftrace_event_field name: %s\n", 4820 field->field->name); 4821 else { 4822 ret = -EINVAL; 4823 goto out; 4824 } 4825 4826 seq_printf(m, " type: %s\n", field->type); 4827 seq_printf(m, " size: %u\n", field->size); 4828 seq_printf(m, " is_signed: %u\n", field->is_signed); 4829 out: 4830 return ret; 4831 } 4832 4833 static int hist_action_debug_show(struct seq_file *m, 4834 struct action_data *data, int i) 4835 { 4836 int ret = 0; 4837 4838 if (data->handler == HANDLER_ONMAX || 4839 data->handler == HANDLER_ONCHANGE) { 4840 seq_printf(m, "\n hist_data->actions[%d].track_data.var_ref:\n", i); 4841 ret = hist_field_debug_show(m, data->track_data.var_ref, 4842 HIST_FIELD_FL_VAR_REF); 4843 if (ret) 4844 goto out; 4845 4846 seq_printf(m, "\n hist_data->actions[%d].track_data.track_var:\n", i); 4847 ret = hist_field_debug_show(m, data->track_data.track_var, 4848 HIST_FIELD_FL_VAR); 4849 if (ret) 4850 goto out; 4851 } 4852 4853 if (data->handler == HANDLER_ONMATCH) { 4854 seq_printf(m, "\n hist_data->actions[%d].match_data.event_system: %s\n", 4855 i, data->match_data.event_system); 4856 seq_printf(m, " hist_data->actions[%d].match_data.event: %s\n", 4857 i, data->match_data.event); 4858 } 4859 out: 4860 return ret; 4861 } 4862 4863 static int hist_actions_debug_show(struct seq_file *m, 4864 struct hist_trigger_data *hist_data) 4865 { 4866 int i, ret = 0; 4867 4868 if (hist_data->n_actions) 4869 seq_puts(m, "\n action tracking variables (for onmax()/onchange()/onmatch()):\n"); 4870 4871 for (i = 0; i < hist_data->n_actions; i++) { 4872 struct action_data *action = hist_data->actions[i]; 4873 4874 ret = hist_action_debug_show(m, action, i); 4875 if (ret) 4876 goto out; 4877 } 4878 4879 if (hist_data->n_save_vars) 4880 seq_puts(m, "\n save action variables (save() params):\n"); 4881 4882 for (i = 0; i < hist_data->n_save_vars; i++) { 4883 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true); 4884 if (ret) 4885 goto out; 4886 } 4887 out: 4888 return ret; 4889 } 4890 4891 static void hist_trigger_debug_show(struct seq_file *m, 4892 struct event_trigger_data *data, int n) 4893 { 4894 struct hist_trigger_data *hist_data; 4895 int i, ret; 4896 4897 if (n > 0) 4898 seq_puts(m, "\n\n"); 4899 4900 seq_puts(m, "# event histogram\n#\n# trigger info: "); 4901 data->ops->print(m, data->ops, data); 4902 seq_puts(m, "#\n\n"); 4903 4904 hist_data = data->private_data; 4905 4906 seq_printf(m, "hist_data: %p\n\n", hist_data); 4907 seq_printf(m, " n_vals: %u\n", hist_data->n_vals); 4908 seq_printf(m, " n_keys: %u\n", hist_data->n_keys); 4909 seq_printf(m, " n_fields: %u\n", hist_data->n_fields); 4910 4911 seq_puts(m, "\n val fields:\n\n"); 4912 4913 seq_puts(m, " hist_data->fields[0]:\n"); 4914 ret = hist_field_debug_show(m, hist_data->fields[0], 4915 HIST_FIELD_FL_HITCOUNT); 4916 if (ret) 4917 return; 4918 4919 for (i = 1; i < hist_data->n_vals; i++) { 4920 seq_printf(m, "\n hist_data->fields[%d]:\n", i); 4921 ret = hist_field_debug_show(m, hist_data->fields[i], 0); 4922 if (ret) 4923 return; 4924 } 4925 4926 seq_puts(m, "\n key fields:\n"); 4927 4928 for (i = hist_data->n_vals; i < hist_data->n_fields; i++) { 4929 seq_printf(m, "\n hist_data->fields[%d]:\n", i); 4930 ret = hist_field_debug_show(m, hist_data->fields[i], 4931 HIST_FIELD_FL_KEY); 4932 if (ret) 4933 return; 4934 } 4935 4936 if (hist_data->n_var_refs) 4937 seq_puts(m, "\n variable reference fields:\n"); 4938 4939 for (i = 0; i < hist_data->n_var_refs; i++) { 4940 seq_printf(m, "\n hist_data->var_refs[%d]:\n", i); 4941 ret = hist_field_debug_show(m, hist_data->var_refs[i], 4942 HIST_FIELD_FL_VAR_REF); 4943 if (ret) 4944 return; 4945 } 4946 4947 if (hist_data->n_field_vars) 4948 seq_puts(m, "\n field variables:\n"); 4949 4950 for (i = 0; i < hist_data->n_field_vars; i++) { 4951 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false); 4952 if (ret) 4953 return; 4954 } 4955 4956 ret = hist_actions_debug_show(m, hist_data); 4957 if (ret) 4958 return; 4959 } 4960 4961 static int hist_debug_show(struct seq_file *m, void *v) 4962 { 4963 struct event_trigger_data *data; 4964 struct trace_event_file *event_file; 4965 int n = 0, ret = 0; 4966 4967 mutex_lock(&event_mutex); 4968 4969 event_file = event_file_data(m->private); 4970 if (unlikely(!event_file)) { 4971 ret = -ENODEV; 4972 goto out_unlock; 4973 } 4974 4975 list_for_each_entry(data, &event_file->triggers, list) { 4976 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 4977 hist_trigger_debug_show(m, data, n++); 4978 } 4979 4980 out_unlock: 4981 mutex_unlock(&event_mutex); 4982 4983 return ret; 4984 } 4985 4986 static int event_hist_debug_open(struct inode *inode, struct file *file) 4987 { 4988 int ret; 4989 4990 ret = security_locked_down(LOCKDOWN_TRACEFS); 4991 if (ret) 4992 return ret; 4993 4994 return single_open(file, hist_debug_show, file); 4995 } 4996 4997 const struct file_operations event_hist_debug_fops = { 4998 .open = event_hist_debug_open, 4999 .read = seq_read, 5000 .llseek = seq_lseek, 5001 .release = single_release, 5002 }; 5003 #endif 5004 5005 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field) 5006 { 5007 const char *field_name = hist_field_name(hist_field, 0); 5008 5009 if (hist_field->var.name) 5010 seq_printf(m, "%s=", hist_field->var.name); 5011 5012 if (hist_field->flags & HIST_FIELD_FL_CPU) 5013 seq_puts(m, "cpu"); 5014 else if (field_name) { 5015 if (hist_field->flags & HIST_FIELD_FL_VAR_REF || 5016 hist_field->flags & HIST_FIELD_FL_ALIAS) 5017 seq_putc(m, '$'); 5018 seq_printf(m, "%s", field_name); 5019 } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP) 5020 seq_puts(m, "common_timestamp"); 5021 5022 if (hist_field->flags) { 5023 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) && 5024 !(hist_field->flags & HIST_FIELD_FL_EXPR)) { 5025 const char *flags = get_hist_field_flags(hist_field); 5026 5027 if (flags) 5028 seq_printf(m, ".%s", flags); 5029 } 5030 } 5031 } 5032 5033 static int event_hist_trigger_print(struct seq_file *m, 5034 struct event_trigger_ops *ops, 5035 struct event_trigger_data *data) 5036 { 5037 struct hist_trigger_data *hist_data = data->private_data; 5038 struct hist_field *field; 5039 bool have_var = false; 5040 unsigned int i; 5041 5042 seq_puts(m, "hist:"); 5043 5044 if (data->name) 5045 seq_printf(m, "%s:", data->name); 5046 5047 seq_puts(m, "keys="); 5048 5049 for_each_hist_key_field(i, hist_data) { 5050 field = hist_data->fields[i]; 5051 5052 if (i > hist_data->n_vals) 5053 seq_puts(m, ","); 5054 5055 if (field->flags & HIST_FIELD_FL_STACKTRACE) 5056 seq_puts(m, "stacktrace"); 5057 else 5058 hist_field_print(m, field); 5059 } 5060 5061 seq_puts(m, ":vals="); 5062 5063 for_each_hist_val_field(i, hist_data) { 5064 field = hist_data->fields[i]; 5065 if (field->flags & HIST_FIELD_FL_VAR) { 5066 have_var = true; 5067 continue; 5068 } 5069 5070 if (i == HITCOUNT_IDX) 5071 seq_puts(m, "hitcount"); 5072 else { 5073 seq_puts(m, ","); 5074 hist_field_print(m, field); 5075 } 5076 } 5077 5078 if (have_var) { 5079 unsigned int n = 0; 5080 5081 seq_puts(m, ":"); 5082 5083 for_each_hist_val_field(i, hist_data) { 5084 field = hist_data->fields[i]; 5085 5086 if (field->flags & HIST_FIELD_FL_VAR) { 5087 if (n++) 5088 seq_puts(m, ","); 5089 hist_field_print(m, field); 5090 } 5091 } 5092 } 5093 5094 seq_puts(m, ":sort="); 5095 5096 for (i = 0; i < hist_data->n_sort_keys; i++) { 5097 struct tracing_map_sort_key *sort_key; 5098 unsigned int idx, first_key_idx; 5099 5100 /* skip VAR vals */ 5101 first_key_idx = hist_data->n_vals - hist_data->n_vars; 5102 5103 sort_key = &hist_data->sort_keys[i]; 5104 idx = sort_key->field_idx; 5105 5106 if (WARN_ON(idx >= HIST_FIELDS_MAX)) 5107 return -EINVAL; 5108 5109 if (i > 0) 5110 seq_puts(m, ","); 5111 5112 if (idx == HITCOUNT_IDX) 5113 seq_puts(m, "hitcount"); 5114 else { 5115 if (idx >= first_key_idx) 5116 idx += hist_data->n_vars; 5117 hist_field_print(m, hist_data->fields[idx]); 5118 } 5119 5120 if (sort_key->descending) 5121 seq_puts(m, ".descending"); 5122 } 5123 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits)); 5124 if (hist_data->enable_timestamps) 5125 seq_printf(m, ":clock=%s", hist_data->attrs->clock); 5126 5127 print_actions_spec(m, hist_data); 5128 5129 if (data->filter_str) 5130 seq_printf(m, " if %s", data->filter_str); 5131 5132 if (data->paused) 5133 seq_puts(m, " [paused]"); 5134 else 5135 seq_puts(m, " [active]"); 5136 5137 seq_putc(m, '\n'); 5138 5139 return 0; 5140 } 5141 5142 static int event_hist_trigger_init(struct event_trigger_ops *ops, 5143 struct event_trigger_data *data) 5144 { 5145 struct hist_trigger_data *hist_data = data->private_data; 5146 5147 if (!data->ref && hist_data->attrs->name) 5148 save_named_trigger(hist_data->attrs->name, data); 5149 5150 data->ref++; 5151 5152 return 0; 5153 } 5154 5155 static void unregister_field_var_hists(struct hist_trigger_data *hist_data) 5156 { 5157 struct trace_event_file *file; 5158 unsigned int i; 5159 char *cmd; 5160 int ret; 5161 5162 for (i = 0; i < hist_data->n_field_var_hists; i++) { 5163 file = hist_data->field_var_hists[i]->hist_data->event_file; 5164 cmd = hist_data->field_var_hists[i]->cmd; 5165 ret = event_hist_trigger_func(&trigger_hist_cmd, file, 5166 "!hist", "hist", cmd); 5167 } 5168 } 5169 5170 static void event_hist_trigger_free(struct event_trigger_ops *ops, 5171 struct event_trigger_data *data) 5172 { 5173 struct hist_trigger_data *hist_data = data->private_data; 5174 5175 if (WARN_ON_ONCE(data->ref <= 0)) 5176 return; 5177 5178 data->ref--; 5179 if (!data->ref) { 5180 if (data->name) 5181 del_named_trigger(data); 5182 5183 trigger_data_free(data); 5184 5185 remove_hist_vars(hist_data); 5186 5187 unregister_field_var_hists(hist_data); 5188 5189 destroy_hist_data(hist_data); 5190 } 5191 } 5192 5193 static struct event_trigger_ops event_hist_trigger_ops = { 5194 .func = event_hist_trigger, 5195 .print = event_hist_trigger_print, 5196 .init = event_hist_trigger_init, 5197 .free = event_hist_trigger_free, 5198 }; 5199 5200 static int event_hist_trigger_named_init(struct event_trigger_ops *ops, 5201 struct event_trigger_data *data) 5202 { 5203 data->ref++; 5204 5205 save_named_trigger(data->named_data->name, data); 5206 5207 event_hist_trigger_init(ops, data->named_data); 5208 5209 return 0; 5210 } 5211 5212 static void event_hist_trigger_named_free(struct event_trigger_ops *ops, 5213 struct event_trigger_data *data) 5214 { 5215 if (WARN_ON_ONCE(data->ref <= 0)) 5216 return; 5217 5218 event_hist_trigger_free(ops, data->named_data); 5219 5220 data->ref--; 5221 if (!data->ref) { 5222 del_named_trigger(data); 5223 trigger_data_free(data); 5224 } 5225 } 5226 5227 static struct event_trigger_ops event_hist_trigger_named_ops = { 5228 .func = event_hist_trigger, 5229 .print = event_hist_trigger_print, 5230 .init = event_hist_trigger_named_init, 5231 .free = event_hist_trigger_named_free, 5232 }; 5233 5234 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd, 5235 char *param) 5236 { 5237 return &event_hist_trigger_ops; 5238 } 5239 5240 static void hist_clear(struct event_trigger_data *data) 5241 { 5242 struct hist_trigger_data *hist_data = data->private_data; 5243 5244 if (data->name) 5245 pause_named_trigger(data); 5246 5247 tracepoint_synchronize_unregister(); 5248 5249 tracing_map_clear(hist_data->map); 5250 5251 if (data->name) 5252 unpause_named_trigger(data); 5253 } 5254 5255 static bool compatible_field(struct ftrace_event_field *field, 5256 struct ftrace_event_field *test_field) 5257 { 5258 if (field == test_field) 5259 return true; 5260 if (field == NULL || test_field == NULL) 5261 return false; 5262 if (strcmp(field->name, test_field->name) != 0) 5263 return false; 5264 if (strcmp(field->type, test_field->type) != 0) 5265 return false; 5266 if (field->size != test_field->size) 5267 return false; 5268 if (field->is_signed != test_field->is_signed) 5269 return false; 5270 5271 return true; 5272 } 5273 5274 static bool hist_trigger_match(struct event_trigger_data *data, 5275 struct event_trigger_data *data_test, 5276 struct event_trigger_data *named_data, 5277 bool ignore_filter) 5278 { 5279 struct tracing_map_sort_key *sort_key, *sort_key_test; 5280 struct hist_trigger_data *hist_data, *hist_data_test; 5281 struct hist_field *key_field, *key_field_test; 5282 unsigned int i; 5283 5284 if (named_data && (named_data != data_test) && 5285 (named_data != data_test->named_data)) 5286 return false; 5287 5288 if (!named_data && is_named_trigger(data_test)) 5289 return false; 5290 5291 hist_data = data->private_data; 5292 hist_data_test = data_test->private_data; 5293 5294 if (hist_data->n_vals != hist_data_test->n_vals || 5295 hist_data->n_fields != hist_data_test->n_fields || 5296 hist_data->n_sort_keys != hist_data_test->n_sort_keys) 5297 return false; 5298 5299 if (!ignore_filter) { 5300 if ((data->filter_str && !data_test->filter_str) || 5301 (!data->filter_str && data_test->filter_str)) 5302 return false; 5303 } 5304 5305 for_each_hist_field(i, hist_data) { 5306 key_field = hist_data->fields[i]; 5307 key_field_test = hist_data_test->fields[i]; 5308 5309 if (key_field->flags != key_field_test->flags) 5310 return false; 5311 if (!compatible_field(key_field->field, key_field_test->field)) 5312 return false; 5313 if (key_field->offset != key_field_test->offset) 5314 return false; 5315 if (key_field->size != key_field_test->size) 5316 return false; 5317 if (key_field->is_signed != key_field_test->is_signed) 5318 return false; 5319 if (!!key_field->var.name != !!key_field_test->var.name) 5320 return false; 5321 if (key_field->var.name && 5322 strcmp(key_field->var.name, key_field_test->var.name) != 0) 5323 return false; 5324 } 5325 5326 for (i = 0; i < hist_data->n_sort_keys; i++) { 5327 sort_key = &hist_data->sort_keys[i]; 5328 sort_key_test = &hist_data_test->sort_keys[i]; 5329 5330 if (sort_key->field_idx != sort_key_test->field_idx || 5331 sort_key->descending != sort_key_test->descending) 5332 return false; 5333 } 5334 5335 if (!ignore_filter && data->filter_str && 5336 (strcmp(data->filter_str, data_test->filter_str) != 0)) 5337 return false; 5338 5339 if (!actions_match(hist_data, hist_data_test)) 5340 return false; 5341 5342 return true; 5343 } 5344 5345 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops, 5346 struct event_trigger_data *data, 5347 struct trace_event_file *file) 5348 { 5349 struct hist_trigger_data *hist_data = data->private_data; 5350 struct event_trigger_data *test, *named_data = NULL; 5351 struct trace_array *tr = file->tr; 5352 int ret = 0; 5353 5354 if (hist_data->attrs->name) { 5355 named_data = find_named_trigger(hist_data->attrs->name); 5356 if (named_data) { 5357 if (!hist_trigger_match(data, named_data, named_data, 5358 true)) { 5359 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name)); 5360 ret = -EINVAL; 5361 goto out; 5362 } 5363 } 5364 } 5365 5366 if (hist_data->attrs->name && !named_data) 5367 goto new; 5368 5369 lockdep_assert_held(&event_mutex); 5370 5371 list_for_each_entry(test, &file->triggers, list) { 5372 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5373 if (!hist_trigger_match(data, test, named_data, false)) 5374 continue; 5375 if (hist_data->attrs->pause) 5376 test->paused = true; 5377 else if (hist_data->attrs->cont) 5378 test->paused = false; 5379 else if (hist_data->attrs->clear) 5380 hist_clear(test); 5381 else { 5382 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0); 5383 ret = -EEXIST; 5384 } 5385 goto out; 5386 } 5387 } 5388 new: 5389 if (hist_data->attrs->cont || hist_data->attrs->clear) { 5390 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0); 5391 ret = -ENOENT; 5392 goto out; 5393 } 5394 5395 if (hist_data->attrs->pause) 5396 data->paused = true; 5397 5398 if (named_data) { 5399 data->private_data = named_data->private_data; 5400 set_named_trigger_data(data, named_data); 5401 data->ops = &event_hist_trigger_named_ops; 5402 } 5403 5404 if (data->ops->init) { 5405 ret = data->ops->init(data->ops, data); 5406 if (ret < 0) 5407 goto out; 5408 } 5409 5410 if (hist_data->enable_timestamps) { 5411 char *clock = hist_data->attrs->clock; 5412 5413 ret = tracing_set_clock(file->tr, hist_data->attrs->clock); 5414 if (ret) { 5415 hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock)); 5416 goto out; 5417 } 5418 5419 tracing_set_time_stamp_abs(file->tr, true); 5420 } 5421 5422 if (named_data) 5423 destroy_hist_data(hist_data); 5424 5425 ret++; 5426 out: 5427 return ret; 5428 } 5429 5430 static int hist_trigger_enable(struct event_trigger_data *data, 5431 struct trace_event_file *file) 5432 { 5433 int ret = 0; 5434 5435 list_add_tail_rcu(&data->list, &file->triggers); 5436 5437 update_cond_flag(file); 5438 5439 if (trace_event_trigger_enable_disable(file, 1) < 0) { 5440 list_del_rcu(&data->list); 5441 update_cond_flag(file); 5442 ret--; 5443 } 5444 5445 return ret; 5446 } 5447 5448 static bool have_hist_trigger_match(struct event_trigger_data *data, 5449 struct trace_event_file *file) 5450 { 5451 struct hist_trigger_data *hist_data = data->private_data; 5452 struct event_trigger_data *test, *named_data = NULL; 5453 bool match = false; 5454 5455 lockdep_assert_held(&event_mutex); 5456 5457 if (hist_data->attrs->name) 5458 named_data = find_named_trigger(hist_data->attrs->name); 5459 5460 list_for_each_entry(test, &file->triggers, list) { 5461 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5462 if (hist_trigger_match(data, test, named_data, false)) { 5463 match = true; 5464 break; 5465 } 5466 } 5467 } 5468 5469 return match; 5470 } 5471 5472 static bool hist_trigger_check_refs(struct event_trigger_data *data, 5473 struct trace_event_file *file) 5474 { 5475 struct hist_trigger_data *hist_data = data->private_data; 5476 struct event_trigger_data *test, *named_data = NULL; 5477 5478 lockdep_assert_held(&event_mutex); 5479 5480 if (hist_data->attrs->name) 5481 named_data = find_named_trigger(hist_data->attrs->name); 5482 5483 list_for_each_entry(test, &file->triggers, list) { 5484 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5485 if (!hist_trigger_match(data, test, named_data, false)) 5486 continue; 5487 hist_data = test->private_data; 5488 if (check_var_refs(hist_data)) 5489 return true; 5490 break; 5491 } 5492 } 5493 5494 return false; 5495 } 5496 5497 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops, 5498 struct event_trigger_data *data, 5499 struct trace_event_file *file) 5500 { 5501 struct hist_trigger_data *hist_data = data->private_data; 5502 struct event_trigger_data *test, *named_data = NULL; 5503 bool unregistered = false; 5504 5505 lockdep_assert_held(&event_mutex); 5506 5507 if (hist_data->attrs->name) 5508 named_data = find_named_trigger(hist_data->attrs->name); 5509 5510 list_for_each_entry(test, &file->triggers, list) { 5511 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5512 if (!hist_trigger_match(data, test, named_data, false)) 5513 continue; 5514 unregistered = true; 5515 list_del_rcu(&test->list); 5516 trace_event_trigger_enable_disable(file, 0); 5517 update_cond_flag(file); 5518 break; 5519 } 5520 } 5521 5522 if (unregistered && test->ops->free) 5523 test->ops->free(test->ops, test); 5524 5525 if (hist_data->enable_timestamps) { 5526 if (!hist_data->remove || unregistered) 5527 tracing_set_time_stamp_abs(file->tr, false); 5528 } 5529 } 5530 5531 static bool hist_file_check_refs(struct trace_event_file *file) 5532 { 5533 struct hist_trigger_data *hist_data; 5534 struct event_trigger_data *test; 5535 5536 lockdep_assert_held(&event_mutex); 5537 5538 list_for_each_entry(test, &file->triggers, list) { 5539 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5540 hist_data = test->private_data; 5541 if (check_var_refs(hist_data)) 5542 return true; 5543 } 5544 } 5545 5546 return false; 5547 } 5548 5549 static void hist_unreg_all(struct trace_event_file *file) 5550 { 5551 struct event_trigger_data *test, *n; 5552 struct hist_trigger_data *hist_data; 5553 struct synth_event *se; 5554 const char *se_name; 5555 5556 lockdep_assert_held(&event_mutex); 5557 5558 if (hist_file_check_refs(file)) 5559 return; 5560 5561 list_for_each_entry_safe(test, n, &file->triggers, list) { 5562 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5563 hist_data = test->private_data; 5564 list_del_rcu(&test->list); 5565 trace_event_trigger_enable_disable(file, 0); 5566 5567 se_name = trace_event_name(file->event_call); 5568 se = find_synth_event(se_name); 5569 if (se) 5570 se->ref--; 5571 5572 update_cond_flag(file); 5573 if (hist_data->enable_timestamps) 5574 tracing_set_time_stamp_abs(file->tr, false); 5575 if (test->ops->free) 5576 test->ops->free(test->ops, test); 5577 } 5578 } 5579 } 5580 5581 static int event_hist_trigger_func(struct event_command *cmd_ops, 5582 struct trace_event_file *file, 5583 char *glob, char *cmd, char *param) 5584 { 5585 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT; 5586 struct event_trigger_data *trigger_data; 5587 struct hist_trigger_attrs *attrs; 5588 struct event_trigger_ops *trigger_ops; 5589 struct hist_trigger_data *hist_data; 5590 struct synth_event *se; 5591 const char *se_name; 5592 bool remove = false; 5593 char *trigger, *p; 5594 int ret = 0; 5595 5596 lockdep_assert_held(&event_mutex); 5597 5598 if (glob && strlen(glob)) { 5599 hist_err_clear(); 5600 last_cmd_set(file, param); 5601 } 5602 5603 if (!param) 5604 return -EINVAL; 5605 5606 if (glob[0] == '!') 5607 remove = true; 5608 5609 /* 5610 * separate the trigger from the filter (k:v [if filter]) 5611 * allowing for whitespace in the trigger 5612 */ 5613 p = trigger = param; 5614 do { 5615 p = strstr(p, "if"); 5616 if (!p) 5617 break; 5618 if (p == param) 5619 return -EINVAL; 5620 if (*(p - 1) != ' ' && *(p - 1) != '\t') { 5621 p++; 5622 continue; 5623 } 5624 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1) 5625 return -EINVAL; 5626 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') { 5627 p++; 5628 continue; 5629 } 5630 break; 5631 } while (p); 5632 5633 if (!p) 5634 param = NULL; 5635 else { 5636 *(p - 1) = '\0'; 5637 param = strstrip(p); 5638 trigger = strstrip(trigger); 5639 } 5640 5641 attrs = parse_hist_trigger_attrs(file->tr, trigger); 5642 if (IS_ERR(attrs)) 5643 return PTR_ERR(attrs); 5644 5645 if (attrs->map_bits) 5646 hist_trigger_bits = attrs->map_bits; 5647 5648 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove); 5649 if (IS_ERR(hist_data)) { 5650 destroy_hist_trigger_attrs(attrs); 5651 return PTR_ERR(hist_data); 5652 } 5653 5654 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger); 5655 5656 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL); 5657 if (!trigger_data) { 5658 ret = -ENOMEM; 5659 goto out_free; 5660 } 5661 5662 trigger_data->count = -1; 5663 trigger_data->ops = trigger_ops; 5664 trigger_data->cmd_ops = cmd_ops; 5665 5666 INIT_LIST_HEAD(&trigger_data->list); 5667 RCU_INIT_POINTER(trigger_data->filter, NULL); 5668 5669 trigger_data->private_data = hist_data; 5670 5671 /* if param is non-empty, it's supposed to be a filter */ 5672 if (param && cmd_ops->set_filter) { 5673 ret = cmd_ops->set_filter(param, trigger_data, file); 5674 if (ret < 0) 5675 goto out_free; 5676 } 5677 5678 if (remove) { 5679 if (!have_hist_trigger_match(trigger_data, file)) 5680 goto out_free; 5681 5682 if (hist_trigger_check_refs(trigger_data, file)) { 5683 ret = -EBUSY; 5684 goto out_free; 5685 } 5686 5687 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file); 5688 se_name = trace_event_name(file->event_call); 5689 se = find_synth_event(se_name); 5690 if (se) 5691 se->ref--; 5692 ret = 0; 5693 goto out_free; 5694 } 5695 5696 ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file); 5697 /* 5698 * The above returns on success the # of triggers registered, 5699 * but if it didn't register any it returns zero. Consider no 5700 * triggers registered a failure too. 5701 */ 5702 if (!ret) { 5703 if (!(attrs->pause || attrs->cont || attrs->clear)) 5704 ret = -ENOENT; 5705 goto out_free; 5706 } else if (ret < 0) 5707 goto out_free; 5708 5709 if (get_named_trigger_data(trigger_data)) 5710 goto enable; 5711 5712 if (has_hist_vars(hist_data)) 5713 save_hist_vars(hist_data); 5714 5715 ret = create_actions(hist_data); 5716 if (ret) 5717 goto out_unreg; 5718 5719 ret = tracing_map_init(hist_data->map); 5720 if (ret) 5721 goto out_unreg; 5722 enable: 5723 ret = hist_trigger_enable(trigger_data, file); 5724 if (ret) 5725 goto out_unreg; 5726 5727 se_name = trace_event_name(file->event_call); 5728 se = find_synth_event(se_name); 5729 if (se) 5730 se->ref++; 5731 /* Just return zero, not the number of registered triggers */ 5732 ret = 0; 5733 out: 5734 if (ret == 0) 5735 hist_err_clear(); 5736 5737 return ret; 5738 out_unreg: 5739 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file); 5740 out_free: 5741 if (cmd_ops->set_filter) 5742 cmd_ops->set_filter(NULL, trigger_data, NULL); 5743 5744 remove_hist_vars(hist_data); 5745 5746 kfree(trigger_data); 5747 5748 destroy_hist_data(hist_data); 5749 goto out; 5750 } 5751 5752 static struct event_command trigger_hist_cmd = { 5753 .name = "hist", 5754 .trigger_type = ETT_EVENT_HIST, 5755 .flags = EVENT_CMD_FL_NEEDS_REC, 5756 .func = event_hist_trigger_func, 5757 .reg = hist_register_trigger, 5758 .unreg = hist_unregister_trigger, 5759 .unreg_all = hist_unreg_all, 5760 .get_trigger_ops = event_hist_get_trigger_ops, 5761 .set_filter = set_trigger_filter, 5762 }; 5763 5764 __init int register_trigger_hist_cmd(void) 5765 { 5766 int ret; 5767 5768 ret = register_event_command(&trigger_hist_cmd); 5769 WARN_ON(ret < 0); 5770 5771 return ret; 5772 } 5773 5774 static void 5775 hist_enable_trigger(struct event_trigger_data *data, void *rec, 5776 struct ring_buffer_event *event) 5777 { 5778 struct enable_trigger_data *enable_data = data->private_data; 5779 struct event_trigger_data *test; 5780 5781 list_for_each_entry_rcu(test, &enable_data->file->triggers, list, 5782 lockdep_is_held(&event_mutex)) { 5783 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5784 if (enable_data->enable) 5785 test->paused = false; 5786 else 5787 test->paused = true; 5788 } 5789 } 5790 } 5791 5792 static void 5793 hist_enable_count_trigger(struct event_trigger_data *data, void *rec, 5794 struct ring_buffer_event *event) 5795 { 5796 if (!data->count) 5797 return; 5798 5799 if (data->count != -1) 5800 (data->count)--; 5801 5802 hist_enable_trigger(data, rec, event); 5803 } 5804 5805 static struct event_trigger_ops hist_enable_trigger_ops = { 5806 .func = hist_enable_trigger, 5807 .print = event_enable_trigger_print, 5808 .init = event_trigger_init, 5809 .free = event_enable_trigger_free, 5810 }; 5811 5812 static struct event_trigger_ops hist_enable_count_trigger_ops = { 5813 .func = hist_enable_count_trigger, 5814 .print = event_enable_trigger_print, 5815 .init = event_trigger_init, 5816 .free = event_enable_trigger_free, 5817 }; 5818 5819 static struct event_trigger_ops hist_disable_trigger_ops = { 5820 .func = hist_enable_trigger, 5821 .print = event_enable_trigger_print, 5822 .init = event_trigger_init, 5823 .free = event_enable_trigger_free, 5824 }; 5825 5826 static struct event_trigger_ops hist_disable_count_trigger_ops = { 5827 .func = hist_enable_count_trigger, 5828 .print = event_enable_trigger_print, 5829 .init = event_trigger_init, 5830 .free = event_enable_trigger_free, 5831 }; 5832 5833 static struct event_trigger_ops * 5834 hist_enable_get_trigger_ops(char *cmd, char *param) 5835 { 5836 struct event_trigger_ops *ops; 5837 bool enable; 5838 5839 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0); 5840 5841 if (enable) 5842 ops = param ? &hist_enable_count_trigger_ops : 5843 &hist_enable_trigger_ops; 5844 else 5845 ops = param ? &hist_disable_count_trigger_ops : 5846 &hist_disable_trigger_ops; 5847 5848 return ops; 5849 } 5850 5851 static void hist_enable_unreg_all(struct trace_event_file *file) 5852 { 5853 struct event_trigger_data *test, *n; 5854 5855 list_for_each_entry_safe(test, n, &file->triggers, list) { 5856 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) { 5857 list_del_rcu(&test->list); 5858 update_cond_flag(file); 5859 trace_event_trigger_enable_disable(file, 0); 5860 if (test->ops->free) 5861 test->ops->free(test->ops, test); 5862 } 5863 } 5864 } 5865 5866 static struct event_command trigger_hist_enable_cmd = { 5867 .name = ENABLE_HIST_STR, 5868 .trigger_type = ETT_HIST_ENABLE, 5869 .func = event_enable_trigger_func, 5870 .reg = event_enable_register_trigger, 5871 .unreg = event_enable_unregister_trigger, 5872 .unreg_all = hist_enable_unreg_all, 5873 .get_trigger_ops = hist_enable_get_trigger_ops, 5874 .set_filter = set_trigger_filter, 5875 }; 5876 5877 static struct event_command trigger_hist_disable_cmd = { 5878 .name = DISABLE_HIST_STR, 5879 .trigger_type = ETT_HIST_ENABLE, 5880 .func = event_enable_trigger_func, 5881 .reg = event_enable_register_trigger, 5882 .unreg = event_enable_unregister_trigger, 5883 .unreg_all = hist_enable_unreg_all, 5884 .get_trigger_ops = hist_enable_get_trigger_ops, 5885 .set_filter = set_trigger_filter, 5886 }; 5887 5888 static __init void unregister_trigger_hist_enable_disable_cmds(void) 5889 { 5890 unregister_event_command(&trigger_hist_enable_cmd); 5891 unregister_event_command(&trigger_hist_disable_cmd); 5892 } 5893 5894 __init int register_trigger_hist_enable_disable_cmds(void) 5895 { 5896 int ret; 5897 5898 ret = register_event_command(&trigger_hist_enable_cmd); 5899 if (WARN_ON(ret < 0)) 5900 return ret; 5901 ret = register_event_command(&trigger_hist_disable_cmd); 5902 if (WARN_ON(ret < 0)) 5903 unregister_trigger_hist_enable_disable_cmds(); 5904 5905 return ret; 5906 } 5907