1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * trace_events_hist - trace event hist triggers 4 * 5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com> 6 */ 7 8 #include <linux/module.h> 9 #include <linux/kallsyms.h> 10 #include <linux/security.h> 11 #include <linux/mutex.h> 12 #include <linux/slab.h> 13 #include <linux/stacktrace.h> 14 #include <linux/rculist.h> 15 #include <linux/tracefs.h> 16 17 /* for gfp flag names */ 18 #include <linux/trace_events.h> 19 #include <trace/events/mmflags.h> 20 21 #include "tracing_map.h" 22 #include "trace_synth.h" 23 24 #define ERRORS \ 25 C(NONE, "No error"), \ 26 C(DUPLICATE_VAR, "Variable already defined"), \ 27 C(VAR_NOT_UNIQUE, "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \ 28 C(TOO_MANY_VARS, "Too many variables defined"), \ 29 C(MALFORMED_ASSIGNMENT, "Malformed assignment"), \ 30 C(NAMED_MISMATCH, "Named hist trigger doesn't match existing named trigger (includes variables)"), \ 31 C(TRIGGER_EEXIST, "Hist trigger already exists"), \ 32 C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \ 33 C(SET_CLOCK_FAIL, "Couldn't set trace_clock"), \ 34 C(BAD_FIELD_MODIFIER, "Invalid field modifier"), \ 35 C(TOO_MANY_SUBEXPR, "Too many subexpressions (3 max)"), \ 36 C(TIMESTAMP_MISMATCH, "Timestamp units in expression don't match"), \ 37 C(TOO_MANY_FIELD_VARS, "Too many field variables defined"), \ 38 C(EVENT_FILE_NOT_FOUND, "Event file not found"), \ 39 C(HIST_NOT_FOUND, "Matching event histogram not found"), \ 40 C(HIST_CREATE_FAIL, "Couldn't create histogram for field"), \ 41 C(SYNTH_VAR_NOT_FOUND, "Couldn't find synthetic variable"), \ 42 C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"), \ 43 C(SYNTH_TYPE_MISMATCH, "Param type doesn't match synthetic event field type"), \ 44 C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \ 45 C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"), \ 46 C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"), \ 47 C(ONX_NOT_VAR, "For onmax(x) or onchange(x), x must be a variable"), \ 48 C(ONX_VAR_NOT_FOUND, "Couldn't find onmax or onchange variable"), \ 49 C(ONX_VAR_CREATE_FAIL, "Couldn't create onmax or onchange variable"), \ 50 C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"), \ 51 C(TOO_MANY_PARAMS, "Too many action params"), \ 52 C(PARAM_NOT_FOUND, "Couldn't find param"), \ 53 C(INVALID_PARAM, "Invalid action param"), \ 54 C(ACTION_NOT_FOUND, "No action found"), \ 55 C(NO_SAVE_PARAMS, "No params found for save()"), \ 56 C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \ 57 C(ACTION_MISMATCH, "Handler doesn't support action"), \ 58 C(NO_CLOSING_PAREN, "No closing paren found"), \ 59 C(SUBSYS_NOT_FOUND, "Missing subsystem"), \ 60 C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"), \ 61 C(INVALID_REF_KEY, "Using variable references in keys not supported"), \ 62 C(VAR_NOT_FOUND, "Couldn't find variable"), \ 63 C(FIELD_NOT_FOUND, "Couldn't find field"), \ 64 C(EMPTY_ASSIGNMENT, "Empty assignment"), \ 65 C(INVALID_SORT_MODIFIER,"Invalid sort modifier"), \ 66 C(EMPTY_SORT_FIELD, "Empty sort field"), \ 67 C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"), \ 68 C(INVALID_SORT_FIELD, "Sort field must be a key or a val"), \ 69 C(INVALID_STR_OPERAND, "String type can not be an operand in expression"), \ 70 C(EXPECT_NUMBER, "Expecting numeric literal"), \ 71 C(UNARY_MINUS_SUBEXPR, "Unary minus not supported in sub-expressions"), \ 72 C(DIVISION_BY_ZERO, "Division by zero"), \ 73 C(NEED_NOHC_VAL, "Non-hitcount value is required for 'nohitcount'"), 74 75 #undef C 76 #define C(a, b) HIST_ERR_##a 77 78 enum { ERRORS }; 79 80 #undef C 81 #define C(a, b) b 82 83 static const char *err_text[] = { ERRORS }; 84 85 struct hist_field; 86 87 typedef u64 (*hist_field_fn_t) (struct hist_field *field, 88 struct tracing_map_elt *elt, 89 struct trace_buffer *buffer, 90 struct ring_buffer_event *rbe, 91 void *event); 92 93 #define HIST_FIELD_OPERANDS_MAX 2 94 #define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX) 95 #define HIST_ACTIONS_MAX 8 96 #define HIST_CONST_DIGITS_MAX 21 97 #define HIST_DIV_SHIFT 20 /* For optimizing division by constants */ 98 99 enum field_op_id { 100 FIELD_OP_NONE, 101 FIELD_OP_PLUS, 102 FIELD_OP_MINUS, 103 FIELD_OP_UNARY_MINUS, 104 FIELD_OP_DIV, 105 FIELD_OP_MULT, 106 }; 107 108 enum hist_field_fn { 109 HIST_FIELD_FN_NOP, 110 HIST_FIELD_FN_VAR_REF, 111 HIST_FIELD_FN_COUNTER, 112 HIST_FIELD_FN_CONST, 113 HIST_FIELD_FN_LOG2, 114 HIST_FIELD_FN_BUCKET, 115 HIST_FIELD_FN_TIMESTAMP, 116 HIST_FIELD_FN_CPU, 117 HIST_FIELD_FN_STRING, 118 HIST_FIELD_FN_DYNSTRING, 119 HIST_FIELD_FN_RELDYNSTRING, 120 HIST_FIELD_FN_PSTRING, 121 HIST_FIELD_FN_S64, 122 HIST_FIELD_FN_U64, 123 HIST_FIELD_FN_S32, 124 HIST_FIELD_FN_U32, 125 HIST_FIELD_FN_S16, 126 HIST_FIELD_FN_U16, 127 HIST_FIELD_FN_S8, 128 HIST_FIELD_FN_U8, 129 HIST_FIELD_FN_UMINUS, 130 HIST_FIELD_FN_MINUS, 131 HIST_FIELD_FN_PLUS, 132 HIST_FIELD_FN_DIV, 133 HIST_FIELD_FN_MULT, 134 HIST_FIELD_FN_DIV_POWER2, 135 HIST_FIELD_FN_DIV_NOT_POWER2, 136 HIST_FIELD_FN_DIV_MULT_SHIFT, 137 HIST_FIELD_FN_EXECNAME, 138 HIST_FIELD_FN_STACK, 139 }; 140 141 /* 142 * A hist_var (histogram variable) contains variable information for 143 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF 144 * flag set. A hist_var has a variable name e.g. ts0, and is 145 * associated with a given histogram trigger, as specified by 146 * hist_data. The hist_var idx is the unique index assigned to the 147 * variable by the hist trigger's tracing_map. The idx is what is 148 * used to set a variable's value and, by a variable reference, to 149 * retrieve it. 150 */ 151 struct hist_var { 152 char *name; 153 struct hist_trigger_data *hist_data; 154 unsigned int idx; 155 }; 156 157 struct hist_field { 158 struct ftrace_event_field *field; 159 unsigned long flags; 160 unsigned long buckets; 161 const char *type; 162 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX]; 163 struct hist_trigger_data *hist_data; 164 enum hist_field_fn fn_num; 165 unsigned int ref; 166 unsigned int size; 167 unsigned int offset; 168 unsigned int is_signed; 169 170 /* 171 * Variable fields contain variable-specific info in var. 172 */ 173 struct hist_var var; 174 enum field_op_id operator; 175 char *system; 176 char *event_name; 177 178 /* 179 * The name field is used for EXPR and VAR_REF fields. VAR 180 * fields contain the variable name in var.name. 181 */ 182 char *name; 183 184 /* 185 * When a histogram trigger is hit, if it has any references 186 * to variables, the values of those variables are collected 187 * into a var_ref_vals array by resolve_var_refs(). The 188 * current value of each variable is read from the tracing_map 189 * using the hist field's hist_var.idx and entered into the 190 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx]. 191 */ 192 unsigned int var_ref_idx; 193 bool read_once; 194 195 unsigned int var_str_idx; 196 197 /* Numeric literals are represented as u64 */ 198 u64 constant; 199 /* Used to optimize division by constants */ 200 u64 div_multiplier; 201 }; 202 203 static u64 hist_fn_call(struct hist_field *hist_field, 204 struct tracing_map_elt *elt, 205 struct trace_buffer *buffer, 206 struct ring_buffer_event *rbe, 207 void *event); 208 hist_field_const(struct hist_field * field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)209 static u64 hist_field_const(struct hist_field *field, 210 struct tracing_map_elt *elt, 211 struct trace_buffer *buffer, 212 struct ring_buffer_event *rbe, 213 void *event) 214 { 215 return field->constant; 216 } 217 hist_field_counter(struct hist_field * field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)218 static u64 hist_field_counter(struct hist_field *field, 219 struct tracing_map_elt *elt, 220 struct trace_buffer *buffer, 221 struct ring_buffer_event *rbe, 222 void *event) 223 { 224 return 1; 225 } 226 hist_field_string(struct hist_field * hist_field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)227 static u64 hist_field_string(struct hist_field *hist_field, 228 struct tracing_map_elt *elt, 229 struct trace_buffer *buffer, 230 struct ring_buffer_event *rbe, 231 void *event) 232 { 233 char *addr = (char *)(event + hist_field->field->offset); 234 235 return (u64)(unsigned long)addr; 236 } 237 hist_field_dynstring(struct hist_field * hist_field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)238 static u64 hist_field_dynstring(struct hist_field *hist_field, 239 struct tracing_map_elt *elt, 240 struct trace_buffer *buffer, 241 struct ring_buffer_event *rbe, 242 void *event) 243 { 244 u32 str_item = *(u32 *)(event + hist_field->field->offset); 245 int str_loc = str_item & 0xffff; 246 char *addr = (char *)(event + str_loc); 247 248 return (u64)(unsigned long)addr; 249 } 250 hist_field_reldynstring(struct hist_field * hist_field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)251 static u64 hist_field_reldynstring(struct hist_field *hist_field, 252 struct tracing_map_elt *elt, 253 struct trace_buffer *buffer, 254 struct ring_buffer_event *rbe, 255 void *event) 256 { 257 u32 *item = event + hist_field->field->offset; 258 u32 str_item = *item; 259 int str_loc = str_item & 0xffff; 260 char *addr = (char *)&item[1] + str_loc; 261 262 return (u64)(unsigned long)addr; 263 } 264 hist_field_pstring(struct hist_field * hist_field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)265 static u64 hist_field_pstring(struct hist_field *hist_field, 266 struct tracing_map_elt *elt, 267 struct trace_buffer *buffer, 268 struct ring_buffer_event *rbe, 269 void *event) 270 { 271 char **addr = (char **)(event + hist_field->field->offset); 272 273 return (u64)(unsigned long)*addr; 274 } 275 hist_field_log2(struct hist_field * hist_field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)276 static u64 hist_field_log2(struct hist_field *hist_field, 277 struct tracing_map_elt *elt, 278 struct trace_buffer *buffer, 279 struct ring_buffer_event *rbe, 280 void *event) 281 { 282 struct hist_field *operand = hist_field->operands[0]; 283 284 u64 val = hist_fn_call(operand, elt, buffer, rbe, event); 285 286 return (u64) ilog2(roundup_pow_of_two(val)); 287 } 288 hist_field_bucket(struct hist_field * hist_field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)289 static u64 hist_field_bucket(struct hist_field *hist_field, 290 struct tracing_map_elt *elt, 291 struct trace_buffer *buffer, 292 struct ring_buffer_event *rbe, 293 void *event) 294 { 295 struct hist_field *operand = hist_field->operands[0]; 296 unsigned long buckets = hist_field->buckets; 297 298 u64 val = hist_fn_call(operand, elt, buffer, rbe, event); 299 300 if (WARN_ON_ONCE(!buckets)) 301 return val; 302 303 if (val >= LONG_MAX) 304 val = div64_ul(val, buckets); 305 else 306 val = (u64)((unsigned long)val / buckets); 307 return val * buckets; 308 } 309 hist_field_plus(struct hist_field * hist_field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)310 static u64 hist_field_plus(struct hist_field *hist_field, 311 struct tracing_map_elt *elt, 312 struct trace_buffer *buffer, 313 struct ring_buffer_event *rbe, 314 void *event) 315 { 316 struct hist_field *operand1 = hist_field->operands[0]; 317 struct hist_field *operand2 = hist_field->operands[1]; 318 319 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 320 u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 321 322 return val1 + val2; 323 } 324 hist_field_minus(struct hist_field * hist_field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)325 static u64 hist_field_minus(struct hist_field *hist_field, 326 struct tracing_map_elt *elt, 327 struct trace_buffer *buffer, 328 struct ring_buffer_event *rbe, 329 void *event) 330 { 331 struct hist_field *operand1 = hist_field->operands[0]; 332 struct hist_field *operand2 = hist_field->operands[1]; 333 334 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 335 u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 336 337 return val1 - val2; 338 } 339 hist_field_div(struct hist_field * hist_field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)340 static u64 hist_field_div(struct hist_field *hist_field, 341 struct tracing_map_elt *elt, 342 struct trace_buffer *buffer, 343 struct ring_buffer_event *rbe, 344 void *event) 345 { 346 struct hist_field *operand1 = hist_field->operands[0]; 347 struct hist_field *operand2 = hist_field->operands[1]; 348 349 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 350 u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 351 352 /* Return -1 for the undefined case */ 353 if (!val2) 354 return -1; 355 356 /* Use shift if the divisor is a power of 2 */ 357 if (!(val2 & (val2 - 1))) 358 return val1 >> __ffs64(val2); 359 360 return div64_u64(val1, val2); 361 } 362 div_by_power_of_two(struct hist_field * hist_field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)363 static u64 div_by_power_of_two(struct hist_field *hist_field, 364 struct tracing_map_elt *elt, 365 struct trace_buffer *buffer, 366 struct ring_buffer_event *rbe, 367 void *event) 368 { 369 struct hist_field *operand1 = hist_field->operands[0]; 370 struct hist_field *operand2 = hist_field->operands[1]; 371 372 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 373 374 return val1 >> __ffs64(operand2->constant); 375 } 376 div_by_not_power_of_two(struct hist_field * hist_field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)377 static u64 div_by_not_power_of_two(struct hist_field *hist_field, 378 struct tracing_map_elt *elt, 379 struct trace_buffer *buffer, 380 struct ring_buffer_event *rbe, 381 void *event) 382 { 383 struct hist_field *operand1 = hist_field->operands[0]; 384 struct hist_field *operand2 = hist_field->operands[1]; 385 386 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 387 388 return div64_u64(val1, operand2->constant); 389 } 390 div_by_mult_and_shift(struct hist_field * hist_field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)391 static u64 div_by_mult_and_shift(struct hist_field *hist_field, 392 struct tracing_map_elt *elt, 393 struct trace_buffer *buffer, 394 struct ring_buffer_event *rbe, 395 void *event) 396 { 397 struct hist_field *operand1 = hist_field->operands[0]; 398 struct hist_field *operand2 = hist_field->operands[1]; 399 400 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 401 402 /* 403 * If the divisor is a constant, do a multiplication and shift instead. 404 * 405 * Choose Z = some power of 2. If Y <= Z, then: 406 * X / Y = (X * (Z / Y)) / Z 407 * 408 * (Z / Y) is a constant (mult) which is calculated at parse time, so: 409 * X / Y = (X * mult) / Z 410 * 411 * The division by Z can be replaced by a shift since Z is a power of 2: 412 * X / Y = (X * mult) >> HIST_DIV_SHIFT 413 * 414 * As long, as X < Z the results will not be off by more than 1. 415 */ 416 if (val1 < (1 << HIST_DIV_SHIFT)) { 417 u64 mult = operand2->div_multiplier; 418 419 return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT; 420 } 421 422 return div64_u64(val1, operand2->constant); 423 } 424 hist_field_mult(struct hist_field * hist_field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)425 static u64 hist_field_mult(struct hist_field *hist_field, 426 struct tracing_map_elt *elt, 427 struct trace_buffer *buffer, 428 struct ring_buffer_event *rbe, 429 void *event) 430 { 431 struct hist_field *operand1 = hist_field->operands[0]; 432 struct hist_field *operand2 = hist_field->operands[1]; 433 434 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 435 u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 436 437 return val1 * val2; 438 } 439 hist_field_unary_minus(struct hist_field * hist_field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)440 static u64 hist_field_unary_minus(struct hist_field *hist_field, 441 struct tracing_map_elt *elt, 442 struct trace_buffer *buffer, 443 struct ring_buffer_event *rbe, 444 void *event) 445 { 446 struct hist_field *operand = hist_field->operands[0]; 447 448 s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event); 449 u64 val = (u64)-sval; 450 451 return val; 452 } 453 454 #define DEFINE_HIST_FIELD_FN(type) \ 455 static u64 hist_field_##type(struct hist_field *hist_field, \ 456 struct tracing_map_elt *elt, \ 457 struct trace_buffer *buffer, \ 458 struct ring_buffer_event *rbe, \ 459 void *event) \ 460 { \ 461 type *addr = (type *)(event + hist_field->field->offset); \ 462 \ 463 return (u64)(unsigned long)*addr; \ 464 } 465 466 DEFINE_HIST_FIELD_FN(s64); 467 DEFINE_HIST_FIELD_FN(u64); 468 DEFINE_HIST_FIELD_FN(s32); 469 DEFINE_HIST_FIELD_FN(u32); 470 DEFINE_HIST_FIELD_FN(s16); 471 DEFINE_HIST_FIELD_FN(u16); 472 DEFINE_HIST_FIELD_FN(s8); 473 DEFINE_HIST_FIELD_FN(u8); 474 475 #define for_each_hist_field(i, hist_data) \ 476 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++) 477 478 #define for_each_hist_val_field(i, hist_data) \ 479 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++) 480 481 #define for_each_hist_key_field(i, hist_data) \ 482 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++) 483 484 #define HITCOUNT_IDX 0 485 #define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE) 486 487 enum hist_field_flags { 488 HIST_FIELD_FL_HITCOUNT = 1 << 0, 489 HIST_FIELD_FL_KEY = 1 << 1, 490 HIST_FIELD_FL_STRING = 1 << 2, 491 HIST_FIELD_FL_HEX = 1 << 3, 492 HIST_FIELD_FL_SYM = 1 << 4, 493 HIST_FIELD_FL_SYM_OFFSET = 1 << 5, 494 HIST_FIELD_FL_EXECNAME = 1 << 6, 495 HIST_FIELD_FL_SYSCALL = 1 << 7, 496 HIST_FIELD_FL_STACKTRACE = 1 << 8, 497 HIST_FIELD_FL_LOG2 = 1 << 9, 498 HIST_FIELD_FL_TIMESTAMP = 1 << 10, 499 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11, 500 HIST_FIELD_FL_VAR = 1 << 12, 501 HIST_FIELD_FL_EXPR = 1 << 13, 502 HIST_FIELD_FL_VAR_REF = 1 << 14, 503 HIST_FIELD_FL_CPU = 1 << 15, 504 HIST_FIELD_FL_ALIAS = 1 << 16, 505 HIST_FIELD_FL_BUCKET = 1 << 17, 506 HIST_FIELD_FL_CONST = 1 << 18, 507 HIST_FIELD_FL_PERCENT = 1 << 19, 508 HIST_FIELD_FL_GRAPH = 1 << 20, 509 }; 510 511 struct var_defs { 512 unsigned int n_vars; 513 char *name[TRACING_MAP_VARS_MAX]; 514 char *expr[TRACING_MAP_VARS_MAX]; 515 }; 516 517 struct hist_trigger_attrs { 518 char *keys_str; 519 char *vals_str; 520 char *sort_key_str; 521 char *name; 522 char *clock; 523 bool pause; 524 bool cont; 525 bool clear; 526 bool ts_in_usecs; 527 bool no_hitcount; 528 unsigned int map_bits; 529 530 char *assignment_str[TRACING_MAP_VARS_MAX]; 531 unsigned int n_assignments; 532 533 char *action_str[HIST_ACTIONS_MAX]; 534 unsigned int n_actions; 535 536 struct var_defs var_defs; 537 }; 538 539 struct field_var { 540 struct hist_field *var; 541 struct hist_field *val; 542 }; 543 544 struct field_var_hist { 545 struct hist_trigger_data *hist_data; 546 char *cmd; 547 }; 548 549 struct hist_trigger_data { 550 struct hist_field *fields[HIST_FIELDS_MAX]; 551 unsigned int n_vals; 552 unsigned int n_keys; 553 unsigned int n_fields; 554 unsigned int n_vars; 555 unsigned int n_var_str; 556 unsigned int key_size; 557 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX]; 558 unsigned int n_sort_keys; 559 struct trace_event_file *event_file; 560 struct hist_trigger_attrs *attrs; 561 struct tracing_map *map; 562 bool enable_timestamps; 563 bool remove; 564 struct hist_field *var_refs[TRACING_MAP_VARS_MAX]; 565 unsigned int n_var_refs; 566 567 struct action_data *actions[HIST_ACTIONS_MAX]; 568 unsigned int n_actions; 569 570 struct field_var *field_vars[SYNTH_FIELDS_MAX]; 571 unsigned int n_field_vars; 572 unsigned int n_field_var_str; 573 struct field_var_hist *field_var_hists[SYNTH_FIELDS_MAX]; 574 unsigned int n_field_var_hists; 575 576 struct field_var *save_vars[SYNTH_FIELDS_MAX]; 577 unsigned int n_save_vars; 578 unsigned int n_save_var_str; 579 }; 580 581 struct action_data; 582 583 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data, 584 struct tracing_map_elt *elt, 585 struct trace_buffer *buffer, void *rec, 586 struct ring_buffer_event *rbe, void *key, 587 struct action_data *data, u64 *var_ref_vals); 588 589 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val); 590 591 enum handler_id { 592 HANDLER_ONMATCH = 1, 593 HANDLER_ONMAX, 594 HANDLER_ONCHANGE, 595 }; 596 597 enum action_id { 598 ACTION_SAVE = 1, 599 ACTION_TRACE, 600 ACTION_SNAPSHOT, 601 }; 602 603 struct action_data { 604 enum handler_id handler; 605 enum action_id action; 606 char *action_name; 607 action_fn_t fn; 608 609 unsigned int n_params; 610 char *params[SYNTH_FIELDS_MAX]; 611 612 /* 613 * When a histogram trigger is hit, the values of any 614 * references to variables, including variables being passed 615 * as parameters to synthetic events, are collected into a 616 * var_ref_vals array. This var_ref_idx array is an array of 617 * indices into the var_ref_vals array, one for each synthetic 618 * event param, and is passed to the synthetic event 619 * invocation. 620 */ 621 unsigned int var_ref_idx[SYNTH_FIELDS_MAX]; 622 struct synth_event *synth_event; 623 bool use_trace_keyword; 624 char *synth_event_name; 625 626 union { 627 struct { 628 char *event; 629 char *event_system; 630 } match_data; 631 632 struct { 633 /* 634 * var_str contains the $-unstripped variable 635 * name referenced by var_ref, and used when 636 * printing the action. Because var_ref 637 * creation is deferred to create_actions(), 638 * we need a per-action way to save it until 639 * then, thus var_str. 640 */ 641 char *var_str; 642 643 /* 644 * var_ref refers to the variable being 645 * tracked e.g onmax($var). 646 */ 647 struct hist_field *var_ref; 648 649 /* 650 * track_var contains the 'invisible' tracking 651 * variable created to keep the current 652 * e.g. max value. 653 */ 654 struct hist_field *track_var; 655 656 check_track_val_fn_t check_val; 657 action_fn_t save_data; 658 } track_data; 659 }; 660 }; 661 662 struct track_data { 663 u64 track_val; 664 bool updated; 665 666 unsigned int key_len; 667 void *key; 668 struct tracing_map_elt elt; 669 670 struct action_data *action_data; 671 struct hist_trigger_data *hist_data; 672 }; 673 674 struct hist_elt_data { 675 char *comm; 676 u64 *var_ref_vals; 677 char **field_var_str; 678 int n_field_var_str; 679 }; 680 681 struct snapshot_context { 682 struct tracing_map_elt *elt; 683 void *key; 684 }; 685 686 /* 687 * Returns the specific division function to use if the divisor 688 * is constant. This avoids extra branches when the trigger is hit. 689 */ hist_field_get_div_fn(struct hist_field * divisor)690 static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor) 691 { 692 u64 div = divisor->constant; 693 694 if (!(div & (div - 1))) 695 return HIST_FIELD_FN_DIV_POWER2; 696 697 /* If the divisor is too large, do a regular division */ 698 if (div > (1 << HIST_DIV_SHIFT)) 699 return HIST_FIELD_FN_DIV_NOT_POWER2; 700 701 divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div); 702 return HIST_FIELD_FN_DIV_MULT_SHIFT; 703 } 704 track_data_free(struct track_data * track_data)705 static void track_data_free(struct track_data *track_data) 706 { 707 struct hist_elt_data *elt_data; 708 709 if (!track_data) 710 return; 711 712 kfree(track_data->key); 713 714 elt_data = track_data->elt.private_data; 715 if (elt_data) { 716 kfree(elt_data->comm); 717 kfree(elt_data); 718 } 719 720 kfree(track_data); 721 } 722 track_data_alloc(unsigned int key_len,struct action_data * action_data,struct hist_trigger_data * hist_data)723 static struct track_data *track_data_alloc(unsigned int key_len, 724 struct action_data *action_data, 725 struct hist_trigger_data *hist_data) 726 { 727 struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL); 728 struct hist_elt_data *elt_data; 729 730 if (!data) 731 return ERR_PTR(-ENOMEM); 732 733 data->key = kzalloc(key_len, GFP_KERNEL); 734 if (!data->key) { 735 track_data_free(data); 736 return ERR_PTR(-ENOMEM); 737 } 738 739 data->key_len = key_len; 740 data->action_data = action_data; 741 data->hist_data = hist_data; 742 743 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL); 744 if (!elt_data) { 745 track_data_free(data); 746 return ERR_PTR(-ENOMEM); 747 } 748 749 data->elt.private_data = elt_data; 750 751 elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL); 752 if (!elt_data->comm) { 753 track_data_free(data); 754 return ERR_PTR(-ENOMEM); 755 } 756 757 return data; 758 } 759 760 #define HIST_PREFIX "hist:" 761 762 static char *last_cmd; 763 static char last_cmd_loc[MAX_FILTER_STR_VAL]; 764 errpos(char * str)765 static int errpos(char *str) 766 { 767 if (!str || !last_cmd) 768 return 0; 769 770 return err_pos(last_cmd, str); 771 } 772 last_cmd_set(struct trace_event_file * file,char * str)773 static void last_cmd_set(struct trace_event_file *file, char *str) 774 { 775 const char *system = NULL, *name = NULL; 776 struct trace_event_call *call; 777 int len; 778 779 if (!str) 780 return; 781 782 /* sizeof() contains the nul byte */ 783 len = sizeof(HIST_PREFIX) + strlen(str); 784 kfree(last_cmd); 785 last_cmd = kzalloc(len, GFP_KERNEL); 786 if (!last_cmd) 787 return; 788 789 strcpy(last_cmd, HIST_PREFIX); 790 /* Again, sizeof() contains the nul byte */ 791 len -= sizeof(HIST_PREFIX); 792 strncat(last_cmd, str, len); 793 794 if (file) { 795 call = file->event_call; 796 system = call->class->system; 797 if (system) { 798 name = trace_event_name(call); 799 if (!name) 800 system = NULL; 801 } 802 } 803 804 if (system) 805 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, HIST_PREFIX "%s:%s", system, name); 806 } 807 hist_err(struct trace_array * tr,u8 err_type,u16 err_pos)808 static void hist_err(struct trace_array *tr, u8 err_type, u16 err_pos) 809 { 810 if (!last_cmd) 811 return; 812 813 tracing_log_err(tr, last_cmd_loc, last_cmd, err_text, 814 err_type, err_pos); 815 } 816 hist_err_clear(void)817 static void hist_err_clear(void) 818 { 819 if (last_cmd) 820 last_cmd[0] = '\0'; 821 last_cmd_loc[0] = '\0'; 822 } 823 824 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals, 825 unsigned int *var_ref_idx); 826 trace_synth(struct synth_event * event,u64 * var_ref_vals,unsigned int * var_ref_idx)827 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals, 828 unsigned int *var_ref_idx) 829 { 830 struct tracepoint *tp = event->tp; 831 832 if (unlikely(atomic_read(&tp->key.enabled) > 0)) { 833 struct tracepoint_func *probe_func_ptr; 834 synth_probe_func_t probe_func; 835 void *__data; 836 837 if (!(cpu_online(raw_smp_processor_id()))) 838 return; 839 840 probe_func_ptr = rcu_dereference_sched((tp)->funcs); 841 if (probe_func_ptr) { 842 do { 843 probe_func = probe_func_ptr->func; 844 __data = probe_func_ptr->data; 845 probe_func(__data, var_ref_vals, var_ref_idx); 846 } while ((++probe_func_ptr)->func); 847 } 848 } 849 } 850 action_trace(struct hist_trigger_data * hist_data,struct tracing_map_elt * elt,struct trace_buffer * buffer,void * rec,struct ring_buffer_event * rbe,void * key,struct action_data * data,u64 * var_ref_vals)851 static void action_trace(struct hist_trigger_data *hist_data, 852 struct tracing_map_elt *elt, 853 struct trace_buffer *buffer, void *rec, 854 struct ring_buffer_event *rbe, void *key, 855 struct action_data *data, u64 *var_ref_vals) 856 { 857 struct synth_event *event = data->synth_event; 858 859 trace_synth(event, var_ref_vals, data->var_ref_idx); 860 } 861 862 struct hist_var_data { 863 struct list_head list; 864 struct hist_trigger_data *hist_data; 865 }; 866 hist_field_timestamp(struct hist_field * hist_field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)867 static u64 hist_field_timestamp(struct hist_field *hist_field, 868 struct tracing_map_elt *elt, 869 struct trace_buffer *buffer, 870 struct ring_buffer_event *rbe, 871 void *event) 872 { 873 struct hist_trigger_data *hist_data = hist_field->hist_data; 874 struct trace_array *tr = hist_data->event_file->tr; 875 876 u64 ts = ring_buffer_event_time_stamp(buffer, rbe); 877 878 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr)) 879 ts = ns2usecs(ts); 880 881 return ts; 882 } 883 hist_field_cpu(struct hist_field * hist_field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)884 static u64 hist_field_cpu(struct hist_field *hist_field, 885 struct tracing_map_elt *elt, 886 struct trace_buffer *buffer, 887 struct ring_buffer_event *rbe, 888 void *event) 889 { 890 int cpu = smp_processor_id(); 891 892 return cpu; 893 } 894 895 /** 896 * check_field_for_var_ref - Check if a VAR_REF field references a variable 897 * @hist_field: The VAR_REF field to check 898 * @var_data: The hist trigger that owns the variable 899 * @var_idx: The trigger variable identifier 900 * 901 * Check the given VAR_REF field to see whether or not it references 902 * the given variable associated with the given trigger. 903 * 904 * Return: The VAR_REF field if it does reference the variable, NULL if not 905 */ 906 static struct hist_field * check_field_for_var_ref(struct hist_field * hist_field,struct hist_trigger_data * var_data,unsigned int var_idx)907 check_field_for_var_ref(struct hist_field *hist_field, 908 struct hist_trigger_data *var_data, 909 unsigned int var_idx) 910 { 911 WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF)); 912 913 if (hist_field && hist_field->var.idx == var_idx && 914 hist_field->var.hist_data == var_data) 915 return hist_field; 916 917 return NULL; 918 } 919 920 /** 921 * find_var_ref - Check if a trigger has a reference to a trigger variable 922 * @hist_data: The hist trigger that might have a reference to the variable 923 * @var_data: The hist trigger that owns the variable 924 * @var_idx: The trigger variable identifier 925 * 926 * Check the list of var_refs[] on the first hist trigger to see 927 * whether any of them are references to the variable on the second 928 * trigger. 929 * 930 * Return: The VAR_REF field referencing the variable if so, NULL if not 931 */ find_var_ref(struct hist_trigger_data * hist_data,struct hist_trigger_data * var_data,unsigned int var_idx)932 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data, 933 struct hist_trigger_data *var_data, 934 unsigned int var_idx) 935 { 936 struct hist_field *hist_field; 937 unsigned int i; 938 939 for (i = 0; i < hist_data->n_var_refs; i++) { 940 hist_field = hist_data->var_refs[i]; 941 if (check_field_for_var_ref(hist_field, var_data, var_idx)) 942 return hist_field; 943 } 944 945 return NULL; 946 } 947 948 /** 949 * find_any_var_ref - Check if there is a reference to a given trigger variable 950 * @hist_data: The hist trigger 951 * @var_idx: The trigger variable identifier 952 * 953 * Check to see whether the given variable is currently referenced by 954 * any other trigger. 955 * 956 * The trigger the variable is defined on is explicitly excluded - the 957 * assumption being that a self-reference doesn't prevent a trigger 958 * from being removed. 959 * 960 * Return: The VAR_REF field referencing the variable if so, NULL if not 961 */ find_any_var_ref(struct hist_trigger_data * hist_data,unsigned int var_idx)962 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data, 963 unsigned int var_idx) 964 { 965 struct trace_array *tr = hist_data->event_file->tr; 966 struct hist_field *found = NULL; 967 struct hist_var_data *var_data; 968 969 list_for_each_entry(var_data, &tr->hist_vars, list) { 970 if (var_data->hist_data == hist_data) 971 continue; 972 found = find_var_ref(var_data->hist_data, hist_data, var_idx); 973 if (found) 974 break; 975 } 976 977 return found; 978 } 979 980 /** 981 * check_var_refs - Check if there is a reference to any of trigger's variables 982 * @hist_data: The hist trigger 983 * 984 * A trigger can define one or more variables. If any one of them is 985 * currently referenced by any other trigger, this function will 986 * determine that. 987 * 988 * Typically used to determine whether or not a trigger can be removed 989 * - if there are any references to a trigger's variables, it cannot. 990 * 991 * Return: True if there is a reference to any of trigger's variables 992 */ check_var_refs(struct hist_trigger_data * hist_data)993 static bool check_var_refs(struct hist_trigger_data *hist_data) 994 { 995 struct hist_field *field; 996 bool found = false; 997 int i; 998 999 for_each_hist_field(i, hist_data) { 1000 field = hist_data->fields[i]; 1001 if (field && field->flags & HIST_FIELD_FL_VAR) { 1002 if (find_any_var_ref(hist_data, field->var.idx)) { 1003 found = true; 1004 break; 1005 } 1006 } 1007 } 1008 1009 return found; 1010 } 1011 find_hist_vars(struct hist_trigger_data * hist_data)1012 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data) 1013 { 1014 struct trace_array *tr = hist_data->event_file->tr; 1015 struct hist_var_data *var_data, *found = NULL; 1016 1017 list_for_each_entry(var_data, &tr->hist_vars, list) { 1018 if (var_data->hist_data == hist_data) { 1019 found = var_data; 1020 break; 1021 } 1022 } 1023 1024 return found; 1025 } 1026 field_has_hist_vars(struct hist_field * hist_field,unsigned int level)1027 static bool field_has_hist_vars(struct hist_field *hist_field, 1028 unsigned int level) 1029 { 1030 int i; 1031 1032 if (level > 3) 1033 return false; 1034 1035 if (!hist_field) 1036 return false; 1037 1038 if (hist_field->flags & HIST_FIELD_FL_VAR || 1039 hist_field->flags & HIST_FIELD_FL_VAR_REF) 1040 return true; 1041 1042 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) { 1043 struct hist_field *operand; 1044 1045 operand = hist_field->operands[i]; 1046 if (field_has_hist_vars(operand, level + 1)) 1047 return true; 1048 } 1049 1050 return false; 1051 } 1052 has_hist_vars(struct hist_trigger_data * hist_data)1053 static bool has_hist_vars(struct hist_trigger_data *hist_data) 1054 { 1055 struct hist_field *hist_field; 1056 int i; 1057 1058 for_each_hist_field(i, hist_data) { 1059 hist_field = hist_data->fields[i]; 1060 if (field_has_hist_vars(hist_field, 0)) 1061 return true; 1062 } 1063 1064 return false; 1065 } 1066 save_hist_vars(struct hist_trigger_data * hist_data)1067 static int save_hist_vars(struct hist_trigger_data *hist_data) 1068 { 1069 struct trace_array *tr = hist_data->event_file->tr; 1070 struct hist_var_data *var_data; 1071 1072 var_data = find_hist_vars(hist_data); 1073 if (var_data) 1074 return 0; 1075 1076 if (tracing_check_open_get_tr(tr)) 1077 return -ENODEV; 1078 1079 var_data = kzalloc(sizeof(*var_data), GFP_KERNEL); 1080 if (!var_data) { 1081 trace_array_put(tr); 1082 return -ENOMEM; 1083 } 1084 1085 var_data->hist_data = hist_data; 1086 list_add(&var_data->list, &tr->hist_vars); 1087 1088 return 0; 1089 } 1090 remove_hist_vars(struct hist_trigger_data * hist_data)1091 static void remove_hist_vars(struct hist_trigger_data *hist_data) 1092 { 1093 struct trace_array *tr = hist_data->event_file->tr; 1094 struct hist_var_data *var_data; 1095 1096 var_data = find_hist_vars(hist_data); 1097 if (!var_data) 1098 return; 1099 1100 if (WARN_ON(check_var_refs(hist_data))) 1101 return; 1102 1103 list_del(&var_data->list); 1104 1105 kfree(var_data); 1106 1107 trace_array_put(tr); 1108 } 1109 find_var_field(struct hist_trigger_data * hist_data,const char * var_name)1110 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data, 1111 const char *var_name) 1112 { 1113 struct hist_field *hist_field, *found = NULL; 1114 int i; 1115 1116 for_each_hist_field(i, hist_data) { 1117 hist_field = hist_data->fields[i]; 1118 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR && 1119 strcmp(hist_field->var.name, var_name) == 0) { 1120 found = hist_field; 1121 break; 1122 } 1123 } 1124 1125 return found; 1126 } 1127 find_var(struct hist_trigger_data * hist_data,struct trace_event_file * file,const char * var_name)1128 static struct hist_field *find_var(struct hist_trigger_data *hist_data, 1129 struct trace_event_file *file, 1130 const char *var_name) 1131 { 1132 struct hist_trigger_data *test_data; 1133 struct event_trigger_data *test; 1134 struct hist_field *hist_field; 1135 1136 lockdep_assert_held(&event_mutex); 1137 1138 hist_field = find_var_field(hist_data, var_name); 1139 if (hist_field) 1140 return hist_field; 1141 1142 list_for_each_entry(test, &file->triggers, list) { 1143 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1144 test_data = test->private_data; 1145 hist_field = find_var_field(test_data, var_name); 1146 if (hist_field) 1147 return hist_field; 1148 } 1149 } 1150 1151 return NULL; 1152 } 1153 find_var_file(struct trace_array * tr,char * system,char * event_name,char * var_name)1154 static struct trace_event_file *find_var_file(struct trace_array *tr, 1155 char *system, 1156 char *event_name, 1157 char *var_name) 1158 { 1159 struct hist_trigger_data *var_hist_data; 1160 struct hist_var_data *var_data; 1161 struct trace_event_file *file, *found = NULL; 1162 1163 if (system) 1164 return find_event_file(tr, system, event_name); 1165 1166 list_for_each_entry(var_data, &tr->hist_vars, list) { 1167 var_hist_data = var_data->hist_data; 1168 file = var_hist_data->event_file; 1169 if (file == found) 1170 continue; 1171 1172 if (find_var_field(var_hist_data, var_name)) { 1173 if (found) { 1174 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name)); 1175 return NULL; 1176 } 1177 1178 found = file; 1179 } 1180 } 1181 1182 return found; 1183 } 1184 find_file_var(struct trace_event_file * file,const char * var_name)1185 static struct hist_field *find_file_var(struct trace_event_file *file, 1186 const char *var_name) 1187 { 1188 struct hist_trigger_data *test_data; 1189 struct event_trigger_data *test; 1190 struct hist_field *hist_field; 1191 1192 lockdep_assert_held(&event_mutex); 1193 1194 list_for_each_entry(test, &file->triggers, list) { 1195 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1196 test_data = test->private_data; 1197 hist_field = find_var_field(test_data, var_name); 1198 if (hist_field) 1199 return hist_field; 1200 } 1201 } 1202 1203 return NULL; 1204 } 1205 1206 static struct hist_field * find_match_var(struct hist_trigger_data * hist_data,char * var_name)1207 find_match_var(struct hist_trigger_data *hist_data, char *var_name) 1208 { 1209 struct trace_array *tr = hist_data->event_file->tr; 1210 struct hist_field *hist_field, *found = NULL; 1211 struct trace_event_file *file; 1212 unsigned int i; 1213 1214 for (i = 0; i < hist_data->n_actions; i++) { 1215 struct action_data *data = hist_data->actions[i]; 1216 1217 if (data->handler == HANDLER_ONMATCH) { 1218 char *system = data->match_data.event_system; 1219 char *event_name = data->match_data.event; 1220 1221 file = find_var_file(tr, system, event_name, var_name); 1222 if (!file) 1223 continue; 1224 hist_field = find_file_var(file, var_name); 1225 if (hist_field) { 1226 if (found) { 1227 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, 1228 errpos(var_name)); 1229 return ERR_PTR(-EINVAL); 1230 } 1231 1232 found = hist_field; 1233 } 1234 } 1235 } 1236 return found; 1237 } 1238 find_event_var(struct hist_trigger_data * hist_data,char * system,char * event_name,char * var_name)1239 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data, 1240 char *system, 1241 char *event_name, 1242 char *var_name) 1243 { 1244 struct trace_array *tr = hist_data->event_file->tr; 1245 struct hist_field *hist_field = NULL; 1246 struct trace_event_file *file; 1247 1248 if (!system || !event_name) { 1249 hist_field = find_match_var(hist_data, var_name); 1250 if (IS_ERR(hist_field)) 1251 return NULL; 1252 if (hist_field) 1253 return hist_field; 1254 } 1255 1256 file = find_var_file(tr, system, event_name, var_name); 1257 if (!file) 1258 return NULL; 1259 1260 hist_field = find_file_var(file, var_name); 1261 1262 return hist_field; 1263 } 1264 hist_field_var_ref(struct hist_field * hist_field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)1265 static u64 hist_field_var_ref(struct hist_field *hist_field, 1266 struct tracing_map_elt *elt, 1267 struct trace_buffer *buffer, 1268 struct ring_buffer_event *rbe, 1269 void *event) 1270 { 1271 struct hist_elt_data *elt_data; 1272 u64 var_val = 0; 1273 1274 if (WARN_ON_ONCE(!elt)) 1275 return var_val; 1276 1277 elt_data = elt->private_data; 1278 var_val = elt_data->var_ref_vals[hist_field->var_ref_idx]; 1279 1280 return var_val; 1281 } 1282 resolve_var_refs(struct hist_trigger_data * hist_data,void * key,u64 * var_ref_vals,bool self)1283 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key, 1284 u64 *var_ref_vals, bool self) 1285 { 1286 struct hist_trigger_data *var_data; 1287 struct tracing_map_elt *var_elt; 1288 struct hist_field *hist_field; 1289 unsigned int i, var_idx; 1290 bool resolved = true; 1291 u64 var_val = 0; 1292 1293 for (i = 0; i < hist_data->n_var_refs; i++) { 1294 hist_field = hist_data->var_refs[i]; 1295 var_idx = hist_field->var.idx; 1296 var_data = hist_field->var.hist_data; 1297 1298 if (var_data == NULL) { 1299 resolved = false; 1300 break; 1301 } 1302 1303 if ((self && var_data != hist_data) || 1304 (!self && var_data == hist_data)) 1305 continue; 1306 1307 var_elt = tracing_map_lookup(var_data->map, key); 1308 if (!var_elt) { 1309 resolved = false; 1310 break; 1311 } 1312 1313 if (!tracing_map_var_set(var_elt, var_idx)) { 1314 resolved = false; 1315 break; 1316 } 1317 1318 if (self || !hist_field->read_once) 1319 var_val = tracing_map_read_var(var_elt, var_idx); 1320 else 1321 var_val = tracing_map_read_var_once(var_elt, var_idx); 1322 1323 var_ref_vals[i] = var_val; 1324 } 1325 1326 return resolved; 1327 } 1328 hist_field_name(struct hist_field * field,unsigned int level)1329 static const char *hist_field_name(struct hist_field *field, 1330 unsigned int level) 1331 { 1332 const char *field_name = ""; 1333 1334 if (WARN_ON_ONCE(!field)) 1335 return field_name; 1336 1337 if (level > 1) 1338 return field_name; 1339 1340 if (field->field) 1341 field_name = field->field->name; 1342 else if (field->flags & HIST_FIELD_FL_LOG2 || 1343 field->flags & HIST_FIELD_FL_ALIAS || 1344 field->flags & HIST_FIELD_FL_BUCKET) 1345 field_name = hist_field_name(field->operands[0], ++level); 1346 else if (field->flags & HIST_FIELD_FL_CPU) 1347 field_name = "common_cpu"; 1348 else if (field->flags & HIST_FIELD_FL_EXPR || 1349 field->flags & HIST_FIELD_FL_VAR_REF) { 1350 if (field->system) { 1351 static char full_name[MAX_FILTER_STR_VAL]; 1352 1353 strcat(full_name, field->system); 1354 strcat(full_name, "."); 1355 strcat(full_name, field->event_name); 1356 strcat(full_name, "."); 1357 strcat(full_name, field->name); 1358 field_name = full_name; 1359 } else 1360 field_name = field->name; 1361 } else if (field->flags & HIST_FIELD_FL_TIMESTAMP) 1362 field_name = "common_timestamp"; 1363 else if (field->flags & HIST_FIELD_FL_STACKTRACE) { 1364 if (field->field) 1365 field_name = field->field->name; 1366 else 1367 field_name = "common_stacktrace"; 1368 } else if (field->flags & HIST_FIELD_FL_HITCOUNT) 1369 field_name = "hitcount"; 1370 1371 if (field_name == NULL) 1372 field_name = ""; 1373 1374 return field_name; 1375 } 1376 select_value_fn(int field_size,int field_is_signed)1377 static enum hist_field_fn select_value_fn(int field_size, int field_is_signed) 1378 { 1379 switch (field_size) { 1380 case 8: 1381 if (field_is_signed) 1382 return HIST_FIELD_FN_S64; 1383 else 1384 return HIST_FIELD_FN_U64; 1385 case 4: 1386 if (field_is_signed) 1387 return HIST_FIELD_FN_S32; 1388 else 1389 return HIST_FIELD_FN_U32; 1390 case 2: 1391 if (field_is_signed) 1392 return HIST_FIELD_FN_S16; 1393 else 1394 return HIST_FIELD_FN_U16; 1395 case 1: 1396 if (field_is_signed) 1397 return HIST_FIELD_FN_S8; 1398 else 1399 return HIST_FIELD_FN_U8; 1400 } 1401 1402 return HIST_FIELD_FN_NOP; 1403 } 1404 parse_map_size(char * str)1405 static int parse_map_size(char *str) 1406 { 1407 unsigned long size, map_bits; 1408 int ret; 1409 1410 ret = kstrtoul(str, 0, &size); 1411 if (ret) 1412 goto out; 1413 1414 map_bits = ilog2(roundup_pow_of_two(size)); 1415 if (map_bits < TRACING_MAP_BITS_MIN || 1416 map_bits > TRACING_MAP_BITS_MAX) 1417 ret = -EINVAL; 1418 else 1419 ret = map_bits; 1420 out: 1421 return ret; 1422 } 1423 destroy_hist_trigger_attrs(struct hist_trigger_attrs * attrs)1424 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs) 1425 { 1426 unsigned int i; 1427 1428 if (!attrs) 1429 return; 1430 1431 for (i = 0; i < attrs->n_assignments; i++) 1432 kfree(attrs->assignment_str[i]); 1433 1434 for (i = 0; i < attrs->n_actions; i++) 1435 kfree(attrs->action_str[i]); 1436 1437 kfree(attrs->name); 1438 kfree(attrs->sort_key_str); 1439 kfree(attrs->keys_str); 1440 kfree(attrs->vals_str); 1441 kfree(attrs->clock); 1442 kfree(attrs); 1443 } 1444 parse_action(char * str,struct hist_trigger_attrs * attrs)1445 static int parse_action(char *str, struct hist_trigger_attrs *attrs) 1446 { 1447 int ret = -EINVAL; 1448 1449 if (attrs->n_actions >= HIST_ACTIONS_MAX) 1450 return ret; 1451 1452 if ((str_has_prefix(str, "onmatch(")) || 1453 (str_has_prefix(str, "onmax(")) || 1454 (str_has_prefix(str, "onchange("))) { 1455 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL); 1456 if (!attrs->action_str[attrs->n_actions]) { 1457 ret = -ENOMEM; 1458 return ret; 1459 } 1460 attrs->n_actions++; 1461 ret = 0; 1462 } 1463 return ret; 1464 } 1465 parse_assignment(struct trace_array * tr,char * str,struct hist_trigger_attrs * attrs)1466 static int parse_assignment(struct trace_array *tr, 1467 char *str, struct hist_trigger_attrs *attrs) 1468 { 1469 int len, ret = 0; 1470 1471 if ((len = str_has_prefix(str, "key=")) || 1472 (len = str_has_prefix(str, "keys="))) { 1473 attrs->keys_str = kstrdup(str + len, GFP_KERNEL); 1474 if (!attrs->keys_str) { 1475 ret = -ENOMEM; 1476 goto out; 1477 } 1478 } else if ((len = str_has_prefix(str, "val=")) || 1479 (len = str_has_prefix(str, "vals=")) || 1480 (len = str_has_prefix(str, "values="))) { 1481 attrs->vals_str = kstrdup(str + len, GFP_KERNEL); 1482 if (!attrs->vals_str) { 1483 ret = -ENOMEM; 1484 goto out; 1485 } 1486 } else if ((len = str_has_prefix(str, "sort="))) { 1487 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL); 1488 if (!attrs->sort_key_str) { 1489 ret = -ENOMEM; 1490 goto out; 1491 } 1492 } else if (str_has_prefix(str, "name=")) { 1493 attrs->name = kstrdup(str, GFP_KERNEL); 1494 if (!attrs->name) { 1495 ret = -ENOMEM; 1496 goto out; 1497 } 1498 } else if ((len = str_has_prefix(str, "clock="))) { 1499 str += len; 1500 1501 str = strstrip(str); 1502 attrs->clock = kstrdup(str, GFP_KERNEL); 1503 if (!attrs->clock) { 1504 ret = -ENOMEM; 1505 goto out; 1506 } 1507 } else if ((len = str_has_prefix(str, "size="))) { 1508 int map_bits = parse_map_size(str + len); 1509 1510 if (map_bits < 0) { 1511 ret = map_bits; 1512 goto out; 1513 } 1514 attrs->map_bits = map_bits; 1515 } else { 1516 char *assignment; 1517 1518 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) { 1519 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str)); 1520 ret = -EINVAL; 1521 goto out; 1522 } 1523 1524 assignment = kstrdup(str, GFP_KERNEL); 1525 if (!assignment) { 1526 ret = -ENOMEM; 1527 goto out; 1528 } 1529 1530 attrs->assignment_str[attrs->n_assignments++] = assignment; 1531 } 1532 out: 1533 return ret; 1534 } 1535 1536 static struct hist_trigger_attrs * parse_hist_trigger_attrs(struct trace_array * tr,char * trigger_str)1537 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str) 1538 { 1539 struct hist_trigger_attrs *attrs; 1540 int ret = 0; 1541 1542 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL); 1543 if (!attrs) 1544 return ERR_PTR(-ENOMEM); 1545 1546 while (trigger_str) { 1547 char *str = strsep(&trigger_str, ":"); 1548 char *rhs; 1549 1550 rhs = strchr(str, '='); 1551 if (rhs) { 1552 if (!strlen(++rhs)) { 1553 ret = -EINVAL; 1554 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str)); 1555 goto free; 1556 } 1557 ret = parse_assignment(tr, str, attrs); 1558 if (ret) 1559 goto free; 1560 } else if (strcmp(str, "nohitcount") == 0 || 1561 strcmp(str, "NOHC") == 0) 1562 attrs->no_hitcount = true; 1563 else if (strcmp(str, "pause") == 0) 1564 attrs->pause = true; 1565 else if ((strcmp(str, "cont") == 0) || 1566 (strcmp(str, "continue") == 0)) 1567 attrs->cont = true; 1568 else if (strcmp(str, "clear") == 0) 1569 attrs->clear = true; 1570 else { 1571 ret = parse_action(str, attrs); 1572 if (ret) 1573 goto free; 1574 } 1575 } 1576 1577 if (!attrs->keys_str) { 1578 ret = -EINVAL; 1579 goto free; 1580 } 1581 1582 if (!attrs->clock) { 1583 attrs->clock = kstrdup("global", GFP_KERNEL); 1584 if (!attrs->clock) { 1585 ret = -ENOMEM; 1586 goto free; 1587 } 1588 } 1589 1590 return attrs; 1591 free: 1592 destroy_hist_trigger_attrs(attrs); 1593 1594 return ERR_PTR(ret); 1595 } 1596 save_comm(char * comm,struct task_struct * task)1597 static inline void save_comm(char *comm, struct task_struct *task) 1598 { 1599 if (!task->pid) { 1600 strcpy(comm, "<idle>"); 1601 return; 1602 } 1603 1604 if (WARN_ON_ONCE(task->pid < 0)) { 1605 strcpy(comm, "<XXX>"); 1606 return; 1607 } 1608 1609 strncpy(comm, task->comm, TASK_COMM_LEN); 1610 } 1611 hist_elt_data_free(struct hist_elt_data * elt_data)1612 static void hist_elt_data_free(struct hist_elt_data *elt_data) 1613 { 1614 unsigned int i; 1615 1616 for (i = 0; i < elt_data->n_field_var_str; i++) 1617 kfree(elt_data->field_var_str[i]); 1618 1619 kfree(elt_data->field_var_str); 1620 1621 kfree(elt_data->comm); 1622 kfree(elt_data); 1623 } 1624 hist_trigger_elt_data_free(struct tracing_map_elt * elt)1625 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt) 1626 { 1627 struct hist_elt_data *elt_data = elt->private_data; 1628 1629 hist_elt_data_free(elt_data); 1630 } 1631 hist_trigger_elt_data_alloc(struct tracing_map_elt * elt)1632 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt) 1633 { 1634 struct hist_trigger_data *hist_data = elt->map->private_data; 1635 unsigned int size = TASK_COMM_LEN; 1636 struct hist_elt_data *elt_data; 1637 struct hist_field *hist_field; 1638 unsigned int i, n_str; 1639 1640 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL); 1641 if (!elt_data) 1642 return -ENOMEM; 1643 1644 for_each_hist_field(i, hist_data) { 1645 hist_field = hist_data->fields[i]; 1646 1647 if (hist_field->flags & HIST_FIELD_FL_EXECNAME) { 1648 elt_data->comm = kzalloc(size, GFP_KERNEL); 1649 if (!elt_data->comm) { 1650 kfree(elt_data); 1651 return -ENOMEM; 1652 } 1653 break; 1654 } 1655 } 1656 1657 n_str = hist_data->n_field_var_str + hist_data->n_save_var_str + 1658 hist_data->n_var_str; 1659 if (n_str > SYNTH_FIELDS_MAX) { 1660 hist_elt_data_free(elt_data); 1661 return -EINVAL; 1662 } 1663 1664 BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1)); 1665 1666 size = STR_VAR_LEN_MAX; 1667 1668 elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL); 1669 if (!elt_data->field_var_str) { 1670 hist_elt_data_free(elt_data); 1671 return -EINVAL; 1672 } 1673 elt_data->n_field_var_str = n_str; 1674 1675 for (i = 0; i < n_str; i++) { 1676 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL); 1677 if (!elt_data->field_var_str[i]) { 1678 hist_elt_data_free(elt_data); 1679 return -ENOMEM; 1680 } 1681 } 1682 1683 elt->private_data = elt_data; 1684 1685 return 0; 1686 } 1687 hist_trigger_elt_data_init(struct tracing_map_elt * elt)1688 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt) 1689 { 1690 struct hist_elt_data *elt_data = elt->private_data; 1691 1692 if (elt_data->comm) 1693 save_comm(elt_data->comm, current); 1694 } 1695 1696 static const struct tracing_map_ops hist_trigger_elt_data_ops = { 1697 .elt_alloc = hist_trigger_elt_data_alloc, 1698 .elt_free = hist_trigger_elt_data_free, 1699 .elt_init = hist_trigger_elt_data_init, 1700 }; 1701 get_hist_field_flags(struct hist_field * hist_field)1702 static const char *get_hist_field_flags(struct hist_field *hist_field) 1703 { 1704 const char *flags_str = NULL; 1705 1706 if (hist_field->flags & HIST_FIELD_FL_HEX) 1707 flags_str = "hex"; 1708 else if (hist_field->flags & HIST_FIELD_FL_SYM) 1709 flags_str = "sym"; 1710 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET) 1711 flags_str = "sym-offset"; 1712 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME) 1713 flags_str = "execname"; 1714 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL) 1715 flags_str = "syscall"; 1716 else if (hist_field->flags & HIST_FIELD_FL_LOG2) 1717 flags_str = "log2"; 1718 else if (hist_field->flags & HIST_FIELD_FL_BUCKET) 1719 flags_str = "buckets"; 1720 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS) 1721 flags_str = "usecs"; 1722 else if (hist_field->flags & HIST_FIELD_FL_PERCENT) 1723 flags_str = "percent"; 1724 else if (hist_field->flags & HIST_FIELD_FL_GRAPH) 1725 flags_str = "graph"; 1726 else if (hist_field->flags & HIST_FIELD_FL_STACKTRACE) 1727 flags_str = "stacktrace"; 1728 1729 return flags_str; 1730 } 1731 expr_field_str(struct hist_field * field,char * expr)1732 static void expr_field_str(struct hist_field *field, char *expr) 1733 { 1734 if (field->flags & HIST_FIELD_FL_VAR_REF) 1735 strcat(expr, "$"); 1736 else if (field->flags & HIST_FIELD_FL_CONST) { 1737 char str[HIST_CONST_DIGITS_MAX]; 1738 1739 snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant); 1740 strcat(expr, str); 1741 } 1742 1743 strcat(expr, hist_field_name(field, 0)); 1744 1745 if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) { 1746 const char *flags_str = get_hist_field_flags(field); 1747 1748 if (flags_str) { 1749 strcat(expr, "."); 1750 strcat(expr, flags_str); 1751 } 1752 } 1753 } 1754 expr_str(struct hist_field * field,unsigned int level)1755 static char *expr_str(struct hist_field *field, unsigned int level) 1756 { 1757 char *expr; 1758 1759 if (level > 1) 1760 return NULL; 1761 1762 expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 1763 if (!expr) 1764 return NULL; 1765 1766 if (!field->operands[0]) { 1767 expr_field_str(field, expr); 1768 return expr; 1769 } 1770 1771 if (field->operator == FIELD_OP_UNARY_MINUS) { 1772 char *subexpr; 1773 1774 strcat(expr, "-("); 1775 subexpr = expr_str(field->operands[0], ++level); 1776 if (!subexpr) { 1777 kfree(expr); 1778 return NULL; 1779 } 1780 strcat(expr, subexpr); 1781 strcat(expr, ")"); 1782 1783 kfree(subexpr); 1784 1785 return expr; 1786 } 1787 1788 expr_field_str(field->operands[0], expr); 1789 1790 switch (field->operator) { 1791 case FIELD_OP_MINUS: 1792 strcat(expr, "-"); 1793 break; 1794 case FIELD_OP_PLUS: 1795 strcat(expr, "+"); 1796 break; 1797 case FIELD_OP_DIV: 1798 strcat(expr, "/"); 1799 break; 1800 case FIELD_OP_MULT: 1801 strcat(expr, "*"); 1802 break; 1803 default: 1804 kfree(expr); 1805 return NULL; 1806 } 1807 1808 expr_field_str(field->operands[1], expr); 1809 1810 return expr; 1811 } 1812 1813 /* 1814 * If field_op != FIELD_OP_NONE, *sep points to the root operator 1815 * of the expression tree to be evaluated. 1816 */ contains_operator(char * str,char ** sep)1817 static int contains_operator(char *str, char **sep) 1818 { 1819 enum field_op_id field_op = FIELD_OP_NONE; 1820 char *minus_op, *plus_op, *div_op, *mult_op; 1821 1822 1823 /* 1824 * Report the last occurrence of the operators first, so that the 1825 * expression is evaluated left to right. This is important since 1826 * subtraction and division are not associative. 1827 * 1828 * e.g 1829 * 64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2 1830 * 14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2 1831 */ 1832 1833 /* 1834 * First, find lower precedence addition and subtraction 1835 * since the expression will be evaluated recursively. 1836 */ 1837 minus_op = strrchr(str, '-'); 1838 if (minus_op) { 1839 /* 1840 * Unary minus is not supported in sub-expressions. If 1841 * present, it is always the next root operator. 1842 */ 1843 if (minus_op == str) { 1844 field_op = FIELD_OP_UNARY_MINUS; 1845 goto out; 1846 } 1847 1848 field_op = FIELD_OP_MINUS; 1849 } 1850 1851 plus_op = strrchr(str, '+'); 1852 if (plus_op || minus_op) { 1853 /* 1854 * For operators of the same precedence use to rightmost as the 1855 * root, so that the expression is evaluated left to right. 1856 */ 1857 if (plus_op > minus_op) 1858 field_op = FIELD_OP_PLUS; 1859 goto out; 1860 } 1861 1862 /* 1863 * Multiplication and division have higher precedence than addition and 1864 * subtraction. 1865 */ 1866 div_op = strrchr(str, '/'); 1867 if (div_op) 1868 field_op = FIELD_OP_DIV; 1869 1870 mult_op = strrchr(str, '*'); 1871 /* 1872 * For operators of the same precedence use to rightmost as the 1873 * root, so that the expression is evaluated left to right. 1874 */ 1875 if (mult_op > div_op) 1876 field_op = FIELD_OP_MULT; 1877 1878 out: 1879 if (sep) { 1880 switch (field_op) { 1881 case FIELD_OP_UNARY_MINUS: 1882 case FIELD_OP_MINUS: 1883 *sep = minus_op; 1884 break; 1885 case FIELD_OP_PLUS: 1886 *sep = plus_op; 1887 break; 1888 case FIELD_OP_DIV: 1889 *sep = div_op; 1890 break; 1891 case FIELD_OP_MULT: 1892 *sep = mult_op; 1893 break; 1894 case FIELD_OP_NONE: 1895 default: 1896 *sep = NULL; 1897 break; 1898 } 1899 } 1900 1901 return field_op; 1902 } 1903 get_hist_field(struct hist_field * hist_field)1904 static void get_hist_field(struct hist_field *hist_field) 1905 { 1906 hist_field->ref++; 1907 } 1908 __destroy_hist_field(struct hist_field * hist_field)1909 static void __destroy_hist_field(struct hist_field *hist_field) 1910 { 1911 if (--hist_field->ref > 1) 1912 return; 1913 1914 kfree(hist_field->var.name); 1915 kfree(hist_field->name); 1916 1917 /* Can likely be a const */ 1918 kfree_const(hist_field->type); 1919 1920 kfree(hist_field->system); 1921 kfree(hist_field->event_name); 1922 1923 kfree(hist_field); 1924 } 1925 destroy_hist_field(struct hist_field * hist_field,unsigned int level)1926 static void destroy_hist_field(struct hist_field *hist_field, 1927 unsigned int level) 1928 { 1929 unsigned int i; 1930 1931 if (level > 3) 1932 return; 1933 1934 if (!hist_field) 1935 return; 1936 1937 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) 1938 return; /* var refs will be destroyed separately */ 1939 1940 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) 1941 destroy_hist_field(hist_field->operands[i], level + 1); 1942 1943 __destroy_hist_field(hist_field); 1944 } 1945 create_hist_field(struct hist_trigger_data * hist_data,struct ftrace_event_field * field,unsigned long flags,char * var_name)1946 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, 1947 struct ftrace_event_field *field, 1948 unsigned long flags, 1949 char *var_name) 1950 { 1951 struct hist_field *hist_field; 1952 1953 if (field && is_function_field(field)) 1954 return NULL; 1955 1956 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL); 1957 if (!hist_field) 1958 return NULL; 1959 1960 hist_field->ref = 1; 1961 1962 hist_field->hist_data = hist_data; 1963 1964 if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS) 1965 goto out; /* caller will populate */ 1966 1967 if (flags & HIST_FIELD_FL_VAR_REF) { 1968 hist_field->fn_num = HIST_FIELD_FN_VAR_REF; 1969 goto out; 1970 } 1971 1972 if (flags & HIST_FIELD_FL_HITCOUNT) { 1973 hist_field->fn_num = HIST_FIELD_FN_COUNTER; 1974 hist_field->size = sizeof(u64); 1975 hist_field->type = "u64"; 1976 goto out; 1977 } 1978 1979 if (flags & HIST_FIELD_FL_CONST) { 1980 hist_field->fn_num = HIST_FIELD_FN_CONST; 1981 hist_field->size = sizeof(u64); 1982 hist_field->type = kstrdup("u64", GFP_KERNEL); 1983 if (!hist_field->type) 1984 goto free; 1985 goto out; 1986 } 1987 1988 if (flags & HIST_FIELD_FL_STACKTRACE) { 1989 if (field) 1990 hist_field->fn_num = HIST_FIELD_FN_STACK; 1991 else 1992 hist_field->fn_num = HIST_FIELD_FN_NOP; 1993 hist_field->size = HIST_STACKTRACE_SIZE; 1994 hist_field->type = kstrdup_const("unsigned long[]", GFP_KERNEL); 1995 if (!hist_field->type) 1996 goto free; 1997 goto out; 1998 } 1999 2000 if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) { 2001 unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET); 2002 hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 : 2003 HIST_FIELD_FN_BUCKET; 2004 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL); 2005 if (!hist_field->operands[0]) 2006 goto free; 2007 hist_field->size = hist_field->operands[0]->size; 2008 hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL); 2009 if (!hist_field->type) 2010 goto free; 2011 goto out; 2012 } 2013 2014 if (flags & HIST_FIELD_FL_TIMESTAMP) { 2015 hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP; 2016 hist_field->size = sizeof(u64); 2017 hist_field->type = "u64"; 2018 goto out; 2019 } 2020 2021 if (flags & HIST_FIELD_FL_CPU) { 2022 hist_field->fn_num = HIST_FIELD_FN_CPU; 2023 hist_field->size = sizeof(int); 2024 hist_field->type = "unsigned int"; 2025 goto out; 2026 } 2027 2028 if (WARN_ON_ONCE(!field)) 2029 goto out; 2030 2031 /* Pointers to strings are just pointers and dangerous to dereference */ 2032 if (is_string_field(field) && 2033 (field->filter_type != FILTER_PTR_STRING)) { 2034 flags |= HIST_FIELD_FL_STRING; 2035 2036 hist_field->size = MAX_FILTER_STR_VAL; 2037 hist_field->type = kstrdup_const(field->type, GFP_KERNEL); 2038 if (!hist_field->type) 2039 goto free; 2040 2041 if (field->filter_type == FILTER_STATIC_STRING) { 2042 hist_field->fn_num = HIST_FIELD_FN_STRING; 2043 hist_field->size = field->size; 2044 } else if (field->filter_type == FILTER_DYN_STRING) { 2045 hist_field->fn_num = HIST_FIELD_FN_DYNSTRING; 2046 } else if (field->filter_type == FILTER_RDYN_STRING) 2047 hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING; 2048 else 2049 hist_field->fn_num = HIST_FIELD_FN_PSTRING; 2050 } else { 2051 hist_field->size = field->size; 2052 hist_field->is_signed = field->is_signed; 2053 hist_field->type = kstrdup_const(field->type, GFP_KERNEL); 2054 if (!hist_field->type) 2055 goto free; 2056 2057 hist_field->fn_num = select_value_fn(field->size, 2058 field->is_signed); 2059 if (hist_field->fn_num == HIST_FIELD_FN_NOP) { 2060 destroy_hist_field(hist_field, 0); 2061 return NULL; 2062 } 2063 } 2064 out: 2065 hist_field->field = field; 2066 hist_field->flags = flags; 2067 2068 if (var_name) { 2069 hist_field->var.name = kstrdup(var_name, GFP_KERNEL); 2070 if (!hist_field->var.name) 2071 goto free; 2072 } 2073 2074 return hist_field; 2075 free: 2076 destroy_hist_field(hist_field, 0); 2077 return NULL; 2078 } 2079 destroy_hist_fields(struct hist_trigger_data * hist_data)2080 static void destroy_hist_fields(struct hist_trigger_data *hist_data) 2081 { 2082 unsigned int i; 2083 2084 for (i = 0; i < HIST_FIELDS_MAX; i++) { 2085 if (hist_data->fields[i]) { 2086 destroy_hist_field(hist_data->fields[i], 0); 2087 hist_data->fields[i] = NULL; 2088 } 2089 } 2090 2091 for (i = 0; i < hist_data->n_var_refs; i++) { 2092 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF)); 2093 __destroy_hist_field(hist_data->var_refs[i]); 2094 hist_data->var_refs[i] = NULL; 2095 } 2096 } 2097 init_var_ref(struct hist_field * ref_field,struct hist_field * var_field,char * system,char * event_name)2098 static int init_var_ref(struct hist_field *ref_field, 2099 struct hist_field *var_field, 2100 char *system, char *event_name) 2101 { 2102 int err = 0; 2103 2104 ref_field->var.idx = var_field->var.idx; 2105 ref_field->var.hist_data = var_field->hist_data; 2106 ref_field->size = var_field->size; 2107 ref_field->is_signed = var_field->is_signed; 2108 ref_field->flags |= var_field->flags & 2109 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2110 2111 if (system) { 2112 ref_field->system = kstrdup(system, GFP_KERNEL); 2113 if (!ref_field->system) 2114 return -ENOMEM; 2115 } 2116 2117 if (event_name) { 2118 ref_field->event_name = kstrdup(event_name, GFP_KERNEL); 2119 if (!ref_field->event_name) { 2120 err = -ENOMEM; 2121 goto free; 2122 } 2123 } 2124 2125 if (var_field->var.name) { 2126 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL); 2127 if (!ref_field->name) { 2128 err = -ENOMEM; 2129 goto free; 2130 } 2131 } else if (var_field->name) { 2132 ref_field->name = kstrdup(var_field->name, GFP_KERNEL); 2133 if (!ref_field->name) { 2134 err = -ENOMEM; 2135 goto free; 2136 } 2137 } 2138 2139 ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL); 2140 if (!ref_field->type) { 2141 err = -ENOMEM; 2142 goto free; 2143 } 2144 out: 2145 return err; 2146 free: 2147 kfree(ref_field->system); 2148 ref_field->system = NULL; 2149 kfree(ref_field->event_name); 2150 ref_field->event_name = NULL; 2151 kfree(ref_field->name); 2152 ref_field->name = NULL; 2153 2154 goto out; 2155 } 2156 find_var_ref_idx(struct hist_trigger_data * hist_data,struct hist_field * var_field)2157 static int find_var_ref_idx(struct hist_trigger_data *hist_data, 2158 struct hist_field *var_field) 2159 { 2160 struct hist_field *ref_field; 2161 int i; 2162 2163 for (i = 0; i < hist_data->n_var_refs; i++) { 2164 ref_field = hist_data->var_refs[i]; 2165 if (ref_field->var.idx == var_field->var.idx && 2166 ref_field->var.hist_data == var_field->hist_data) 2167 return i; 2168 } 2169 2170 return -ENOENT; 2171 } 2172 2173 /** 2174 * create_var_ref - Create a variable reference and attach it to trigger 2175 * @hist_data: The trigger that will be referencing the variable 2176 * @var_field: The VAR field to create a reference to 2177 * @system: The optional system string 2178 * @event_name: The optional event_name string 2179 * 2180 * Given a variable hist_field, create a VAR_REF hist_field that 2181 * represents a reference to it. 2182 * 2183 * This function also adds the reference to the trigger that 2184 * now references the variable. 2185 * 2186 * Return: The VAR_REF field if successful, NULL if not 2187 */ create_var_ref(struct hist_trigger_data * hist_data,struct hist_field * var_field,char * system,char * event_name)2188 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data, 2189 struct hist_field *var_field, 2190 char *system, char *event_name) 2191 { 2192 unsigned long flags = HIST_FIELD_FL_VAR_REF; 2193 struct hist_field *ref_field; 2194 int i; 2195 2196 /* Check if the variable already exists */ 2197 for (i = 0; i < hist_data->n_var_refs; i++) { 2198 ref_field = hist_data->var_refs[i]; 2199 if (ref_field->var.idx == var_field->var.idx && 2200 ref_field->var.hist_data == var_field->hist_data) { 2201 get_hist_field(ref_field); 2202 return ref_field; 2203 } 2204 } 2205 /* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */ 2206 if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX) 2207 return NULL; 2208 ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL); 2209 if (ref_field) { 2210 if (init_var_ref(ref_field, var_field, system, event_name)) { 2211 destroy_hist_field(ref_field, 0); 2212 return NULL; 2213 } 2214 2215 hist_data->var_refs[hist_data->n_var_refs] = ref_field; 2216 ref_field->var_ref_idx = hist_data->n_var_refs++; 2217 } 2218 2219 return ref_field; 2220 } 2221 is_var_ref(char * var_name)2222 static bool is_var_ref(char *var_name) 2223 { 2224 if (!var_name || strlen(var_name) < 2 || var_name[0] != '$') 2225 return false; 2226 2227 return true; 2228 } 2229 field_name_from_var(struct hist_trigger_data * hist_data,char * var_name)2230 static char *field_name_from_var(struct hist_trigger_data *hist_data, 2231 char *var_name) 2232 { 2233 char *name, *field; 2234 unsigned int i; 2235 2236 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 2237 name = hist_data->attrs->var_defs.name[i]; 2238 2239 if (strcmp(var_name, name) == 0) { 2240 field = hist_data->attrs->var_defs.expr[i]; 2241 if (contains_operator(field, NULL) || is_var_ref(field)) 2242 continue; 2243 return field; 2244 } 2245 } 2246 2247 return NULL; 2248 } 2249 local_field_var_ref(struct hist_trigger_data * hist_data,char * system,char * event_name,char * var_name)2250 static char *local_field_var_ref(struct hist_trigger_data *hist_data, 2251 char *system, char *event_name, 2252 char *var_name) 2253 { 2254 struct trace_event_call *call; 2255 2256 if (system && event_name) { 2257 call = hist_data->event_file->event_call; 2258 2259 if (strcmp(system, call->class->system) != 0) 2260 return NULL; 2261 2262 if (strcmp(event_name, trace_event_name(call)) != 0) 2263 return NULL; 2264 } 2265 2266 if (!!system != !!event_name) 2267 return NULL; 2268 2269 if (!is_var_ref(var_name)) 2270 return NULL; 2271 2272 var_name++; 2273 2274 return field_name_from_var(hist_data, var_name); 2275 } 2276 parse_var_ref(struct hist_trigger_data * hist_data,char * system,char * event_name,char * var_name)2277 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data, 2278 char *system, char *event_name, 2279 char *var_name) 2280 { 2281 struct hist_field *var_field = NULL, *ref_field = NULL; 2282 struct trace_array *tr = hist_data->event_file->tr; 2283 2284 if (!is_var_ref(var_name)) 2285 return NULL; 2286 2287 var_name++; 2288 2289 var_field = find_event_var(hist_data, system, event_name, var_name); 2290 if (var_field) 2291 ref_field = create_var_ref(hist_data, var_field, 2292 system, event_name); 2293 2294 if (!ref_field) 2295 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name)); 2296 2297 return ref_field; 2298 } 2299 2300 static struct ftrace_event_field * parse_field(struct hist_trigger_data * hist_data,struct trace_event_file * file,char * field_str,unsigned long * flags,unsigned long * buckets)2301 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, 2302 char *field_str, unsigned long *flags, unsigned long *buckets) 2303 { 2304 struct ftrace_event_field *field = NULL; 2305 char *field_name, *modifier, *str; 2306 struct trace_array *tr = file->tr; 2307 2308 modifier = str = kstrdup(field_str, GFP_KERNEL); 2309 if (!modifier) 2310 return ERR_PTR(-ENOMEM); 2311 2312 field_name = strsep(&modifier, "."); 2313 if (modifier) { 2314 if (strcmp(modifier, "hex") == 0) 2315 *flags |= HIST_FIELD_FL_HEX; 2316 else if (strcmp(modifier, "sym") == 0) 2317 *flags |= HIST_FIELD_FL_SYM; 2318 /* 2319 * 'sym-offset' occurrences in the trigger string are modified 2320 * to 'symXoffset' to simplify arithmetic expression parsing. 2321 */ 2322 else if (strcmp(modifier, "symXoffset") == 0) 2323 *flags |= HIST_FIELD_FL_SYM_OFFSET; 2324 else if ((strcmp(modifier, "execname") == 0) && 2325 (strcmp(field_name, "common_pid") == 0)) 2326 *flags |= HIST_FIELD_FL_EXECNAME; 2327 else if (strcmp(modifier, "syscall") == 0) 2328 *flags |= HIST_FIELD_FL_SYSCALL; 2329 else if (strcmp(modifier, "stacktrace") == 0) 2330 *flags |= HIST_FIELD_FL_STACKTRACE; 2331 else if (strcmp(modifier, "log2") == 0) 2332 *flags |= HIST_FIELD_FL_LOG2; 2333 else if (strcmp(modifier, "usecs") == 0) 2334 *flags |= HIST_FIELD_FL_TIMESTAMP_USECS; 2335 else if (strncmp(modifier, "bucket", 6) == 0) { 2336 int ret; 2337 2338 modifier += 6; 2339 2340 if (*modifier == 's') 2341 modifier++; 2342 if (*modifier != '=') 2343 goto error; 2344 modifier++; 2345 ret = kstrtoul(modifier, 0, buckets); 2346 if (ret || !(*buckets)) 2347 goto error; 2348 *flags |= HIST_FIELD_FL_BUCKET; 2349 } else if (strncmp(modifier, "percent", 7) == 0) { 2350 if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY)) 2351 goto error; 2352 *flags |= HIST_FIELD_FL_PERCENT; 2353 } else if (strncmp(modifier, "graph", 5) == 0) { 2354 if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY)) 2355 goto error; 2356 *flags |= HIST_FIELD_FL_GRAPH; 2357 } else { 2358 error: 2359 hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier)); 2360 field = ERR_PTR(-EINVAL); 2361 goto out; 2362 } 2363 } 2364 2365 if (strcmp(field_name, "common_timestamp") == 0) { 2366 *flags |= HIST_FIELD_FL_TIMESTAMP; 2367 hist_data->enable_timestamps = true; 2368 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS) 2369 hist_data->attrs->ts_in_usecs = true; 2370 } else if (strcmp(field_name, "common_stacktrace") == 0) { 2371 *flags |= HIST_FIELD_FL_STACKTRACE; 2372 } else if (strcmp(field_name, "common_cpu") == 0) 2373 *flags |= HIST_FIELD_FL_CPU; 2374 else if (strcmp(field_name, "hitcount") == 0) 2375 *flags |= HIST_FIELD_FL_HITCOUNT; 2376 else { 2377 field = trace_find_event_field(file->event_call, field_name); 2378 if (!field || !field->size) { 2379 /* 2380 * For backward compatibility, if field_name 2381 * was "cpu" or "stacktrace", then we treat this 2382 * the same as common_cpu and common_stacktrace 2383 * respectively. This also works for "CPU", and 2384 * "STACKTRACE". 2385 */ 2386 if (field && field->filter_type == FILTER_CPU) { 2387 *flags |= HIST_FIELD_FL_CPU; 2388 } else if (field && field->filter_type == FILTER_STACKTRACE) { 2389 *flags |= HIST_FIELD_FL_STACKTRACE; 2390 } else { 2391 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, 2392 errpos(field_name)); 2393 field = ERR_PTR(-EINVAL); 2394 goto out; 2395 } 2396 } 2397 } 2398 out: 2399 kfree(str); 2400 2401 return field; 2402 } 2403 create_alias(struct hist_trigger_data * hist_data,struct hist_field * var_ref,char * var_name)2404 static struct hist_field *create_alias(struct hist_trigger_data *hist_data, 2405 struct hist_field *var_ref, 2406 char *var_name) 2407 { 2408 struct hist_field *alias = NULL; 2409 unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR; 2410 2411 alias = create_hist_field(hist_data, NULL, flags, var_name); 2412 if (!alias) 2413 return NULL; 2414 2415 alias->fn_num = var_ref->fn_num; 2416 alias->operands[0] = var_ref; 2417 2418 if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) { 2419 destroy_hist_field(alias, 0); 2420 return NULL; 2421 } 2422 2423 alias->var_ref_idx = var_ref->var_ref_idx; 2424 2425 return alias; 2426 } 2427 parse_const(struct hist_trigger_data * hist_data,char * str,char * var_name,unsigned long * flags)2428 static struct hist_field *parse_const(struct hist_trigger_data *hist_data, 2429 char *str, char *var_name, 2430 unsigned long *flags) 2431 { 2432 struct trace_array *tr = hist_data->event_file->tr; 2433 struct hist_field *field = NULL; 2434 u64 constant; 2435 2436 if (kstrtoull(str, 0, &constant)) { 2437 hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str)); 2438 return NULL; 2439 } 2440 2441 *flags |= HIST_FIELD_FL_CONST; 2442 field = create_hist_field(hist_data, NULL, *flags, var_name); 2443 if (!field) 2444 return NULL; 2445 2446 field->constant = constant; 2447 2448 return field; 2449 } 2450 parse_atom(struct hist_trigger_data * hist_data,struct trace_event_file * file,char * str,unsigned long * flags,char * var_name)2451 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data, 2452 struct trace_event_file *file, char *str, 2453 unsigned long *flags, char *var_name) 2454 { 2455 char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str; 2456 struct ftrace_event_field *field = NULL; 2457 struct hist_field *hist_field = NULL; 2458 unsigned long buckets = 0; 2459 int ret = 0; 2460 2461 if (isdigit(str[0])) { 2462 hist_field = parse_const(hist_data, str, var_name, flags); 2463 if (!hist_field) { 2464 ret = -EINVAL; 2465 goto out; 2466 } 2467 return hist_field; 2468 } 2469 2470 s = strchr(str, '.'); 2471 if (s) { 2472 s = strchr(++s, '.'); 2473 if (s) { 2474 ref_system = strsep(&str, "."); 2475 if (!str) { 2476 ret = -EINVAL; 2477 goto out; 2478 } 2479 ref_event = strsep(&str, "."); 2480 if (!str) { 2481 ret = -EINVAL; 2482 goto out; 2483 } 2484 ref_var = str; 2485 } 2486 } 2487 2488 s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var); 2489 if (!s) { 2490 hist_field = parse_var_ref(hist_data, ref_system, 2491 ref_event, ref_var); 2492 if (hist_field) { 2493 if (var_name) { 2494 hist_field = create_alias(hist_data, hist_field, var_name); 2495 if (!hist_field) { 2496 ret = -ENOMEM; 2497 goto out; 2498 } 2499 } 2500 return hist_field; 2501 } 2502 } else 2503 str = s; 2504 2505 field = parse_field(hist_data, file, str, flags, &buckets); 2506 if (IS_ERR(field)) { 2507 ret = PTR_ERR(field); 2508 goto out; 2509 } 2510 2511 hist_field = create_hist_field(hist_data, field, *flags, var_name); 2512 if (!hist_field) { 2513 ret = -ENOMEM; 2514 goto out; 2515 } 2516 hist_field->buckets = buckets; 2517 2518 return hist_field; 2519 out: 2520 return ERR_PTR(ret); 2521 } 2522 2523 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2524 struct trace_event_file *file, 2525 char *str, unsigned long flags, 2526 char *var_name, unsigned int *n_subexprs); 2527 parse_unary(struct hist_trigger_data * hist_data,struct trace_event_file * file,char * str,unsigned long flags,char * var_name,unsigned int * n_subexprs)2528 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data, 2529 struct trace_event_file *file, 2530 char *str, unsigned long flags, 2531 char *var_name, unsigned int *n_subexprs) 2532 { 2533 struct hist_field *operand1, *expr = NULL; 2534 unsigned long operand_flags; 2535 int ret = 0; 2536 char *s; 2537 2538 /* Unary minus operator, increment n_subexprs */ 2539 ++*n_subexprs; 2540 2541 /* we support only -(xxx) i.e. explicit parens required */ 2542 2543 if (*n_subexprs > 3) { 2544 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str)); 2545 ret = -EINVAL; 2546 goto free; 2547 } 2548 2549 str++; /* skip leading '-' */ 2550 2551 s = strchr(str, '('); 2552 if (s) 2553 str++; 2554 else { 2555 ret = -EINVAL; 2556 goto free; 2557 } 2558 2559 s = strrchr(str, ')'); 2560 if (s) { 2561 /* unary minus not supported in sub-expressions */ 2562 if (*(s+1) != '\0') { 2563 hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR, 2564 errpos(str)); 2565 ret = -EINVAL; 2566 goto free; 2567 } 2568 *s = '\0'; 2569 } 2570 else { 2571 ret = -EINVAL; /* no closing ')' */ 2572 goto free; 2573 } 2574 2575 flags |= HIST_FIELD_FL_EXPR; 2576 expr = create_hist_field(hist_data, NULL, flags, var_name); 2577 if (!expr) { 2578 ret = -ENOMEM; 2579 goto free; 2580 } 2581 2582 operand_flags = 0; 2583 operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs); 2584 if (IS_ERR(operand1)) { 2585 ret = PTR_ERR(operand1); 2586 goto free; 2587 } 2588 if (operand1->flags & HIST_FIELD_FL_STRING) { 2589 /* String type can not be the operand of unary operator. */ 2590 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str)); 2591 destroy_hist_field(operand1, 0); 2592 ret = -EINVAL; 2593 goto free; 2594 } 2595 2596 expr->flags |= operand1->flags & 2597 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2598 expr->fn_num = HIST_FIELD_FN_UMINUS; 2599 expr->operands[0] = operand1; 2600 expr->size = operand1->size; 2601 expr->is_signed = operand1->is_signed; 2602 expr->operator = FIELD_OP_UNARY_MINUS; 2603 expr->name = expr_str(expr, 0); 2604 expr->type = kstrdup_const(operand1->type, GFP_KERNEL); 2605 if (!expr->type) { 2606 ret = -ENOMEM; 2607 goto free; 2608 } 2609 2610 return expr; 2611 free: 2612 destroy_hist_field(expr, 0); 2613 return ERR_PTR(ret); 2614 } 2615 2616 /* 2617 * If the operands are var refs, return pointers the 2618 * variable(s) referenced in var1 and var2, else NULL. 2619 */ check_expr_operands(struct trace_array * tr,struct hist_field * operand1,struct hist_field * operand2,struct hist_field ** var1,struct hist_field ** var2)2620 static int check_expr_operands(struct trace_array *tr, 2621 struct hist_field *operand1, 2622 struct hist_field *operand2, 2623 struct hist_field **var1, 2624 struct hist_field **var2) 2625 { 2626 unsigned long operand1_flags = operand1->flags; 2627 unsigned long operand2_flags = operand2->flags; 2628 2629 if ((operand1_flags & HIST_FIELD_FL_VAR_REF) || 2630 (operand1_flags & HIST_FIELD_FL_ALIAS)) { 2631 struct hist_field *var; 2632 2633 var = find_var_field(operand1->var.hist_data, operand1->name); 2634 if (!var) 2635 return -EINVAL; 2636 operand1_flags = var->flags; 2637 *var1 = var; 2638 } 2639 2640 if ((operand2_flags & HIST_FIELD_FL_VAR_REF) || 2641 (operand2_flags & HIST_FIELD_FL_ALIAS)) { 2642 struct hist_field *var; 2643 2644 var = find_var_field(operand2->var.hist_data, operand2->name); 2645 if (!var) 2646 return -EINVAL; 2647 operand2_flags = var->flags; 2648 *var2 = var; 2649 } 2650 2651 if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) != 2652 (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) { 2653 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0); 2654 return -EINVAL; 2655 } 2656 2657 return 0; 2658 } 2659 parse_expr(struct hist_trigger_data * hist_data,struct trace_event_file * file,char * str,unsigned long flags,char * var_name,unsigned int * n_subexprs)2660 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2661 struct trace_event_file *file, 2662 char *str, unsigned long flags, 2663 char *var_name, unsigned int *n_subexprs) 2664 { 2665 struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL; 2666 struct hist_field *var1 = NULL, *var2 = NULL; 2667 unsigned long operand_flags, operand2_flags; 2668 int field_op, ret = -EINVAL; 2669 char *sep, *operand1_str; 2670 enum hist_field_fn op_fn; 2671 bool combine_consts; 2672 2673 if (*n_subexprs > 3) { 2674 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str)); 2675 return ERR_PTR(-EINVAL); 2676 } 2677 2678 field_op = contains_operator(str, &sep); 2679 2680 if (field_op == FIELD_OP_NONE) 2681 return parse_atom(hist_data, file, str, &flags, var_name); 2682 2683 if (field_op == FIELD_OP_UNARY_MINUS) 2684 return parse_unary(hist_data, file, str, flags, var_name, n_subexprs); 2685 2686 /* Binary operator found, increment n_subexprs */ 2687 ++*n_subexprs; 2688 2689 /* Split the expression string at the root operator */ 2690 if (!sep) 2691 return ERR_PTR(-EINVAL); 2692 2693 *sep = '\0'; 2694 operand1_str = str; 2695 str = sep+1; 2696 2697 /* Binary operator requires both operands */ 2698 if (*operand1_str == '\0' || *str == '\0') 2699 return ERR_PTR(-EINVAL); 2700 2701 operand_flags = 0; 2702 2703 /* LHS of string is an expression e.g. a+b in a+b+c */ 2704 operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs); 2705 if (IS_ERR(operand1)) 2706 return ERR_CAST(operand1); 2707 2708 if (operand1->flags & HIST_FIELD_FL_STRING) { 2709 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str)); 2710 ret = -EINVAL; 2711 goto free_op1; 2712 } 2713 2714 /* RHS of string is another expression e.g. c in a+b+c */ 2715 operand_flags = 0; 2716 operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs); 2717 if (IS_ERR(operand2)) { 2718 ret = PTR_ERR(operand2); 2719 goto free_op1; 2720 } 2721 if (operand2->flags & HIST_FIELD_FL_STRING) { 2722 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str)); 2723 ret = -EINVAL; 2724 goto free_operands; 2725 } 2726 2727 switch (field_op) { 2728 case FIELD_OP_MINUS: 2729 op_fn = HIST_FIELD_FN_MINUS; 2730 break; 2731 case FIELD_OP_PLUS: 2732 op_fn = HIST_FIELD_FN_PLUS; 2733 break; 2734 case FIELD_OP_DIV: 2735 op_fn = HIST_FIELD_FN_DIV; 2736 break; 2737 case FIELD_OP_MULT: 2738 op_fn = HIST_FIELD_FN_MULT; 2739 break; 2740 default: 2741 ret = -EINVAL; 2742 goto free_operands; 2743 } 2744 2745 ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2); 2746 if (ret) 2747 goto free_operands; 2748 2749 operand_flags = var1 ? var1->flags : operand1->flags; 2750 operand2_flags = var2 ? var2->flags : operand2->flags; 2751 2752 /* 2753 * If both operands are constant, the expression can be 2754 * collapsed to a single constant. 2755 */ 2756 combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST; 2757 2758 flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR; 2759 2760 flags |= operand1->flags & 2761 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2762 2763 expr = create_hist_field(hist_data, NULL, flags, var_name); 2764 if (!expr) { 2765 ret = -ENOMEM; 2766 goto free_operands; 2767 } 2768 2769 operand1->read_once = true; 2770 operand2->read_once = true; 2771 2772 /* The operands are now owned and free'd by 'expr' */ 2773 expr->operands[0] = operand1; 2774 expr->operands[1] = operand2; 2775 2776 if (field_op == FIELD_OP_DIV && 2777 operand2_flags & HIST_FIELD_FL_CONST) { 2778 u64 divisor = var2 ? var2->constant : operand2->constant; 2779 2780 if (!divisor) { 2781 hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str)); 2782 ret = -EDOM; 2783 goto free_expr; 2784 } 2785 2786 /* 2787 * Copy the divisor here so we don't have to look it up 2788 * later if this is a var ref 2789 */ 2790 operand2->constant = divisor; 2791 op_fn = hist_field_get_div_fn(operand2); 2792 } 2793 2794 expr->fn_num = op_fn; 2795 2796 if (combine_consts) { 2797 if (var1) 2798 expr->operands[0] = var1; 2799 if (var2) 2800 expr->operands[1] = var2; 2801 2802 expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL); 2803 expr->fn_num = HIST_FIELD_FN_CONST; 2804 2805 expr->operands[0] = NULL; 2806 expr->operands[1] = NULL; 2807 2808 /* 2809 * var refs won't be destroyed immediately 2810 * See: destroy_hist_field() 2811 */ 2812 destroy_hist_field(operand2, 0); 2813 destroy_hist_field(operand1, 0); 2814 2815 expr->name = expr_str(expr, 0); 2816 } else { 2817 /* The operand sizes should be the same, so just pick one */ 2818 expr->size = operand1->size; 2819 expr->is_signed = operand1->is_signed; 2820 2821 expr->operator = field_op; 2822 expr->type = kstrdup_const(operand1->type, GFP_KERNEL); 2823 if (!expr->type) { 2824 ret = -ENOMEM; 2825 goto free_expr; 2826 } 2827 2828 expr->name = expr_str(expr, 0); 2829 } 2830 2831 return expr; 2832 2833 free_operands: 2834 destroy_hist_field(operand2, 0); 2835 free_op1: 2836 destroy_hist_field(operand1, 0); 2837 return ERR_PTR(ret); 2838 2839 free_expr: 2840 destroy_hist_field(expr, 0); 2841 return ERR_PTR(ret); 2842 } 2843 find_trigger_filter(struct hist_trigger_data * hist_data,struct trace_event_file * file)2844 static char *find_trigger_filter(struct hist_trigger_data *hist_data, 2845 struct trace_event_file *file) 2846 { 2847 struct event_trigger_data *test; 2848 2849 lockdep_assert_held(&event_mutex); 2850 2851 list_for_each_entry(test, &file->triggers, list) { 2852 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2853 if (test->private_data == hist_data) 2854 return test->filter_str; 2855 } 2856 } 2857 2858 return NULL; 2859 } 2860 2861 static struct event_command trigger_hist_cmd; 2862 static int event_hist_trigger_parse(struct event_command *cmd_ops, 2863 struct trace_event_file *file, 2864 char *glob, char *cmd, 2865 char *param_and_filter); 2866 compatible_keys(struct hist_trigger_data * target_hist_data,struct hist_trigger_data * hist_data,unsigned int n_keys)2867 static bool compatible_keys(struct hist_trigger_data *target_hist_data, 2868 struct hist_trigger_data *hist_data, 2869 unsigned int n_keys) 2870 { 2871 struct hist_field *target_hist_field, *hist_field; 2872 unsigned int n, i, j; 2873 2874 if (hist_data->n_fields - hist_data->n_vals != n_keys) 2875 return false; 2876 2877 i = hist_data->n_vals; 2878 j = target_hist_data->n_vals; 2879 2880 for (n = 0; n < n_keys; n++) { 2881 hist_field = hist_data->fields[i + n]; 2882 target_hist_field = target_hist_data->fields[j + n]; 2883 2884 if (strcmp(hist_field->type, target_hist_field->type) != 0) 2885 return false; 2886 if (hist_field->size != target_hist_field->size) 2887 return false; 2888 if (hist_field->is_signed != target_hist_field->is_signed) 2889 return false; 2890 } 2891 2892 return true; 2893 } 2894 2895 static struct hist_trigger_data * find_compatible_hist(struct hist_trigger_data * target_hist_data,struct trace_event_file * file)2896 find_compatible_hist(struct hist_trigger_data *target_hist_data, 2897 struct trace_event_file *file) 2898 { 2899 struct hist_trigger_data *hist_data; 2900 struct event_trigger_data *test; 2901 unsigned int n_keys; 2902 2903 lockdep_assert_held(&event_mutex); 2904 2905 n_keys = target_hist_data->n_fields - target_hist_data->n_vals; 2906 2907 list_for_each_entry(test, &file->triggers, list) { 2908 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2909 hist_data = test->private_data; 2910 2911 if (compatible_keys(target_hist_data, hist_data, n_keys)) 2912 return hist_data; 2913 } 2914 } 2915 2916 return NULL; 2917 } 2918 event_file(struct trace_array * tr,char * system,char * event_name)2919 static struct trace_event_file *event_file(struct trace_array *tr, 2920 char *system, char *event_name) 2921 { 2922 struct trace_event_file *file; 2923 2924 file = __find_event_file(tr, system, event_name); 2925 if (!file) 2926 return ERR_PTR(-EINVAL); 2927 2928 return file; 2929 } 2930 2931 static struct hist_field * find_synthetic_field_var(struct hist_trigger_data * target_hist_data,char * system,char * event_name,char * field_name)2932 find_synthetic_field_var(struct hist_trigger_data *target_hist_data, 2933 char *system, char *event_name, char *field_name) 2934 { 2935 struct hist_field *event_var; 2936 char *synthetic_name; 2937 2938 synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2939 if (!synthetic_name) 2940 return ERR_PTR(-ENOMEM); 2941 2942 strcpy(synthetic_name, "synthetic_"); 2943 strcat(synthetic_name, field_name); 2944 2945 event_var = find_event_var(target_hist_data, system, event_name, synthetic_name); 2946 2947 kfree(synthetic_name); 2948 2949 return event_var; 2950 } 2951 2952 /** 2953 * create_field_var_hist - Automatically create a histogram and var for a field 2954 * @target_hist_data: The target hist trigger 2955 * @subsys_name: Optional subsystem name 2956 * @event_name: Optional event name 2957 * @field_name: The name of the field (and the resulting variable) 2958 * 2959 * Hist trigger actions fetch data from variables, not directly from 2960 * events. However, for convenience, users are allowed to directly 2961 * specify an event field in an action, which will be automatically 2962 * converted into a variable on their behalf. 2963 * 2964 * If a user specifies a field on an event that isn't the event the 2965 * histogram currently being defined (the target event histogram), the 2966 * only way that can be accomplished is if a new hist trigger is 2967 * created and the field variable defined on that. 2968 * 2969 * This function creates a new histogram compatible with the target 2970 * event (meaning a histogram with the same key as the target 2971 * histogram), and creates a variable for the specified field, but 2972 * with 'synthetic_' prepended to the variable name in order to avoid 2973 * collision with normal field variables. 2974 * 2975 * Return: The variable created for the field. 2976 */ 2977 static struct hist_field * create_field_var_hist(struct hist_trigger_data * target_hist_data,char * subsys_name,char * event_name,char * field_name)2978 create_field_var_hist(struct hist_trigger_data *target_hist_data, 2979 char *subsys_name, char *event_name, char *field_name) 2980 { 2981 struct trace_array *tr = target_hist_data->event_file->tr; 2982 struct hist_trigger_data *hist_data; 2983 unsigned int i, n, first = true; 2984 struct field_var_hist *var_hist; 2985 struct trace_event_file *file; 2986 struct hist_field *key_field; 2987 struct hist_field *event_var; 2988 char *saved_filter; 2989 char *cmd; 2990 int ret; 2991 2992 if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) { 2993 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name)); 2994 return ERR_PTR(-EINVAL); 2995 } 2996 2997 file = event_file(tr, subsys_name, event_name); 2998 2999 if (IS_ERR(file)) { 3000 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name)); 3001 ret = PTR_ERR(file); 3002 return ERR_PTR(ret); 3003 } 3004 3005 /* 3006 * Look for a histogram compatible with target. We'll use the 3007 * found histogram specification to create a new matching 3008 * histogram with our variable on it. target_hist_data is not 3009 * yet a registered histogram so we can't use that. 3010 */ 3011 hist_data = find_compatible_hist(target_hist_data, file); 3012 if (!hist_data) { 3013 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name)); 3014 return ERR_PTR(-EINVAL); 3015 } 3016 3017 /* See if a synthetic field variable has already been created */ 3018 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 3019 event_name, field_name); 3020 if (!IS_ERR_OR_NULL(event_var)) 3021 return event_var; 3022 3023 var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL); 3024 if (!var_hist) 3025 return ERR_PTR(-ENOMEM); 3026 3027 cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 3028 if (!cmd) { 3029 kfree(var_hist); 3030 return ERR_PTR(-ENOMEM); 3031 } 3032 3033 /* Use the same keys as the compatible histogram */ 3034 strcat(cmd, "keys="); 3035 3036 for_each_hist_key_field(i, hist_data) { 3037 key_field = hist_data->fields[i]; 3038 if (!first) 3039 strcat(cmd, ","); 3040 strcat(cmd, key_field->field->name); 3041 first = false; 3042 } 3043 3044 /* Create the synthetic field variable specification */ 3045 strcat(cmd, ":synthetic_"); 3046 strcat(cmd, field_name); 3047 strcat(cmd, "="); 3048 strcat(cmd, field_name); 3049 3050 /* Use the same filter as the compatible histogram */ 3051 saved_filter = find_trigger_filter(hist_data, file); 3052 if (saved_filter) { 3053 strcat(cmd, " if "); 3054 strcat(cmd, saved_filter); 3055 } 3056 3057 var_hist->cmd = kstrdup(cmd, GFP_KERNEL); 3058 if (!var_hist->cmd) { 3059 kfree(cmd); 3060 kfree(var_hist); 3061 return ERR_PTR(-ENOMEM); 3062 } 3063 3064 /* Save the compatible histogram information */ 3065 var_hist->hist_data = hist_data; 3066 3067 /* Create the new histogram with our variable */ 3068 ret = event_hist_trigger_parse(&trigger_hist_cmd, file, 3069 "", "hist", cmd); 3070 if (ret) { 3071 kfree(cmd); 3072 kfree(var_hist->cmd); 3073 kfree(var_hist); 3074 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name)); 3075 return ERR_PTR(ret); 3076 } 3077 3078 kfree(cmd); 3079 3080 /* If we can't find the variable, something went wrong */ 3081 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 3082 event_name, field_name); 3083 if (IS_ERR_OR_NULL(event_var)) { 3084 kfree(var_hist->cmd); 3085 kfree(var_hist); 3086 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name)); 3087 return ERR_PTR(-EINVAL); 3088 } 3089 3090 n = target_hist_data->n_field_var_hists; 3091 target_hist_data->field_var_hists[n] = var_hist; 3092 target_hist_data->n_field_var_hists++; 3093 3094 return event_var; 3095 } 3096 3097 static struct hist_field * find_target_event_var(struct hist_trigger_data * hist_data,char * subsys_name,char * event_name,char * var_name)3098 find_target_event_var(struct hist_trigger_data *hist_data, 3099 char *subsys_name, char *event_name, char *var_name) 3100 { 3101 struct trace_event_file *file = hist_data->event_file; 3102 struct hist_field *hist_field = NULL; 3103 3104 if (subsys_name) { 3105 struct trace_event_call *call; 3106 3107 if (!event_name) 3108 return NULL; 3109 3110 call = file->event_call; 3111 3112 if (strcmp(subsys_name, call->class->system) != 0) 3113 return NULL; 3114 3115 if (strcmp(event_name, trace_event_name(call)) != 0) 3116 return NULL; 3117 } 3118 3119 hist_field = find_var_field(hist_data, var_name); 3120 3121 return hist_field; 3122 } 3123 __update_field_vars(struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * rec,struct field_var ** field_vars,unsigned int n_field_vars,unsigned int field_var_str_start)3124 static inline void __update_field_vars(struct tracing_map_elt *elt, 3125 struct trace_buffer *buffer, 3126 struct ring_buffer_event *rbe, 3127 void *rec, 3128 struct field_var **field_vars, 3129 unsigned int n_field_vars, 3130 unsigned int field_var_str_start) 3131 { 3132 struct hist_elt_data *elt_data = elt->private_data; 3133 unsigned int i, j, var_idx; 3134 u64 var_val; 3135 3136 /* Make sure stacktrace can fit in the string variable length */ 3137 BUILD_BUG_ON((HIST_STACKTRACE_DEPTH + 1) * sizeof(long) >= STR_VAR_LEN_MAX); 3138 3139 for (i = 0, j = field_var_str_start; i < n_field_vars; i++) { 3140 struct field_var *field_var = field_vars[i]; 3141 struct hist_field *var = field_var->var; 3142 struct hist_field *val = field_var->val; 3143 3144 var_val = hist_fn_call(val, elt, buffer, rbe, rec); 3145 var_idx = var->var.idx; 3146 3147 if (val->flags & (HIST_FIELD_FL_STRING | 3148 HIST_FIELD_FL_STACKTRACE)) { 3149 char *str = elt_data->field_var_str[j++]; 3150 char *val_str = (char *)(uintptr_t)var_val; 3151 unsigned int size; 3152 3153 if (val->flags & HIST_FIELD_FL_STRING) { 3154 size = min(val->size, STR_VAR_LEN_MAX); 3155 strscpy(str, val_str, size); 3156 } else { 3157 char *stack_start = str + sizeof(unsigned long); 3158 int e; 3159 3160 e = stack_trace_save((void *)stack_start, 3161 HIST_STACKTRACE_DEPTH, 3162 HIST_STACKTRACE_SKIP); 3163 if (e < HIST_STACKTRACE_DEPTH - 1) 3164 ((unsigned long *)stack_start)[e] = 0; 3165 *((unsigned long *)str) = e; 3166 } 3167 var_val = (u64)(uintptr_t)str; 3168 } 3169 tracing_map_set_var(elt, var_idx, var_val); 3170 } 3171 } 3172 update_field_vars(struct hist_trigger_data * hist_data,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * rec)3173 static void update_field_vars(struct hist_trigger_data *hist_data, 3174 struct tracing_map_elt *elt, 3175 struct trace_buffer *buffer, 3176 struct ring_buffer_event *rbe, 3177 void *rec) 3178 { 3179 __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars, 3180 hist_data->n_field_vars, 0); 3181 } 3182 save_track_data_vars(struct hist_trigger_data * hist_data,struct tracing_map_elt * elt,struct trace_buffer * buffer,void * rec,struct ring_buffer_event * rbe,void * key,struct action_data * data,u64 * var_ref_vals)3183 static void save_track_data_vars(struct hist_trigger_data *hist_data, 3184 struct tracing_map_elt *elt, 3185 struct trace_buffer *buffer, void *rec, 3186 struct ring_buffer_event *rbe, void *key, 3187 struct action_data *data, u64 *var_ref_vals) 3188 { 3189 __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars, 3190 hist_data->n_save_vars, hist_data->n_field_var_str); 3191 } 3192 create_var(struct hist_trigger_data * hist_data,struct trace_event_file * file,char * name,int size,const char * type)3193 static struct hist_field *create_var(struct hist_trigger_data *hist_data, 3194 struct trace_event_file *file, 3195 char *name, int size, const char *type) 3196 { 3197 struct hist_field *var; 3198 int idx; 3199 3200 if (find_var(hist_data, file, name) && !hist_data->remove) { 3201 var = ERR_PTR(-EINVAL); 3202 goto out; 3203 } 3204 3205 var = kzalloc(sizeof(struct hist_field), GFP_KERNEL); 3206 if (!var) { 3207 var = ERR_PTR(-ENOMEM); 3208 goto out; 3209 } 3210 3211 idx = tracing_map_add_var(hist_data->map); 3212 if (idx < 0) { 3213 kfree(var); 3214 var = ERR_PTR(-EINVAL); 3215 goto out; 3216 } 3217 3218 var->ref = 1; 3219 var->flags = HIST_FIELD_FL_VAR; 3220 var->var.idx = idx; 3221 var->var.hist_data = var->hist_data = hist_data; 3222 var->size = size; 3223 var->var.name = kstrdup(name, GFP_KERNEL); 3224 var->type = kstrdup_const(type, GFP_KERNEL); 3225 if (!var->var.name || !var->type) { 3226 kfree_const(var->type); 3227 kfree(var->var.name); 3228 kfree(var); 3229 var = ERR_PTR(-ENOMEM); 3230 } 3231 out: 3232 return var; 3233 } 3234 create_field_var(struct hist_trigger_data * hist_data,struct trace_event_file * file,char * field_name)3235 static struct field_var *create_field_var(struct hist_trigger_data *hist_data, 3236 struct trace_event_file *file, 3237 char *field_name) 3238 { 3239 struct hist_field *val = NULL, *var = NULL; 3240 unsigned long flags = HIST_FIELD_FL_VAR; 3241 struct trace_array *tr = file->tr; 3242 struct field_var *field_var; 3243 int ret = 0; 3244 3245 if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) { 3246 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name)); 3247 ret = -EINVAL; 3248 goto err; 3249 } 3250 3251 val = parse_atom(hist_data, file, field_name, &flags, NULL); 3252 if (IS_ERR(val)) { 3253 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name)); 3254 ret = PTR_ERR(val); 3255 goto err; 3256 } 3257 3258 var = create_var(hist_data, file, field_name, val->size, val->type); 3259 if (IS_ERR(var)) { 3260 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name)); 3261 kfree(val); 3262 ret = PTR_ERR(var); 3263 goto err; 3264 } 3265 3266 field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL); 3267 if (!field_var) { 3268 kfree(val); 3269 kfree(var); 3270 ret = -ENOMEM; 3271 goto err; 3272 } 3273 3274 field_var->var = var; 3275 field_var->val = val; 3276 out: 3277 return field_var; 3278 err: 3279 field_var = ERR_PTR(ret); 3280 goto out; 3281 } 3282 3283 /** 3284 * create_target_field_var - Automatically create a variable for a field 3285 * @target_hist_data: The target hist trigger 3286 * @subsys_name: Optional subsystem name 3287 * @event_name: Optional event name 3288 * @var_name: The name of the field (and the resulting variable) 3289 * 3290 * Hist trigger actions fetch data from variables, not directly from 3291 * events. However, for convenience, users are allowed to directly 3292 * specify an event field in an action, which will be automatically 3293 * converted into a variable on their behalf. 3294 * 3295 * This function creates a field variable with the name var_name on 3296 * the hist trigger currently being defined on the target event. If 3297 * subsys_name and event_name are specified, this function simply 3298 * verifies that they do in fact match the target event subsystem and 3299 * event name. 3300 * 3301 * Return: The variable created for the field. 3302 */ 3303 static struct field_var * create_target_field_var(struct hist_trigger_data * target_hist_data,char * subsys_name,char * event_name,char * var_name)3304 create_target_field_var(struct hist_trigger_data *target_hist_data, 3305 char *subsys_name, char *event_name, char *var_name) 3306 { 3307 struct trace_event_file *file = target_hist_data->event_file; 3308 3309 if (subsys_name) { 3310 struct trace_event_call *call; 3311 3312 if (!event_name) 3313 return NULL; 3314 3315 call = file->event_call; 3316 3317 if (strcmp(subsys_name, call->class->system) != 0) 3318 return NULL; 3319 3320 if (strcmp(event_name, trace_event_name(call)) != 0) 3321 return NULL; 3322 } 3323 3324 return create_field_var(target_hist_data, file, var_name); 3325 } 3326 check_track_val_max(u64 track_val,u64 var_val)3327 static bool check_track_val_max(u64 track_val, u64 var_val) 3328 { 3329 if (var_val <= track_val) 3330 return false; 3331 3332 return true; 3333 } 3334 check_track_val_changed(u64 track_val,u64 var_val)3335 static bool check_track_val_changed(u64 track_val, u64 var_val) 3336 { 3337 if (var_val == track_val) 3338 return false; 3339 3340 return true; 3341 } 3342 get_track_val(struct hist_trigger_data * hist_data,struct tracing_map_elt * elt,struct action_data * data)3343 static u64 get_track_val(struct hist_trigger_data *hist_data, 3344 struct tracing_map_elt *elt, 3345 struct action_data *data) 3346 { 3347 unsigned int track_var_idx = data->track_data.track_var->var.idx; 3348 u64 track_val; 3349 3350 track_val = tracing_map_read_var(elt, track_var_idx); 3351 3352 return track_val; 3353 } 3354 save_track_val(struct hist_trigger_data * hist_data,struct tracing_map_elt * elt,struct action_data * data,u64 var_val)3355 static void save_track_val(struct hist_trigger_data *hist_data, 3356 struct tracing_map_elt *elt, 3357 struct action_data *data, u64 var_val) 3358 { 3359 unsigned int track_var_idx = data->track_data.track_var->var.idx; 3360 3361 tracing_map_set_var(elt, track_var_idx, var_val); 3362 } 3363 save_track_data(struct hist_trigger_data * hist_data,struct tracing_map_elt * elt,struct trace_buffer * buffer,void * rec,struct ring_buffer_event * rbe,void * key,struct action_data * data,u64 * var_ref_vals)3364 static void save_track_data(struct hist_trigger_data *hist_data, 3365 struct tracing_map_elt *elt, 3366 struct trace_buffer *buffer, void *rec, 3367 struct ring_buffer_event *rbe, void *key, 3368 struct action_data *data, u64 *var_ref_vals) 3369 { 3370 if (data->track_data.save_data) 3371 data->track_data.save_data(hist_data, elt, buffer, rec, rbe, 3372 key, data, var_ref_vals); 3373 } 3374 check_track_val(struct tracing_map_elt * elt,struct action_data * data,u64 var_val)3375 static bool check_track_val(struct tracing_map_elt *elt, 3376 struct action_data *data, 3377 u64 var_val) 3378 { 3379 struct hist_trigger_data *hist_data; 3380 u64 track_val; 3381 3382 hist_data = data->track_data.track_var->hist_data; 3383 track_val = get_track_val(hist_data, elt, data); 3384 3385 return data->track_data.check_val(track_val, var_val); 3386 } 3387 3388 #ifdef CONFIG_TRACER_SNAPSHOT cond_snapshot_update(struct trace_array * tr,void * cond_data)3389 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data) 3390 { 3391 /* called with tr->max_lock held */ 3392 struct track_data *track_data = tr->cond_snapshot->cond_data; 3393 struct hist_elt_data *elt_data, *track_elt_data; 3394 struct snapshot_context *context = cond_data; 3395 struct action_data *action; 3396 u64 track_val; 3397 3398 if (!track_data) 3399 return false; 3400 3401 action = track_data->action_data; 3402 3403 track_val = get_track_val(track_data->hist_data, context->elt, 3404 track_data->action_data); 3405 3406 if (!action->track_data.check_val(track_data->track_val, track_val)) 3407 return false; 3408 3409 track_data->track_val = track_val; 3410 memcpy(track_data->key, context->key, track_data->key_len); 3411 3412 elt_data = context->elt->private_data; 3413 track_elt_data = track_data->elt.private_data; 3414 if (elt_data->comm) 3415 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN); 3416 3417 track_data->updated = true; 3418 3419 return true; 3420 } 3421 save_track_data_snapshot(struct hist_trigger_data * hist_data,struct tracing_map_elt * elt,struct trace_buffer * buffer,void * rec,struct ring_buffer_event * rbe,void * key,struct action_data * data,u64 * var_ref_vals)3422 static void save_track_data_snapshot(struct hist_trigger_data *hist_data, 3423 struct tracing_map_elt *elt, 3424 struct trace_buffer *buffer, void *rec, 3425 struct ring_buffer_event *rbe, void *key, 3426 struct action_data *data, 3427 u64 *var_ref_vals) 3428 { 3429 struct trace_event_file *file = hist_data->event_file; 3430 struct snapshot_context context; 3431 3432 context.elt = elt; 3433 context.key = key; 3434 3435 tracing_snapshot_cond(file->tr, &context); 3436 } 3437 3438 static void hist_trigger_print_key(struct seq_file *m, 3439 struct hist_trigger_data *hist_data, 3440 void *key, 3441 struct tracing_map_elt *elt); 3442 snapshot_action(struct hist_trigger_data * hist_data)3443 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data) 3444 { 3445 unsigned int i; 3446 3447 if (!hist_data->n_actions) 3448 return NULL; 3449 3450 for (i = 0; i < hist_data->n_actions; i++) { 3451 struct action_data *data = hist_data->actions[i]; 3452 3453 if (data->action == ACTION_SNAPSHOT) 3454 return data; 3455 } 3456 3457 return NULL; 3458 } 3459 track_data_snapshot_print(struct seq_file * m,struct hist_trigger_data * hist_data)3460 static void track_data_snapshot_print(struct seq_file *m, 3461 struct hist_trigger_data *hist_data) 3462 { 3463 struct trace_event_file *file = hist_data->event_file; 3464 struct track_data *track_data; 3465 struct action_data *action; 3466 3467 track_data = tracing_cond_snapshot_data(file->tr); 3468 if (!track_data) 3469 return; 3470 3471 if (!track_data->updated) 3472 return; 3473 3474 action = snapshot_action(hist_data); 3475 if (!action) 3476 return; 3477 3478 seq_puts(m, "\nSnapshot taken (see tracing/snapshot). Details:\n"); 3479 seq_printf(m, "\ttriggering value { %s(%s) }: %10llu", 3480 action->handler == HANDLER_ONMAX ? "onmax" : "onchange", 3481 action->track_data.var_str, track_data->track_val); 3482 3483 seq_puts(m, "\ttriggered by event with key: "); 3484 hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt); 3485 seq_putc(m, '\n'); 3486 } 3487 #else cond_snapshot_update(struct trace_array * tr,void * cond_data)3488 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data) 3489 { 3490 return false; 3491 } save_track_data_snapshot(struct hist_trigger_data * hist_data,struct tracing_map_elt * elt,struct trace_buffer * buffer,void * rec,struct ring_buffer_event * rbe,void * key,struct action_data * data,u64 * var_ref_vals)3492 static void save_track_data_snapshot(struct hist_trigger_data *hist_data, 3493 struct tracing_map_elt *elt, 3494 struct trace_buffer *buffer, void *rec, 3495 struct ring_buffer_event *rbe, void *key, 3496 struct action_data *data, 3497 u64 *var_ref_vals) {} track_data_snapshot_print(struct seq_file * m,struct hist_trigger_data * hist_data)3498 static void track_data_snapshot_print(struct seq_file *m, 3499 struct hist_trigger_data *hist_data) {} 3500 #endif /* CONFIG_TRACER_SNAPSHOT */ 3501 track_data_print(struct seq_file * m,struct hist_trigger_data * hist_data,struct tracing_map_elt * elt,struct action_data * data)3502 static void track_data_print(struct seq_file *m, 3503 struct hist_trigger_data *hist_data, 3504 struct tracing_map_elt *elt, 3505 struct action_data *data) 3506 { 3507 u64 track_val = get_track_val(hist_data, elt, data); 3508 unsigned int i, save_var_idx; 3509 3510 if (data->handler == HANDLER_ONMAX) 3511 seq_printf(m, "\n\tmax: %10llu", track_val); 3512 else if (data->handler == HANDLER_ONCHANGE) 3513 seq_printf(m, "\n\tchanged: %10llu", track_val); 3514 3515 if (data->action == ACTION_SNAPSHOT) 3516 return; 3517 3518 for (i = 0; i < hist_data->n_save_vars; i++) { 3519 struct hist_field *save_val = hist_data->save_vars[i]->val; 3520 struct hist_field *save_var = hist_data->save_vars[i]->var; 3521 u64 val; 3522 3523 save_var_idx = save_var->var.idx; 3524 3525 val = tracing_map_read_var(elt, save_var_idx); 3526 3527 if (save_val->flags & HIST_FIELD_FL_STRING) { 3528 seq_printf(m, " %s: %-32s", save_var->var.name, 3529 (char *)(uintptr_t)(val)); 3530 } else 3531 seq_printf(m, " %s: %10llu", save_var->var.name, val); 3532 } 3533 } 3534 ontrack_action(struct hist_trigger_data * hist_data,struct tracing_map_elt * elt,struct trace_buffer * buffer,void * rec,struct ring_buffer_event * rbe,void * key,struct action_data * data,u64 * var_ref_vals)3535 static void ontrack_action(struct hist_trigger_data *hist_data, 3536 struct tracing_map_elt *elt, 3537 struct trace_buffer *buffer, void *rec, 3538 struct ring_buffer_event *rbe, void *key, 3539 struct action_data *data, u64 *var_ref_vals) 3540 { 3541 u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx]; 3542 3543 if (check_track_val(elt, data, var_val)) { 3544 save_track_val(hist_data, elt, data, var_val); 3545 save_track_data(hist_data, elt, buffer, rec, rbe, 3546 key, data, var_ref_vals); 3547 } 3548 } 3549 action_data_destroy(struct action_data * data)3550 static void action_data_destroy(struct action_data *data) 3551 { 3552 unsigned int i; 3553 3554 lockdep_assert_held(&event_mutex); 3555 3556 kfree(data->action_name); 3557 3558 for (i = 0; i < data->n_params; i++) 3559 kfree(data->params[i]); 3560 3561 if (data->synth_event) 3562 data->synth_event->ref--; 3563 3564 kfree(data->synth_event_name); 3565 3566 kfree(data); 3567 } 3568 track_data_destroy(struct hist_trigger_data * hist_data,struct action_data * data)3569 static void track_data_destroy(struct hist_trigger_data *hist_data, 3570 struct action_data *data) 3571 { 3572 struct trace_event_file *file = hist_data->event_file; 3573 3574 destroy_hist_field(data->track_data.track_var, 0); 3575 3576 if (data->action == ACTION_SNAPSHOT) { 3577 struct track_data *track_data; 3578 3579 track_data = tracing_cond_snapshot_data(file->tr); 3580 if (track_data && track_data->hist_data == hist_data) { 3581 tracing_snapshot_cond_disable(file->tr); 3582 track_data_free(track_data); 3583 } 3584 } 3585 3586 kfree(data->track_data.var_str); 3587 3588 action_data_destroy(data); 3589 } 3590 3591 static int action_create(struct hist_trigger_data *hist_data, 3592 struct action_data *data); 3593 track_data_create(struct hist_trigger_data * hist_data,struct action_data * data)3594 static int track_data_create(struct hist_trigger_data *hist_data, 3595 struct action_data *data) 3596 { 3597 struct hist_field *var_field, *ref_field, *track_var = NULL; 3598 struct trace_event_file *file = hist_data->event_file; 3599 struct trace_array *tr = file->tr; 3600 char *track_data_var_str; 3601 int ret = 0; 3602 3603 track_data_var_str = data->track_data.var_str; 3604 if (track_data_var_str[0] != '$') { 3605 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str)); 3606 return -EINVAL; 3607 } 3608 track_data_var_str++; 3609 3610 var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str); 3611 if (!var_field) { 3612 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str)); 3613 return -EINVAL; 3614 } 3615 3616 ref_field = create_var_ref(hist_data, var_field, NULL, NULL); 3617 if (!ref_field) 3618 return -ENOMEM; 3619 3620 data->track_data.var_ref = ref_field; 3621 3622 if (data->handler == HANDLER_ONMAX) 3623 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64"); 3624 if (IS_ERR(track_var)) { 3625 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0); 3626 ret = PTR_ERR(track_var); 3627 goto out; 3628 } 3629 3630 if (data->handler == HANDLER_ONCHANGE) 3631 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64"); 3632 if (IS_ERR(track_var)) { 3633 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0); 3634 ret = PTR_ERR(track_var); 3635 goto out; 3636 } 3637 data->track_data.track_var = track_var; 3638 3639 ret = action_create(hist_data, data); 3640 out: 3641 return ret; 3642 } 3643 parse_action_params(struct trace_array * tr,char * params,struct action_data * data)3644 static int parse_action_params(struct trace_array *tr, char *params, 3645 struct action_data *data) 3646 { 3647 char *param, *saved_param; 3648 bool first_param = true; 3649 int ret = 0; 3650 3651 while (params) { 3652 if (data->n_params >= SYNTH_FIELDS_MAX) { 3653 hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0); 3654 ret = -EINVAL; 3655 goto out; 3656 } 3657 3658 param = strsep(¶ms, ","); 3659 if (!param) { 3660 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0); 3661 ret = -EINVAL; 3662 goto out; 3663 } 3664 3665 param = strstrip(param); 3666 if (strlen(param) < 2) { 3667 hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param)); 3668 ret = -EINVAL; 3669 goto out; 3670 } 3671 3672 saved_param = kstrdup(param, GFP_KERNEL); 3673 if (!saved_param) { 3674 ret = -ENOMEM; 3675 goto out; 3676 } 3677 3678 if (first_param && data->use_trace_keyword) { 3679 data->synth_event_name = saved_param; 3680 first_param = false; 3681 continue; 3682 } 3683 first_param = false; 3684 3685 data->params[data->n_params++] = saved_param; 3686 } 3687 out: 3688 return ret; 3689 } 3690 action_parse(struct trace_array * tr,char * str,struct action_data * data,enum handler_id handler)3691 static int action_parse(struct trace_array *tr, char *str, struct action_data *data, 3692 enum handler_id handler) 3693 { 3694 char *action_name; 3695 int ret = 0; 3696 3697 strsep(&str, "."); 3698 if (!str) { 3699 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0); 3700 ret = -EINVAL; 3701 goto out; 3702 } 3703 3704 action_name = strsep(&str, "("); 3705 if (!action_name || !str) { 3706 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0); 3707 ret = -EINVAL; 3708 goto out; 3709 } 3710 3711 if (str_has_prefix(action_name, "save")) { 3712 char *params = strsep(&str, ")"); 3713 3714 if (!params) { 3715 hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0); 3716 ret = -EINVAL; 3717 goto out; 3718 } 3719 3720 ret = parse_action_params(tr, params, data); 3721 if (ret) 3722 goto out; 3723 3724 if (handler == HANDLER_ONMAX) 3725 data->track_data.check_val = check_track_val_max; 3726 else if (handler == HANDLER_ONCHANGE) 3727 data->track_data.check_val = check_track_val_changed; 3728 else { 3729 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name)); 3730 ret = -EINVAL; 3731 goto out; 3732 } 3733 3734 data->track_data.save_data = save_track_data_vars; 3735 data->fn = ontrack_action; 3736 data->action = ACTION_SAVE; 3737 } else if (str_has_prefix(action_name, "snapshot")) { 3738 char *params = strsep(&str, ")"); 3739 3740 if (!str) { 3741 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params)); 3742 ret = -EINVAL; 3743 goto out; 3744 } 3745 3746 if (handler == HANDLER_ONMAX) 3747 data->track_data.check_val = check_track_val_max; 3748 else if (handler == HANDLER_ONCHANGE) 3749 data->track_data.check_val = check_track_val_changed; 3750 else { 3751 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name)); 3752 ret = -EINVAL; 3753 goto out; 3754 } 3755 3756 data->track_data.save_data = save_track_data_snapshot; 3757 data->fn = ontrack_action; 3758 data->action = ACTION_SNAPSHOT; 3759 } else { 3760 char *params = strsep(&str, ")"); 3761 3762 if (str_has_prefix(action_name, "trace")) 3763 data->use_trace_keyword = true; 3764 3765 if (params) { 3766 ret = parse_action_params(tr, params, data); 3767 if (ret) 3768 goto out; 3769 } 3770 3771 if (handler == HANDLER_ONMAX) 3772 data->track_data.check_val = check_track_val_max; 3773 else if (handler == HANDLER_ONCHANGE) 3774 data->track_data.check_val = check_track_val_changed; 3775 3776 if (handler != HANDLER_ONMATCH) { 3777 data->track_data.save_data = action_trace; 3778 data->fn = ontrack_action; 3779 } else 3780 data->fn = action_trace; 3781 3782 data->action = ACTION_TRACE; 3783 } 3784 3785 data->action_name = kstrdup(action_name, GFP_KERNEL); 3786 if (!data->action_name) { 3787 ret = -ENOMEM; 3788 goto out; 3789 } 3790 3791 data->handler = handler; 3792 out: 3793 return ret; 3794 } 3795 track_data_parse(struct hist_trigger_data * hist_data,char * str,enum handler_id handler)3796 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data, 3797 char *str, enum handler_id handler) 3798 { 3799 struct action_data *data; 3800 int ret = -EINVAL; 3801 char *var_str; 3802 3803 data = kzalloc(sizeof(*data), GFP_KERNEL); 3804 if (!data) 3805 return ERR_PTR(-ENOMEM); 3806 3807 var_str = strsep(&str, ")"); 3808 if (!var_str || !str) { 3809 ret = -EINVAL; 3810 goto free; 3811 } 3812 3813 data->track_data.var_str = kstrdup(var_str, GFP_KERNEL); 3814 if (!data->track_data.var_str) { 3815 ret = -ENOMEM; 3816 goto free; 3817 } 3818 3819 ret = action_parse(hist_data->event_file->tr, str, data, handler); 3820 if (ret) 3821 goto free; 3822 out: 3823 return data; 3824 free: 3825 track_data_destroy(hist_data, data); 3826 data = ERR_PTR(ret); 3827 goto out; 3828 } 3829 onmatch_destroy(struct action_data * data)3830 static void onmatch_destroy(struct action_data *data) 3831 { 3832 kfree(data->match_data.event); 3833 kfree(data->match_data.event_system); 3834 3835 action_data_destroy(data); 3836 } 3837 destroy_field_var(struct field_var * field_var)3838 static void destroy_field_var(struct field_var *field_var) 3839 { 3840 if (!field_var) 3841 return; 3842 3843 destroy_hist_field(field_var->var, 0); 3844 destroy_hist_field(field_var->val, 0); 3845 3846 kfree(field_var); 3847 } 3848 destroy_field_vars(struct hist_trigger_data * hist_data)3849 static void destroy_field_vars(struct hist_trigger_data *hist_data) 3850 { 3851 unsigned int i; 3852 3853 for (i = 0; i < hist_data->n_field_vars; i++) 3854 destroy_field_var(hist_data->field_vars[i]); 3855 3856 for (i = 0; i < hist_data->n_save_vars; i++) 3857 destroy_field_var(hist_data->save_vars[i]); 3858 } 3859 save_field_var(struct hist_trigger_data * hist_data,struct field_var * field_var)3860 static void save_field_var(struct hist_trigger_data *hist_data, 3861 struct field_var *field_var) 3862 { 3863 hist_data->field_vars[hist_data->n_field_vars++] = field_var; 3864 3865 /* Stack traces are saved in the string storage too */ 3866 if (field_var->val->flags & (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) 3867 hist_data->n_field_var_str++; 3868 } 3869 3870 check_synth_field(struct synth_event * event,struct hist_field * hist_field,unsigned int field_pos)3871 static int check_synth_field(struct synth_event *event, 3872 struct hist_field *hist_field, 3873 unsigned int field_pos) 3874 { 3875 struct synth_field *field; 3876 3877 if (field_pos >= event->n_fields) 3878 return -EINVAL; 3879 3880 field = event->fields[field_pos]; 3881 3882 /* 3883 * A dynamic string synth field can accept static or 3884 * dynamic. A static string synth field can only accept a 3885 * same-sized static string, which is checked for later. 3886 */ 3887 if (strstr(hist_field->type, "char[") && field->is_string 3888 && field->is_dynamic) 3889 return 0; 3890 3891 if (strstr(hist_field->type, "long[") && field->is_stack) 3892 return 0; 3893 3894 if (strcmp(field->type, hist_field->type) != 0) { 3895 if (field->size != hist_field->size || 3896 (!field->is_string && field->is_signed != hist_field->is_signed)) 3897 return -EINVAL; 3898 } 3899 3900 return 0; 3901 } 3902 3903 static struct hist_field * trace_action_find_var(struct hist_trigger_data * hist_data,struct action_data * data,char * system,char * event,char * var)3904 trace_action_find_var(struct hist_trigger_data *hist_data, 3905 struct action_data *data, 3906 char *system, char *event, char *var) 3907 { 3908 struct trace_array *tr = hist_data->event_file->tr; 3909 struct hist_field *hist_field; 3910 3911 var++; /* skip '$' */ 3912 3913 hist_field = find_target_event_var(hist_data, system, event, var); 3914 if (!hist_field) { 3915 if (!system && data->handler == HANDLER_ONMATCH) { 3916 system = data->match_data.event_system; 3917 event = data->match_data.event; 3918 } 3919 3920 hist_field = find_event_var(hist_data, system, event, var); 3921 } 3922 3923 if (!hist_field) 3924 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var)); 3925 3926 return hist_field; 3927 } 3928 3929 static struct hist_field * trace_action_create_field_var(struct hist_trigger_data * hist_data,struct action_data * data,char * system,char * event,char * var)3930 trace_action_create_field_var(struct hist_trigger_data *hist_data, 3931 struct action_data *data, char *system, 3932 char *event, char *var) 3933 { 3934 struct hist_field *hist_field = NULL; 3935 struct field_var *field_var; 3936 3937 /* 3938 * First try to create a field var on the target event (the 3939 * currently being defined). This will create a variable for 3940 * unqualified fields on the target event, or if qualified, 3941 * target fields that have qualified names matching the target. 3942 */ 3943 field_var = create_target_field_var(hist_data, system, event, var); 3944 3945 if (field_var && !IS_ERR(field_var)) { 3946 save_field_var(hist_data, field_var); 3947 hist_field = field_var->var; 3948 } else { 3949 field_var = NULL; 3950 /* 3951 * If no explicit system.event is specified, default to 3952 * looking for fields on the onmatch(system.event.xxx) 3953 * event. 3954 */ 3955 if (!system && data->handler == HANDLER_ONMATCH) { 3956 system = data->match_data.event_system; 3957 event = data->match_data.event; 3958 } 3959 3960 if (!event) 3961 goto free; 3962 /* 3963 * At this point, we're looking at a field on another 3964 * event. Because we can't modify a hist trigger on 3965 * another event to add a variable for a field, we need 3966 * to create a new trigger on that event and create the 3967 * variable at the same time. 3968 */ 3969 hist_field = create_field_var_hist(hist_data, system, event, var); 3970 if (IS_ERR(hist_field)) 3971 goto free; 3972 } 3973 out: 3974 return hist_field; 3975 free: 3976 destroy_field_var(field_var); 3977 hist_field = NULL; 3978 goto out; 3979 } 3980 trace_action_create(struct hist_trigger_data * hist_data,struct action_data * data)3981 static int trace_action_create(struct hist_trigger_data *hist_data, 3982 struct action_data *data) 3983 { 3984 struct trace_array *tr = hist_data->event_file->tr; 3985 char *event_name, *param, *system = NULL; 3986 struct hist_field *hist_field, *var_ref; 3987 unsigned int i; 3988 unsigned int field_pos = 0; 3989 struct synth_event *event; 3990 char *synth_event_name; 3991 int var_ref_idx, ret = 0; 3992 3993 lockdep_assert_held(&event_mutex); 3994 3995 /* Sanity check to avoid out-of-bound write on 'data->var_ref_idx' */ 3996 if (data->n_params > SYNTH_FIELDS_MAX) 3997 return -EINVAL; 3998 3999 if (data->use_trace_keyword) 4000 synth_event_name = data->synth_event_name; 4001 else 4002 synth_event_name = data->action_name; 4003 4004 event = find_synth_event(synth_event_name); 4005 if (!event) { 4006 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name)); 4007 return -EINVAL; 4008 } 4009 4010 event->ref++; 4011 4012 for (i = 0; i < data->n_params; i++) { 4013 char *p; 4014 4015 p = param = kstrdup(data->params[i], GFP_KERNEL); 4016 if (!param) { 4017 ret = -ENOMEM; 4018 goto err; 4019 } 4020 4021 system = strsep(¶m, "."); 4022 if (!param) { 4023 param = (char *)system; 4024 system = event_name = NULL; 4025 } else { 4026 event_name = strsep(¶m, "."); 4027 if (!param) { 4028 kfree(p); 4029 ret = -EINVAL; 4030 goto err; 4031 } 4032 } 4033 4034 if (param[0] == '$') 4035 hist_field = trace_action_find_var(hist_data, data, 4036 system, event_name, 4037 param); 4038 else 4039 hist_field = trace_action_create_field_var(hist_data, 4040 data, 4041 system, 4042 event_name, 4043 param); 4044 4045 if (!hist_field) { 4046 kfree(p); 4047 ret = -EINVAL; 4048 goto err; 4049 } 4050 4051 if (check_synth_field(event, hist_field, field_pos) == 0) { 4052 var_ref = create_var_ref(hist_data, hist_field, 4053 system, event_name); 4054 if (!var_ref) { 4055 kfree(p); 4056 ret = -ENOMEM; 4057 goto err; 4058 } 4059 4060 var_ref_idx = find_var_ref_idx(hist_data, var_ref); 4061 if (WARN_ON(var_ref_idx < 0)) { 4062 kfree(p); 4063 ret = var_ref_idx; 4064 goto err; 4065 } 4066 4067 data->var_ref_idx[i] = var_ref_idx; 4068 4069 field_pos++; 4070 kfree(p); 4071 continue; 4072 } 4073 4074 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param)); 4075 kfree(p); 4076 ret = -EINVAL; 4077 goto err; 4078 } 4079 4080 if (field_pos != event->n_fields) { 4081 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name)); 4082 ret = -EINVAL; 4083 goto err; 4084 } 4085 4086 data->synth_event = event; 4087 out: 4088 return ret; 4089 err: 4090 event->ref--; 4091 4092 goto out; 4093 } 4094 action_create(struct hist_trigger_data * hist_data,struct action_data * data)4095 static int action_create(struct hist_trigger_data *hist_data, 4096 struct action_data *data) 4097 { 4098 struct trace_event_file *file = hist_data->event_file; 4099 struct trace_array *tr = file->tr; 4100 struct track_data *track_data; 4101 struct field_var *field_var; 4102 unsigned int i; 4103 char *param; 4104 int ret = 0; 4105 4106 if (data->action == ACTION_TRACE) 4107 return trace_action_create(hist_data, data); 4108 4109 if (data->action == ACTION_SNAPSHOT) { 4110 track_data = track_data_alloc(hist_data->key_size, data, hist_data); 4111 if (IS_ERR(track_data)) { 4112 ret = PTR_ERR(track_data); 4113 goto out; 4114 } 4115 4116 ret = tracing_snapshot_cond_enable(file->tr, track_data, 4117 cond_snapshot_update); 4118 if (ret) 4119 track_data_free(track_data); 4120 4121 goto out; 4122 } 4123 4124 if (data->action == ACTION_SAVE) { 4125 if (hist_data->n_save_vars) { 4126 ret = -EEXIST; 4127 hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0); 4128 goto out; 4129 } 4130 4131 for (i = 0; i < data->n_params; i++) { 4132 param = kstrdup(data->params[i], GFP_KERNEL); 4133 if (!param) { 4134 ret = -ENOMEM; 4135 goto out; 4136 } 4137 4138 field_var = create_target_field_var(hist_data, NULL, NULL, param); 4139 if (IS_ERR(field_var)) { 4140 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL, 4141 errpos(param)); 4142 ret = PTR_ERR(field_var); 4143 kfree(param); 4144 goto out; 4145 } 4146 4147 hist_data->save_vars[hist_data->n_save_vars++] = field_var; 4148 if (field_var->val->flags & 4149 (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) 4150 hist_data->n_save_var_str++; 4151 kfree(param); 4152 } 4153 } 4154 out: 4155 return ret; 4156 } 4157 onmatch_create(struct hist_trigger_data * hist_data,struct action_data * data)4158 static int onmatch_create(struct hist_trigger_data *hist_data, 4159 struct action_data *data) 4160 { 4161 return action_create(hist_data, data); 4162 } 4163 onmatch_parse(struct trace_array * tr,char * str)4164 static struct action_data *onmatch_parse(struct trace_array *tr, char *str) 4165 { 4166 char *match_event, *match_event_system; 4167 struct action_data *data; 4168 int ret = -EINVAL; 4169 4170 data = kzalloc(sizeof(*data), GFP_KERNEL); 4171 if (!data) 4172 return ERR_PTR(-ENOMEM); 4173 4174 match_event = strsep(&str, ")"); 4175 if (!match_event || !str) { 4176 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event)); 4177 goto free; 4178 } 4179 4180 match_event_system = strsep(&match_event, "."); 4181 if (!match_event) { 4182 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system)); 4183 goto free; 4184 } 4185 4186 if (IS_ERR(event_file(tr, match_event_system, match_event))) { 4187 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event)); 4188 goto free; 4189 } 4190 4191 data->match_data.event = kstrdup(match_event, GFP_KERNEL); 4192 if (!data->match_data.event) { 4193 ret = -ENOMEM; 4194 goto free; 4195 } 4196 4197 data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL); 4198 if (!data->match_data.event_system) { 4199 ret = -ENOMEM; 4200 goto free; 4201 } 4202 4203 ret = action_parse(tr, str, data, HANDLER_ONMATCH); 4204 if (ret) 4205 goto free; 4206 out: 4207 return data; 4208 free: 4209 onmatch_destroy(data); 4210 data = ERR_PTR(ret); 4211 goto out; 4212 } 4213 create_hitcount_val(struct hist_trigger_data * hist_data)4214 static int create_hitcount_val(struct hist_trigger_data *hist_data) 4215 { 4216 hist_data->fields[HITCOUNT_IDX] = 4217 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL); 4218 if (!hist_data->fields[HITCOUNT_IDX]) 4219 return -ENOMEM; 4220 4221 hist_data->n_vals++; 4222 hist_data->n_fields++; 4223 4224 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX)) 4225 return -EINVAL; 4226 4227 return 0; 4228 } 4229 __create_val_field(struct hist_trigger_data * hist_data,unsigned int val_idx,struct trace_event_file * file,char * var_name,char * field_str,unsigned long flags)4230 static int __create_val_field(struct hist_trigger_data *hist_data, 4231 unsigned int val_idx, 4232 struct trace_event_file *file, 4233 char *var_name, char *field_str, 4234 unsigned long flags) 4235 { 4236 struct hist_field *hist_field; 4237 int ret = 0, n_subexprs = 0; 4238 4239 hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs); 4240 if (IS_ERR(hist_field)) { 4241 ret = PTR_ERR(hist_field); 4242 goto out; 4243 } 4244 4245 /* values and variables should not have some modifiers */ 4246 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4247 /* Variable */ 4248 if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT | 4249 HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2)) 4250 goto err; 4251 } else { 4252 /* Value */ 4253 if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT | 4254 HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2 | 4255 HIST_FIELD_FL_SYM | HIST_FIELD_FL_SYM_OFFSET | 4256 HIST_FIELD_FL_SYSCALL | HIST_FIELD_FL_STACKTRACE)) 4257 goto err; 4258 } 4259 4260 hist_data->fields[val_idx] = hist_field; 4261 4262 ++hist_data->n_vals; 4263 ++hist_data->n_fields; 4264 4265 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 4266 ret = -EINVAL; 4267 out: 4268 return ret; 4269 err: 4270 hist_err(file->tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str)); 4271 return -EINVAL; 4272 } 4273 create_val_field(struct hist_trigger_data * hist_data,unsigned int val_idx,struct trace_event_file * file,char * field_str)4274 static int create_val_field(struct hist_trigger_data *hist_data, 4275 unsigned int val_idx, 4276 struct trace_event_file *file, 4277 char *field_str) 4278 { 4279 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX)) 4280 return -EINVAL; 4281 4282 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0); 4283 } 4284 4285 static const char no_comm[] = "(no comm)"; 4286 hist_field_execname(struct hist_field * hist_field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)4287 static u64 hist_field_execname(struct hist_field *hist_field, 4288 struct tracing_map_elt *elt, 4289 struct trace_buffer *buffer, 4290 struct ring_buffer_event *rbe, 4291 void *event) 4292 { 4293 struct hist_elt_data *elt_data; 4294 4295 if (WARN_ON_ONCE(!elt)) 4296 return (u64)(unsigned long)no_comm; 4297 4298 elt_data = elt->private_data; 4299 4300 if (WARN_ON_ONCE(!elt_data->comm)) 4301 return (u64)(unsigned long)no_comm; 4302 4303 return (u64)(unsigned long)(elt_data->comm); 4304 } 4305 hist_field_stack(struct hist_field * hist_field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)4306 static u64 hist_field_stack(struct hist_field *hist_field, 4307 struct tracing_map_elt *elt, 4308 struct trace_buffer *buffer, 4309 struct ring_buffer_event *rbe, 4310 void *event) 4311 { 4312 u32 str_item = *(u32 *)(event + hist_field->field->offset); 4313 int str_loc = str_item & 0xffff; 4314 char *addr = (char *)(event + str_loc); 4315 4316 return (u64)(unsigned long)addr; 4317 } 4318 hist_fn_call(struct hist_field * hist_field,struct tracing_map_elt * elt,struct trace_buffer * buffer,struct ring_buffer_event * rbe,void * event)4319 static u64 hist_fn_call(struct hist_field *hist_field, 4320 struct tracing_map_elt *elt, 4321 struct trace_buffer *buffer, 4322 struct ring_buffer_event *rbe, 4323 void *event) 4324 { 4325 switch (hist_field->fn_num) { 4326 case HIST_FIELD_FN_VAR_REF: 4327 return hist_field_var_ref(hist_field, elt, buffer, rbe, event); 4328 case HIST_FIELD_FN_COUNTER: 4329 return hist_field_counter(hist_field, elt, buffer, rbe, event); 4330 case HIST_FIELD_FN_CONST: 4331 return hist_field_const(hist_field, elt, buffer, rbe, event); 4332 case HIST_FIELD_FN_LOG2: 4333 return hist_field_log2(hist_field, elt, buffer, rbe, event); 4334 case HIST_FIELD_FN_BUCKET: 4335 return hist_field_bucket(hist_field, elt, buffer, rbe, event); 4336 case HIST_FIELD_FN_TIMESTAMP: 4337 return hist_field_timestamp(hist_field, elt, buffer, rbe, event); 4338 case HIST_FIELD_FN_CPU: 4339 return hist_field_cpu(hist_field, elt, buffer, rbe, event); 4340 case HIST_FIELD_FN_STRING: 4341 return hist_field_string(hist_field, elt, buffer, rbe, event); 4342 case HIST_FIELD_FN_DYNSTRING: 4343 return hist_field_dynstring(hist_field, elt, buffer, rbe, event); 4344 case HIST_FIELD_FN_RELDYNSTRING: 4345 return hist_field_reldynstring(hist_field, elt, buffer, rbe, event); 4346 case HIST_FIELD_FN_PSTRING: 4347 return hist_field_pstring(hist_field, elt, buffer, rbe, event); 4348 case HIST_FIELD_FN_S64: 4349 return hist_field_s64(hist_field, elt, buffer, rbe, event); 4350 case HIST_FIELD_FN_U64: 4351 return hist_field_u64(hist_field, elt, buffer, rbe, event); 4352 case HIST_FIELD_FN_S32: 4353 return hist_field_s32(hist_field, elt, buffer, rbe, event); 4354 case HIST_FIELD_FN_U32: 4355 return hist_field_u32(hist_field, elt, buffer, rbe, event); 4356 case HIST_FIELD_FN_S16: 4357 return hist_field_s16(hist_field, elt, buffer, rbe, event); 4358 case HIST_FIELD_FN_U16: 4359 return hist_field_u16(hist_field, elt, buffer, rbe, event); 4360 case HIST_FIELD_FN_S8: 4361 return hist_field_s8(hist_field, elt, buffer, rbe, event); 4362 case HIST_FIELD_FN_U8: 4363 return hist_field_u8(hist_field, elt, buffer, rbe, event); 4364 case HIST_FIELD_FN_UMINUS: 4365 return hist_field_unary_minus(hist_field, elt, buffer, rbe, event); 4366 case HIST_FIELD_FN_MINUS: 4367 return hist_field_minus(hist_field, elt, buffer, rbe, event); 4368 case HIST_FIELD_FN_PLUS: 4369 return hist_field_plus(hist_field, elt, buffer, rbe, event); 4370 case HIST_FIELD_FN_DIV: 4371 return hist_field_div(hist_field, elt, buffer, rbe, event); 4372 case HIST_FIELD_FN_MULT: 4373 return hist_field_mult(hist_field, elt, buffer, rbe, event); 4374 case HIST_FIELD_FN_DIV_POWER2: 4375 return div_by_power_of_two(hist_field, elt, buffer, rbe, event); 4376 case HIST_FIELD_FN_DIV_NOT_POWER2: 4377 return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event); 4378 case HIST_FIELD_FN_DIV_MULT_SHIFT: 4379 return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event); 4380 case HIST_FIELD_FN_EXECNAME: 4381 return hist_field_execname(hist_field, elt, buffer, rbe, event); 4382 case HIST_FIELD_FN_STACK: 4383 return hist_field_stack(hist_field, elt, buffer, rbe, event); 4384 default: 4385 return 0; 4386 } 4387 } 4388 4389 /* Convert a var that points to common_pid.execname to a string */ update_var_execname(struct hist_field * hist_field)4390 static void update_var_execname(struct hist_field *hist_field) 4391 { 4392 hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR | 4393 HIST_FIELD_FL_EXECNAME; 4394 hist_field->size = MAX_FILTER_STR_VAL; 4395 hist_field->is_signed = 0; 4396 4397 kfree_const(hist_field->type); 4398 hist_field->type = "char[]"; 4399 4400 hist_field->fn_num = HIST_FIELD_FN_EXECNAME; 4401 } 4402 create_var_field(struct hist_trigger_data * hist_data,unsigned int val_idx,struct trace_event_file * file,char * var_name,char * expr_str)4403 static int create_var_field(struct hist_trigger_data *hist_data, 4404 unsigned int val_idx, 4405 struct trace_event_file *file, 4406 char *var_name, char *expr_str) 4407 { 4408 struct trace_array *tr = hist_data->event_file->tr; 4409 unsigned long flags = 0; 4410 int ret; 4411 4412 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 4413 return -EINVAL; 4414 4415 if (find_var(hist_data, file, var_name) && !hist_data->remove) { 4416 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name)); 4417 return -EINVAL; 4418 } 4419 4420 flags |= HIST_FIELD_FL_VAR; 4421 hist_data->n_vars++; 4422 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX)) 4423 return -EINVAL; 4424 4425 ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags); 4426 4427 if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME) 4428 update_var_execname(hist_data->fields[val_idx]); 4429 4430 if (!ret && hist_data->fields[val_idx]->flags & 4431 (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) 4432 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++; 4433 4434 return ret; 4435 } 4436 create_val_fields(struct hist_trigger_data * hist_data,struct trace_event_file * file)4437 static int create_val_fields(struct hist_trigger_data *hist_data, 4438 struct trace_event_file *file) 4439 { 4440 unsigned int i, j = 1, n_hitcount = 0; 4441 char *fields_str, *field_str; 4442 int ret; 4443 4444 ret = create_hitcount_val(hist_data); 4445 if (ret) 4446 goto out; 4447 4448 fields_str = hist_data->attrs->vals_str; 4449 if (!fields_str) 4450 goto out; 4451 4452 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX && 4453 j < TRACING_MAP_VALS_MAX; i++) { 4454 field_str = strsep(&fields_str, ","); 4455 if (!field_str) 4456 break; 4457 4458 if (strcmp(field_str, "hitcount") == 0) { 4459 if (!n_hitcount++) 4460 continue; 4461 } 4462 4463 ret = create_val_field(hist_data, j++, file, field_str); 4464 if (ret) 4465 goto out; 4466 } 4467 4468 if (fields_str && (strcmp(fields_str, "hitcount") != 0)) 4469 ret = -EINVAL; 4470 out: 4471 /* There is only raw hitcount but nohitcount suppresses it. */ 4472 if (j == 1 && hist_data->attrs->no_hitcount) { 4473 hist_err(hist_data->event_file->tr, HIST_ERR_NEED_NOHC_VAL, 0); 4474 ret = -ENOENT; 4475 } 4476 4477 return ret; 4478 } 4479 create_key_field(struct hist_trigger_data * hist_data,unsigned int key_idx,unsigned int key_offset,struct trace_event_file * file,char * field_str)4480 static int create_key_field(struct hist_trigger_data *hist_data, 4481 unsigned int key_idx, 4482 unsigned int key_offset, 4483 struct trace_event_file *file, 4484 char *field_str) 4485 { 4486 struct trace_array *tr = hist_data->event_file->tr; 4487 struct hist_field *hist_field = NULL; 4488 unsigned long flags = 0; 4489 unsigned int key_size; 4490 int ret = 0, n_subexprs = 0; 4491 4492 if (WARN_ON(key_idx >= HIST_FIELDS_MAX)) 4493 return -EINVAL; 4494 4495 flags |= HIST_FIELD_FL_KEY; 4496 4497 if (strcmp(field_str, "stacktrace") == 0) { 4498 flags |= HIST_FIELD_FL_STACKTRACE; 4499 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH; 4500 hist_field = create_hist_field(hist_data, NULL, flags, NULL); 4501 } else { 4502 hist_field = parse_expr(hist_data, file, field_str, flags, 4503 NULL, &n_subexprs); 4504 if (IS_ERR(hist_field)) { 4505 ret = PTR_ERR(hist_field); 4506 goto out; 4507 } 4508 4509 if (field_has_hist_vars(hist_field, 0)) { 4510 hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str)); 4511 destroy_hist_field(hist_field, 0); 4512 ret = -EINVAL; 4513 goto out; 4514 } 4515 4516 key_size = hist_field->size; 4517 } 4518 4519 hist_data->fields[key_idx] = hist_field; 4520 4521 key_size = ALIGN(key_size, sizeof(u64)); 4522 hist_data->fields[key_idx]->size = key_size; 4523 hist_data->fields[key_idx]->offset = key_offset; 4524 4525 hist_data->key_size += key_size; 4526 4527 if (hist_data->key_size > HIST_KEY_SIZE_MAX) { 4528 ret = -EINVAL; 4529 goto out; 4530 } 4531 4532 hist_data->n_keys++; 4533 hist_data->n_fields++; 4534 4535 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX)) 4536 return -EINVAL; 4537 4538 ret = key_size; 4539 out: 4540 return ret; 4541 } 4542 create_key_fields(struct hist_trigger_data * hist_data,struct trace_event_file * file)4543 static int create_key_fields(struct hist_trigger_data *hist_data, 4544 struct trace_event_file *file) 4545 { 4546 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals; 4547 char *fields_str, *field_str; 4548 int ret = -EINVAL; 4549 4550 fields_str = hist_data->attrs->keys_str; 4551 if (!fields_str) 4552 goto out; 4553 4554 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) { 4555 field_str = strsep(&fields_str, ","); 4556 if (!field_str) 4557 break; 4558 ret = create_key_field(hist_data, i, key_offset, 4559 file, field_str); 4560 if (ret < 0) 4561 goto out; 4562 key_offset += ret; 4563 } 4564 if (fields_str) { 4565 ret = -EINVAL; 4566 goto out; 4567 } 4568 ret = 0; 4569 out: 4570 return ret; 4571 } 4572 create_var_fields(struct hist_trigger_data * hist_data,struct trace_event_file * file)4573 static int create_var_fields(struct hist_trigger_data *hist_data, 4574 struct trace_event_file *file) 4575 { 4576 unsigned int i, j = hist_data->n_vals; 4577 int ret = 0; 4578 4579 unsigned int n_vars = hist_data->attrs->var_defs.n_vars; 4580 4581 for (i = 0; i < n_vars; i++) { 4582 char *var_name = hist_data->attrs->var_defs.name[i]; 4583 char *expr = hist_data->attrs->var_defs.expr[i]; 4584 4585 ret = create_var_field(hist_data, j++, file, var_name, expr); 4586 if (ret) 4587 goto out; 4588 } 4589 out: 4590 return ret; 4591 } 4592 free_var_defs(struct hist_trigger_data * hist_data)4593 static void free_var_defs(struct hist_trigger_data *hist_data) 4594 { 4595 unsigned int i; 4596 4597 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 4598 kfree(hist_data->attrs->var_defs.name[i]); 4599 kfree(hist_data->attrs->var_defs.expr[i]); 4600 } 4601 4602 hist_data->attrs->var_defs.n_vars = 0; 4603 } 4604 parse_var_defs(struct hist_trigger_data * hist_data)4605 static int parse_var_defs(struct hist_trigger_data *hist_data) 4606 { 4607 struct trace_array *tr = hist_data->event_file->tr; 4608 char *s, *str, *var_name, *field_str; 4609 unsigned int i, j, n_vars = 0; 4610 int ret = 0; 4611 4612 for (i = 0; i < hist_data->attrs->n_assignments; i++) { 4613 str = hist_data->attrs->assignment_str[i]; 4614 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) { 4615 field_str = strsep(&str, ","); 4616 if (!field_str) 4617 break; 4618 4619 var_name = strsep(&field_str, "="); 4620 if (!var_name || !field_str) { 4621 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT, 4622 errpos(var_name)); 4623 ret = -EINVAL; 4624 goto free; 4625 } 4626 4627 if (n_vars == TRACING_MAP_VARS_MAX) { 4628 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name)); 4629 ret = -EINVAL; 4630 goto free; 4631 } 4632 4633 s = kstrdup(var_name, GFP_KERNEL); 4634 if (!s) { 4635 ret = -ENOMEM; 4636 goto free; 4637 } 4638 hist_data->attrs->var_defs.name[n_vars] = s; 4639 4640 s = kstrdup(field_str, GFP_KERNEL); 4641 if (!s) { 4642 kfree(hist_data->attrs->var_defs.name[n_vars]); 4643 hist_data->attrs->var_defs.name[n_vars] = NULL; 4644 ret = -ENOMEM; 4645 goto free; 4646 } 4647 hist_data->attrs->var_defs.expr[n_vars++] = s; 4648 4649 hist_data->attrs->var_defs.n_vars = n_vars; 4650 } 4651 } 4652 4653 return ret; 4654 free: 4655 free_var_defs(hist_data); 4656 4657 return ret; 4658 } 4659 create_hist_fields(struct hist_trigger_data * hist_data,struct trace_event_file * file)4660 static int create_hist_fields(struct hist_trigger_data *hist_data, 4661 struct trace_event_file *file) 4662 { 4663 int ret; 4664 4665 ret = parse_var_defs(hist_data); 4666 if (ret) 4667 return ret; 4668 4669 ret = create_val_fields(hist_data, file); 4670 if (ret) 4671 goto out; 4672 4673 ret = create_var_fields(hist_data, file); 4674 if (ret) 4675 goto out; 4676 4677 ret = create_key_fields(hist_data, file); 4678 4679 out: 4680 free_var_defs(hist_data); 4681 4682 return ret; 4683 } 4684 is_descending(struct trace_array * tr,const char * str)4685 static int is_descending(struct trace_array *tr, const char *str) 4686 { 4687 if (!str) 4688 return 0; 4689 4690 if (strcmp(str, "descending") == 0) 4691 return 1; 4692 4693 if (strcmp(str, "ascending") == 0) 4694 return 0; 4695 4696 hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str)); 4697 4698 return -EINVAL; 4699 } 4700 create_sort_keys(struct hist_trigger_data * hist_data)4701 static int create_sort_keys(struct hist_trigger_data *hist_data) 4702 { 4703 struct trace_array *tr = hist_data->event_file->tr; 4704 char *fields_str = hist_data->attrs->sort_key_str; 4705 struct tracing_map_sort_key *sort_key; 4706 int descending, ret = 0; 4707 unsigned int i, j, k; 4708 4709 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */ 4710 4711 if (!fields_str) 4712 goto out; 4713 4714 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) { 4715 struct hist_field *hist_field; 4716 char *field_str, *field_name; 4717 const char *test_name; 4718 4719 sort_key = &hist_data->sort_keys[i]; 4720 4721 field_str = strsep(&fields_str, ","); 4722 if (!field_str) 4723 break; 4724 4725 if (!*field_str) { 4726 ret = -EINVAL; 4727 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort=")); 4728 break; 4729 } 4730 4731 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) { 4732 hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort=")); 4733 ret = -EINVAL; 4734 break; 4735 } 4736 4737 field_name = strsep(&field_str, "."); 4738 if (!field_name || !*field_name) { 4739 ret = -EINVAL; 4740 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort=")); 4741 break; 4742 } 4743 4744 if (strcmp(field_name, "hitcount") == 0) { 4745 descending = is_descending(tr, field_str); 4746 if (descending < 0) { 4747 ret = descending; 4748 break; 4749 } 4750 sort_key->descending = descending; 4751 continue; 4752 } 4753 4754 for (j = 1, k = 1; j < hist_data->n_fields; j++) { 4755 unsigned int idx; 4756 4757 hist_field = hist_data->fields[j]; 4758 if (hist_field->flags & HIST_FIELD_FL_VAR) 4759 continue; 4760 4761 idx = k++; 4762 4763 test_name = hist_field_name(hist_field, 0); 4764 4765 if (strcmp(field_name, test_name) == 0) { 4766 sort_key->field_idx = idx; 4767 descending = is_descending(tr, field_str); 4768 if (descending < 0) { 4769 ret = descending; 4770 goto out; 4771 } 4772 sort_key->descending = descending; 4773 break; 4774 } 4775 } 4776 if (j == hist_data->n_fields) { 4777 ret = -EINVAL; 4778 hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name)); 4779 break; 4780 } 4781 } 4782 4783 hist_data->n_sort_keys = i; 4784 out: 4785 return ret; 4786 } 4787 destroy_actions(struct hist_trigger_data * hist_data)4788 static void destroy_actions(struct hist_trigger_data *hist_data) 4789 { 4790 unsigned int i; 4791 4792 for (i = 0; i < hist_data->n_actions; i++) { 4793 struct action_data *data = hist_data->actions[i]; 4794 4795 if (data->handler == HANDLER_ONMATCH) 4796 onmatch_destroy(data); 4797 else if (data->handler == HANDLER_ONMAX || 4798 data->handler == HANDLER_ONCHANGE) 4799 track_data_destroy(hist_data, data); 4800 else 4801 kfree(data); 4802 } 4803 } 4804 parse_actions(struct hist_trigger_data * hist_data)4805 static int parse_actions(struct hist_trigger_data *hist_data) 4806 { 4807 struct trace_array *tr = hist_data->event_file->tr; 4808 struct action_data *data; 4809 unsigned int i; 4810 int ret = 0; 4811 char *str; 4812 int len; 4813 4814 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4815 str = hist_data->attrs->action_str[i]; 4816 4817 if ((len = str_has_prefix(str, "onmatch("))) { 4818 char *action_str = str + len; 4819 4820 data = onmatch_parse(tr, action_str); 4821 if (IS_ERR(data)) { 4822 ret = PTR_ERR(data); 4823 break; 4824 } 4825 } else if ((len = str_has_prefix(str, "onmax("))) { 4826 char *action_str = str + len; 4827 4828 data = track_data_parse(hist_data, action_str, 4829 HANDLER_ONMAX); 4830 if (IS_ERR(data)) { 4831 ret = PTR_ERR(data); 4832 break; 4833 } 4834 } else if ((len = str_has_prefix(str, "onchange("))) { 4835 char *action_str = str + len; 4836 4837 data = track_data_parse(hist_data, action_str, 4838 HANDLER_ONCHANGE); 4839 if (IS_ERR(data)) { 4840 ret = PTR_ERR(data); 4841 break; 4842 } 4843 } else { 4844 ret = -EINVAL; 4845 break; 4846 } 4847 4848 hist_data->actions[hist_data->n_actions++] = data; 4849 } 4850 4851 return ret; 4852 } 4853 create_actions(struct hist_trigger_data * hist_data)4854 static int create_actions(struct hist_trigger_data *hist_data) 4855 { 4856 struct action_data *data; 4857 unsigned int i; 4858 int ret = 0; 4859 4860 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4861 data = hist_data->actions[i]; 4862 4863 if (data->handler == HANDLER_ONMATCH) { 4864 ret = onmatch_create(hist_data, data); 4865 if (ret) 4866 break; 4867 } else if (data->handler == HANDLER_ONMAX || 4868 data->handler == HANDLER_ONCHANGE) { 4869 ret = track_data_create(hist_data, data); 4870 if (ret) 4871 break; 4872 } else { 4873 ret = -EINVAL; 4874 break; 4875 } 4876 } 4877 4878 return ret; 4879 } 4880 print_actions(struct seq_file * m,struct hist_trigger_data * hist_data,struct tracing_map_elt * elt)4881 static void print_actions(struct seq_file *m, 4882 struct hist_trigger_data *hist_data, 4883 struct tracing_map_elt *elt) 4884 { 4885 unsigned int i; 4886 4887 for (i = 0; i < hist_data->n_actions; i++) { 4888 struct action_data *data = hist_data->actions[i]; 4889 4890 if (data->action == ACTION_SNAPSHOT) 4891 continue; 4892 4893 if (data->handler == HANDLER_ONMAX || 4894 data->handler == HANDLER_ONCHANGE) 4895 track_data_print(m, hist_data, elt, data); 4896 } 4897 } 4898 print_action_spec(struct seq_file * m,struct hist_trigger_data * hist_data,struct action_data * data)4899 static void print_action_spec(struct seq_file *m, 4900 struct hist_trigger_data *hist_data, 4901 struct action_data *data) 4902 { 4903 unsigned int i; 4904 4905 if (data->action == ACTION_SAVE) { 4906 for (i = 0; i < hist_data->n_save_vars; i++) { 4907 seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name); 4908 if (i < hist_data->n_save_vars - 1) 4909 seq_puts(m, ","); 4910 } 4911 } else if (data->action == ACTION_TRACE) { 4912 if (data->use_trace_keyword) 4913 seq_printf(m, "%s", data->synth_event_name); 4914 for (i = 0; i < data->n_params; i++) { 4915 if (i || data->use_trace_keyword) 4916 seq_puts(m, ","); 4917 seq_printf(m, "%s", data->params[i]); 4918 } 4919 } 4920 } 4921 print_track_data_spec(struct seq_file * m,struct hist_trigger_data * hist_data,struct action_data * data)4922 static void print_track_data_spec(struct seq_file *m, 4923 struct hist_trigger_data *hist_data, 4924 struct action_data *data) 4925 { 4926 if (data->handler == HANDLER_ONMAX) 4927 seq_puts(m, ":onmax("); 4928 else if (data->handler == HANDLER_ONCHANGE) 4929 seq_puts(m, ":onchange("); 4930 seq_printf(m, "%s", data->track_data.var_str); 4931 seq_printf(m, ").%s(", data->action_name); 4932 4933 print_action_spec(m, hist_data, data); 4934 4935 seq_puts(m, ")"); 4936 } 4937 print_onmatch_spec(struct seq_file * m,struct hist_trigger_data * hist_data,struct action_data * data)4938 static void print_onmatch_spec(struct seq_file *m, 4939 struct hist_trigger_data *hist_data, 4940 struct action_data *data) 4941 { 4942 seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system, 4943 data->match_data.event); 4944 4945 seq_printf(m, "%s(", data->action_name); 4946 4947 print_action_spec(m, hist_data, data); 4948 4949 seq_puts(m, ")"); 4950 } 4951 actions_match(struct hist_trigger_data * hist_data,struct hist_trigger_data * hist_data_test)4952 static bool actions_match(struct hist_trigger_data *hist_data, 4953 struct hist_trigger_data *hist_data_test) 4954 { 4955 unsigned int i, j; 4956 4957 if (hist_data->n_actions != hist_data_test->n_actions) 4958 return false; 4959 4960 for (i = 0; i < hist_data->n_actions; i++) { 4961 struct action_data *data = hist_data->actions[i]; 4962 struct action_data *data_test = hist_data_test->actions[i]; 4963 char *action_name, *action_name_test; 4964 4965 if (data->handler != data_test->handler) 4966 return false; 4967 if (data->action != data_test->action) 4968 return false; 4969 4970 if (data->n_params != data_test->n_params) 4971 return false; 4972 4973 for (j = 0; j < data->n_params; j++) { 4974 if (strcmp(data->params[j], data_test->params[j]) != 0) 4975 return false; 4976 } 4977 4978 if (data->use_trace_keyword) 4979 action_name = data->synth_event_name; 4980 else 4981 action_name = data->action_name; 4982 4983 if (data_test->use_trace_keyword) 4984 action_name_test = data_test->synth_event_name; 4985 else 4986 action_name_test = data_test->action_name; 4987 4988 if (strcmp(action_name, action_name_test) != 0) 4989 return false; 4990 4991 if (data->handler == HANDLER_ONMATCH) { 4992 if (strcmp(data->match_data.event_system, 4993 data_test->match_data.event_system) != 0) 4994 return false; 4995 if (strcmp(data->match_data.event, 4996 data_test->match_data.event) != 0) 4997 return false; 4998 } else if (data->handler == HANDLER_ONMAX || 4999 data->handler == HANDLER_ONCHANGE) { 5000 if (strcmp(data->track_data.var_str, 5001 data_test->track_data.var_str) != 0) 5002 return false; 5003 } 5004 } 5005 5006 return true; 5007 } 5008 5009 print_actions_spec(struct seq_file * m,struct hist_trigger_data * hist_data)5010 static void print_actions_spec(struct seq_file *m, 5011 struct hist_trigger_data *hist_data) 5012 { 5013 unsigned int i; 5014 5015 for (i = 0; i < hist_data->n_actions; i++) { 5016 struct action_data *data = hist_data->actions[i]; 5017 5018 if (data->handler == HANDLER_ONMATCH) 5019 print_onmatch_spec(m, hist_data, data); 5020 else if (data->handler == HANDLER_ONMAX || 5021 data->handler == HANDLER_ONCHANGE) 5022 print_track_data_spec(m, hist_data, data); 5023 } 5024 } 5025 destroy_field_var_hists(struct hist_trigger_data * hist_data)5026 static void destroy_field_var_hists(struct hist_trigger_data *hist_data) 5027 { 5028 unsigned int i; 5029 5030 for (i = 0; i < hist_data->n_field_var_hists; i++) { 5031 kfree(hist_data->field_var_hists[i]->cmd); 5032 kfree(hist_data->field_var_hists[i]); 5033 } 5034 } 5035 destroy_hist_data(struct hist_trigger_data * hist_data)5036 static void destroy_hist_data(struct hist_trigger_data *hist_data) 5037 { 5038 if (!hist_data) 5039 return; 5040 5041 destroy_hist_trigger_attrs(hist_data->attrs); 5042 destroy_hist_fields(hist_data); 5043 tracing_map_destroy(hist_data->map); 5044 5045 destroy_actions(hist_data); 5046 destroy_field_vars(hist_data); 5047 destroy_field_var_hists(hist_data); 5048 5049 kfree(hist_data); 5050 } 5051 create_tracing_map_fields(struct hist_trigger_data * hist_data)5052 static int create_tracing_map_fields(struct hist_trigger_data *hist_data) 5053 { 5054 struct tracing_map *map = hist_data->map; 5055 struct ftrace_event_field *field; 5056 struct hist_field *hist_field; 5057 int i, idx = 0; 5058 5059 for_each_hist_field(i, hist_data) { 5060 hist_field = hist_data->fields[i]; 5061 if (hist_field->flags & HIST_FIELD_FL_KEY) { 5062 tracing_map_cmp_fn_t cmp_fn; 5063 5064 field = hist_field->field; 5065 5066 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE) 5067 cmp_fn = tracing_map_cmp_none; 5068 else if (!field || hist_field->flags & HIST_FIELD_FL_CPU) 5069 cmp_fn = tracing_map_cmp_num(hist_field->size, 5070 hist_field->is_signed); 5071 else if (is_string_field(field)) 5072 cmp_fn = tracing_map_cmp_string; 5073 else 5074 cmp_fn = tracing_map_cmp_num(field->size, 5075 field->is_signed); 5076 idx = tracing_map_add_key_field(map, 5077 hist_field->offset, 5078 cmp_fn); 5079 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR)) 5080 idx = tracing_map_add_sum_field(map); 5081 5082 if (idx < 0) 5083 return idx; 5084 5085 if (hist_field->flags & HIST_FIELD_FL_VAR) { 5086 idx = tracing_map_add_var(map); 5087 if (idx < 0) 5088 return idx; 5089 hist_field->var.idx = idx; 5090 hist_field->var.hist_data = hist_data; 5091 } 5092 } 5093 5094 return 0; 5095 } 5096 5097 static struct hist_trigger_data * create_hist_data(unsigned int map_bits,struct hist_trigger_attrs * attrs,struct trace_event_file * file,bool remove)5098 create_hist_data(unsigned int map_bits, 5099 struct hist_trigger_attrs *attrs, 5100 struct trace_event_file *file, 5101 bool remove) 5102 { 5103 const struct tracing_map_ops *map_ops = NULL; 5104 struct hist_trigger_data *hist_data; 5105 int ret = 0; 5106 5107 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL); 5108 if (!hist_data) 5109 return ERR_PTR(-ENOMEM); 5110 5111 hist_data->attrs = attrs; 5112 hist_data->remove = remove; 5113 hist_data->event_file = file; 5114 5115 ret = parse_actions(hist_data); 5116 if (ret) 5117 goto free; 5118 5119 ret = create_hist_fields(hist_data, file); 5120 if (ret) 5121 goto free; 5122 5123 ret = create_sort_keys(hist_data); 5124 if (ret) 5125 goto free; 5126 5127 map_ops = &hist_trigger_elt_data_ops; 5128 5129 hist_data->map = tracing_map_create(map_bits, hist_data->key_size, 5130 map_ops, hist_data); 5131 if (IS_ERR(hist_data->map)) { 5132 ret = PTR_ERR(hist_data->map); 5133 hist_data->map = NULL; 5134 goto free; 5135 } 5136 5137 ret = create_tracing_map_fields(hist_data); 5138 if (ret) 5139 goto free; 5140 out: 5141 return hist_data; 5142 free: 5143 hist_data->attrs = NULL; 5144 5145 destroy_hist_data(hist_data); 5146 5147 hist_data = ERR_PTR(ret); 5148 5149 goto out; 5150 } 5151 hist_trigger_elt_update(struct hist_trigger_data * hist_data,struct tracing_map_elt * elt,struct trace_buffer * buffer,void * rec,struct ring_buffer_event * rbe,u64 * var_ref_vals)5152 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, 5153 struct tracing_map_elt *elt, 5154 struct trace_buffer *buffer, void *rec, 5155 struct ring_buffer_event *rbe, 5156 u64 *var_ref_vals) 5157 { 5158 struct hist_elt_data *elt_data; 5159 struct hist_field *hist_field; 5160 unsigned int i, var_idx; 5161 u64 hist_val; 5162 5163 elt_data = elt->private_data; 5164 elt_data->var_ref_vals = var_ref_vals; 5165 5166 for_each_hist_val_field(i, hist_data) { 5167 hist_field = hist_data->fields[i]; 5168 hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec); 5169 if (hist_field->flags & HIST_FIELD_FL_VAR) { 5170 var_idx = hist_field->var.idx; 5171 5172 if (hist_field->flags & 5173 (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) { 5174 unsigned int str_start, var_str_idx, idx; 5175 char *str, *val_str; 5176 unsigned int size; 5177 5178 str_start = hist_data->n_field_var_str + 5179 hist_data->n_save_var_str; 5180 var_str_idx = hist_field->var_str_idx; 5181 idx = str_start + var_str_idx; 5182 5183 str = elt_data->field_var_str[idx]; 5184 val_str = (char *)(uintptr_t)hist_val; 5185 5186 if (hist_field->flags & HIST_FIELD_FL_STRING) { 5187 size = min(hist_field->size, STR_VAR_LEN_MAX); 5188 strscpy(str, val_str, size); 5189 } else { 5190 char *stack_start = str + sizeof(unsigned long); 5191 int e; 5192 5193 e = stack_trace_save((void *)stack_start, 5194 HIST_STACKTRACE_DEPTH, 5195 HIST_STACKTRACE_SKIP); 5196 if (e < HIST_STACKTRACE_DEPTH - 1) 5197 ((unsigned long *)stack_start)[e] = 0; 5198 *((unsigned long *)str) = e; 5199 } 5200 hist_val = (u64)(uintptr_t)str; 5201 } 5202 tracing_map_set_var(elt, var_idx, hist_val); 5203 continue; 5204 } 5205 tracing_map_update_sum(elt, i, hist_val); 5206 } 5207 5208 for_each_hist_key_field(i, hist_data) { 5209 hist_field = hist_data->fields[i]; 5210 if (hist_field->flags & HIST_FIELD_FL_VAR) { 5211 hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec); 5212 var_idx = hist_field->var.idx; 5213 tracing_map_set_var(elt, var_idx, hist_val); 5214 } 5215 } 5216 5217 update_field_vars(hist_data, elt, buffer, rbe, rec); 5218 } 5219 add_to_key(char * compound_key,void * key,struct hist_field * key_field,void * rec)5220 static inline void add_to_key(char *compound_key, void *key, 5221 struct hist_field *key_field, void *rec) 5222 { 5223 size_t size = key_field->size; 5224 5225 if (key_field->flags & HIST_FIELD_FL_STRING) { 5226 struct ftrace_event_field *field; 5227 5228 field = key_field->field; 5229 if (field->filter_type == FILTER_DYN_STRING || 5230 field->filter_type == FILTER_RDYN_STRING) 5231 size = *(u32 *)(rec + field->offset) >> 16; 5232 else if (field->filter_type == FILTER_STATIC_STRING) 5233 size = field->size; 5234 5235 /* ensure NULL-termination */ 5236 if (size > key_field->size - 1) 5237 size = key_field->size - 1; 5238 5239 strncpy(compound_key + key_field->offset, (char *)key, size); 5240 } else 5241 memcpy(compound_key + key_field->offset, key, size); 5242 } 5243 5244 static void hist_trigger_actions(struct hist_trigger_data * hist_data,struct tracing_map_elt * elt,struct trace_buffer * buffer,void * rec,struct ring_buffer_event * rbe,void * key,u64 * var_ref_vals)5245 hist_trigger_actions(struct hist_trigger_data *hist_data, 5246 struct tracing_map_elt *elt, 5247 struct trace_buffer *buffer, void *rec, 5248 struct ring_buffer_event *rbe, void *key, 5249 u64 *var_ref_vals) 5250 { 5251 struct action_data *data; 5252 unsigned int i; 5253 5254 for (i = 0; i < hist_data->n_actions; i++) { 5255 data = hist_data->actions[i]; 5256 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals); 5257 } 5258 } 5259 event_hist_trigger(struct event_trigger_data * data,struct trace_buffer * buffer,void * rec,struct ring_buffer_event * rbe)5260 static void event_hist_trigger(struct event_trigger_data *data, 5261 struct trace_buffer *buffer, void *rec, 5262 struct ring_buffer_event *rbe) 5263 { 5264 struct hist_trigger_data *hist_data = data->private_data; 5265 bool use_compound_key = (hist_data->n_keys > 1); 5266 unsigned long entries[HIST_STACKTRACE_DEPTH]; 5267 u64 var_ref_vals[TRACING_MAP_VARS_MAX]; 5268 char compound_key[HIST_KEY_SIZE_MAX]; 5269 struct tracing_map_elt *elt = NULL; 5270 struct hist_field *key_field; 5271 u64 field_contents; 5272 void *key = NULL; 5273 unsigned int i; 5274 5275 if (unlikely(!rbe)) 5276 return; 5277 5278 memset(compound_key, 0, hist_data->key_size); 5279 5280 for_each_hist_key_field(i, hist_data) { 5281 key_field = hist_data->fields[i]; 5282 5283 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 5284 memset(entries, 0, HIST_STACKTRACE_SIZE); 5285 if (key_field->field) { 5286 unsigned long *stack, n_entries; 5287 5288 field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec); 5289 stack = (unsigned long *)(long)field_contents; 5290 n_entries = *stack; 5291 memcpy(entries, ++stack, n_entries * sizeof(unsigned long)); 5292 } else { 5293 stack_trace_save(entries, HIST_STACKTRACE_DEPTH, 5294 HIST_STACKTRACE_SKIP); 5295 } 5296 key = entries; 5297 } else { 5298 field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec); 5299 if (key_field->flags & HIST_FIELD_FL_STRING) { 5300 key = (void *)(unsigned long)field_contents; 5301 use_compound_key = true; 5302 } else 5303 key = (void *)&field_contents; 5304 } 5305 5306 if (use_compound_key) 5307 add_to_key(compound_key, key, key_field, rec); 5308 } 5309 5310 if (use_compound_key) 5311 key = compound_key; 5312 5313 if (hist_data->n_var_refs && 5314 !resolve_var_refs(hist_data, key, var_ref_vals, false)) 5315 return; 5316 5317 elt = tracing_map_insert(hist_data->map, key); 5318 if (!elt) 5319 return; 5320 5321 hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals); 5322 5323 if (resolve_var_refs(hist_data, key, var_ref_vals, true)) 5324 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals); 5325 } 5326 hist_trigger_stacktrace_print(struct seq_file * m,unsigned long * stacktrace_entries,unsigned int max_entries)5327 static void hist_trigger_stacktrace_print(struct seq_file *m, 5328 unsigned long *stacktrace_entries, 5329 unsigned int max_entries) 5330 { 5331 unsigned int spaces = 8; 5332 unsigned int i; 5333 5334 for (i = 0; i < max_entries; i++) { 5335 if (!stacktrace_entries[i]) 5336 return; 5337 5338 seq_printf(m, "%*c", 1 + spaces, ' '); 5339 seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]); 5340 } 5341 } 5342 hist_trigger_print_key(struct seq_file * m,struct hist_trigger_data * hist_data,void * key,struct tracing_map_elt * elt)5343 static void hist_trigger_print_key(struct seq_file *m, 5344 struct hist_trigger_data *hist_data, 5345 void *key, 5346 struct tracing_map_elt *elt) 5347 { 5348 struct hist_field *key_field; 5349 bool multiline = false; 5350 const char *field_name; 5351 unsigned int i; 5352 u64 uval; 5353 5354 seq_puts(m, "{ "); 5355 5356 for_each_hist_key_field(i, hist_data) { 5357 key_field = hist_data->fields[i]; 5358 5359 if (i > hist_data->n_vals) 5360 seq_puts(m, ", "); 5361 5362 field_name = hist_field_name(key_field, 0); 5363 5364 if (key_field->flags & HIST_FIELD_FL_HEX) { 5365 uval = *(u64 *)(key + key_field->offset); 5366 seq_printf(m, "%s: %llx", field_name, uval); 5367 } else if (key_field->flags & HIST_FIELD_FL_SYM) { 5368 uval = *(u64 *)(key + key_field->offset); 5369 seq_printf(m, "%s: [%llx] %-45ps", field_name, 5370 uval, (void *)(uintptr_t)uval); 5371 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) { 5372 uval = *(u64 *)(key + key_field->offset); 5373 seq_printf(m, "%s: [%llx] %-55pS", field_name, 5374 uval, (void *)(uintptr_t)uval); 5375 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 5376 struct hist_elt_data *elt_data = elt->private_data; 5377 char *comm; 5378 5379 if (WARN_ON_ONCE(!elt_data)) 5380 return; 5381 5382 comm = elt_data->comm; 5383 5384 uval = *(u64 *)(key + key_field->offset); 5385 seq_printf(m, "%s: %-16s[%10llu]", field_name, 5386 comm, uval); 5387 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) { 5388 const char *syscall_name; 5389 5390 uval = *(u64 *)(key + key_field->offset); 5391 syscall_name = get_syscall_name(uval); 5392 if (!syscall_name) 5393 syscall_name = "unknown_syscall"; 5394 5395 seq_printf(m, "%s: %-30s[%3llu]", field_name, 5396 syscall_name, uval); 5397 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 5398 if (key_field->field) 5399 seq_printf(m, "%s.stacktrace", key_field->field->name); 5400 else 5401 seq_puts(m, "common_stacktrace:\n"); 5402 hist_trigger_stacktrace_print(m, 5403 key + key_field->offset, 5404 HIST_STACKTRACE_DEPTH); 5405 multiline = true; 5406 } else if (key_field->flags & HIST_FIELD_FL_LOG2) { 5407 seq_printf(m, "%s: ~ 2^%-2llu", field_name, 5408 *(u64 *)(key + key_field->offset)); 5409 } else if (key_field->flags & HIST_FIELD_FL_BUCKET) { 5410 unsigned long buckets = key_field->buckets; 5411 uval = *(u64 *)(key + key_field->offset); 5412 seq_printf(m, "%s: ~ %llu-%llu", field_name, 5413 uval, uval + buckets -1); 5414 } else if (key_field->flags & HIST_FIELD_FL_STRING) { 5415 seq_printf(m, "%s: %-50s", field_name, 5416 (char *)(key + key_field->offset)); 5417 } else { 5418 uval = *(u64 *)(key + key_field->offset); 5419 seq_printf(m, "%s: %10llu", field_name, uval); 5420 } 5421 } 5422 5423 if (!multiline) 5424 seq_puts(m, " "); 5425 5426 seq_puts(m, "}"); 5427 } 5428 5429 /* Get the 100 times of the percentage of @val in @total */ __get_percentage(u64 val,u64 total)5430 static inline unsigned int __get_percentage(u64 val, u64 total) 5431 { 5432 if (!total) 5433 goto div0; 5434 5435 if (val < (U64_MAX / 10000)) 5436 return (unsigned int)div64_ul(val * 10000, total); 5437 5438 total = div64_u64(total, 10000); 5439 if (!total) 5440 goto div0; 5441 5442 return (unsigned int)div64_ul(val, total); 5443 div0: 5444 return val ? UINT_MAX : 0; 5445 } 5446 5447 #define BAR_CHAR '#' 5448 __fill_bar_str(char * buf,int size,u64 val,u64 max)5449 static inline const char *__fill_bar_str(char *buf, int size, u64 val, u64 max) 5450 { 5451 unsigned int len = __get_percentage(val, max); 5452 int i; 5453 5454 if (len == UINT_MAX) { 5455 snprintf(buf, size, "[ERROR]"); 5456 return buf; 5457 } 5458 5459 len = len * size / 10000; 5460 for (i = 0; i < len && i < size; i++) 5461 buf[i] = BAR_CHAR; 5462 while (i < size) 5463 buf[i++] = ' '; 5464 buf[size] = '\0'; 5465 5466 return buf; 5467 } 5468 5469 struct hist_val_stat { 5470 u64 max; 5471 u64 total; 5472 }; 5473 hist_trigger_print_val(struct seq_file * m,unsigned int idx,const char * field_name,unsigned long flags,struct hist_val_stat * stats,struct tracing_map_elt * elt)5474 static void hist_trigger_print_val(struct seq_file *m, unsigned int idx, 5475 const char *field_name, unsigned long flags, 5476 struct hist_val_stat *stats, 5477 struct tracing_map_elt *elt) 5478 { 5479 u64 val = tracing_map_read_sum(elt, idx); 5480 unsigned int pc; 5481 char bar[21]; 5482 5483 if (flags & HIST_FIELD_FL_PERCENT) { 5484 pc = __get_percentage(val, stats[idx].total); 5485 if (pc == UINT_MAX) 5486 seq_printf(m, " %s (%%):[ERROR]", field_name); 5487 else 5488 seq_printf(m, " %s (%%): %3u.%02u", field_name, 5489 pc / 100, pc % 100); 5490 } else if (flags & HIST_FIELD_FL_GRAPH) { 5491 seq_printf(m, " %s: %20s", field_name, 5492 __fill_bar_str(bar, 20, val, stats[idx].max)); 5493 } else if (flags & HIST_FIELD_FL_HEX) { 5494 seq_printf(m, " %s: %10llx", field_name, val); 5495 } else { 5496 seq_printf(m, " %s: %10llu", field_name, val); 5497 } 5498 } 5499 hist_trigger_entry_print(struct seq_file * m,struct hist_trigger_data * hist_data,struct hist_val_stat * stats,void * key,struct tracing_map_elt * elt)5500 static void hist_trigger_entry_print(struct seq_file *m, 5501 struct hist_trigger_data *hist_data, 5502 struct hist_val_stat *stats, 5503 void *key, 5504 struct tracing_map_elt *elt) 5505 { 5506 const char *field_name; 5507 unsigned int i = HITCOUNT_IDX; 5508 unsigned long flags; 5509 5510 hist_trigger_print_key(m, hist_data, key, elt); 5511 5512 /* At first, show the raw hitcount if !nohitcount */ 5513 if (!hist_data->attrs->no_hitcount) 5514 hist_trigger_print_val(m, i, "hitcount", 0, stats, elt); 5515 5516 for (i = 1; i < hist_data->n_vals; i++) { 5517 field_name = hist_field_name(hist_data->fields[i], 0); 5518 flags = hist_data->fields[i]->flags; 5519 if (flags & HIST_FIELD_FL_VAR || flags & HIST_FIELD_FL_EXPR) 5520 continue; 5521 5522 seq_puts(m, " "); 5523 hist_trigger_print_val(m, i, field_name, flags, stats, elt); 5524 } 5525 5526 print_actions(m, hist_data, elt); 5527 5528 seq_puts(m, "\n"); 5529 } 5530 print_entries(struct seq_file * m,struct hist_trigger_data * hist_data)5531 static int print_entries(struct seq_file *m, 5532 struct hist_trigger_data *hist_data) 5533 { 5534 struct tracing_map_sort_entry **sort_entries = NULL; 5535 struct tracing_map *map = hist_data->map; 5536 int i, j, n_entries; 5537 struct hist_val_stat *stats = NULL; 5538 u64 val; 5539 5540 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys, 5541 hist_data->n_sort_keys, 5542 &sort_entries); 5543 if (n_entries < 0) 5544 return n_entries; 5545 5546 /* Calculate the max and the total for each field if needed. */ 5547 for (j = 0; j < hist_data->n_vals; j++) { 5548 if (!(hist_data->fields[j]->flags & 5549 (HIST_FIELD_FL_PERCENT | HIST_FIELD_FL_GRAPH))) 5550 continue; 5551 if (!stats) { 5552 stats = kcalloc(hist_data->n_vals, sizeof(*stats), 5553 GFP_KERNEL); 5554 if (!stats) { 5555 n_entries = -ENOMEM; 5556 goto out; 5557 } 5558 } 5559 for (i = 0; i < n_entries; i++) { 5560 val = tracing_map_read_sum(sort_entries[i]->elt, j); 5561 stats[j].total += val; 5562 if (stats[j].max < val) 5563 stats[j].max = val; 5564 } 5565 } 5566 5567 for (i = 0; i < n_entries; i++) 5568 hist_trigger_entry_print(m, hist_data, stats, 5569 sort_entries[i]->key, 5570 sort_entries[i]->elt); 5571 5572 kfree(stats); 5573 out: 5574 tracing_map_destroy_sort_entries(sort_entries, n_entries); 5575 5576 return n_entries; 5577 } 5578 hist_trigger_show(struct seq_file * m,struct event_trigger_data * data,int n)5579 static void hist_trigger_show(struct seq_file *m, 5580 struct event_trigger_data *data, int n) 5581 { 5582 struct hist_trigger_data *hist_data; 5583 int n_entries; 5584 5585 if (n > 0) 5586 seq_puts(m, "\n\n"); 5587 5588 seq_puts(m, "# event histogram\n#\n# trigger info: "); 5589 data->ops->print(m, data); 5590 seq_puts(m, "#\n\n"); 5591 5592 hist_data = data->private_data; 5593 n_entries = print_entries(m, hist_data); 5594 if (n_entries < 0) 5595 n_entries = 0; 5596 5597 track_data_snapshot_print(m, hist_data); 5598 5599 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n", 5600 (u64)atomic64_read(&hist_data->map->hits), 5601 n_entries, (u64)atomic64_read(&hist_data->map->drops)); 5602 } 5603 hist_show(struct seq_file * m,void * v)5604 static int hist_show(struct seq_file *m, void *v) 5605 { 5606 struct event_trigger_data *data; 5607 struct trace_event_file *event_file; 5608 int n = 0, ret = 0; 5609 5610 mutex_lock(&event_mutex); 5611 5612 event_file = event_file_file(m->private); 5613 if (unlikely(!event_file)) { 5614 ret = -ENODEV; 5615 goto out_unlock; 5616 } 5617 5618 list_for_each_entry(data, &event_file->triggers, list) { 5619 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 5620 hist_trigger_show(m, data, n++); 5621 } 5622 5623 out_unlock: 5624 mutex_unlock(&event_mutex); 5625 5626 return ret; 5627 } 5628 event_hist_open(struct inode * inode,struct file * file)5629 static int event_hist_open(struct inode *inode, struct file *file) 5630 { 5631 int ret; 5632 5633 ret = tracing_open_file_tr(inode, file); 5634 if (ret) 5635 return ret; 5636 5637 /* Clear private_data to avoid warning in single_open() */ 5638 file->private_data = NULL; 5639 return single_open(file, hist_show, file); 5640 } 5641 5642 const struct file_operations event_hist_fops = { 5643 .open = event_hist_open, 5644 .read = seq_read, 5645 .llseek = seq_lseek, 5646 .release = tracing_single_release_file_tr, 5647 }; 5648 5649 #ifdef CONFIG_HIST_TRIGGERS_DEBUG hist_field_debug_show_flags(struct seq_file * m,unsigned long flags)5650 static void hist_field_debug_show_flags(struct seq_file *m, 5651 unsigned long flags) 5652 { 5653 seq_puts(m, " flags:\n"); 5654 5655 if (flags & HIST_FIELD_FL_KEY) 5656 seq_puts(m, " HIST_FIELD_FL_KEY\n"); 5657 else if (flags & HIST_FIELD_FL_HITCOUNT) 5658 seq_puts(m, " VAL: HIST_FIELD_FL_HITCOUNT\n"); 5659 else if (flags & HIST_FIELD_FL_VAR) 5660 seq_puts(m, " HIST_FIELD_FL_VAR\n"); 5661 else if (flags & HIST_FIELD_FL_VAR_REF) 5662 seq_puts(m, " HIST_FIELD_FL_VAR_REF\n"); 5663 else 5664 seq_puts(m, " VAL: normal u64 value\n"); 5665 5666 if (flags & HIST_FIELD_FL_ALIAS) 5667 seq_puts(m, " HIST_FIELD_FL_ALIAS\n"); 5668 else if (flags & HIST_FIELD_FL_CONST) 5669 seq_puts(m, " HIST_FIELD_FL_CONST\n"); 5670 } 5671 hist_field_debug_show(struct seq_file * m,struct hist_field * field,unsigned long flags)5672 static int hist_field_debug_show(struct seq_file *m, 5673 struct hist_field *field, unsigned long flags) 5674 { 5675 if ((field->flags & flags) != flags) { 5676 seq_printf(m, "ERROR: bad flags - %lx\n", flags); 5677 return -EINVAL; 5678 } 5679 5680 hist_field_debug_show_flags(m, field->flags); 5681 if (field->field) 5682 seq_printf(m, " ftrace_event_field name: %s\n", 5683 field->field->name); 5684 5685 if (field->flags & HIST_FIELD_FL_VAR) { 5686 seq_printf(m, " var.name: %s\n", field->var.name); 5687 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 5688 field->var.idx); 5689 } 5690 5691 if (field->flags & HIST_FIELD_FL_CONST) 5692 seq_printf(m, " constant: %llu\n", field->constant); 5693 5694 if (field->flags & HIST_FIELD_FL_ALIAS) 5695 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n", 5696 field->var_ref_idx); 5697 5698 if (field->flags & HIST_FIELD_FL_VAR_REF) { 5699 seq_printf(m, " name: %s\n", field->name); 5700 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 5701 field->var.idx); 5702 seq_printf(m, " var.hist_data: %p\n", field->var.hist_data); 5703 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n", 5704 field->var_ref_idx); 5705 if (field->system) 5706 seq_printf(m, " system: %s\n", field->system); 5707 if (field->event_name) 5708 seq_printf(m, " event_name: %s\n", field->event_name); 5709 } 5710 5711 seq_printf(m, " type: %s\n", field->type); 5712 seq_printf(m, " size: %u\n", field->size); 5713 seq_printf(m, " is_signed: %u\n", field->is_signed); 5714 5715 return 0; 5716 } 5717 field_var_debug_show(struct seq_file * m,struct field_var * field_var,unsigned int i,bool save_vars)5718 static int field_var_debug_show(struct seq_file *m, 5719 struct field_var *field_var, unsigned int i, 5720 bool save_vars) 5721 { 5722 const char *vars_name = save_vars ? "save_vars" : "field_vars"; 5723 struct hist_field *field; 5724 int ret = 0; 5725 5726 seq_printf(m, "\n hist_data->%s[%d]:\n", vars_name, i); 5727 5728 field = field_var->var; 5729 5730 seq_printf(m, "\n %s[%d].var:\n", vars_name, i); 5731 5732 hist_field_debug_show_flags(m, field->flags); 5733 seq_printf(m, " var.name: %s\n", field->var.name); 5734 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 5735 field->var.idx); 5736 5737 field = field_var->val; 5738 5739 seq_printf(m, "\n %s[%d].val:\n", vars_name, i); 5740 if (field->field) 5741 seq_printf(m, " ftrace_event_field name: %s\n", 5742 field->field->name); 5743 else { 5744 ret = -EINVAL; 5745 goto out; 5746 } 5747 5748 seq_printf(m, " type: %s\n", field->type); 5749 seq_printf(m, " size: %u\n", field->size); 5750 seq_printf(m, " is_signed: %u\n", field->is_signed); 5751 out: 5752 return ret; 5753 } 5754 hist_action_debug_show(struct seq_file * m,struct action_data * data,int i)5755 static int hist_action_debug_show(struct seq_file *m, 5756 struct action_data *data, int i) 5757 { 5758 int ret = 0; 5759 5760 if (data->handler == HANDLER_ONMAX || 5761 data->handler == HANDLER_ONCHANGE) { 5762 seq_printf(m, "\n hist_data->actions[%d].track_data.var_ref:\n", i); 5763 ret = hist_field_debug_show(m, data->track_data.var_ref, 5764 HIST_FIELD_FL_VAR_REF); 5765 if (ret) 5766 goto out; 5767 5768 seq_printf(m, "\n hist_data->actions[%d].track_data.track_var:\n", i); 5769 ret = hist_field_debug_show(m, data->track_data.track_var, 5770 HIST_FIELD_FL_VAR); 5771 if (ret) 5772 goto out; 5773 } 5774 5775 if (data->handler == HANDLER_ONMATCH) { 5776 seq_printf(m, "\n hist_data->actions[%d].match_data.event_system: %s\n", 5777 i, data->match_data.event_system); 5778 seq_printf(m, " hist_data->actions[%d].match_data.event: %s\n", 5779 i, data->match_data.event); 5780 } 5781 out: 5782 return ret; 5783 } 5784 hist_actions_debug_show(struct seq_file * m,struct hist_trigger_data * hist_data)5785 static int hist_actions_debug_show(struct seq_file *m, 5786 struct hist_trigger_data *hist_data) 5787 { 5788 int i, ret = 0; 5789 5790 if (hist_data->n_actions) 5791 seq_puts(m, "\n action tracking variables (for onmax()/onchange()/onmatch()):\n"); 5792 5793 for (i = 0; i < hist_data->n_actions; i++) { 5794 struct action_data *action = hist_data->actions[i]; 5795 5796 ret = hist_action_debug_show(m, action, i); 5797 if (ret) 5798 goto out; 5799 } 5800 5801 if (hist_data->n_save_vars) 5802 seq_puts(m, "\n save action variables (save() params):\n"); 5803 5804 for (i = 0; i < hist_data->n_save_vars; i++) { 5805 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true); 5806 if (ret) 5807 goto out; 5808 } 5809 out: 5810 return ret; 5811 } 5812 hist_trigger_debug_show(struct seq_file * m,struct event_trigger_data * data,int n)5813 static void hist_trigger_debug_show(struct seq_file *m, 5814 struct event_trigger_data *data, int n) 5815 { 5816 struct hist_trigger_data *hist_data; 5817 int i, ret; 5818 5819 if (n > 0) 5820 seq_puts(m, "\n\n"); 5821 5822 seq_puts(m, "# event histogram\n#\n# trigger info: "); 5823 data->ops->print(m, data); 5824 seq_puts(m, "#\n\n"); 5825 5826 hist_data = data->private_data; 5827 5828 seq_printf(m, "hist_data: %p\n\n", hist_data); 5829 seq_printf(m, " n_vals: %u\n", hist_data->n_vals); 5830 seq_printf(m, " n_keys: %u\n", hist_data->n_keys); 5831 seq_printf(m, " n_fields: %u\n", hist_data->n_fields); 5832 5833 seq_puts(m, "\n val fields:\n\n"); 5834 5835 seq_puts(m, " hist_data->fields[0]:\n"); 5836 ret = hist_field_debug_show(m, hist_data->fields[0], 5837 HIST_FIELD_FL_HITCOUNT); 5838 if (ret) 5839 return; 5840 5841 for (i = 1; i < hist_data->n_vals; i++) { 5842 seq_printf(m, "\n hist_data->fields[%d]:\n", i); 5843 ret = hist_field_debug_show(m, hist_data->fields[i], 0); 5844 if (ret) 5845 return; 5846 } 5847 5848 seq_puts(m, "\n key fields:\n"); 5849 5850 for (i = hist_data->n_vals; i < hist_data->n_fields; i++) { 5851 seq_printf(m, "\n hist_data->fields[%d]:\n", i); 5852 ret = hist_field_debug_show(m, hist_data->fields[i], 5853 HIST_FIELD_FL_KEY); 5854 if (ret) 5855 return; 5856 } 5857 5858 if (hist_data->n_var_refs) 5859 seq_puts(m, "\n variable reference fields:\n"); 5860 5861 for (i = 0; i < hist_data->n_var_refs; i++) { 5862 seq_printf(m, "\n hist_data->var_refs[%d]:\n", i); 5863 ret = hist_field_debug_show(m, hist_data->var_refs[i], 5864 HIST_FIELD_FL_VAR_REF); 5865 if (ret) 5866 return; 5867 } 5868 5869 if (hist_data->n_field_vars) 5870 seq_puts(m, "\n field variables:\n"); 5871 5872 for (i = 0; i < hist_data->n_field_vars; i++) { 5873 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false); 5874 if (ret) 5875 return; 5876 } 5877 5878 ret = hist_actions_debug_show(m, hist_data); 5879 if (ret) 5880 return; 5881 } 5882 hist_debug_show(struct seq_file * m,void * v)5883 static int hist_debug_show(struct seq_file *m, void *v) 5884 { 5885 struct event_trigger_data *data; 5886 struct trace_event_file *event_file; 5887 int n = 0, ret = 0; 5888 5889 mutex_lock(&event_mutex); 5890 5891 event_file = event_file_file(m->private); 5892 if (unlikely(!event_file)) { 5893 ret = -ENODEV; 5894 goto out_unlock; 5895 } 5896 5897 list_for_each_entry(data, &event_file->triggers, list) { 5898 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 5899 hist_trigger_debug_show(m, data, n++); 5900 } 5901 5902 out_unlock: 5903 mutex_unlock(&event_mutex); 5904 5905 return ret; 5906 } 5907 event_hist_debug_open(struct inode * inode,struct file * file)5908 static int event_hist_debug_open(struct inode *inode, struct file *file) 5909 { 5910 int ret; 5911 5912 ret = tracing_open_file_tr(inode, file); 5913 if (ret) 5914 return ret; 5915 5916 /* Clear private_data to avoid warning in single_open() */ 5917 file->private_data = NULL; 5918 return single_open(file, hist_debug_show, file); 5919 } 5920 5921 const struct file_operations event_hist_debug_fops = { 5922 .open = event_hist_debug_open, 5923 .read = seq_read, 5924 .llseek = seq_lseek, 5925 .release = tracing_single_release_file_tr, 5926 }; 5927 #endif 5928 hist_field_print(struct seq_file * m,struct hist_field * hist_field)5929 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field) 5930 { 5931 const char *field_name = hist_field_name(hist_field, 0); 5932 5933 if (hist_field->var.name) 5934 seq_printf(m, "%s=", hist_field->var.name); 5935 5936 if (hist_field->flags & HIST_FIELD_FL_CPU) 5937 seq_puts(m, "common_cpu"); 5938 else if (hist_field->flags & HIST_FIELD_FL_CONST) 5939 seq_printf(m, "%llu", hist_field->constant); 5940 else if (field_name) { 5941 if (hist_field->flags & HIST_FIELD_FL_VAR_REF || 5942 hist_field->flags & HIST_FIELD_FL_ALIAS) 5943 seq_putc(m, '$'); 5944 seq_printf(m, "%s", field_name); 5945 } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP) 5946 seq_puts(m, "common_timestamp"); 5947 5948 if (hist_field->flags) { 5949 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) && 5950 !(hist_field->flags & HIST_FIELD_FL_EXPR) && 5951 !(hist_field->flags & HIST_FIELD_FL_STACKTRACE)) { 5952 const char *flags = get_hist_field_flags(hist_field); 5953 5954 if (flags) 5955 seq_printf(m, ".%s", flags); 5956 } 5957 } 5958 if (hist_field->buckets) 5959 seq_printf(m, "=%ld", hist_field->buckets); 5960 } 5961 event_hist_trigger_print(struct seq_file * m,struct event_trigger_data * data)5962 static int event_hist_trigger_print(struct seq_file *m, 5963 struct event_trigger_data *data) 5964 { 5965 struct hist_trigger_data *hist_data = data->private_data; 5966 struct hist_field *field; 5967 bool have_var = false; 5968 bool show_val = false; 5969 unsigned int i; 5970 5971 seq_puts(m, HIST_PREFIX); 5972 5973 if (data->name) 5974 seq_printf(m, "%s:", data->name); 5975 5976 seq_puts(m, "keys="); 5977 5978 for_each_hist_key_field(i, hist_data) { 5979 field = hist_data->fields[i]; 5980 5981 if (i > hist_data->n_vals) 5982 seq_puts(m, ","); 5983 5984 if (field->flags & HIST_FIELD_FL_STACKTRACE) { 5985 if (field->field) 5986 seq_printf(m, "%s.stacktrace", field->field->name); 5987 else 5988 seq_puts(m, "common_stacktrace"); 5989 } else 5990 hist_field_print(m, field); 5991 } 5992 5993 seq_puts(m, ":vals="); 5994 5995 for_each_hist_val_field(i, hist_data) { 5996 field = hist_data->fields[i]; 5997 if (field->flags & HIST_FIELD_FL_VAR) { 5998 have_var = true; 5999 continue; 6000 } 6001 6002 if (i == HITCOUNT_IDX) { 6003 if (hist_data->attrs->no_hitcount) 6004 continue; 6005 seq_puts(m, "hitcount"); 6006 } else { 6007 if (show_val) 6008 seq_puts(m, ","); 6009 hist_field_print(m, field); 6010 } 6011 show_val = true; 6012 } 6013 6014 if (have_var) { 6015 unsigned int n = 0; 6016 6017 seq_puts(m, ":"); 6018 6019 for_each_hist_val_field(i, hist_data) { 6020 field = hist_data->fields[i]; 6021 6022 if (field->flags & HIST_FIELD_FL_VAR) { 6023 if (n++) 6024 seq_puts(m, ","); 6025 hist_field_print(m, field); 6026 } 6027 } 6028 } 6029 6030 seq_puts(m, ":sort="); 6031 6032 for (i = 0; i < hist_data->n_sort_keys; i++) { 6033 struct tracing_map_sort_key *sort_key; 6034 unsigned int idx, first_key_idx; 6035 6036 /* skip VAR vals */ 6037 first_key_idx = hist_data->n_vals - hist_data->n_vars; 6038 6039 sort_key = &hist_data->sort_keys[i]; 6040 idx = sort_key->field_idx; 6041 6042 if (WARN_ON(idx >= HIST_FIELDS_MAX)) 6043 return -EINVAL; 6044 6045 if (i > 0) 6046 seq_puts(m, ","); 6047 6048 if (idx == HITCOUNT_IDX) 6049 seq_puts(m, "hitcount"); 6050 else { 6051 if (idx >= first_key_idx) 6052 idx += hist_data->n_vars; 6053 hist_field_print(m, hist_data->fields[idx]); 6054 } 6055 6056 if (sort_key->descending) 6057 seq_puts(m, ".descending"); 6058 } 6059 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits)); 6060 if (hist_data->enable_timestamps) 6061 seq_printf(m, ":clock=%s", hist_data->attrs->clock); 6062 if (hist_data->attrs->no_hitcount) 6063 seq_puts(m, ":nohitcount"); 6064 6065 print_actions_spec(m, hist_data); 6066 6067 if (data->filter_str) 6068 seq_printf(m, " if %s", data->filter_str); 6069 6070 if (data->paused) 6071 seq_puts(m, " [paused]"); 6072 else 6073 seq_puts(m, " [active]"); 6074 6075 seq_putc(m, '\n'); 6076 6077 return 0; 6078 } 6079 event_hist_trigger_init(struct event_trigger_data * data)6080 static int event_hist_trigger_init(struct event_trigger_data *data) 6081 { 6082 struct hist_trigger_data *hist_data = data->private_data; 6083 6084 if (!data->ref && hist_data->attrs->name) 6085 save_named_trigger(hist_data->attrs->name, data); 6086 6087 data->ref++; 6088 6089 return 0; 6090 } 6091 unregister_field_var_hists(struct hist_trigger_data * hist_data)6092 static void unregister_field_var_hists(struct hist_trigger_data *hist_data) 6093 { 6094 struct trace_event_file *file; 6095 unsigned int i; 6096 char *cmd; 6097 int ret; 6098 6099 for (i = 0; i < hist_data->n_field_var_hists; i++) { 6100 file = hist_data->field_var_hists[i]->hist_data->event_file; 6101 cmd = hist_data->field_var_hists[i]->cmd; 6102 ret = event_hist_trigger_parse(&trigger_hist_cmd, file, 6103 "!hist", "hist", cmd); 6104 WARN_ON_ONCE(ret < 0); 6105 } 6106 } 6107 event_hist_trigger_free(struct event_trigger_data * data)6108 static void event_hist_trigger_free(struct event_trigger_data *data) 6109 { 6110 struct hist_trigger_data *hist_data = data->private_data; 6111 6112 if (WARN_ON_ONCE(data->ref <= 0)) 6113 return; 6114 6115 data->ref--; 6116 if (!data->ref) { 6117 if (data->name) 6118 del_named_trigger(data); 6119 6120 trigger_data_free(data); 6121 6122 remove_hist_vars(hist_data); 6123 6124 unregister_field_var_hists(hist_data); 6125 6126 destroy_hist_data(hist_data); 6127 } 6128 } 6129 6130 static struct event_trigger_ops event_hist_trigger_ops = { 6131 .trigger = event_hist_trigger, 6132 .print = event_hist_trigger_print, 6133 .init = event_hist_trigger_init, 6134 .free = event_hist_trigger_free, 6135 }; 6136 event_hist_trigger_named_init(struct event_trigger_data * data)6137 static int event_hist_trigger_named_init(struct event_trigger_data *data) 6138 { 6139 data->ref++; 6140 6141 save_named_trigger(data->named_data->name, data); 6142 6143 event_hist_trigger_init(data->named_data); 6144 6145 return 0; 6146 } 6147 event_hist_trigger_named_free(struct event_trigger_data * data)6148 static void event_hist_trigger_named_free(struct event_trigger_data *data) 6149 { 6150 if (WARN_ON_ONCE(data->ref <= 0)) 6151 return; 6152 6153 event_hist_trigger_free(data->named_data); 6154 6155 data->ref--; 6156 if (!data->ref) { 6157 del_named_trigger(data); 6158 trigger_data_free(data); 6159 } 6160 } 6161 6162 static struct event_trigger_ops event_hist_trigger_named_ops = { 6163 .trigger = event_hist_trigger, 6164 .print = event_hist_trigger_print, 6165 .init = event_hist_trigger_named_init, 6166 .free = event_hist_trigger_named_free, 6167 }; 6168 event_hist_get_trigger_ops(char * cmd,char * param)6169 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd, 6170 char *param) 6171 { 6172 return &event_hist_trigger_ops; 6173 } 6174 hist_clear(struct event_trigger_data * data)6175 static void hist_clear(struct event_trigger_data *data) 6176 { 6177 struct hist_trigger_data *hist_data = data->private_data; 6178 6179 if (data->name) 6180 pause_named_trigger(data); 6181 6182 tracepoint_synchronize_unregister(); 6183 6184 tracing_map_clear(hist_data->map); 6185 6186 if (data->name) 6187 unpause_named_trigger(data); 6188 } 6189 compatible_field(struct ftrace_event_field * field,struct ftrace_event_field * test_field)6190 static bool compatible_field(struct ftrace_event_field *field, 6191 struct ftrace_event_field *test_field) 6192 { 6193 if (field == test_field) 6194 return true; 6195 if (field == NULL || test_field == NULL) 6196 return false; 6197 if (strcmp(field->name, test_field->name) != 0) 6198 return false; 6199 if (strcmp(field->type, test_field->type) != 0) 6200 return false; 6201 if (field->size != test_field->size) 6202 return false; 6203 if (field->is_signed != test_field->is_signed) 6204 return false; 6205 6206 return true; 6207 } 6208 hist_trigger_match(struct event_trigger_data * data,struct event_trigger_data * data_test,struct event_trigger_data * named_data,bool ignore_filter)6209 static bool hist_trigger_match(struct event_trigger_data *data, 6210 struct event_trigger_data *data_test, 6211 struct event_trigger_data *named_data, 6212 bool ignore_filter) 6213 { 6214 struct tracing_map_sort_key *sort_key, *sort_key_test; 6215 struct hist_trigger_data *hist_data, *hist_data_test; 6216 struct hist_field *key_field, *key_field_test; 6217 unsigned int i; 6218 6219 if (named_data && (named_data != data_test) && 6220 (named_data != data_test->named_data)) 6221 return false; 6222 6223 if (!named_data && is_named_trigger(data_test)) 6224 return false; 6225 6226 hist_data = data->private_data; 6227 hist_data_test = data_test->private_data; 6228 6229 if (hist_data->n_vals != hist_data_test->n_vals || 6230 hist_data->n_fields != hist_data_test->n_fields || 6231 hist_data->n_sort_keys != hist_data_test->n_sort_keys) 6232 return false; 6233 6234 if (!ignore_filter) { 6235 if ((data->filter_str && !data_test->filter_str) || 6236 (!data->filter_str && data_test->filter_str)) 6237 return false; 6238 } 6239 6240 for_each_hist_field(i, hist_data) { 6241 key_field = hist_data->fields[i]; 6242 key_field_test = hist_data_test->fields[i]; 6243 6244 if (key_field->flags != key_field_test->flags) 6245 return false; 6246 if (!compatible_field(key_field->field, key_field_test->field)) 6247 return false; 6248 if (key_field->offset != key_field_test->offset) 6249 return false; 6250 if (key_field->size != key_field_test->size) 6251 return false; 6252 if (key_field->is_signed != key_field_test->is_signed) 6253 return false; 6254 if (!!key_field->var.name != !!key_field_test->var.name) 6255 return false; 6256 if (key_field->var.name && 6257 strcmp(key_field->var.name, key_field_test->var.name) != 0) 6258 return false; 6259 } 6260 6261 for (i = 0; i < hist_data->n_sort_keys; i++) { 6262 sort_key = &hist_data->sort_keys[i]; 6263 sort_key_test = &hist_data_test->sort_keys[i]; 6264 6265 if (sort_key->field_idx != sort_key_test->field_idx || 6266 sort_key->descending != sort_key_test->descending) 6267 return false; 6268 } 6269 6270 if (!ignore_filter && data->filter_str && 6271 (strcmp(data->filter_str, data_test->filter_str) != 0)) 6272 return false; 6273 6274 if (!actions_match(hist_data, hist_data_test)) 6275 return false; 6276 6277 return true; 6278 } 6279 existing_hist_update_only(char * glob,struct event_trigger_data * data,struct trace_event_file * file)6280 static bool existing_hist_update_only(char *glob, 6281 struct event_trigger_data *data, 6282 struct trace_event_file *file) 6283 { 6284 struct hist_trigger_data *hist_data = data->private_data; 6285 struct event_trigger_data *test, *named_data = NULL; 6286 bool updated = false; 6287 6288 if (!hist_data->attrs->pause && !hist_data->attrs->cont && 6289 !hist_data->attrs->clear) 6290 goto out; 6291 6292 if (hist_data->attrs->name) { 6293 named_data = find_named_trigger(hist_data->attrs->name); 6294 if (named_data) { 6295 if (!hist_trigger_match(data, named_data, named_data, 6296 true)) 6297 goto out; 6298 } 6299 } 6300 6301 if (hist_data->attrs->name && !named_data) 6302 goto out; 6303 6304 list_for_each_entry(test, &file->triggers, list) { 6305 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6306 if (!hist_trigger_match(data, test, named_data, false)) 6307 continue; 6308 if (hist_data->attrs->pause) 6309 test->paused = true; 6310 else if (hist_data->attrs->cont) 6311 test->paused = false; 6312 else if (hist_data->attrs->clear) 6313 hist_clear(test); 6314 updated = true; 6315 goto out; 6316 } 6317 } 6318 out: 6319 return updated; 6320 } 6321 hist_register_trigger(char * glob,struct event_trigger_data * data,struct trace_event_file * file)6322 static int hist_register_trigger(char *glob, 6323 struct event_trigger_data *data, 6324 struct trace_event_file *file) 6325 { 6326 struct hist_trigger_data *hist_data = data->private_data; 6327 struct event_trigger_data *test, *named_data = NULL; 6328 struct trace_array *tr = file->tr; 6329 int ret = 0; 6330 6331 if (hist_data->attrs->name) { 6332 named_data = find_named_trigger(hist_data->attrs->name); 6333 if (named_data) { 6334 if (!hist_trigger_match(data, named_data, named_data, 6335 true)) { 6336 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name)); 6337 ret = -EINVAL; 6338 goto out; 6339 } 6340 } 6341 } 6342 6343 if (hist_data->attrs->name && !named_data) 6344 goto new; 6345 6346 lockdep_assert_held(&event_mutex); 6347 6348 list_for_each_entry(test, &file->triggers, list) { 6349 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6350 if (hist_trigger_match(data, test, named_data, false)) { 6351 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0); 6352 ret = -EEXIST; 6353 goto out; 6354 } 6355 } 6356 } 6357 new: 6358 if (hist_data->attrs->cont || hist_data->attrs->clear) { 6359 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0); 6360 ret = -ENOENT; 6361 goto out; 6362 } 6363 6364 if (hist_data->attrs->pause) 6365 data->paused = true; 6366 6367 if (named_data) { 6368 data->private_data = named_data->private_data; 6369 set_named_trigger_data(data, named_data); 6370 data->ops = &event_hist_trigger_named_ops; 6371 } 6372 6373 if (data->ops->init) { 6374 ret = data->ops->init(data); 6375 if (ret < 0) 6376 goto out; 6377 } 6378 6379 if (hist_data->enable_timestamps) { 6380 char *clock = hist_data->attrs->clock; 6381 6382 ret = tracing_set_clock(file->tr, hist_data->attrs->clock); 6383 if (ret) { 6384 hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock)); 6385 goto out; 6386 } 6387 6388 tracing_set_filter_buffering(file->tr, true); 6389 } 6390 6391 if (named_data) 6392 destroy_hist_data(hist_data); 6393 out: 6394 return ret; 6395 } 6396 hist_trigger_enable(struct event_trigger_data * data,struct trace_event_file * file)6397 static int hist_trigger_enable(struct event_trigger_data *data, 6398 struct trace_event_file *file) 6399 { 6400 int ret = 0; 6401 6402 list_add_tail_rcu(&data->list, &file->triggers); 6403 6404 update_cond_flag(file); 6405 6406 if (trace_event_trigger_enable_disable(file, 1) < 0) { 6407 list_del_rcu(&data->list); 6408 update_cond_flag(file); 6409 ret--; 6410 } 6411 6412 return ret; 6413 } 6414 have_hist_trigger_match(struct event_trigger_data * data,struct trace_event_file * file)6415 static bool have_hist_trigger_match(struct event_trigger_data *data, 6416 struct trace_event_file *file) 6417 { 6418 struct hist_trigger_data *hist_data = data->private_data; 6419 struct event_trigger_data *test, *named_data = NULL; 6420 bool match = false; 6421 6422 lockdep_assert_held(&event_mutex); 6423 6424 if (hist_data->attrs->name) 6425 named_data = find_named_trigger(hist_data->attrs->name); 6426 6427 list_for_each_entry(test, &file->triggers, list) { 6428 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6429 if (hist_trigger_match(data, test, named_data, false)) { 6430 match = true; 6431 break; 6432 } 6433 } 6434 } 6435 6436 return match; 6437 } 6438 hist_trigger_check_refs(struct event_trigger_data * data,struct trace_event_file * file)6439 static bool hist_trigger_check_refs(struct event_trigger_data *data, 6440 struct trace_event_file *file) 6441 { 6442 struct hist_trigger_data *hist_data = data->private_data; 6443 struct event_trigger_data *test, *named_data = NULL; 6444 6445 lockdep_assert_held(&event_mutex); 6446 6447 if (hist_data->attrs->name) 6448 named_data = find_named_trigger(hist_data->attrs->name); 6449 6450 list_for_each_entry(test, &file->triggers, list) { 6451 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6452 if (!hist_trigger_match(data, test, named_data, false)) 6453 continue; 6454 hist_data = test->private_data; 6455 if (check_var_refs(hist_data)) 6456 return true; 6457 break; 6458 } 6459 } 6460 6461 return false; 6462 } 6463 hist_unregister_trigger(char * glob,struct event_trigger_data * data,struct trace_event_file * file)6464 static void hist_unregister_trigger(char *glob, 6465 struct event_trigger_data *data, 6466 struct trace_event_file *file) 6467 { 6468 struct event_trigger_data *test = NULL, *iter, *named_data = NULL; 6469 struct hist_trigger_data *hist_data = data->private_data; 6470 6471 lockdep_assert_held(&event_mutex); 6472 6473 if (hist_data->attrs->name) 6474 named_data = find_named_trigger(hist_data->attrs->name); 6475 6476 list_for_each_entry(iter, &file->triggers, list) { 6477 if (iter->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6478 if (!hist_trigger_match(data, iter, named_data, false)) 6479 continue; 6480 test = iter; 6481 list_del_rcu(&test->list); 6482 trace_event_trigger_enable_disable(file, 0); 6483 update_cond_flag(file); 6484 break; 6485 } 6486 } 6487 6488 if (test && test->ops->free) 6489 test->ops->free(test); 6490 6491 if (hist_data->enable_timestamps) { 6492 if (!hist_data->remove || test) 6493 tracing_set_filter_buffering(file->tr, false); 6494 } 6495 } 6496 hist_file_check_refs(struct trace_event_file * file)6497 static bool hist_file_check_refs(struct trace_event_file *file) 6498 { 6499 struct hist_trigger_data *hist_data; 6500 struct event_trigger_data *test; 6501 6502 lockdep_assert_held(&event_mutex); 6503 6504 list_for_each_entry(test, &file->triggers, list) { 6505 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6506 hist_data = test->private_data; 6507 if (check_var_refs(hist_data)) 6508 return true; 6509 } 6510 } 6511 6512 return false; 6513 } 6514 hist_unreg_all(struct trace_event_file * file)6515 static void hist_unreg_all(struct trace_event_file *file) 6516 { 6517 struct event_trigger_data *test, *n; 6518 struct hist_trigger_data *hist_data; 6519 struct synth_event *se; 6520 const char *se_name; 6521 6522 lockdep_assert_held(&event_mutex); 6523 6524 if (hist_file_check_refs(file)) 6525 return; 6526 6527 list_for_each_entry_safe(test, n, &file->triggers, list) { 6528 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6529 hist_data = test->private_data; 6530 list_del_rcu(&test->list); 6531 trace_event_trigger_enable_disable(file, 0); 6532 6533 se_name = trace_event_name(file->event_call); 6534 se = find_synth_event(se_name); 6535 if (se) 6536 se->ref--; 6537 6538 update_cond_flag(file); 6539 if (hist_data->enable_timestamps) 6540 tracing_set_filter_buffering(file->tr, false); 6541 if (test->ops->free) 6542 test->ops->free(test); 6543 } 6544 } 6545 } 6546 event_hist_trigger_parse(struct event_command * cmd_ops,struct trace_event_file * file,char * glob,char * cmd,char * param_and_filter)6547 static int event_hist_trigger_parse(struct event_command *cmd_ops, 6548 struct trace_event_file *file, 6549 char *glob, char *cmd, 6550 char *param_and_filter) 6551 { 6552 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT; 6553 struct event_trigger_data *trigger_data; 6554 struct hist_trigger_attrs *attrs; 6555 struct hist_trigger_data *hist_data; 6556 char *param, *filter, *p, *start; 6557 struct synth_event *se; 6558 const char *se_name; 6559 bool remove; 6560 int ret = 0; 6561 6562 lockdep_assert_held(&event_mutex); 6563 6564 if (WARN_ON(!glob)) 6565 return -EINVAL; 6566 6567 if (glob[0]) { 6568 hist_err_clear(); 6569 last_cmd_set(file, param_and_filter); 6570 } 6571 6572 remove = event_trigger_check_remove(glob); 6573 6574 if (event_trigger_empty_param(param_and_filter)) 6575 return -EINVAL; 6576 6577 /* 6578 * separate the trigger from the filter (k:v [if filter]) 6579 * allowing for whitespace in the trigger 6580 */ 6581 p = param = param_and_filter; 6582 do { 6583 p = strstr(p, "if"); 6584 if (!p) 6585 break; 6586 if (p == param_and_filter) 6587 return -EINVAL; 6588 if (*(p - 1) != ' ' && *(p - 1) != '\t') { 6589 p++; 6590 continue; 6591 } 6592 if (p >= param_and_filter + strlen(param_and_filter) - (sizeof("if") - 1) - 1) 6593 return -EINVAL; 6594 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') { 6595 p++; 6596 continue; 6597 } 6598 break; 6599 } while (1); 6600 6601 if (!p) 6602 filter = NULL; 6603 else { 6604 *(p - 1) = '\0'; 6605 filter = strstrip(p); 6606 param = strstrip(param); 6607 } 6608 6609 /* 6610 * To simplify arithmetic expression parsing, replace occurrences of 6611 * '.sym-offset' modifier with '.symXoffset' 6612 */ 6613 start = strstr(param, ".sym-offset"); 6614 while (start) { 6615 *(start + 4) = 'X'; 6616 start = strstr(start + 11, ".sym-offset"); 6617 } 6618 6619 attrs = parse_hist_trigger_attrs(file->tr, param); 6620 if (IS_ERR(attrs)) 6621 return PTR_ERR(attrs); 6622 6623 if (attrs->map_bits) 6624 hist_trigger_bits = attrs->map_bits; 6625 6626 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove); 6627 if (IS_ERR(hist_data)) { 6628 destroy_hist_trigger_attrs(attrs); 6629 return PTR_ERR(hist_data); 6630 } 6631 6632 trigger_data = event_trigger_alloc(cmd_ops, cmd, param, hist_data); 6633 if (!trigger_data) { 6634 ret = -ENOMEM; 6635 goto out_free; 6636 } 6637 6638 ret = event_trigger_set_filter(cmd_ops, file, filter, trigger_data); 6639 if (ret < 0) 6640 goto out_free; 6641 6642 if (remove) { 6643 if (!have_hist_trigger_match(trigger_data, file)) 6644 goto out_free; 6645 6646 if (hist_trigger_check_refs(trigger_data, file)) { 6647 ret = -EBUSY; 6648 goto out_free; 6649 } 6650 6651 event_trigger_unregister(cmd_ops, file, glob+1, trigger_data); 6652 se_name = trace_event_name(file->event_call); 6653 se = find_synth_event(se_name); 6654 if (se) 6655 se->ref--; 6656 ret = 0; 6657 goto out_free; 6658 } 6659 6660 if (existing_hist_update_only(glob, trigger_data, file)) 6661 goto out_free; 6662 6663 if (!get_named_trigger_data(trigger_data)) { 6664 6665 ret = create_actions(hist_data); 6666 if (ret) 6667 goto out_free; 6668 6669 if (has_hist_vars(hist_data) || hist_data->n_var_refs) { 6670 ret = save_hist_vars(hist_data); 6671 if (ret) 6672 goto out_free; 6673 } 6674 6675 ret = tracing_map_init(hist_data->map); 6676 if (ret) 6677 goto out_free; 6678 } 6679 6680 ret = event_trigger_register(cmd_ops, file, glob, trigger_data); 6681 if (ret < 0) 6682 goto out_free; 6683 6684 ret = hist_trigger_enable(trigger_data, file); 6685 if (ret) 6686 goto out_unreg; 6687 6688 se_name = trace_event_name(file->event_call); 6689 se = find_synth_event(se_name); 6690 if (se) 6691 se->ref++; 6692 out: 6693 if (ret == 0 && glob[0]) 6694 hist_err_clear(); 6695 6696 return ret; 6697 out_unreg: 6698 event_trigger_unregister(cmd_ops, file, glob+1, trigger_data); 6699 out_free: 6700 event_trigger_reset_filter(cmd_ops, trigger_data); 6701 6702 remove_hist_vars(hist_data); 6703 6704 kfree(trigger_data); 6705 6706 destroy_hist_data(hist_data); 6707 goto out; 6708 } 6709 6710 static struct event_command trigger_hist_cmd = { 6711 .name = "hist", 6712 .trigger_type = ETT_EVENT_HIST, 6713 .flags = EVENT_CMD_FL_NEEDS_REC, 6714 .parse = event_hist_trigger_parse, 6715 .reg = hist_register_trigger, 6716 .unreg = hist_unregister_trigger, 6717 .unreg_all = hist_unreg_all, 6718 .get_trigger_ops = event_hist_get_trigger_ops, 6719 .set_filter = set_trigger_filter, 6720 }; 6721 register_trigger_hist_cmd(void)6722 __init int register_trigger_hist_cmd(void) 6723 { 6724 int ret; 6725 6726 ret = register_event_command(&trigger_hist_cmd); 6727 WARN_ON(ret < 0); 6728 6729 return ret; 6730 } 6731 6732 static void hist_enable_trigger(struct event_trigger_data * data,struct trace_buffer * buffer,void * rec,struct ring_buffer_event * event)6733 hist_enable_trigger(struct event_trigger_data *data, 6734 struct trace_buffer *buffer, void *rec, 6735 struct ring_buffer_event *event) 6736 { 6737 struct enable_trigger_data *enable_data = data->private_data; 6738 struct event_trigger_data *test; 6739 6740 list_for_each_entry_rcu(test, &enable_data->file->triggers, list, 6741 lockdep_is_held(&event_mutex)) { 6742 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6743 if (enable_data->enable) 6744 test->paused = false; 6745 else 6746 test->paused = true; 6747 } 6748 } 6749 } 6750 6751 static void hist_enable_count_trigger(struct event_trigger_data * data,struct trace_buffer * buffer,void * rec,struct ring_buffer_event * event)6752 hist_enable_count_trigger(struct event_trigger_data *data, 6753 struct trace_buffer *buffer, void *rec, 6754 struct ring_buffer_event *event) 6755 { 6756 if (!data->count) 6757 return; 6758 6759 if (data->count != -1) 6760 (data->count)--; 6761 6762 hist_enable_trigger(data, buffer, rec, event); 6763 } 6764 6765 static struct event_trigger_ops hist_enable_trigger_ops = { 6766 .trigger = hist_enable_trigger, 6767 .print = event_enable_trigger_print, 6768 .init = event_trigger_init, 6769 .free = event_enable_trigger_free, 6770 }; 6771 6772 static struct event_trigger_ops hist_enable_count_trigger_ops = { 6773 .trigger = hist_enable_count_trigger, 6774 .print = event_enable_trigger_print, 6775 .init = event_trigger_init, 6776 .free = event_enable_trigger_free, 6777 }; 6778 6779 static struct event_trigger_ops hist_disable_trigger_ops = { 6780 .trigger = hist_enable_trigger, 6781 .print = event_enable_trigger_print, 6782 .init = event_trigger_init, 6783 .free = event_enable_trigger_free, 6784 }; 6785 6786 static struct event_trigger_ops hist_disable_count_trigger_ops = { 6787 .trigger = hist_enable_count_trigger, 6788 .print = event_enable_trigger_print, 6789 .init = event_trigger_init, 6790 .free = event_enable_trigger_free, 6791 }; 6792 6793 static struct event_trigger_ops * hist_enable_get_trigger_ops(char * cmd,char * param)6794 hist_enable_get_trigger_ops(char *cmd, char *param) 6795 { 6796 struct event_trigger_ops *ops; 6797 bool enable; 6798 6799 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0); 6800 6801 if (enable) 6802 ops = param ? &hist_enable_count_trigger_ops : 6803 &hist_enable_trigger_ops; 6804 else 6805 ops = param ? &hist_disable_count_trigger_ops : 6806 &hist_disable_trigger_ops; 6807 6808 return ops; 6809 } 6810 hist_enable_unreg_all(struct trace_event_file * file)6811 static void hist_enable_unreg_all(struct trace_event_file *file) 6812 { 6813 struct event_trigger_data *test, *n; 6814 6815 list_for_each_entry_safe(test, n, &file->triggers, list) { 6816 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) { 6817 list_del_rcu(&test->list); 6818 update_cond_flag(file); 6819 trace_event_trigger_enable_disable(file, 0); 6820 if (test->ops->free) 6821 test->ops->free(test); 6822 } 6823 } 6824 } 6825 6826 static struct event_command trigger_hist_enable_cmd = { 6827 .name = ENABLE_HIST_STR, 6828 .trigger_type = ETT_HIST_ENABLE, 6829 .parse = event_enable_trigger_parse, 6830 .reg = event_enable_register_trigger, 6831 .unreg = event_enable_unregister_trigger, 6832 .unreg_all = hist_enable_unreg_all, 6833 .get_trigger_ops = hist_enable_get_trigger_ops, 6834 .set_filter = set_trigger_filter, 6835 }; 6836 6837 static struct event_command trigger_hist_disable_cmd = { 6838 .name = DISABLE_HIST_STR, 6839 .trigger_type = ETT_HIST_ENABLE, 6840 .parse = event_enable_trigger_parse, 6841 .reg = event_enable_register_trigger, 6842 .unreg = event_enable_unregister_trigger, 6843 .unreg_all = hist_enable_unreg_all, 6844 .get_trigger_ops = hist_enable_get_trigger_ops, 6845 .set_filter = set_trigger_filter, 6846 }; 6847 unregister_trigger_hist_enable_disable_cmds(void)6848 static __init void unregister_trigger_hist_enable_disable_cmds(void) 6849 { 6850 unregister_event_command(&trigger_hist_enable_cmd); 6851 unregister_event_command(&trigger_hist_disable_cmd); 6852 } 6853 register_trigger_hist_enable_disable_cmds(void)6854 __init int register_trigger_hist_enable_disable_cmds(void) 6855 { 6856 int ret; 6857 6858 ret = register_event_command(&trigger_hist_enable_cmd); 6859 if (WARN_ON(ret < 0)) 6860 return ret; 6861 ret = register_event_command(&trigger_hist_disable_cmd); 6862 if (WARN_ON(ret < 0)) 6863 unregister_trigger_hist_enable_disable_cmds(); 6864 6865 return ret; 6866 } 6867