1 /* 2 * trace_events_hist - trace event hist triggers 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com> 15 */ 16 17 #include <linux/module.h> 18 #include <linux/kallsyms.h> 19 #include <linux/mutex.h> 20 #include <linux/slab.h> 21 #include <linux/stacktrace.h> 22 #include <linux/rculist.h> 23 24 #include "tracing_map.h" 25 #include "trace.h" 26 27 struct hist_field; 28 29 typedef u64 (*hist_field_fn_t) (struct hist_field *field, void *event); 30 31 #define HIST_FIELD_OPERANDS_MAX 2 32 33 struct hist_field { 34 struct ftrace_event_field *field; 35 unsigned long flags; 36 hist_field_fn_t fn; 37 unsigned int size; 38 unsigned int offset; 39 unsigned int is_signed; 40 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX]; 41 }; 42 43 static u64 hist_field_none(struct hist_field *field, void *event) 44 { 45 return 0; 46 } 47 48 static u64 hist_field_counter(struct hist_field *field, void *event) 49 { 50 return 1; 51 } 52 53 static u64 hist_field_string(struct hist_field *hist_field, void *event) 54 { 55 char *addr = (char *)(event + hist_field->field->offset); 56 57 return (u64)(unsigned long)addr; 58 } 59 60 static u64 hist_field_dynstring(struct hist_field *hist_field, void *event) 61 { 62 u32 str_item = *(u32 *)(event + hist_field->field->offset); 63 int str_loc = str_item & 0xffff; 64 char *addr = (char *)(event + str_loc); 65 66 return (u64)(unsigned long)addr; 67 } 68 69 static u64 hist_field_pstring(struct hist_field *hist_field, void *event) 70 { 71 char **addr = (char **)(event + hist_field->field->offset); 72 73 return (u64)(unsigned long)*addr; 74 } 75 76 static u64 hist_field_log2(struct hist_field *hist_field, void *event) 77 { 78 struct hist_field *operand = hist_field->operands[0]; 79 80 u64 val = operand->fn(operand, event); 81 82 return (u64) ilog2(roundup_pow_of_two(val)); 83 } 84 85 #define DEFINE_HIST_FIELD_FN(type) \ 86 static u64 hist_field_##type(struct hist_field *hist_field, void *event)\ 87 { \ 88 type *addr = (type *)(event + hist_field->field->offset); \ 89 \ 90 return (u64)(unsigned long)*addr; \ 91 } 92 93 DEFINE_HIST_FIELD_FN(s64); 94 DEFINE_HIST_FIELD_FN(u64); 95 DEFINE_HIST_FIELD_FN(s32); 96 DEFINE_HIST_FIELD_FN(u32); 97 DEFINE_HIST_FIELD_FN(s16); 98 DEFINE_HIST_FIELD_FN(u16); 99 DEFINE_HIST_FIELD_FN(s8); 100 DEFINE_HIST_FIELD_FN(u8); 101 102 #define for_each_hist_field(i, hist_data) \ 103 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++) 104 105 #define for_each_hist_val_field(i, hist_data) \ 106 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++) 107 108 #define for_each_hist_key_field(i, hist_data) \ 109 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++) 110 111 #define HIST_STACKTRACE_DEPTH 16 112 #define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long)) 113 #define HIST_STACKTRACE_SKIP 5 114 115 #define HITCOUNT_IDX 0 116 #define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE) 117 118 enum hist_field_flags { 119 HIST_FIELD_FL_HITCOUNT = 1 << 0, 120 HIST_FIELD_FL_KEY = 1 << 1, 121 HIST_FIELD_FL_STRING = 1 << 2, 122 HIST_FIELD_FL_HEX = 1 << 3, 123 HIST_FIELD_FL_SYM = 1 << 4, 124 HIST_FIELD_FL_SYM_OFFSET = 1 << 5, 125 HIST_FIELD_FL_EXECNAME = 1 << 6, 126 HIST_FIELD_FL_SYSCALL = 1 << 7, 127 HIST_FIELD_FL_STACKTRACE = 1 << 8, 128 HIST_FIELD_FL_LOG2 = 1 << 9, 129 }; 130 131 struct hist_trigger_attrs { 132 char *keys_str; 133 char *vals_str; 134 char *sort_key_str; 135 char *name; 136 bool pause; 137 bool cont; 138 bool clear; 139 unsigned int map_bits; 140 }; 141 142 struct hist_trigger_data { 143 struct hist_field *fields[TRACING_MAP_FIELDS_MAX]; 144 unsigned int n_vals; 145 unsigned int n_keys; 146 unsigned int n_fields; 147 unsigned int key_size; 148 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX]; 149 unsigned int n_sort_keys; 150 struct trace_event_file *event_file; 151 struct hist_trigger_attrs *attrs; 152 struct tracing_map *map; 153 }; 154 155 static const char *hist_field_name(struct hist_field *field, 156 unsigned int level) 157 { 158 const char *field_name = ""; 159 160 if (level > 1) 161 return field_name; 162 163 if (field->field) 164 field_name = field->field->name; 165 else if (field->flags & HIST_FIELD_FL_LOG2) 166 field_name = hist_field_name(field->operands[0], ++level); 167 168 if (field_name == NULL) 169 field_name = ""; 170 171 return field_name; 172 } 173 174 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed) 175 { 176 hist_field_fn_t fn = NULL; 177 178 switch (field_size) { 179 case 8: 180 if (field_is_signed) 181 fn = hist_field_s64; 182 else 183 fn = hist_field_u64; 184 break; 185 case 4: 186 if (field_is_signed) 187 fn = hist_field_s32; 188 else 189 fn = hist_field_u32; 190 break; 191 case 2: 192 if (field_is_signed) 193 fn = hist_field_s16; 194 else 195 fn = hist_field_u16; 196 break; 197 case 1: 198 if (field_is_signed) 199 fn = hist_field_s8; 200 else 201 fn = hist_field_u8; 202 break; 203 } 204 205 return fn; 206 } 207 208 static int parse_map_size(char *str) 209 { 210 unsigned long size, map_bits; 211 int ret; 212 213 strsep(&str, "="); 214 if (!str) { 215 ret = -EINVAL; 216 goto out; 217 } 218 219 ret = kstrtoul(str, 0, &size); 220 if (ret) 221 goto out; 222 223 map_bits = ilog2(roundup_pow_of_two(size)); 224 if (map_bits < TRACING_MAP_BITS_MIN || 225 map_bits > TRACING_MAP_BITS_MAX) 226 ret = -EINVAL; 227 else 228 ret = map_bits; 229 out: 230 return ret; 231 } 232 233 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs) 234 { 235 if (!attrs) 236 return; 237 238 kfree(attrs->name); 239 kfree(attrs->sort_key_str); 240 kfree(attrs->keys_str); 241 kfree(attrs->vals_str); 242 kfree(attrs); 243 } 244 245 static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str) 246 { 247 struct hist_trigger_attrs *attrs; 248 int ret = 0; 249 250 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL); 251 if (!attrs) 252 return ERR_PTR(-ENOMEM); 253 254 while (trigger_str) { 255 char *str = strsep(&trigger_str, ":"); 256 257 if ((strncmp(str, "key=", strlen("key=")) == 0) || 258 (strncmp(str, "keys=", strlen("keys=")) == 0)) 259 attrs->keys_str = kstrdup(str, GFP_KERNEL); 260 else if ((strncmp(str, "val=", strlen("val=")) == 0) || 261 (strncmp(str, "vals=", strlen("vals=")) == 0) || 262 (strncmp(str, "values=", strlen("values=")) == 0)) 263 attrs->vals_str = kstrdup(str, GFP_KERNEL); 264 else if (strncmp(str, "sort=", strlen("sort=")) == 0) 265 attrs->sort_key_str = kstrdup(str, GFP_KERNEL); 266 else if (strncmp(str, "name=", strlen("name=")) == 0) 267 attrs->name = kstrdup(str, GFP_KERNEL); 268 else if (strcmp(str, "pause") == 0) 269 attrs->pause = true; 270 else if ((strcmp(str, "cont") == 0) || 271 (strcmp(str, "continue") == 0)) 272 attrs->cont = true; 273 else if (strcmp(str, "clear") == 0) 274 attrs->clear = true; 275 else if (strncmp(str, "size=", strlen("size=")) == 0) { 276 int map_bits = parse_map_size(str); 277 278 if (map_bits < 0) { 279 ret = map_bits; 280 goto free; 281 } 282 attrs->map_bits = map_bits; 283 } else { 284 ret = -EINVAL; 285 goto free; 286 } 287 } 288 289 if (!attrs->keys_str) { 290 ret = -EINVAL; 291 goto free; 292 } 293 294 return attrs; 295 free: 296 destroy_hist_trigger_attrs(attrs); 297 298 return ERR_PTR(ret); 299 } 300 301 static inline void save_comm(char *comm, struct task_struct *task) 302 { 303 if (!task->pid) { 304 strcpy(comm, "<idle>"); 305 return; 306 } 307 308 if (WARN_ON_ONCE(task->pid < 0)) { 309 strcpy(comm, "<XXX>"); 310 return; 311 } 312 313 memcpy(comm, task->comm, TASK_COMM_LEN); 314 } 315 316 static void hist_trigger_elt_comm_free(struct tracing_map_elt *elt) 317 { 318 kfree((char *)elt->private_data); 319 } 320 321 static int hist_trigger_elt_comm_alloc(struct tracing_map_elt *elt) 322 { 323 struct hist_trigger_data *hist_data = elt->map->private_data; 324 struct hist_field *key_field; 325 unsigned int i; 326 327 for_each_hist_key_field(i, hist_data) { 328 key_field = hist_data->fields[i]; 329 330 if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 331 unsigned int size = TASK_COMM_LEN + 1; 332 333 elt->private_data = kzalloc(size, GFP_KERNEL); 334 if (!elt->private_data) 335 return -ENOMEM; 336 break; 337 } 338 } 339 340 return 0; 341 } 342 343 static void hist_trigger_elt_comm_copy(struct tracing_map_elt *to, 344 struct tracing_map_elt *from) 345 { 346 char *comm_from = from->private_data; 347 char *comm_to = to->private_data; 348 349 if (comm_from) 350 memcpy(comm_to, comm_from, TASK_COMM_LEN + 1); 351 } 352 353 static void hist_trigger_elt_comm_init(struct tracing_map_elt *elt) 354 { 355 char *comm = elt->private_data; 356 357 if (comm) 358 save_comm(comm, current); 359 } 360 361 static const struct tracing_map_ops hist_trigger_elt_comm_ops = { 362 .elt_alloc = hist_trigger_elt_comm_alloc, 363 .elt_copy = hist_trigger_elt_comm_copy, 364 .elt_free = hist_trigger_elt_comm_free, 365 .elt_init = hist_trigger_elt_comm_init, 366 }; 367 368 static void destroy_hist_field(struct hist_field *hist_field, 369 unsigned int level) 370 { 371 unsigned int i; 372 373 if (level > 2) 374 return; 375 376 if (!hist_field) 377 return; 378 379 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) 380 destroy_hist_field(hist_field->operands[i], level + 1); 381 382 kfree(hist_field); 383 } 384 385 static struct hist_field *create_hist_field(struct ftrace_event_field *field, 386 unsigned long flags) 387 { 388 struct hist_field *hist_field; 389 390 if (field && is_function_field(field)) 391 return NULL; 392 393 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL); 394 if (!hist_field) 395 return NULL; 396 397 if (flags & HIST_FIELD_FL_HITCOUNT) { 398 hist_field->fn = hist_field_counter; 399 goto out; 400 } 401 402 if (flags & HIST_FIELD_FL_STACKTRACE) { 403 hist_field->fn = hist_field_none; 404 goto out; 405 } 406 407 if (flags & HIST_FIELD_FL_LOG2) { 408 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2; 409 hist_field->fn = hist_field_log2; 410 hist_field->operands[0] = create_hist_field(field, fl); 411 hist_field->size = hist_field->operands[0]->size; 412 goto out; 413 } 414 415 if (WARN_ON_ONCE(!field)) 416 goto out; 417 418 if (is_string_field(field)) { 419 flags |= HIST_FIELD_FL_STRING; 420 421 if (field->filter_type == FILTER_STATIC_STRING) 422 hist_field->fn = hist_field_string; 423 else if (field->filter_type == FILTER_DYN_STRING) 424 hist_field->fn = hist_field_dynstring; 425 else 426 hist_field->fn = hist_field_pstring; 427 } else { 428 hist_field->fn = select_value_fn(field->size, 429 field->is_signed); 430 if (!hist_field->fn) { 431 destroy_hist_field(hist_field, 0); 432 return NULL; 433 } 434 } 435 out: 436 hist_field->field = field; 437 hist_field->flags = flags; 438 439 return hist_field; 440 } 441 442 static void destroy_hist_fields(struct hist_trigger_data *hist_data) 443 { 444 unsigned int i; 445 446 for (i = 0; i < TRACING_MAP_FIELDS_MAX; i++) { 447 if (hist_data->fields[i]) { 448 destroy_hist_field(hist_data->fields[i], 0); 449 hist_data->fields[i] = NULL; 450 } 451 } 452 } 453 454 static int create_hitcount_val(struct hist_trigger_data *hist_data) 455 { 456 hist_data->fields[HITCOUNT_IDX] = 457 create_hist_field(NULL, HIST_FIELD_FL_HITCOUNT); 458 if (!hist_data->fields[HITCOUNT_IDX]) 459 return -ENOMEM; 460 461 hist_data->n_vals++; 462 463 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX)) 464 return -EINVAL; 465 466 return 0; 467 } 468 469 static int create_val_field(struct hist_trigger_data *hist_data, 470 unsigned int val_idx, 471 struct trace_event_file *file, 472 char *field_str) 473 { 474 struct ftrace_event_field *field = NULL; 475 unsigned long flags = 0; 476 char *field_name; 477 int ret = 0; 478 479 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX)) 480 return -EINVAL; 481 482 field_name = strsep(&field_str, "."); 483 if (field_str) { 484 if (strcmp(field_str, "hex") == 0) 485 flags |= HIST_FIELD_FL_HEX; 486 else { 487 ret = -EINVAL; 488 goto out; 489 } 490 } 491 492 field = trace_find_event_field(file->event_call, field_name); 493 if (!field || !field->size) { 494 ret = -EINVAL; 495 goto out; 496 } 497 498 hist_data->fields[val_idx] = create_hist_field(field, flags); 499 if (!hist_data->fields[val_idx]) { 500 ret = -ENOMEM; 501 goto out; 502 } 503 504 ++hist_data->n_vals; 505 506 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX)) 507 ret = -EINVAL; 508 out: 509 return ret; 510 } 511 512 static int create_val_fields(struct hist_trigger_data *hist_data, 513 struct trace_event_file *file) 514 { 515 char *fields_str, *field_str; 516 unsigned int i, j; 517 int ret; 518 519 ret = create_hitcount_val(hist_data); 520 if (ret) 521 goto out; 522 523 fields_str = hist_data->attrs->vals_str; 524 if (!fields_str) 525 goto out; 526 527 strsep(&fields_str, "="); 528 if (!fields_str) 529 goto out; 530 531 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX && 532 j < TRACING_MAP_VALS_MAX; i++) { 533 field_str = strsep(&fields_str, ","); 534 if (!field_str) 535 break; 536 if (strcmp(field_str, "hitcount") == 0) 537 continue; 538 ret = create_val_field(hist_data, j++, file, field_str); 539 if (ret) 540 goto out; 541 } 542 if (fields_str && (strcmp(fields_str, "hitcount") != 0)) 543 ret = -EINVAL; 544 out: 545 return ret; 546 } 547 548 static int create_key_field(struct hist_trigger_data *hist_data, 549 unsigned int key_idx, 550 unsigned int key_offset, 551 struct trace_event_file *file, 552 char *field_str) 553 { 554 struct ftrace_event_field *field = NULL; 555 unsigned long flags = 0; 556 unsigned int key_size; 557 int ret = 0; 558 559 if (WARN_ON(key_idx >= TRACING_MAP_FIELDS_MAX)) 560 return -EINVAL; 561 562 flags |= HIST_FIELD_FL_KEY; 563 564 if (strcmp(field_str, "stacktrace") == 0) { 565 flags |= HIST_FIELD_FL_STACKTRACE; 566 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH; 567 } else { 568 char *field_name = strsep(&field_str, "."); 569 570 if (field_str) { 571 if (strcmp(field_str, "hex") == 0) 572 flags |= HIST_FIELD_FL_HEX; 573 else if (strcmp(field_str, "sym") == 0) 574 flags |= HIST_FIELD_FL_SYM; 575 else if (strcmp(field_str, "sym-offset") == 0) 576 flags |= HIST_FIELD_FL_SYM_OFFSET; 577 else if ((strcmp(field_str, "execname") == 0) && 578 (strcmp(field_name, "common_pid") == 0)) 579 flags |= HIST_FIELD_FL_EXECNAME; 580 else if (strcmp(field_str, "syscall") == 0) 581 flags |= HIST_FIELD_FL_SYSCALL; 582 else if (strcmp(field_str, "log2") == 0) 583 flags |= HIST_FIELD_FL_LOG2; 584 else { 585 ret = -EINVAL; 586 goto out; 587 } 588 } 589 590 field = trace_find_event_field(file->event_call, field_name); 591 if (!field || !field->size) { 592 ret = -EINVAL; 593 goto out; 594 } 595 596 if (is_string_field(field)) 597 key_size = MAX_FILTER_STR_VAL; 598 else 599 key_size = field->size; 600 } 601 602 hist_data->fields[key_idx] = create_hist_field(field, flags); 603 if (!hist_data->fields[key_idx]) { 604 ret = -ENOMEM; 605 goto out; 606 } 607 608 key_size = ALIGN(key_size, sizeof(u64)); 609 hist_data->fields[key_idx]->size = key_size; 610 hist_data->fields[key_idx]->offset = key_offset; 611 hist_data->key_size += key_size; 612 if (hist_data->key_size > HIST_KEY_SIZE_MAX) { 613 ret = -EINVAL; 614 goto out; 615 } 616 617 hist_data->n_keys++; 618 619 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX)) 620 return -EINVAL; 621 622 ret = key_size; 623 out: 624 return ret; 625 } 626 627 static int create_key_fields(struct hist_trigger_data *hist_data, 628 struct trace_event_file *file) 629 { 630 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals; 631 char *fields_str, *field_str; 632 int ret = -EINVAL; 633 634 fields_str = hist_data->attrs->keys_str; 635 if (!fields_str) 636 goto out; 637 638 strsep(&fields_str, "="); 639 if (!fields_str) 640 goto out; 641 642 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) { 643 field_str = strsep(&fields_str, ","); 644 if (!field_str) 645 break; 646 ret = create_key_field(hist_data, i, key_offset, 647 file, field_str); 648 if (ret < 0) 649 goto out; 650 key_offset += ret; 651 } 652 if (fields_str) { 653 ret = -EINVAL; 654 goto out; 655 } 656 ret = 0; 657 out: 658 return ret; 659 } 660 661 static int create_hist_fields(struct hist_trigger_data *hist_data, 662 struct trace_event_file *file) 663 { 664 int ret; 665 666 ret = create_val_fields(hist_data, file); 667 if (ret) 668 goto out; 669 670 ret = create_key_fields(hist_data, file); 671 if (ret) 672 goto out; 673 674 hist_data->n_fields = hist_data->n_vals + hist_data->n_keys; 675 out: 676 return ret; 677 } 678 679 static int is_descending(const char *str) 680 { 681 if (!str) 682 return 0; 683 684 if (strcmp(str, "descending") == 0) 685 return 1; 686 687 if (strcmp(str, "ascending") == 0) 688 return 0; 689 690 return -EINVAL; 691 } 692 693 static int create_sort_keys(struct hist_trigger_data *hist_data) 694 { 695 char *fields_str = hist_data->attrs->sort_key_str; 696 struct tracing_map_sort_key *sort_key; 697 int descending, ret = 0; 698 unsigned int i, j; 699 700 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */ 701 702 if (!fields_str) 703 goto out; 704 705 strsep(&fields_str, "="); 706 if (!fields_str) { 707 ret = -EINVAL; 708 goto out; 709 } 710 711 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) { 712 struct hist_field *hist_field; 713 char *field_str, *field_name; 714 const char *test_name; 715 716 sort_key = &hist_data->sort_keys[i]; 717 718 field_str = strsep(&fields_str, ","); 719 if (!field_str) { 720 if (i == 0) 721 ret = -EINVAL; 722 break; 723 } 724 725 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) { 726 ret = -EINVAL; 727 break; 728 } 729 730 field_name = strsep(&field_str, "."); 731 if (!field_name) { 732 ret = -EINVAL; 733 break; 734 } 735 736 if (strcmp(field_name, "hitcount") == 0) { 737 descending = is_descending(field_str); 738 if (descending < 0) { 739 ret = descending; 740 break; 741 } 742 sort_key->descending = descending; 743 continue; 744 } 745 746 for (j = 1; j < hist_data->n_fields; j++) { 747 hist_field = hist_data->fields[j]; 748 test_name = hist_field_name(hist_field, 0); 749 750 if (strcmp(field_name, test_name) == 0) { 751 sort_key->field_idx = j; 752 descending = is_descending(field_str); 753 if (descending < 0) { 754 ret = descending; 755 goto out; 756 } 757 sort_key->descending = descending; 758 break; 759 } 760 } 761 if (j == hist_data->n_fields) { 762 ret = -EINVAL; 763 break; 764 } 765 } 766 hist_data->n_sort_keys = i; 767 out: 768 return ret; 769 } 770 771 static void destroy_hist_data(struct hist_trigger_data *hist_data) 772 { 773 destroy_hist_trigger_attrs(hist_data->attrs); 774 destroy_hist_fields(hist_data); 775 tracing_map_destroy(hist_data->map); 776 kfree(hist_data); 777 } 778 779 static int create_tracing_map_fields(struct hist_trigger_data *hist_data) 780 { 781 struct tracing_map *map = hist_data->map; 782 struct ftrace_event_field *field; 783 struct hist_field *hist_field; 784 int i, idx; 785 786 for_each_hist_field(i, hist_data) { 787 hist_field = hist_data->fields[i]; 788 if (hist_field->flags & HIST_FIELD_FL_KEY) { 789 tracing_map_cmp_fn_t cmp_fn; 790 791 field = hist_field->field; 792 793 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE) 794 cmp_fn = tracing_map_cmp_none; 795 else if (is_string_field(field)) 796 cmp_fn = tracing_map_cmp_string; 797 else 798 cmp_fn = tracing_map_cmp_num(field->size, 799 field->is_signed); 800 idx = tracing_map_add_key_field(map, 801 hist_field->offset, 802 cmp_fn); 803 804 } else 805 idx = tracing_map_add_sum_field(map); 806 807 if (idx < 0) 808 return idx; 809 } 810 811 return 0; 812 } 813 814 static bool need_tracing_map_ops(struct hist_trigger_data *hist_data) 815 { 816 struct hist_field *key_field; 817 unsigned int i; 818 819 for_each_hist_key_field(i, hist_data) { 820 key_field = hist_data->fields[i]; 821 822 if (key_field->flags & HIST_FIELD_FL_EXECNAME) 823 return true; 824 } 825 826 return false; 827 } 828 829 static struct hist_trigger_data * 830 create_hist_data(unsigned int map_bits, 831 struct hist_trigger_attrs *attrs, 832 struct trace_event_file *file) 833 { 834 const struct tracing_map_ops *map_ops = NULL; 835 struct hist_trigger_data *hist_data; 836 int ret = 0; 837 838 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL); 839 if (!hist_data) 840 return ERR_PTR(-ENOMEM); 841 842 hist_data->attrs = attrs; 843 844 ret = create_hist_fields(hist_data, file); 845 if (ret) 846 goto free; 847 848 ret = create_sort_keys(hist_data); 849 if (ret) 850 goto free; 851 852 if (need_tracing_map_ops(hist_data)) 853 map_ops = &hist_trigger_elt_comm_ops; 854 855 hist_data->map = tracing_map_create(map_bits, hist_data->key_size, 856 map_ops, hist_data); 857 if (IS_ERR(hist_data->map)) { 858 ret = PTR_ERR(hist_data->map); 859 hist_data->map = NULL; 860 goto free; 861 } 862 863 ret = create_tracing_map_fields(hist_data); 864 if (ret) 865 goto free; 866 867 ret = tracing_map_init(hist_data->map); 868 if (ret) 869 goto free; 870 871 hist_data->event_file = file; 872 out: 873 return hist_data; 874 free: 875 hist_data->attrs = NULL; 876 877 destroy_hist_data(hist_data); 878 879 hist_data = ERR_PTR(ret); 880 881 goto out; 882 } 883 884 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, 885 struct tracing_map_elt *elt, 886 void *rec) 887 { 888 struct hist_field *hist_field; 889 unsigned int i; 890 u64 hist_val; 891 892 for_each_hist_val_field(i, hist_data) { 893 hist_field = hist_data->fields[i]; 894 hist_val = hist_field->fn(hist_field, rec); 895 tracing_map_update_sum(elt, i, hist_val); 896 } 897 } 898 899 static inline void add_to_key(char *compound_key, void *key, 900 struct hist_field *key_field, void *rec) 901 { 902 size_t size = key_field->size; 903 904 if (key_field->flags & HIST_FIELD_FL_STRING) { 905 struct ftrace_event_field *field; 906 907 field = key_field->field; 908 if (field->filter_type == FILTER_DYN_STRING) 909 size = *(u32 *)(rec + field->offset) >> 16; 910 else if (field->filter_type == FILTER_PTR_STRING) 911 size = strlen(key); 912 else if (field->filter_type == FILTER_STATIC_STRING) 913 size = field->size; 914 915 /* ensure NULL-termination */ 916 if (size > key_field->size - 1) 917 size = key_field->size - 1; 918 } 919 920 memcpy(compound_key + key_field->offset, key, size); 921 } 922 923 static void event_hist_trigger(struct event_trigger_data *data, void *rec) 924 { 925 struct hist_trigger_data *hist_data = data->private_data; 926 bool use_compound_key = (hist_data->n_keys > 1); 927 unsigned long entries[HIST_STACKTRACE_DEPTH]; 928 char compound_key[HIST_KEY_SIZE_MAX]; 929 struct stack_trace stacktrace; 930 struct hist_field *key_field; 931 struct tracing_map_elt *elt; 932 u64 field_contents; 933 void *key = NULL; 934 unsigned int i; 935 936 memset(compound_key, 0, hist_data->key_size); 937 938 for_each_hist_key_field(i, hist_data) { 939 key_field = hist_data->fields[i]; 940 941 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 942 stacktrace.max_entries = HIST_STACKTRACE_DEPTH; 943 stacktrace.entries = entries; 944 stacktrace.nr_entries = 0; 945 stacktrace.skip = HIST_STACKTRACE_SKIP; 946 947 memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE); 948 save_stack_trace(&stacktrace); 949 950 key = entries; 951 } else { 952 field_contents = key_field->fn(key_field, rec); 953 if (key_field->flags & HIST_FIELD_FL_STRING) { 954 key = (void *)(unsigned long)field_contents; 955 use_compound_key = true; 956 } else 957 key = (void *)&field_contents; 958 } 959 960 if (use_compound_key) 961 add_to_key(compound_key, key, key_field, rec); 962 } 963 964 if (use_compound_key) 965 key = compound_key; 966 967 elt = tracing_map_insert(hist_data->map, key); 968 if (elt) 969 hist_trigger_elt_update(hist_data, elt, rec); 970 } 971 972 static void hist_trigger_stacktrace_print(struct seq_file *m, 973 unsigned long *stacktrace_entries, 974 unsigned int max_entries) 975 { 976 char str[KSYM_SYMBOL_LEN]; 977 unsigned int spaces = 8; 978 unsigned int i; 979 980 for (i = 0; i < max_entries; i++) { 981 if (stacktrace_entries[i] == ULONG_MAX) 982 return; 983 984 seq_printf(m, "%*c", 1 + spaces, ' '); 985 sprint_symbol(str, stacktrace_entries[i]); 986 seq_printf(m, "%s\n", str); 987 } 988 } 989 990 static void 991 hist_trigger_entry_print(struct seq_file *m, 992 struct hist_trigger_data *hist_data, void *key, 993 struct tracing_map_elt *elt) 994 { 995 struct hist_field *key_field; 996 char str[KSYM_SYMBOL_LEN]; 997 bool multiline = false; 998 const char *field_name; 999 unsigned int i; 1000 u64 uval; 1001 1002 seq_puts(m, "{ "); 1003 1004 for_each_hist_key_field(i, hist_data) { 1005 key_field = hist_data->fields[i]; 1006 1007 if (i > hist_data->n_vals) 1008 seq_puts(m, ", "); 1009 1010 field_name = hist_field_name(key_field, 0); 1011 1012 if (key_field->flags & HIST_FIELD_FL_HEX) { 1013 uval = *(u64 *)(key + key_field->offset); 1014 seq_printf(m, "%s: %llx", field_name, uval); 1015 } else if (key_field->flags & HIST_FIELD_FL_SYM) { 1016 uval = *(u64 *)(key + key_field->offset); 1017 sprint_symbol_no_offset(str, uval); 1018 seq_printf(m, "%s: [%llx] %-45s", field_name, 1019 uval, str); 1020 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) { 1021 uval = *(u64 *)(key + key_field->offset); 1022 sprint_symbol(str, uval); 1023 seq_printf(m, "%s: [%llx] %-55s", field_name, 1024 uval, str); 1025 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 1026 char *comm = elt->private_data; 1027 1028 uval = *(u64 *)(key + key_field->offset); 1029 seq_printf(m, "%s: %-16s[%10llu]", field_name, 1030 comm, uval); 1031 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) { 1032 const char *syscall_name; 1033 1034 uval = *(u64 *)(key + key_field->offset); 1035 syscall_name = get_syscall_name(uval); 1036 if (!syscall_name) 1037 syscall_name = "unknown_syscall"; 1038 1039 seq_printf(m, "%s: %-30s[%3llu]", field_name, 1040 syscall_name, uval); 1041 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 1042 seq_puts(m, "stacktrace:\n"); 1043 hist_trigger_stacktrace_print(m, 1044 key + key_field->offset, 1045 HIST_STACKTRACE_DEPTH); 1046 multiline = true; 1047 } else if (key_field->flags & HIST_FIELD_FL_LOG2) { 1048 seq_printf(m, "%s: ~ 2^%-2llu", field_name, 1049 *(u64 *)(key + key_field->offset)); 1050 } else if (key_field->flags & HIST_FIELD_FL_STRING) { 1051 seq_printf(m, "%s: %-50s", field_name, 1052 (char *)(key + key_field->offset)); 1053 } else { 1054 uval = *(u64 *)(key + key_field->offset); 1055 seq_printf(m, "%s: %10llu", field_name, uval); 1056 } 1057 } 1058 1059 if (!multiline) 1060 seq_puts(m, " "); 1061 1062 seq_puts(m, "}"); 1063 1064 seq_printf(m, " hitcount: %10llu", 1065 tracing_map_read_sum(elt, HITCOUNT_IDX)); 1066 1067 for (i = 1; i < hist_data->n_vals; i++) { 1068 field_name = hist_field_name(hist_data->fields[i], 0); 1069 1070 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) { 1071 seq_printf(m, " %s: %10llx", field_name, 1072 tracing_map_read_sum(elt, i)); 1073 } else { 1074 seq_printf(m, " %s: %10llu", field_name, 1075 tracing_map_read_sum(elt, i)); 1076 } 1077 } 1078 1079 seq_puts(m, "\n"); 1080 } 1081 1082 static int print_entries(struct seq_file *m, 1083 struct hist_trigger_data *hist_data) 1084 { 1085 struct tracing_map_sort_entry **sort_entries = NULL; 1086 struct tracing_map *map = hist_data->map; 1087 int i, n_entries; 1088 1089 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys, 1090 hist_data->n_sort_keys, 1091 &sort_entries); 1092 if (n_entries < 0) 1093 return n_entries; 1094 1095 for (i = 0; i < n_entries; i++) 1096 hist_trigger_entry_print(m, hist_data, 1097 sort_entries[i]->key, 1098 sort_entries[i]->elt); 1099 1100 tracing_map_destroy_sort_entries(sort_entries, n_entries); 1101 1102 return n_entries; 1103 } 1104 1105 static void hist_trigger_show(struct seq_file *m, 1106 struct event_trigger_data *data, int n) 1107 { 1108 struct hist_trigger_data *hist_data; 1109 int n_entries; 1110 1111 if (n > 0) 1112 seq_puts(m, "\n\n"); 1113 1114 seq_puts(m, "# event histogram\n#\n# trigger info: "); 1115 data->ops->print(m, data->ops, data); 1116 seq_puts(m, "#\n\n"); 1117 1118 hist_data = data->private_data; 1119 n_entries = print_entries(m, hist_data); 1120 if (n_entries < 0) 1121 n_entries = 0; 1122 1123 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n", 1124 (u64)atomic64_read(&hist_data->map->hits), 1125 n_entries, (u64)atomic64_read(&hist_data->map->drops)); 1126 } 1127 1128 static int hist_show(struct seq_file *m, void *v) 1129 { 1130 struct event_trigger_data *data; 1131 struct trace_event_file *event_file; 1132 int n = 0, ret = 0; 1133 1134 mutex_lock(&event_mutex); 1135 1136 event_file = event_file_data(m->private); 1137 if (unlikely(!event_file)) { 1138 ret = -ENODEV; 1139 goto out_unlock; 1140 } 1141 1142 list_for_each_entry_rcu(data, &event_file->triggers, list) { 1143 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 1144 hist_trigger_show(m, data, n++); 1145 } 1146 1147 out_unlock: 1148 mutex_unlock(&event_mutex); 1149 1150 return ret; 1151 } 1152 1153 static int event_hist_open(struct inode *inode, struct file *file) 1154 { 1155 return single_open(file, hist_show, file); 1156 } 1157 1158 const struct file_operations event_hist_fops = { 1159 .open = event_hist_open, 1160 .read = seq_read, 1161 .llseek = seq_lseek, 1162 .release = single_release, 1163 }; 1164 1165 static const char *get_hist_field_flags(struct hist_field *hist_field) 1166 { 1167 const char *flags_str = NULL; 1168 1169 if (hist_field->flags & HIST_FIELD_FL_HEX) 1170 flags_str = "hex"; 1171 else if (hist_field->flags & HIST_FIELD_FL_SYM) 1172 flags_str = "sym"; 1173 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET) 1174 flags_str = "sym-offset"; 1175 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME) 1176 flags_str = "execname"; 1177 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL) 1178 flags_str = "syscall"; 1179 else if (hist_field->flags & HIST_FIELD_FL_LOG2) 1180 flags_str = "log2"; 1181 1182 return flags_str; 1183 } 1184 1185 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field) 1186 { 1187 const char *field_name = hist_field_name(hist_field, 0); 1188 1189 seq_printf(m, "%s", field_name); 1190 if (hist_field->flags) { 1191 const char *flags_str = get_hist_field_flags(hist_field); 1192 1193 if (flags_str) 1194 seq_printf(m, ".%s", flags_str); 1195 } 1196 } 1197 1198 static int event_hist_trigger_print(struct seq_file *m, 1199 struct event_trigger_ops *ops, 1200 struct event_trigger_data *data) 1201 { 1202 struct hist_trigger_data *hist_data = data->private_data; 1203 struct hist_field *key_field; 1204 unsigned int i; 1205 1206 seq_puts(m, "hist:"); 1207 1208 if (data->name) 1209 seq_printf(m, "%s:", data->name); 1210 1211 seq_puts(m, "keys="); 1212 1213 for_each_hist_key_field(i, hist_data) { 1214 key_field = hist_data->fields[i]; 1215 1216 if (i > hist_data->n_vals) 1217 seq_puts(m, ","); 1218 1219 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) 1220 seq_puts(m, "stacktrace"); 1221 else 1222 hist_field_print(m, key_field); 1223 } 1224 1225 seq_puts(m, ":vals="); 1226 1227 for_each_hist_val_field(i, hist_data) { 1228 if (i == HITCOUNT_IDX) 1229 seq_puts(m, "hitcount"); 1230 else { 1231 seq_puts(m, ","); 1232 hist_field_print(m, hist_data->fields[i]); 1233 } 1234 } 1235 1236 seq_puts(m, ":sort="); 1237 1238 for (i = 0; i < hist_data->n_sort_keys; i++) { 1239 struct tracing_map_sort_key *sort_key; 1240 1241 sort_key = &hist_data->sort_keys[i]; 1242 1243 if (i > 0) 1244 seq_puts(m, ","); 1245 1246 if (sort_key->field_idx == HITCOUNT_IDX) 1247 seq_puts(m, "hitcount"); 1248 else { 1249 unsigned int idx = sort_key->field_idx; 1250 1251 if (WARN_ON(idx >= TRACING_MAP_FIELDS_MAX)) 1252 return -EINVAL; 1253 1254 hist_field_print(m, hist_data->fields[idx]); 1255 } 1256 1257 if (sort_key->descending) 1258 seq_puts(m, ".descending"); 1259 } 1260 1261 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits)); 1262 1263 if (data->filter_str) 1264 seq_printf(m, " if %s", data->filter_str); 1265 1266 if (data->paused) 1267 seq_puts(m, " [paused]"); 1268 else 1269 seq_puts(m, " [active]"); 1270 1271 seq_putc(m, '\n'); 1272 1273 return 0; 1274 } 1275 1276 static int event_hist_trigger_init(struct event_trigger_ops *ops, 1277 struct event_trigger_data *data) 1278 { 1279 struct hist_trigger_data *hist_data = data->private_data; 1280 1281 if (!data->ref && hist_data->attrs->name) 1282 save_named_trigger(hist_data->attrs->name, data); 1283 1284 data->ref++; 1285 1286 return 0; 1287 } 1288 1289 static void event_hist_trigger_free(struct event_trigger_ops *ops, 1290 struct event_trigger_data *data) 1291 { 1292 struct hist_trigger_data *hist_data = data->private_data; 1293 1294 if (WARN_ON_ONCE(data->ref <= 0)) 1295 return; 1296 1297 data->ref--; 1298 if (!data->ref) { 1299 if (data->name) 1300 del_named_trigger(data); 1301 trigger_data_free(data); 1302 destroy_hist_data(hist_data); 1303 } 1304 } 1305 1306 static struct event_trigger_ops event_hist_trigger_ops = { 1307 .func = event_hist_trigger, 1308 .print = event_hist_trigger_print, 1309 .init = event_hist_trigger_init, 1310 .free = event_hist_trigger_free, 1311 }; 1312 1313 static int event_hist_trigger_named_init(struct event_trigger_ops *ops, 1314 struct event_trigger_data *data) 1315 { 1316 data->ref++; 1317 1318 save_named_trigger(data->named_data->name, data); 1319 1320 event_hist_trigger_init(ops, data->named_data); 1321 1322 return 0; 1323 } 1324 1325 static void event_hist_trigger_named_free(struct event_trigger_ops *ops, 1326 struct event_trigger_data *data) 1327 { 1328 if (WARN_ON_ONCE(data->ref <= 0)) 1329 return; 1330 1331 event_hist_trigger_free(ops, data->named_data); 1332 1333 data->ref--; 1334 if (!data->ref) { 1335 del_named_trigger(data); 1336 trigger_data_free(data); 1337 } 1338 } 1339 1340 static struct event_trigger_ops event_hist_trigger_named_ops = { 1341 .func = event_hist_trigger, 1342 .print = event_hist_trigger_print, 1343 .init = event_hist_trigger_named_init, 1344 .free = event_hist_trigger_named_free, 1345 }; 1346 1347 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd, 1348 char *param) 1349 { 1350 return &event_hist_trigger_ops; 1351 } 1352 1353 static void hist_clear(struct event_trigger_data *data) 1354 { 1355 struct hist_trigger_data *hist_data = data->private_data; 1356 1357 if (data->name) 1358 pause_named_trigger(data); 1359 1360 synchronize_sched(); 1361 1362 tracing_map_clear(hist_data->map); 1363 1364 if (data->name) 1365 unpause_named_trigger(data); 1366 } 1367 1368 static bool compatible_field(struct ftrace_event_field *field, 1369 struct ftrace_event_field *test_field) 1370 { 1371 if (field == test_field) 1372 return true; 1373 if (field == NULL || test_field == NULL) 1374 return false; 1375 if (strcmp(field->name, test_field->name) != 0) 1376 return false; 1377 if (strcmp(field->type, test_field->type) != 0) 1378 return false; 1379 if (field->size != test_field->size) 1380 return false; 1381 if (field->is_signed != test_field->is_signed) 1382 return false; 1383 1384 return true; 1385 } 1386 1387 static bool hist_trigger_match(struct event_trigger_data *data, 1388 struct event_trigger_data *data_test, 1389 struct event_trigger_data *named_data, 1390 bool ignore_filter) 1391 { 1392 struct tracing_map_sort_key *sort_key, *sort_key_test; 1393 struct hist_trigger_data *hist_data, *hist_data_test; 1394 struct hist_field *key_field, *key_field_test; 1395 unsigned int i; 1396 1397 if (named_data && (named_data != data_test) && 1398 (named_data != data_test->named_data)) 1399 return false; 1400 1401 if (!named_data && is_named_trigger(data_test)) 1402 return false; 1403 1404 hist_data = data->private_data; 1405 hist_data_test = data_test->private_data; 1406 1407 if (hist_data->n_vals != hist_data_test->n_vals || 1408 hist_data->n_fields != hist_data_test->n_fields || 1409 hist_data->n_sort_keys != hist_data_test->n_sort_keys) 1410 return false; 1411 1412 if (!ignore_filter) { 1413 if ((data->filter_str && !data_test->filter_str) || 1414 (!data->filter_str && data_test->filter_str)) 1415 return false; 1416 } 1417 1418 for_each_hist_field(i, hist_data) { 1419 key_field = hist_data->fields[i]; 1420 key_field_test = hist_data_test->fields[i]; 1421 1422 if (key_field->flags != key_field_test->flags) 1423 return false; 1424 if (!compatible_field(key_field->field, key_field_test->field)) 1425 return false; 1426 if (key_field->offset != key_field_test->offset) 1427 return false; 1428 } 1429 1430 for (i = 0; i < hist_data->n_sort_keys; i++) { 1431 sort_key = &hist_data->sort_keys[i]; 1432 sort_key_test = &hist_data_test->sort_keys[i]; 1433 1434 if (sort_key->field_idx != sort_key_test->field_idx || 1435 sort_key->descending != sort_key_test->descending) 1436 return false; 1437 } 1438 1439 if (!ignore_filter && data->filter_str && 1440 (strcmp(data->filter_str, data_test->filter_str) != 0)) 1441 return false; 1442 1443 return true; 1444 } 1445 1446 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops, 1447 struct event_trigger_data *data, 1448 struct trace_event_file *file) 1449 { 1450 struct hist_trigger_data *hist_data = data->private_data; 1451 struct event_trigger_data *test, *named_data = NULL; 1452 int ret = 0; 1453 1454 if (hist_data->attrs->name) { 1455 named_data = find_named_trigger(hist_data->attrs->name); 1456 if (named_data) { 1457 if (!hist_trigger_match(data, named_data, named_data, 1458 true)) { 1459 ret = -EINVAL; 1460 goto out; 1461 } 1462 } 1463 } 1464 1465 if (hist_data->attrs->name && !named_data) 1466 goto new; 1467 1468 list_for_each_entry_rcu(test, &file->triggers, list) { 1469 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1470 if (!hist_trigger_match(data, test, named_data, false)) 1471 continue; 1472 if (hist_data->attrs->pause) 1473 test->paused = true; 1474 else if (hist_data->attrs->cont) 1475 test->paused = false; 1476 else if (hist_data->attrs->clear) 1477 hist_clear(test); 1478 else 1479 ret = -EEXIST; 1480 goto out; 1481 } 1482 } 1483 new: 1484 if (hist_data->attrs->cont || hist_data->attrs->clear) { 1485 ret = -ENOENT; 1486 goto out; 1487 } 1488 1489 if (hist_data->attrs->pause) 1490 data->paused = true; 1491 1492 if (named_data) { 1493 destroy_hist_data(data->private_data); 1494 data->private_data = named_data->private_data; 1495 set_named_trigger_data(data, named_data); 1496 data->ops = &event_hist_trigger_named_ops; 1497 } 1498 1499 if (data->ops->init) { 1500 ret = data->ops->init(data->ops, data); 1501 if (ret < 0) 1502 goto out; 1503 } 1504 1505 list_add_rcu(&data->list, &file->triggers); 1506 ret++; 1507 1508 update_cond_flag(file); 1509 1510 if (trace_event_trigger_enable_disable(file, 1) < 0) { 1511 list_del_rcu(&data->list); 1512 update_cond_flag(file); 1513 ret--; 1514 } 1515 out: 1516 return ret; 1517 } 1518 1519 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops, 1520 struct event_trigger_data *data, 1521 struct trace_event_file *file) 1522 { 1523 struct hist_trigger_data *hist_data = data->private_data; 1524 struct event_trigger_data *test, *named_data = NULL; 1525 bool unregistered = false; 1526 1527 if (hist_data->attrs->name) 1528 named_data = find_named_trigger(hist_data->attrs->name); 1529 1530 list_for_each_entry_rcu(test, &file->triggers, list) { 1531 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1532 if (!hist_trigger_match(data, test, named_data, false)) 1533 continue; 1534 unregistered = true; 1535 list_del_rcu(&test->list); 1536 trace_event_trigger_enable_disable(file, 0); 1537 update_cond_flag(file); 1538 break; 1539 } 1540 } 1541 1542 if (unregistered && test->ops->free) 1543 test->ops->free(test->ops, test); 1544 } 1545 1546 static void hist_unreg_all(struct trace_event_file *file) 1547 { 1548 struct event_trigger_data *test, *n; 1549 1550 list_for_each_entry_safe(test, n, &file->triggers, list) { 1551 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1552 list_del_rcu(&test->list); 1553 trace_event_trigger_enable_disable(file, 0); 1554 update_cond_flag(file); 1555 if (test->ops->free) 1556 test->ops->free(test->ops, test); 1557 } 1558 } 1559 } 1560 1561 static int event_hist_trigger_func(struct event_command *cmd_ops, 1562 struct trace_event_file *file, 1563 char *glob, char *cmd, char *param) 1564 { 1565 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT; 1566 struct event_trigger_data *trigger_data; 1567 struct hist_trigger_attrs *attrs; 1568 struct event_trigger_ops *trigger_ops; 1569 struct hist_trigger_data *hist_data; 1570 char *trigger; 1571 int ret = 0; 1572 1573 if (!param) 1574 return -EINVAL; 1575 1576 /* separate the trigger from the filter (k:v [if filter]) */ 1577 trigger = strsep(¶m, " \t"); 1578 if (!trigger) 1579 return -EINVAL; 1580 1581 attrs = parse_hist_trigger_attrs(trigger); 1582 if (IS_ERR(attrs)) 1583 return PTR_ERR(attrs); 1584 1585 if (attrs->map_bits) 1586 hist_trigger_bits = attrs->map_bits; 1587 1588 hist_data = create_hist_data(hist_trigger_bits, attrs, file); 1589 if (IS_ERR(hist_data)) { 1590 destroy_hist_trigger_attrs(attrs); 1591 return PTR_ERR(hist_data); 1592 } 1593 1594 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger); 1595 1596 ret = -ENOMEM; 1597 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL); 1598 if (!trigger_data) 1599 goto out_free; 1600 1601 trigger_data->count = -1; 1602 trigger_data->ops = trigger_ops; 1603 trigger_data->cmd_ops = cmd_ops; 1604 1605 INIT_LIST_HEAD(&trigger_data->list); 1606 RCU_INIT_POINTER(trigger_data->filter, NULL); 1607 1608 trigger_data->private_data = hist_data; 1609 1610 /* if param is non-empty, it's supposed to be a filter */ 1611 if (param && cmd_ops->set_filter) { 1612 ret = cmd_ops->set_filter(param, trigger_data, file); 1613 if (ret < 0) 1614 goto out_free; 1615 } 1616 1617 if (glob[0] == '!') { 1618 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file); 1619 ret = 0; 1620 goto out_free; 1621 } 1622 1623 ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file); 1624 /* 1625 * The above returns on success the # of triggers registered, 1626 * but if it didn't register any it returns zero. Consider no 1627 * triggers registered a failure too. 1628 */ 1629 if (!ret) { 1630 if (!(attrs->pause || attrs->cont || attrs->clear)) 1631 ret = -ENOENT; 1632 goto out_free; 1633 } else if (ret < 0) 1634 goto out_free; 1635 /* Just return zero, not the number of registered triggers */ 1636 ret = 0; 1637 out: 1638 return ret; 1639 out_free: 1640 if (cmd_ops->set_filter) 1641 cmd_ops->set_filter(NULL, trigger_data, NULL); 1642 1643 kfree(trigger_data); 1644 1645 destroy_hist_data(hist_data); 1646 goto out; 1647 } 1648 1649 static struct event_command trigger_hist_cmd = { 1650 .name = "hist", 1651 .trigger_type = ETT_EVENT_HIST, 1652 .flags = EVENT_CMD_FL_NEEDS_REC, 1653 .func = event_hist_trigger_func, 1654 .reg = hist_register_trigger, 1655 .unreg = hist_unregister_trigger, 1656 .unreg_all = hist_unreg_all, 1657 .get_trigger_ops = event_hist_get_trigger_ops, 1658 .set_filter = set_trigger_filter, 1659 }; 1660 1661 __init int register_trigger_hist_cmd(void) 1662 { 1663 int ret; 1664 1665 ret = register_event_command(&trigger_hist_cmd); 1666 WARN_ON(ret < 0); 1667 1668 return ret; 1669 } 1670 1671 static void 1672 hist_enable_trigger(struct event_trigger_data *data, void *rec) 1673 { 1674 struct enable_trigger_data *enable_data = data->private_data; 1675 struct event_trigger_data *test; 1676 1677 list_for_each_entry_rcu(test, &enable_data->file->triggers, list) { 1678 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1679 if (enable_data->enable) 1680 test->paused = false; 1681 else 1682 test->paused = true; 1683 } 1684 } 1685 } 1686 1687 static void 1688 hist_enable_count_trigger(struct event_trigger_data *data, void *rec) 1689 { 1690 if (!data->count) 1691 return; 1692 1693 if (data->count != -1) 1694 (data->count)--; 1695 1696 hist_enable_trigger(data, rec); 1697 } 1698 1699 static struct event_trigger_ops hist_enable_trigger_ops = { 1700 .func = hist_enable_trigger, 1701 .print = event_enable_trigger_print, 1702 .init = event_trigger_init, 1703 .free = event_enable_trigger_free, 1704 }; 1705 1706 static struct event_trigger_ops hist_enable_count_trigger_ops = { 1707 .func = hist_enable_count_trigger, 1708 .print = event_enable_trigger_print, 1709 .init = event_trigger_init, 1710 .free = event_enable_trigger_free, 1711 }; 1712 1713 static struct event_trigger_ops hist_disable_trigger_ops = { 1714 .func = hist_enable_trigger, 1715 .print = event_enable_trigger_print, 1716 .init = event_trigger_init, 1717 .free = event_enable_trigger_free, 1718 }; 1719 1720 static struct event_trigger_ops hist_disable_count_trigger_ops = { 1721 .func = hist_enable_count_trigger, 1722 .print = event_enable_trigger_print, 1723 .init = event_trigger_init, 1724 .free = event_enable_trigger_free, 1725 }; 1726 1727 static struct event_trigger_ops * 1728 hist_enable_get_trigger_ops(char *cmd, char *param) 1729 { 1730 struct event_trigger_ops *ops; 1731 bool enable; 1732 1733 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0); 1734 1735 if (enable) 1736 ops = param ? &hist_enable_count_trigger_ops : 1737 &hist_enable_trigger_ops; 1738 else 1739 ops = param ? &hist_disable_count_trigger_ops : 1740 &hist_disable_trigger_ops; 1741 1742 return ops; 1743 } 1744 1745 static void hist_enable_unreg_all(struct trace_event_file *file) 1746 { 1747 struct event_trigger_data *test, *n; 1748 1749 list_for_each_entry_safe(test, n, &file->triggers, list) { 1750 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) { 1751 list_del_rcu(&test->list); 1752 update_cond_flag(file); 1753 trace_event_trigger_enable_disable(file, 0); 1754 if (test->ops->free) 1755 test->ops->free(test->ops, test); 1756 } 1757 } 1758 } 1759 1760 static struct event_command trigger_hist_enable_cmd = { 1761 .name = ENABLE_HIST_STR, 1762 .trigger_type = ETT_HIST_ENABLE, 1763 .func = event_enable_trigger_func, 1764 .reg = event_enable_register_trigger, 1765 .unreg = event_enable_unregister_trigger, 1766 .unreg_all = hist_enable_unreg_all, 1767 .get_trigger_ops = hist_enable_get_trigger_ops, 1768 .set_filter = set_trigger_filter, 1769 }; 1770 1771 static struct event_command trigger_hist_disable_cmd = { 1772 .name = DISABLE_HIST_STR, 1773 .trigger_type = ETT_HIST_ENABLE, 1774 .func = event_enable_trigger_func, 1775 .reg = event_enable_register_trigger, 1776 .unreg = event_enable_unregister_trigger, 1777 .unreg_all = hist_enable_unreg_all, 1778 .get_trigger_ops = hist_enable_get_trigger_ops, 1779 .set_filter = set_trigger_filter, 1780 }; 1781 1782 static __init void unregister_trigger_hist_enable_disable_cmds(void) 1783 { 1784 unregister_event_command(&trigger_hist_enable_cmd); 1785 unregister_event_command(&trigger_hist_disable_cmd); 1786 } 1787 1788 __init int register_trigger_hist_enable_disable_cmds(void) 1789 { 1790 int ret; 1791 1792 ret = register_event_command(&trigger_hist_enable_cmd); 1793 if (WARN_ON(ret < 0)) 1794 return ret; 1795 ret = register_event_command(&trigger_hist_disable_cmd); 1796 if (WARN_ON(ret < 0)) 1797 unregister_trigger_hist_enable_disable_cmds(); 1798 1799 return ret; 1800 } 1801