1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * trace_events_hist - trace event hist triggers
4  *
5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
6  */
7 
8 #include <linux/module.h>
9 #include <linux/kallsyms.h>
10 #include <linux/mutex.h>
11 #include <linux/slab.h>
12 #include <linux/stacktrace.h>
13 #include <linux/rculist.h>
14 #include <linux/tracefs.h>
15 
16 #include "tracing_map.h"
17 #include "trace.h"
18 #include "trace_dynevent.h"
19 
20 #define SYNTH_SYSTEM		"synthetic"
21 #define SYNTH_FIELDS_MAX	16
22 
23 #define STR_VAR_LEN_MAX		32 /* must be multiple of sizeof(u64) */
24 
25 struct hist_field;
26 
27 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
28 				struct tracing_map_elt *elt,
29 				struct ring_buffer_event *rbe,
30 				void *event);
31 
32 #define HIST_FIELD_OPERANDS_MAX	2
33 #define HIST_FIELDS_MAX		(TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
34 #define HIST_ACTIONS_MAX	8
35 
36 enum field_op_id {
37 	FIELD_OP_NONE,
38 	FIELD_OP_PLUS,
39 	FIELD_OP_MINUS,
40 	FIELD_OP_UNARY_MINUS,
41 };
42 
43 /*
44  * A hist_var (histogram variable) contains variable information for
45  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
46  * flag set.  A hist_var has a variable name e.g. ts0, and is
47  * associated with a given histogram trigger, as specified by
48  * hist_data.  The hist_var idx is the unique index assigned to the
49  * variable by the hist trigger's tracing_map.  The idx is what is
50  * used to set a variable's value and, by a variable reference, to
51  * retrieve it.
52  */
53 struct hist_var {
54 	char				*name;
55 	struct hist_trigger_data	*hist_data;
56 	unsigned int			idx;
57 };
58 
59 struct hist_field {
60 	struct ftrace_event_field	*field;
61 	unsigned long			flags;
62 	hist_field_fn_t			fn;
63 	unsigned int			size;
64 	unsigned int			offset;
65 	unsigned int                    is_signed;
66 	const char			*type;
67 	struct hist_field		*operands[HIST_FIELD_OPERANDS_MAX];
68 	struct hist_trigger_data	*hist_data;
69 
70 	/*
71 	 * Variable fields contain variable-specific info in var.
72 	 */
73 	struct hist_var			var;
74 	enum field_op_id		operator;
75 	char				*system;
76 	char				*event_name;
77 
78 	/*
79 	 * The name field is used for EXPR and VAR_REF fields.  VAR
80 	 * fields contain the variable name in var.name.
81 	 */
82 	char				*name;
83 
84 	/*
85 	 * When a histogram trigger is hit, if it has any references
86 	 * to variables, the values of those variables are collected
87 	 * into a var_ref_vals array by resolve_var_refs().  The
88 	 * current value of each variable is read from the tracing_map
89 	 * using the hist field's hist_var.idx and entered into the
90 	 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
91 	 */
92 	unsigned int			var_ref_idx;
93 	bool                            read_once;
94 };
95 
96 static u64 hist_field_none(struct hist_field *field,
97 			   struct tracing_map_elt *elt,
98 			   struct ring_buffer_event *rbe,
99 			   void *event)
100 {
101 	return 0;
102 }
103 
104 static u64 hist_field_counter(struct hist_field *field,
105 			      struct tracing_map_elt *elt,
106 			      struct ring_buffer_event *rbe,
107 			      void *event)
108 {
109 	return 1;
110 }
111 
112 static u64 hist_field_string(struct hist_field *hist_field,
113 			     struct tracing_map_elt *elt,
114 			     struct ring_buffer_event *rbe,
115 			     void *event)
116 {
117 	char *addr = (char *)(event + hist_field->field->offset);
118 
119 	return (u64)(unsigned long)addr;
120 }
121 
122 static u64 hist_field_dynstring(struct hist_field *hist_field,
123 				struct tracing_map_elt *elt,
124 				struct ring_buffer_event *rbe,
125 				void *event)
126 {
127 	u32 str_item = *(u32 *)(event + hist_field->field->offset);
128 	int str_loc = str_item & 0xffff;
129 	char *addr = (char *)(event + str_loc);
130 
131 	return (u64)(unsigned long)addr;
132 }
133 
134 static u64 hist_field_pstring(struct hist_field *hist_field,
135 			      struct tracing_map_elt *elt,
136 			      struct ring_buffer_event *rbe,
137 			      void *event)
138 {
139 	char **addr = (char **)(event + hist_field->field->offset);
140 
141 	return (u64)(unsigned long)*addr;
142 }
143 
144 static u64 hist_field_log2(struct hist_field *hist_field,
145 			   struct tracing_map_elt *elt,
146 			   struct ring_buffer_event *rbe,
147 			   void *event)
148 {
149 	struct hist_field *operand = hist_field->operands[0];
150 
151 	u64 val = operand->fn(operand, elt, rbe, event);
152 
153 	return (u64) ilog2(roundup_pow_of_two(val));
154 }
155 
156 static u64 hist_field_plus(struct hist_field *hist_field,
157 			   struct tracing_map_elt *elt,
158 			   struct ring_buffer_event *rbe,
159 			   void *event)
160 {
161 	struct hist_field *operand1 = hist_field->operands[0];
162 	struct hist_field *operand2 = hist_field->operands[1];
163 
164 	u64 val1 = operand1->fn(operand1, elt, rbe, event);
165 	u64 val2 = operand2->fn(operand2, elt, rbe, event);
166 
167 	return val1 + val2;
168 }
169 
170 static u64 hist_field_minus(struct hist_field *hist_field,
171 			    struct tracing_map_elt *elt,
172 			    struct ring_buffer_event *rbe,
173 			    void *event)
174 {
175 	struct hist_field *operand1 = hist_field->operands[0];
176 	struct hist_field *operand2 = hist_field->operands[1];
177 
178 	u64 val1 = operand1->fn(operand1, elt, rbe, event);
179 	u64 val2 = operand2->fn(operand2, elt, rbe, event);
180 
181 	return val1 - val2;
182 }
183 
184 static u64 hist_field_unary_minus(struct hist_field *hist_field,
185 				  struct tracing_map_elt *elt,
186 				  struct ring_buffer_event *rbe,
187 				  void *event)
188 {
189 	struct hist_field *operand = hist_field->operands[0];
190 
191 	s64 sval = (s64)operand->fn(operand, elt, rbe, event);
192 	u64 val = (u64)-sval;
193 
194 	return val;
195 }
196 
197 #define DEFINE_HIST_FIELD_FN(type)					\
198 	static u64 hist_field_##type(struct hist_field *hist_field,	\
199 				     struct tracing_map_elt *elt,	\
200 				     struct ring_buffer_event *rbe,	\
201 				     void *event)			\
202 {									\
203 	type *addr = (type *)(event + hist_field->field->offset);	\
204 									\
205 	return (u64)(unsigned long)*addr;				\
206 }
207 
208 DEFINE_HIST_FIELD_FN(s64);
209 DEFINE_HIST_FIELD_FN(u64);
210 DEFINE_HIST_FIELD_FN(s32);
211 DEFINE_HIST_FIELD_FN(u32);
212 DEFINE_HIST_FIELD_FN(s16);
213 DEFINE_HIST_FIELD_FN(u16);
214 DEFINE_HIST_FIELD_FN(s8);
215 DEFINE_HIST_FIELD_FN(u8);
216 
217 #define for_each_hist_field(i, hist_data)	\
218 	for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
219 
220 #define for_each_hist_val_field(i, hist_data)	\
221 	for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
222 
223 #define for_each_hist_key_field(i, hist_data)	\
224 	for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
225 
226 #define HIST_STACKTRACE_DEPTH	16
227 #define HIST_STACKTRACE_SIZE	(HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
228 #define HIST_STACKTRACE_SKIP	5
229 
230 #define HITCOUNT_IDX		0
231 #define HIST_KEY_SIZE_MAX	(MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
232 
233 enum hist_field_flags {
234 	HIST_FIELD_FL_HITCOUNT		= 1 << 0,
235 	HIST_FIELD_FL_KEY		= 1 << 1,
236 	HIST_FIELD_FL_STRING		= 1 << 2,
237 	HIST_FIELD_FL_HEX		= 1 << 3,
238 	HIST_FIELD_FL_SYM		= 1 << 4,
239 	HIST_FIELD_FL_SYM_OFFSET	= 1 << 5,
240 	HIST_FIELD_FL_EXECNAME		= 1 << 6,
241 	HIST_FIELD_FL_SYSCALL		= 1 << 7,
242 	HIST_FIELD_FL_STACKTRACE	= 1 << 8,
243 	HIST_FIELD_FL_LOG2		= 1 << 9,
244 	HIST_FIELD_FL_TIMESTAMP		= 1 << 10,
245 	HIST_FIELD_FL_TIMESTAMP_USECS	= 1 << 11,
246 	HIST_FIELD_FL_VAR		= 1 << 12,
247 	HIST_FIELD_FL_EXPR		= 1 << 13,
248 	HIST_FIELD_FL_VAR_REF		= 1 << 14,
249 	HIST_FIELD_FL_CPU		= 1 << 15,
250 	HIST_FIELD_FL_ALIAS		= 1 << 16,
251 };
252 
253 struct var_defs {
254 	unsigned int	n_vars;
255 	char		*name[TRACING_MAP_VARS_MAX];
256 	char		*expr[TRACING_MAP_VARS_MAX];
257 };
258 
259 struct hist_trigger_attrs {
260 	char		*keys_str;
261 	char		*vals_str;
262 	char		*sort_key_str;
263 	char		*name;
264 	char		*clock;
265 	bool		pause;
266 	bool		cont;
267 	bool		clear;
268 	bool		ts_in_usecs;
269 	unsigned int	map_bits;
270 
271 	char		*assignment_str[TRACING_MAP_VARS_MAX];
272 	unsigned int	n_assignments;
273 
274 	char		*action_str[HIST_ACTIONS_MAX];
275 	unsigned int	n_actions;
276 
277 	struct var_defs	var_defs;
278 };
279 
280 struct field_var {
281 	struct hist_field	*var;
282 	struct hist_field	*val;
283 };
284 
285 struct field_var_hist {
286 	struct hist_trigger_data	*hist_data;
287 	char				*cmd;
288 };
289 
290 struct hist_trigger_data {
291 	struct hist_field               *fields[HIST_FIELDS_MAX];
292 	unsigned int			n_vals;
293 	unsigned int			n_keys;
294 	unsigned int			n_fields;
295 	unsigned int			n_vars;
296 	unsigned int			key_size;
297 	struct tracing_map_sort_key	sort_keys[TRACING_MAP_SORT_KEYS_MAX];
298 	unsigned int			n_sort_keys;
299 	struct trace_event_file		*event_file;
300 	struct hist_trigger_attrs	*attrs;
301 	struct tracing_map		*map;
302 	bool				enable_timestamps;
303 	bool				remove;
304 	struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
305 	unsigned int			n_var_refs;
306 
307 	struct action_data		*actions[HIST_ACTIONS_MAX];
308 	unsigned int			n_actions;
309 
310 	struct field_var		*field_vars[SYNTH_FIELDS_MAX];
311 	unsigned int			n_field_vars;
312 	unsigned int			n_field_var_str;
313 	struct field_var_hist		*field_var_hists[SYNTH_FIELDS_MAX];
314 	unsigned int			n_field_var_hists;
315 
316 	struct field_var		*save_vars[SYNTH_FIELDS_MAX];
317 	unsigned int			n_save_vars;
318 	unsigned int			n_save_var_str;
319 };
320 
321 static int synth_event_create(int argc, const char **argv);
322 static int synth_event_show(struct seq_file *m, struct dyn_event *ev);
323 static int synth_event_release(struct dyn_event *ev);
324 static bool synth_event_is_busy(struct dyn_event *ev);
325 static bool synth_event_match(const char *system, const char *event,
326 			      struct dyn_event *ev);
327 
328 static struct dyn_event_operations synth_event_ops = {
329 	.create = synth_event_create,
330 	.show = synth_event_show,
331 	.is_busy = synth_event_is_busy,
332 	.free = synth_event_release,
333 	.match = synth_event_match,
334 };
335 
336 struct synth_field {
337 	char *type;
338 	char *name;
339 	size_t size;
340 	bool is_signed;
341 	bool is_string;
342 };
343 
344 struct synth_event {
345 	struct dyn_event			devent;
346 	int					ref;
347 	char					*name;
348 	struct synth_field			**fields;
349 	unsigned int				n_fields;
350 	unsigned int				n_u64;
351 	struct trace_event_class		class;
352 	struct trace_event_call			call;
353 	struct tracepoint			*tp;
354 };
355 
356 static bool is_synth_event(struct dyn_event *ev)
357 {
358 	return ev->ops == &synth_event_ops;
359 }
360 
361 static struct synth_event *to_synth_event(struct dyn_event *ev)
362 {
363 	return container_of(ev, struct synth_event, devent);
364 }
365 
366 static bool synth_event_is_busy(struct dyn_event *ev)
367 {
368 	struct synth_event *event = to_synth_event(ev);
369 
370 	return event->ref != 0;
371 }
372 
373 static bool synth_event_match(const char *system, const char *event,
374 			      struct dyn_event *ev)
375 {
376 	struct synth_event *sev = to_synth_event(ev);
377 
378 	return strcmp(sev->name, event) == 0 &&
379 		(!system || strcmp(system, SYNTH_SYSTEM) == 0);
380 }
381 
382 struct action_data;
383 
384 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
385 			     struct tracing_map_elt *elt, void *rec,
386 			     struct ring_buffer_event *rbe, void *key,
387 			     struct action_data *data, u64 *var_ref_vals);
388 
389 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
390 
391 enum handler_id {
392 	HANDLER_ONMATCH = 1,
393 	HANDLER_ONMAX,
394 	HANDLER_ONCHANGE,
395 };
396 
397 enum action_id {
398 	ACTION_SAVE = 1,
399 	ACTION_TRACE,
400 	ACTION_SNAPSHOT,
401 };
402 
403 struct action_data {
404 	enum handler_id		handler;
405 	enum action_id		action;
406 	char			*action_name;
407 	action_fn_t		fn;
408 
409 	unsigned int		n_params;
410 	char			*params[SYNTH_FIELDS_MAX];
411 
412 	/*
413 	 * When a histogram trigger is hit, the values of any
414 	 * references to variables, including variables being passed
415 	 * as parameters to synthetic events, are collected into a
416 	 * var_ref_vals array.  This var_ref_idx is the index of the
417 	 * first param in the array to be passed to the synthetic
418 	 * event invocation.
419 	 */
420 	unsigned int		var_ref_idx;
421 	struct synth_event	*synth_event;
422 	bool			use_trace_keyword;
423 	char			*synth_event_name;
424 
425 	union {
426 		struct {
427 			char			*event;
428 			char			*event_system;
429 		} match_data;
430 
431 		struct {
432 			/*
433 			 * var_str contains the $-unstripped variable
434 			 * name referenced by var_ref, and used when
435 			 * printing the action.  Because var_ref
436 			 * creation is deferred to create_actions(),
437 			 * we need a per-action way to save it until
438 			 * then, thus var_str.
439 			 */
440 			char			*var_str;
441 
442 			/*
443 			 * var_ref refers to the variable being
444 			 * tracked e.g onmax($var).
445 			 */
446 			struct hist_field	*var_ref;
447 
448 			/*
449 			 * track_var contains the 'invisible' tracking
450 			 * variable created to keep the current
451 			 * e.g. max value.
452 			 */
453 			struct hist_field	*track_var;
454 
455 			check_track_val_fn_t	check_val;
456 			action_fn_t		save_data;
457 		} track_data;
458 	};
459 };
460 
461 struct track_data {
462 	u64				track_val;
463 	bool				updated;
464 
465 	unsigned int			key_len;
466 	void				*key;
467 	struct tracing_map_elt		elt;
468 
469 	struct action_data		*action_data;
470 	struct hist_trigger_data	*hist_data;
471 };
472 
473 struct hist_elt_data {
474 	char *comm;
475 	u64 *var_ref_vals;
476 	char *field_var_str[SYNTH_FIELDS_MAX];
477 };
478 
479 struct snapshot_context {
480 	struct tracing_map_elt	*elt;
481 	void			*key;
482 };
483 
484 static void track_data_free(struct track_data *track_data)
485 {
486 	struct hist_elt_data *elt_data;
487 
488 	if (!track_data)
489 		return;
490 
491 	kfree(track_data->key);
492 
493 	elt_data = track_data->elt.private_data;
494 	if (elt_data) {
495 		kfree(elt_data->comm);
496 		kfree(elt_data);
497 	}
498 
499 	kfree(track_data);
500 }
501 
502 static struct track_data *track_data_alloc(unsigned int key_len,
503 					   struct action_data *action_data,
504 					   struct hist_trigger_data *hist_data)
505 {
506 	struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
507 	struct hist_elt_data *elt_data;
508 
509 	if (!data)
510 		return ERR_PTR(-ENOMEM);
511 
512 	data->key = kzalloc(key_len, GFP_KERNEL);
513 	if (!data->key) {
514 		track_data_free(data);
515 		return ERR_PTR(-ENOMEM);
516 	}
517 
518 	data->key_len = key_len;
519 	data->action_data = action_data;
520 	data->hist_data = hist_data;
521 
522 	elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
523 	if (!elt_data) {
524 		track_data_free(data);
525 		return ERR_PTR(-ENOMEM);
526 	}
527 	data->elt.private_data = elt_data;
528 
529 	elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
530 	if (!elt_data->comm) {
531 		track_data_free(data);
532 		return ERR_PTR(-ENOMEM);
533 	}
534 
535 	return data;
536 }
537 
538 static char last_hist_cmd[MAX_FILTER_STR_VAL];
539 static char hist_err_str[MAX_FILTER_STR_VAL];
540 
541 static void last_cmd_set(char *str)
542 {
543 	if (!str)
544 		return;
545 
546 	strncpy(last_hist_cmd, str, MAX_FILTER_STR_VAL - 1);
547 }
548 
549 static void hist_err(char *str, char *var)
550 {
551 	int maxlen = MAX_FILTER_STR_VAL - 1;
552 
553 	if (!str)
554 		return;
555 
556 	if (strlen(hist_err_str))
557 		return;
558 
559 	if (!var)
560 		var = "";
561 
562 	if (strlen(hist_err_str) + strlen(str) + strlen(var) > maxlen)
563 		return;
564 
565 	strcat(hist_err_str, str);
566 	strcat(hist_err_str, var);
567 }
568 
569 static void hist_err_event(char *str, char *system, char *event, char *var)
570 {
571 	char err[MAX_FILTER_STR_VAL];
572 
573 	if (system && var)
574 		snprintf(err, MAX_FILTER_STR_VAL, "%s.%s.%s", system, event, var);
575 	else if (system)
576 		snprintf(err, MAX_FILTER_STR_VAL, "%s.%s", system, event);
577 	else
578 		strscpy(err, var, MAX_FILTER_STR_VAL);
579 
580 	hist_err(str, err);
581 }
582 
583 static void hist_err_clear(void)
584 {
585 	hist_err_str[0] = '\0';
586 }
587 
588 static bool have_hist_err(void)
589 {
590 	if (strlen(hist_err_str))
591 		return true;
592 
593 	return false;
594 }
595 
596 struct synth_trace_event {
597 	struct trace_entry	ent;
598 	u64			fields[];
599 };
600 
601 static int synth_event_define_fields(struct trace_event_call *call)
602 {
603 	struct synth_trace_event trace;
604 	int offset = offsetof(typeof(trace), fields);
605 	struct synth_event *event = call->data;
606 	unsigned int i, size, n_u64;
607 	char *name, *type;
608 	bool is_signed;
609 	int ret = 0;
610 
611 	for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
612 		size = event->fields[i]->size;
613 		is_signed = event->fields[i]->is_signed;
614 		type = event->fields[i]->type;
615 		name = event->fields[i]->name;
616 		ret = trace_define_field(call, type, name, offset, size,
617 					 is_signed, FILTER_OTHER);
618 		if (ret)
619 			break;
620 
621 		if (event->fields[i]->is_string) {
622 			offset += STR_VAR_LEN_MAX;
623 			n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
624 		} else {
625 			offset += sizeof(u64);
626 			n_u64++;
627 		}
628 	}
629 
630 	event->n_u64 = n_u64;
631 
632 	return ret;
633 }
634 
635 static bool synth_field_signed(char *type)
636 {
637 	if (str_has_prefix(type, "u"))
638 		return false;
639 
640 	return true;
641 }
642 
643 static int synth_field_is_string(char *type)
644 {
645 	if (strstr(type, "char[") != NULL)
646 		return true;
647 
648 	return false;
649 }
650 
651 static int synth_field_string_size(char *type)
652 {
653 	char buf[4], *end, *start;
654 	unsigned int len;
655 	int size, err;
656 
657 	start = strstr(type, "char[");
658 	if (start == NULL)
659 		return -EINVAL;
660 	start += sizeof("char[") - 1;
661 
662 	end = strchr(type, ']');
663 	if (!end || end < start)
664 		return -EINVAL;
665 
666 	len = end - start;
667 	if (len > 3)
668 		return -EINVAL;
669 
670 	strncpy(buf, start, len);
671 	buf[len] = '\0';
672 
673 	err = kstrtouint(buf, 0, &size);
674 	if (err)
675 		return err;
676 
677 	if (size > STR_VAR_LEN_MAX)
678 		return -EINVAL;
679 
680 	return size;
681 }
682 
683 static int synth_field_size(char *type)
684 {
685 	int size = 0;
686 
687 	if (strcmp(type, "s64") == 0)
688 		size = sizeof(s64);
689 	else if (strcmp(type, "u64") == 0)
690 		size = sizeof(u64);
691 	else if (strcmp(type, "s32") == 0)
692 		size = sizeof(s32);
693 	else if (strcmp(type, "u32") == 0)
694 		size = sizeof(u32);
695 	else if (strcmp(type, "s16") == 0)
696 		size = sizeof(s16);
697 	else if (strcmp(type, "u16") == 0)
698 		size = sizeof(u16);
699 	else if (strcmp(type, "s8") == 0)
700 		size = sizeof(s8);
701 	else if (strcmp(type, "u8") == 0)
702 		size = sizeof(u8);
703 	else if (strcmp(type, "char") == 0)
704 		size = sizeof(char);
705 	else if (strcmp(type, "unsigned char") == 0)
706 		size = sizeof(unsigned char);
707 	else if (strcmp(type, "int") == 0)
708 		size = sizeof(int);
709 	else if (strcmp(type, "unsigned int") == 0)
710 		size = sizeof(unsigned int);
711 	else if (strcmp(type, "long") == 0)
712 		size = sizeof(long);
713 	else if (strcmp(type, "unsigned long") == 0)
714 		size = sizeof(unsigned long);
715 	else if (strcmp(type, "pid_t") == 0)
716 		size = sizeof(pid_t);
717 	else if (synth_field_is_string(type))
718 		size = synth_field_string_size(type);
719 
720 	return size;
721 }
722 
723 static const char *synth_field_fmt(char *type)
724 {
725 	const char *fmt = "%llu";
726 
727 	if (strcmp(type, "s64") == 0)
728 		fmt = "%lld";
729 	else if (strcmp(type, "u64") == 0)
730 		fmt = "%llu";
731 	else if (strcmp(type, "s32") == 0)
732 		fmt = "%d";
733 	else if (strcmp(type, "u32") == 0)
734 		fmt = "%u";
735 	else if (strcmp(type, "s16") == 0)
736 		fmt = "%d";
737 	else if (strcmp(type, "u16") == 0)
738 		fmt = "%u";
739 	else if (strcmp(type, "s8") == 0)
740 		fmt = "%d";
741 	else if (strcmp(type, "u8") == 0)
742 		fmt = "%u";
743 	else if (strcmp(type, "char") == 0)
744 		fmt = "%d";
745 	else if (strcmp(type, "unsigned char") == 0)
746 		fmt = "%u";
747 	else if (strcmp(type, "int") == 0)
748 		fmt = "%d";
749 	else if (strcmp(type, "unsigned int") == 0)
750 		fmt = "%u";
751 	else if (strcmp(type, "long") == 0)
752 		fmt = "%ld";
753 	else if (strcmp(type, "unsigned long") == 0)
754 		fmt = "%lu";
755 	else if (strcmp(type, "pid_t") == 0)
756 		fmt = "%d";
757 	else if (synth_field_is_string(type))
758 		fmt = "%s";
759 
760 	return fmt;
761 }
762 
763 static enum print_line_t print_synth_event(struct trace_iterator *iter,
764 					   int flags,
765 					   struct trace_event *event)
766 {
767 	struct trace_array *tr = iter->tr;
768 	struct trace_seq *s = &iter->seq;
769 	struct synth_trace_event *entry;
770 	struct synth_event *se;
771 	unsigned int i, n_u64;
772 	char print_fmt[32];
773 	const char *fmt;
774 
775 	entry = (struct synth_trace_event *)iter->ent;
776 	se = container_of(event, struct synth_event, call.event);
777 
778 	trace_seq_printf(s, "%s: ", se->name);
779 
780 	for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
781 		if (trace_seq_has_overflowed(s))
782 			goto end;
783 
784 		fmt = synth_field_fmt(se->fields[i]->type);
785 
786 		/* parameter types */
787 		if (tr->trace_flags & TRACE_ITER_VERBOSE)
788 			trace_seq_printf(s, "%s ", fmt);
789 
790 		snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
791 
792 		/* parameter values */
793 		if (se->fields[i]->is_string) {
794 			trace_seq_printf(s, print_fmt, se->fields[i]->name,
795 					 (char *)&entry->fields[n_u64],
796 					 i == se->n_fields - 1 ? "" : " ");
797 			n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
798 		} else {
799 			trace_seq_printf(s, print_fmt, se->fields[i]->name,
800 					 entry->fields[n_u64],
801 					 i == se->n_fields - 1 ? "" : " ");
802 			n_u64++;
803 		}
804 	}
805 end:
806 	trace_seq_putc(s, '\n');
807 
808 	return trace_handle_return(s);
809 }
810 
811 static struct trace_event_functions synth_event_funcs = {
812 	.trace		= print_synth_event
813 };
814 
815 static notrace void trace_event_raw_event_synth(void *__data,
816 						u64 *var_ref_vals,
817 						unsigned int var_ref_idx)
818 {
819 	struct trace_event_file *trace_file = __data;
820 	struct synth_trace_event *entry;
821 	struct trace_event_buffer fbuffer;
822 	struct ring_buffer *buffer;
823 	struct synth_event *event;
824 	unsigned int i, n_u64;
825 	int fields_size = 0;
826 
827 	event = trace_file->event_call->data;
828 
829 	if (trace_trigger_soft_disabled(trace_file))
830 		return;
831 
832 	fields_size = event->n_u64 * sizeof(u64);
833 
834 	/*
835 	 * Avoid ring buffer recursion detection, as this event
836 	 * is being performed within another event.
837 	 */
838 	buffer = trace_file->tr->trace_buffer.buffer;
839 	ring_buffer_nest_start(buffer);
840 
841 	entry = trace_event_buffer_reserve(&fbuffer, trace_file,
842 					   sizeof(*entry) + fields_size);
843 	if (!entry)
844 		goto out;
845 
846 	for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
847 		if (event->fields[i]->is_string) {
848 			char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i];
849 			char *str_field = (char *)&entry->fields[n_u64];
850 
851 			strscpy(str_field, str_val, STR_VAR_LEN_MAX);
852 			n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
853 		} else {
854 			entry->fields[n_u64] = var_ref_vals[var_ref_idx + i];
855 			n_u64++;
856 		}
857 	}
858 
859 	trace_event_buffer_commit(&fbuffer);
860 out:
861 	ring_buffer_nest_end(buffer);
862 }
863 
864 static void free_synth_event_print_fmt(struct trace_event_call *call)
865 {
866 	if (call) {
867 		kfree(call->print_fmt);
868 		call->print_fmt = NULL;
869 	}
870 }
871 
872 static int __set_synth_event_print_fmt(struct synth_event *event,
873 				       char *buf, int len)
874 {
875 	const char *fmt;
876 	int pos = 0;
877 	int i;
878 
879 	/* When len=0, we just calculate the needed length */
880 #define LEN_OR_ZERO (len ? len - pos : 0)
881 
882 	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
883 	for (i = 0; i < event->n_fields; i++) {
884 		fmt = synth_field_fmt(event->fields[i]->type);
885 		pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
886 				event->fields[i]->name, fmt,
887 				i == event->n_fields - 1 ? "" : ", ");
888 	}
889 	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
890 
891 	for (i = 0; i < event->n_fields; i++) {
892 		pos += snprintf(buf + pos, LEN_OR_ZERO,
893 				", REC->%s", event->fields[i]->name);
894 	}
895 
896 #undef LEN_OR_ZERO
897 
898 	/* return the length of print_fmt */
899 	return pos;
900 }
901 
902 static int set_synth_event_print_fmt(struct trace_event_call *call)
903 {
904 	struct synth_event *event = call->data;
905 	char *print_fmt;
906 	int len;
907 
908 	/* First: called with 0 length to calculate the needed length */
909 	len = __set_synth_event_print_fmt(event, NULL, 0);
910 
911 	print_fmt = kmalloc(len + 1, GFP_KERNEL);
912 	if (!print_fmt)
913 		return -ENOMEM;
914 
915 	/* Second: actually write the @print_fmt */
916 	__set_synth_event_print_fmt(event, print_fmt, len + 1);
917 	call->print_fmt = print_fmt;
918 
919 	return 0;
920 }
921 
922 static void free_synth_field(struct synth_field *field)
923 {
924 	kfree(field->type);
925 	kfree(field->name);
926 	kfree(field);
927 }
928 
929 static struct synth_field *parse_synth_field(int argc, const char **argv,
930 					     int *consumed)
931 {
932 	struct synth_field *field;
933 	const char *prefix = NULL, *field_type = argv[0], *field_name, *array;
934 	int len, ret = 0;
935 
936 	if (field_type[0] == ';')
937 		field_type++;
938 
939 	if (!strcmp(field_type, "unsigned")) {
940 		if (argc < 3)
941 			return ERR_PTR(-EINVAL);
942 		prefix = "unsigned ";
943 		field_type = argv[1];
944 		field_name = argv[2];
945 		*consumed = 3;
946 	} else {
947 		field_name = argv[1];
948 		*consumed = 2;
949 	}
950 
951 	field = kzalloc(sizeof(*field), GFP_KERNEL);
952 	if (!field)
953 		return ERR_PTR(-ENOMEM);
954 
955 	len = strlen(field_name);
956 	array = strchr(field_name, '[');
957 	if (array)
958 		len -= strlen(array);
959 	else if (field_name[len - 1] == ';')
960 		len--;
961 
962 	field->name = kmemdup_nul(field_name, len, GFP_KERNEL);
963 	if (!field->name) {
964 		ret = -ENOMEM;
965 		goto free;
966 	}
967 
968 	if (field_type[0] == ';')
969 		field_type++;
970 	len = strlen(field_type) + 1;
971 	if (array)
972 		len += strlen(array);
973 	if (prefix)
974 		len += strlen(prefix);
975 
976 	field->type = kzalloc(len, GFP_KERNEL);
977 	if (!field->type) {
978 		ret = -ENOMEM;
979 		goto free;
980 	}
981 	if (prefix)
982 		strcat(field->type, prefix);
983 	strcat(field->type, field_type);
984 	if (array) {
985 		strcat(field->type, array);
986 		if (field->type[len - 1] == ';')
987 			field->type[len - 1] = '\0';
988 	}
989 
990 	field->size = synth_field_size(field->type);
991 	if (!field->size) {
992 		ret = -EINVAL;
993 		goto free;
994 	}
995 
996 	if (synth_field_is_string(field->type))
997 		field->is_string = true;
998 
999 	field->is_signed = synth_field_signed(field->type);
1000 
1001  out:
1002 	return field;
1003  free:
1004 	free_synth_field(field);
1005 	field = ERR_PTR(ret);
1006 	goto out;
1007 }
1008 
1009 static void free_synth_tracepoint(struct tracepoint *tp)
1010 {
1011 	if (!tp)
1012 		return;
1013 
1014 	kfree(tp->name);
1015 	kfree(tp);
1016 }
1017 
1018 static struct tracepoint *alloc_synth_tracepoint(char *name)
1019 {
1020 	struct tracepoint *tp;
1021 
1022 	tp = kzalloc(sizeof(*tp), GFP_KERNEL);
1023 	if (!tp)
1024 		return ERR_PTR(-ENOMEM);
1025 
1026 	tp->name = kstrdup(name, GFP_KERNEL);
1027 	if (!tp->name) {
1028 		kfree(tp);
1029 		return ERR_PTR(-ENOMEM);
1030 	}
1031 
1032 	return tp;
1033 }
1034 
1035 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
1036 				    unsigned int var_ref_idx);
1037 
1038 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
1039 			       unsigned int var_ref_idx)
1040 {
1041 	struct tracepoint *tp = event->tp;
1042 
1043 	if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
1044 		struct tracepoint_func *probe_func_ptr;
1045 		synth_probe_func_t probe_func;
1046 		void *__data;
1047 
1048 		if (!(cpu_online(raw_smp_processor_id())))
1049 			return;
1050 
1051 		probe_func_ptr = rcu_dereference_sched((tp)->funcs);
1052 		if (probe_func_ptr) {
1053 			do {
1054 				probe_func = probe_func_ptr->func;
1055 				__data = probe_func_ptr->data;
1056 				probe_func(__data, var_ref_vals, var_ref_idx);
1057 			} while ((++probe_func_ptr)->func);
1058 		}
1059 	}
1060 }
1061 
1062 static struct synth_event *find_synth_event(const char *name)
1063 {
1064 	struct dyn_event *pos;
1065 	struct synth_event *event;
1066 
1067 	for_each_dyn_event(pos) {
1068 		if (!is_synth_event(pos))
1069 			continue;
1070 		event = to_synth_event(pos);
1071 		if (strcmp(event->name, name) == 0)
1072 			return event;
1073 	}
1074 
1075 	return NULL;
1076 }
1077 
1078 static int register_synth_event(struct synth_event *event)
1079 {
1080 	struct trace_event_call *call = &event->call;
1081 	int ret = 0;
1082 
1083 	event->call.class = &event->class;
1084 	event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
1085 	if (!event->class.system) {
1086 		ret = -ENOMEM;
1087 		goto out;
1088 	}
1089 
1090 	event->tp = alloc_synth_tracepoint(event->name);
1091 	if (IS_ERR(event->tp)) {
1092 		ret = PTR_ERR(event->tp);
1093 		event->tp = NULL;
1094 		goto out;
1095 	}
1096 
1097 	INIT_LIST_HEAD(&call->class->fields);
1098 	call->event.funcs = &synth_event_funcs;
1099 	call->class->define_fields = synth_event_define_fields;
1100 
1101 	ret = register_trace_event(&call->event);
1102 	if (!ret) {
1103 		ret = -ENODEV;
1104 		goto out;
1105 	}
1106 	call->flags = TRACE_EVENT_FL_TRACEPOINT;
1107 	call->class->reg = trace_event_reg;
1108 	call->class->probe = trace_event_raw_event_synth;
1109 	call->data = event;
1110 	call->tp = event->tp;
1111 
1112 	ret = trace_add_event_call(call);
1113 	if (ret) {
1114 		pr_warn("Failed to register synthetic event: %s\n",
1115 			trace_event_name(call));
1116 		goto err;
1117 	}
1118 
1119 	ret = set_synth_event_print_fmt(call);
1120 	if (ret < 0) {
1121 		trace_remove_event_call(call);
1122 		goto err;
1123 	}
1124  out:
1125 	return ret;
1126  err:
1127 	unregister_trace_event(&call->event);
1128 	goto out;
1129 }
1130 
1131 static int unregister_synth_event(struct synth_event *event)
1132 {
1133 	struct trace_event_call *call = &event->call;
1134 	int ret;
1135 
1136 	ret = trace_remove_event_call(call);
1137 
1138 	return ret;
1139 }
1140 
1141 static void free_synth_event(struct synth_event *event)
1142 {
1143 	unsigned int i;
1144 
1145 	if (!event)
1146 		return;
1147 
1148 	for (i = 0; i < event->n_fields; i++)
1149 		free_synth_field(event->fields[i]);
1150 
1151 	kfree(event->fields);
1152 	kfree(event->name);
1153 	kfree(event->class.system);
1154 	free_synth_tracepoint(event->tp);
1155 	free_synth_event_print_fmt(&event->call);
1156 	kfree(event);
1157 }
1158 
1159 static struct synth_event *alloc_synth_event(const char *name, int n_fields,
1160 					     struct synth_field **fields)
1161 {
1162 	struct synth_event *event;
1163 	unsigned int i;
1164 
1165 	event = kzalloc(sizeof(*event), GFP_KERNEL);
1166 	if (!event) {
1167 		event = ERR_PTR(-ENOMEM);
1168 		goto out;
1169 	}
1170 
1171 	event->name = kstrdup(name, GFP_KERNEL);
1172 	if (!event->name) {
1173 		kfree(event);
1174 		event = ERR_PTR(-ENOMEM);
1175 		goto out;
1176 	}
1177 
1178 	event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
1179 	if (!event->fields) {
1180 		free_synth_event(event);
1181 		event = ERR_PTR(-ENOMEM);
1182 		goto out;
1183 	}
1184 
1185 	dyn_event_init(&event->devent, &synth_event_ops);
1186 
1187 	for (i = 0; i < n_fields; i++)
1188 		event->fields[i] = fields[i];
1189 
1190 	event->n_fields = n_fields;
1191  out:
1192 	return event;
1193 }
1194 
1195 static void action_trace(struct hist_trigger_data *hist_data,
1196 			 struct tracing_map_elt *elt, void *rec,
1197 			 struct ring_buffer_event *rbe, void *key,
1198 			 struct action_data *data, u64 *var_ref_vals)
1199 {
1200 	struct synth_event *event = data->synth_event;
1201 
1202 	trace_synth(event, var_ref_vals, data->var_ref_idx);
1203 }
1204 
1205 struct hist_var_data {
1206 	struct list_head list;
1207 	struct hist_trigger_data *hist_data;
1208 };
1209 
1210 static int __create_synth_event(int argc, const char *name, const char **argv)
1211 {
1212 	struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1213 	struct synth_event *event = NULL;
1214 	int i, consumed = 0, n_fields = 0, ret = 0;
1215 
1216 	/*
1217 	 * Argument syntax:
1218 	 *  - Add synthetic event: <event_name> field[;field] ...
1219 	 *  - Remove synthetic event: !<event_name> field[;field] ...
1220 	 *      where 'field' = type field_name
1221 	 */
1222 
1223 	if (name[0] == '\0' || argc < 1)
1224 		return -EINVAL;
1225 
1226 	mutex_lock(&event_mutex);
1227 
1228 	event = find_synth_event(name);
1229 	if (event) {
1230 		ret = -EEXIST;
1231 		goto out;
1232 	}
1233 
1234 	for (i = 0; i < argc - 1; i++) {
1235 		if (strcmp(argv[i], ";") == 0)
1236 			continue;
1237 		if (n_fields == SYNTH_FIELDS_MAX) {
1238 			ret = -EINVAL;
1239 			goto err;
1240 		}
1241 
1242 		field = parse_synth_field(argc - i, &argv[i], &consumed);
1243 		if (IS_ERR(field)) {
1244 			ret = PTR_ERR(field);
1245 			goto err;
1246 		}
1247 		fields[n_fields++] = field;
1248 		i += consumed - 1;
1249 	}
1250 
1251 	if (i < argc && strcmp(argv[i], ";") != 0) {
1252 		ret = -EINVAL;
1253 		goto err;
1254 	}
1255 
1256 	event = alloc_synth_event(name, n_fields, fields);
1257 	if (IS_ERR(event)) {
1258 		ret = PTR_ERR(event);
1259 		event = NULL;
1260 		goto err;
1261 	}
1262 	ret = register_synth_event(event);
1263 	if (!ret)
1264 		dyn_event_add(&event->devent);
1265 	else
1266 		free_synth_event(event);
1267  out:
1268 	mutex_unlock(&event_mutex);
1269 
1270 	return ret;
1271  err:
1272 	for (i = 0; i < n_fields; i++)
1273 		free_synth_field(fields[i]);
1274 
1275 	goto out;
1276 }
1277 
1278 static int create_or_delete_synth_event(int argc, char **argv)
1279 {
1280 	const char *name = argv[0];
1281 	struct synth_event *event = NULL;
1282 	int ret;
1283 
1284 	/* trace_run_command() ensures argc != 0 */
1285 	if (name[0] == '!') {
1286 		mutex_lock(&event_mutex);
1287 		event = find_synth_event(name + 1);
1288 		if (event) {
1289 			if (event->ref)
1290 				ret = -EBUSY;
1291 			else {
1292 				ret = unregister_synth_event(event);
1293 				if (!ret) {
1294 					dyn_event_remove(&event->devent);
1295 					free_synth_event(event);
1296 				}
1297 			}
1298 		} else
1299 			ret = -ENOENT;
1300 		mutex_unlock(&event_mutex);
1301 		return ret;
1302 	}
1303 
1304 	ret = __create_synth_event(argc - 1, name, (const char **)argv + 1);
1305 	return ret == -ECANCELED ? -EINVAL : ret;
1306 }
1307 
1308 static int synth_event_create(int argc, const char **argv)
1309 {
1310 	const char *name = argv[0];
1311 	int len;
1312 
1313 	if (name[0] != 's' || name[1] != ':')
1314 		return -ECANCELED;
1315 	name += 2;
1316 
1317 	/* This interface accepts group name prefix */
1318 	if (strchr(name, '/')) {
1319 		len = str_has_prefix(name, SYNTH_SYSTEM "/");
1320 		if (len == 0)
1321 			return -EINVAL;
1322 		name += len;
1323 	}
1324 	return __create_synth_event(argc - 1, name, argv + 1);
1325 }
1326 
1327 static int synth_event_release(struct dyn_event *ev)
1328 {
1329 	struct synth_event *event = to_synth_event(ev);
1330 	int ret;
1331 
1332 	if (event->ref)
1333 		return -EBUSY;
1334 
1335 	ret = unregister_synth_event(event);
1336 	if (ret)
1337 		return ret;
1338 
1339 	dyn_event_remove(ev);
1340 	free_synth_event(event);
1341 	return 0;
1342 }
1343 
1344 static int __synth_event_show(struct seq_file *m, struct synth_event *event)
1345 {
1346 	struct synth_field *field;
1347 	unsigned int i;
1348 
1349 	seq_printf(m, "%s\t", event->name);
1350 
1351 	for (i = 0; i < event->n_fields; i++) {
1352 		field = event->fields[i];
1353 
1354 		/* parameter values */
1355 		seq_printf(m, "%s %s%s", field->type, field->name,
1356 			   i == event->n_fields - 1 ? "" : "; ");
1357 	}
1358 
1359 	seq_putc(m, '\n');
1360 
1361 	return 0;
1362 }
1363 
1364 static int synth_event_show(struct seq_file *m, struct dyn_event *ev)
1365 {
1366 	struct synth_event *event = to_synth_event(ev);
1367 
1368 	seq_printf(m, "s:%s/", event->class.system);
1369 
1370 	return __synth_event_show(m, event);
1371 }
1372 
1373 static int synth_events_seq_show(struct seq_file *m, void *v)
1374 {
1375 	struct dyn_event *ev = v;
1376 
1377 	if (!is_synth_event(ev))
1378 		return 0;
1379 
1380 	return __synth_event_show(m, to_synth_event(ev));
1381 }
1382 
1383 static const struct seq_operations synth_events_seq_op = {
1384 	.start	= dyn_event_seq_start,
1385 	.next	= dyn_event_seq_next,
1386 	.stop	= dyn_event_seq_stop,
1387 	.show	= synth_events_seq_show,
1388 };
1389 
1390 static int synth_events_open(struct inode *inode, struct file *file)
1391 {
1392 	int ret;
1393 
1394 	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1395 		ret = dyn_events_release_all(&synth_event_ops);
1396 		if (ret < 0)
1397 			return ret;
1398 	}
1399 
1400 	return seq_open(file, &synth_events_seq_op);
1401 }
1402 
1403 static ssize_t synth_events_write(struct file *file,
1404 				  const char __user *buffer,
1405 				  size_t count, loff_t *ppos)
1406 {
1407 	return trace_parse_run_command(file, buffer, count, ppos,
1408 				       create_or_delete_synth_event);
1409 }
1410 
1411 static const struct file_operations synth_events_fops = {
1412 	.open           = synth_events_open,
1413 	.write		= synth_events_write,
1414 	.read           = seq_read,
1415 	.llseek         = seq_lseek,
1416 	.release        = seq_release,
1417 };
1418 
1419 static u64 hist_field_timestamp(struct hist_field *hist_field,
1420 				struct tracing_map_elt *elt,
1421 				struct ring_buffer_event *rbe,
1422 				void *event)
1423 {
1424 	struct hist_trigger_data *hist_data = hist_field->hist_data;
1425 	struct trace_array *tr = hist_data->event_file->tr;
1426 
1427 	u64 ts = ring_buffer_event_time_stamp(rbe);
1428 
1429 	if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
1430 		ts = ns2usecs(ts);
1431 
1432 	return ts;
1433 }
1434 
1435 static u64 hist_field_cpu(struct hist_field *hist_field,
1436 			  struct tracing_map_elt *elt,
1437 			  struct ring_buffer_event *rbe,
1438 			  void *event)
1439 {
1440 	int cpu = smp_processor_id();
1441 
1442 	return cpu;
1443 }
1444 
1445 /**
1446  * check_field_for_var_ref - Check if a VAR_REF field references a variable
1447  * @hist_field: The VAR_REF field to check
1448  * @var_data: The hist trigger that owns the variable
1449  * @var_idx: The trigger variable identifier
1450  *
1451  * Check the given VAR_REF field to see whether or not it references
1452  * the given variable associated with the given trigger.
1453  *
1454  * Return: The VAR_REF field if it does reference the variable, NULL if not
1455  */
1456 static struct hist_field *
1457 check_field_for_var_ref(struct hist_field *hist_field,
1458 			struct hist_trigger_data *var_data,
1459 			unsigned int var_idx)
1460 {
1461 	WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
1462 
1463 	if (hist_field && hist_field->var.idx == var_idx &&
1464 	    hist_field->var.hist_data == var_data)
1465 		return hist_field;
1466 
1467 	return NULL;
1468 }
1469 
1470 /**
1471  * find_var_ref - Check if a trigger has a reference to a trigger variable
1472  * @hist_data: The hist trigger that might have a reference to the variable
1473  * @var_data: The hist trigger that owns the variable
1474  * @var_idx: The trigger variable identifier
1475  *
1476  * Check the list of var_refs[] on the first hist trigger to see
1477  * whether any of them are references to the variable on the second
1478  * trigger.
1479  *
1480  * Return: The VAR_REF field referencing the variable if so, NULL if not
1481  */
1482 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1483 				       struct hist_trigger_data *var_data,
1484 				       unsigned int var_idx)
1485 {
1486 	struct hist_field *hist_field;
1487 	unsigned int i;
1488 
1489 	for (i = 0; i < hist_data->n_var_refs; i++) {
1490 		hist_field = hist_data->var_refs[i];
1491 		if (check_field_for_var_ref(hist_field, var_data, var_idx))
1492 			return hist_field;
1493 	}
1494 
1495 	return NULL;
1496 }
1497 
1498 /**
1499  * find_any_var_ref - Check if there is a reference to a given trigger variable
1500  * @hist_data: The hist trigger
1501  * @var_idx: The trigger variable identifier
1502  *
1503  * Check to see whether the given variable is currently referenced by
1504  * any other trigger.
1505  *
1506  * The trigger the variable is defined on is explicitly excluded - the
1507  * assumption being that a self-reference doesn't prevent a trigger
1508  * from being removed.
1509  *
1510  * Return: The VAR_REF field referencing the variable if so, NULL if not
1511  */
1512 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1513 					   unsigned int var_idx)
1514 {
1515 	struct trace_array *tr = hist_data->event_file->tr;
1516 	struct hist_field *found = NULL;
1517 	struct hist_var_data *var_data;
1518 
1519 	list_for_each_entry(var_data, &tr->hist_vars, list) {
1520 		if (var_data->hist_data == hist_data)
1521 			continue;
1522 		found = find_var_ref(var_data->hist_data, hist_data, var_idx);
1523 		if (found)
1524 			break;
1525 	}
1526 
1527 	return found;
1528 }
1529 
1530 /**
1531  * check_var_refs - Check if there is a reference to any of trigger's variables
1532  * @hist_data: The hist trigger
1533  *
1534  * A trigger can define one or more variables.  If any one of them is
1535  * currently referenced by any other trigger, this function will
1536  * determine that.
1537 
1538  * Typically used to determine whether or not a trigger can be removed
1539  * - if there are any references to a trigger's variables, it cannot.
1540  *
1541  * Return: True if there is a reference to any of trigger's variables
1542  */
1543 static bool check_var_refs(struct hist_trigger_data *hist_data)
1544 {
1545 	struct hist_field *field;
1546 	bool found = false;
1547 	int i;
1548 
1549 	for_each_hist_field(i, hist_data) {
1550 		field = hist_data->fields[i];
1551 		if (field && field->flags & HIST_FIELD_FL_VAR) {
1552 			if (find_any_var_ref(hist_data, field->var.idx)) {
1553 				found = true;
1554 				break;
1555 			}
1556 		}
1557 	}
1558 
1559 	return found;
1560 }
1561 
1562 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1563 {
1564 	struct trace_array *tr = hist_data->event_file->tr;
1565 	struct hist_var_data *var_data, *found = NULL;
1566 
1567 	list_for_each_entry(var_data, &tr->hist_vars, list) {
1568 		if (var_data->hist_data == hist_data) {
1569 			found = var_data;
1570 			break;
1571 		}
1572 	}
1573 
1574 	return found;
1575 }
1576 
1577 static bool field_has_hist_vars(struct hist_field *hist_field,
1578 				unsigned int level)
1579 {
1580 	int i;
1581 
1582 	if (level > 3)
1583 		return false;
1584 
1585 	if (!hist_field)
1586 		return false;
1587 
1588 	if (hist_field->flags & HIST_FIELD_FL_VAR ||
1589 	    hist_field->flags & HIST_FIELD_FL_VAR_REF)
1590 		return true;
1591 
1592 	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1593 		struct hist_field *operand;
1594 
1595 		operand = hist_field->operands[i];
1596 		if (field_has_hist_vars(operand, level + 1))
1597 			return true;
1598 	}
1599 
1600 	return false;
1601 }
1602 
1603 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1604 {
1605 	struct hist_field *hist_field;
1606 	int i;
1607 
1608 	for_each_hist_field(i, hist_data) {
1609 		hist_field = hist_data->fields[i];
1610 		if (field_has_hist_vars(hist_field, 0))
1611 			return true;
1612 	}
1613 
1614 	return false;
1615 }
1616 
1617 static int save_hist_vars(struct hist_trigger_data *hist_data)
1618 {
1619 	struct trace_array *tr = hist_data->event_file->tr;
1620 	struct hist_var_data *var_data;
1621 
1622 	var_data = find_hist_vars(hist_data);
1623 	if (var_data)
1624 		return 0;
1625 
1626 	if (trace_array_get(tr) < 0)
1627 		return -ENODEV;
1628 
1629 	var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1630 	if (!var_data) {
1631 		trace_array_put(tr);
1632 		return -ENOMEM;
1633 	}
1634 
1635 	var_data->hist_data = hist_data;
1636 	list_add(&var_data->list, &tr->hist_vars);
1637 
1638 	return 0;
1639 }
1640 
1641 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1642 {
1643 	struct trace_array *tr = hist_data->event_file->tr;
1644 	struct hist_var_data *var_data;
1645 
1646 	var_data = find_hist_vars(hist_data);
1647 	if (!var_data)
1648 		return;
1649 
1650 	if (WARN_ON(check_var_refs(hist_data)))
1651 		return;
1652 
1653 	list_del(&var_data->list);
1654 
1655 	kfree(var_data);
1656 
1657 	trace_array_put(tr);
1658 }
1659 
1660 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1661 					 const char *var_name)
1662 {
1663 	struct hist_field *hist_field, *found = NULL;
1664 	int i;
1665 
1666 	for_each_hist_field(i, hist_data) {
1667 		hist_field = hist_data->fields[i];
1668 		if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1669 		    strcmp(hist_field->var.name, var_name) == 0) {
1670 			found = hist_field;
1671 			break;
1672 		}
1673 	}
1674 
1675 	return found;
1676 }
1677 
1678 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1679 				   struct trace_event_file *file,
1680 				   const char *var_name)
1681 {
1682 	struct hist_trigger_data *test_data;
1683 	struct event_trigger_data *test;
1684 	struct hist_field *hist_field;
1685 
1686 	hist_field = find_var_field(hist_data, var_name);
1687 	if (hist_field)
1688 		return hist_field;
1689 
1690 	list_for_each_entry_rcu(test, &file->triggers, list) {
1691 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1692 			test_data = test->private_data;
1693 			hist_field = find_var_field(test_data, var_name);
1694 			if (hist_field)
1695 				return hist_field;
1696 		}
1697 	}
1698 
1699 	return NULL;
1700 }
1701 
1702 static struct trace_event_file *find_var_file(struct trace_array *tr,
1703 					      char *system,
1704 					      char *event_name,
1705 					      char *var_name)
1706 {
1707 	struct hist_trigger_data *var_hist_data;
1708 	struct hist_var_data *var_data;
1709 	struct trace_event_file *file, *found = NULL;
1710 
1711 	if (system)
1712 		return find_event_file(tr, system, event_name);
1713 
1714 	list_for_each_entry(var_data, &tr->hist_vars, list) {
1715 		var_hist_data = var_data->hist_data;
1716 		file = var_hist_data->event_file;
1717 		if (file == found)
1718 			continue;
1719 
1720 		if (find_var_field(var_hist_data, var_name)) {
1721 			if (found) {
1722 				hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
1723 				return NULL;
1724 			}
1725 
1726 			found = file;
1727 		}
1728 	}
1729 
1730 	return found;
1731 }
1732 
1733 static struct hist_field *find_file_var(struct trace_event_file *file,
1734 					const char *var_name)
1735 {
1736 	struct hist_trigger_data *test_data;
1737 	struct event_trigger_data *test;
1738 	struct hist_field *hist_field;
1739 
1740 	list_for_each_entry_rcu(test, &file->triggers, list) {
1741 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1742 			test_data = test->private_data;
1743 			hist_field = find_var_field(test_data, var_name);
1744 			if (hist_field)
1745 				return hist_field;
1746 		}
1747 	}
1748 
1749 	return NULL;
1750 }
1751 
1752 static struct hist_field *
1753 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1754 {
1755 	struct trace_array *tr = hist_data->event_file->tr;
1756 	struct hist_field *hist_field, *found = NULL;
1757 	struct trace_event_file *file;
1758 	unsigned int i;
1759 
1760 	for (i = 0; i < hist_data->n_actions; i++) {
1761 		struct action_data *data = hist_data->actions[i];
1762 
1763 		if (data->handler == HANDLER_ONMATCH) {
1764 			char *system = data->match_data.event_system;
1765 			char *event_name = data->match_data.event;
1766 
1767 			file = find_var_file(tr, system, event_name, var_name);
1768 			if (!file)
1769 				continue;
1770 			hist_field = find_file_var(file, var_name);
1771 			if (hist_field) {
1772 				if (found) {
1773 					hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
1774 					return ERR_PTR(-EINVAL);
1775 				}
1776 
1777 				found = hist_field;
1778 			}
1779 		}
1780 	}
1781 	return found;
1782 }
1783 
1784 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1785 					 char *system,
1786 					 char *event_name,
1787 					 char *var_name)
1788 {
1789 	struct trace_array *tr = hist_data->event_file->tr;
1790 	struct hist_field *hist_field = NULL;
1791 	struct trace_event_file *file;
1792 
1793 	if (!system || !event_name) {
1794 		hist_field = find_match_var(hist_data, var_name);
1795 		if (IS_ERR(hist_field))
1796 			return NULL;
1797 		if (hist_field)
1798 			return hist_field;
1799 	}
1800 
1801 	file = find_var_file(tr, system, event_name, var_name);
1802 	if (!file)
1803 		return NULL;
1804 
1805 	hist_field = find_file_var(file, var_name);
1806 
1807 	return hist_field;
1808 }
1809 
1810 static u64 hist_field_var_ref(struct hist_field *hist_field,
1811 			      struct tracing_map_elt *elt,
1812 			      struct ring_buffer_event *rbe,
1813 			      void *event)
1814 {
1815 	struct hist_elt_data *elt_data;
1816 	u64 var_val = 0;
1817 
1818 	elt_data = elt->private_data;
1819 	var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1820 
1821 	return var_val;
1822 }
1823 
1824 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1825 			     u64 *var_ref_vals, bool self)
1826 {
1827 	struct hist_trigger_data *var_data;
1828 	struct tracing_map_elt *var_elt;
1829 	struct hist_field *hist_field;
1830 	unsigned int i, var_idx;
1831 	bool resolved = true;
1832 	u64 var_val = 0;
1833 
1834 	for (i = 0; i < hist_data->n_var_refs; i++) {
1835 		hist_field = hist_data->var_refs[i];
1836 		var_idx = hist_field->var.idx;
1837 		var_data = hist_field->var.hist_data;
1838 
1839 		if (var_data == NULL) {
1840 			resolved = false;
1841 			break;
1842 		}
1843 
1844 		if ((self && var_data != hist_data) ||
1845 		    (!self && var_data == hist_data))
1846 			continue;
1847 
1848 		var_elt = tracing_map_lookup(var_data->map, key);
1849 		if (!var_elt) {
1850 			resolved = false;
1851 			break;
1852 		}
1853 
1854 		if (!tracing_map_var_set(var_elt, var_idx)) {
1855 			resolved = false;
1856 			break;
1857 		}
1858 
1859 		if (self || !hist_field->read_once)
1860 			var_val = tracing_map_read_var(var_elt, var_idx);
1861 		else
1862 			var_val = tracing_map_read_var_once(var_elt, var_idx);
1863 
1864 		var_ref_vals[i] = var_val;
1865 	}
1866 
1867 	return resolved;
1868 }
1869 
1870 static const char *hist_field_name(struct hist_field *field,
1871 				   unsigned int level)
1872 {
1873 	const char *field_name = "";
1874 
1875 	if (level > 1)
1876 		return field_name;
1877 
1878 	if (field->field)
1879 		field_name = field->field->name;
1880 	else if (field->flags & HIST_FIELD_FL_LOG2 ||
1881 		 field->flags & HIST_FIELD_FL_ALIAS)
1882 		field_name = hist_field_name(field->operands[0], ++level);
1883 	else if (field->flags & HIST_FIELD_FL_CPU)
1884 		field_name = "cpu";
1885 	else if (field->flags & HIST_FIELD_FL_EXPR ||
1886 		 field->flags & HIST_FIELD_FL_VAR_REF) {
1887 		if (field->system) {
1888 			static char full_name[MAX_FILTER_STR_VAL];
1889 
1890 			strcat(full_name, field->system);
1891 			strcat(full_name, ".");
1892 			strcat(full_name, field->event_name);
1893 			strcat(full_name, ".");
1894 			strcat(full_name, field->name);
1895 			field_name = full_name;
1896 		} else
1897 			field_name = field->name;
1898 	} else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1899 		field_name = "common_timestamp";
1900 
1901 	if (field_name == NULL)
1902 		field_name = "";
1903 
1904 	return field_name;
1905 }
1906 
1907 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1908 {
1909 	hist_field_fn_t fn = NULL;
1910 
1911 	switch (field_size) {
1912 	case 8:
1913 		if (field_is_signed)
1914 			fn = hist_field_s64;
1915 		else
1916 			fn = hist_field_u64;
1917 		break;
1918 	case 4:
1919 		if (field_is_signed)
1920 			fn = hist_field_s32;
1921 		else
1922 			fn = hist_field_u32;
1923 		break;
1924 	case 2:
1925 		if (field_is_signed)
1926 			fn = hist_field_s16;
1927 		else
1928 			fn = hist_field_u16;
1929 		break;
1930 	case 1:
1931 		if (field_is_signed)
1932 			fn = hist_field_s8;
1933 		else
1934 			fn = hist_field_u8;
1935 		break;
1936 	}
1937 
1938 	return fn;
1939 }
1940 
1941 static int parse_map_size(char *str)
1942 {
1943 	unsigned long size, map_bits;
1944 	int ret;
1945 
1946 	strsep(&str, "=");
1947 	if (!str) {
1948 		ret = -EINVAL;
1949 		goto out;
1950 	}
1951 
1952 	ret = kstrtoul(str, 0, &size);
1953 	if (ret)
1954 		goto out;
1955 
1956 	map_bits = ilog2(roundup_pow_of_two(size));
1957 	if (map_bits < TRACING_MAP_BITS_MIN ||
1958 	    map_bits > TRACING_MAP_BITS_MAX)
1959 		ret = -EINVAL;
1960 	else
1961 		ret = map_bits;
1962  out:
1963 	return ret;
1964 }
1965 
1966 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1967 {
1968 	unsigned int i;
1969 
1970 	if (!attrs)
1971 		return;
1972 
1973 	for (i = 0; i < attrs->n_assignments; i++)
1974 		kfree(attrs->assignment_str[i]);
1975 
1976 	for (i = 0; i < attrs->n_actions; i++)
1977 		kfree(attrs->action_str[i]);
1978 
1979 	kfree(attrs->name);
1980 	kfree(attrs->sort_key_str);
1981 	kfree(attrs->keys_str);
1982 	kfree(attrs->vals_str);
1983 	kfree(attrs->clock);
1984 	kfree(attrs);
1985 }
1986 
1987 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1988 {
1989 	int ret = -EINVAL;
1990 
1991 	if (attrs->n_actions >= HIST_ACTIONS_MAX)
1992 		return ret;
1993 
1994 	if ((str_has_prefix(str, "onmatch(")) ||
1995 	    (str_has_prefix(str, "onmax(")) ||
1996 	    (str_has_prefix(str, "onchange("))) {
1997 		attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1998 		if (!attrs->action_str[attrs->n_actions]) {
1999 			ret = -ENOMEM;
2000 			return ret;
2001 		}
2002 		attrs->n_actions++;
2003 		ret = 0;
2004 	}
2005 
2006 	return ret;
2007 }
2008 
2009 static int parse_assignment(char *str, struct hist_trigger_attrs *attrs)
2010 {
2011 	int ret = 0;
2012 
2013 	if ((str_has_prefix(str, "key=")) ||
2014 	    (str_has_prefix(str, "keys="))) {
2015 		attrs->keys_str = kstrdup(str, GFP_KERNEL);
2016 		if (!attrs->keys_str) {
2017 			ret = -ENOMEM;
2018 			goto out;
2019 		}
2020 	} else if ((str_has_prefix(str, "val=")) ||
2021 		   (str_has_prefix(str, "vals=")) ||
2022 		   (str_has_prefix(str, "values="))) {
2023 		attrs->vals_str = kstrdup(str, GFP_KERNEL);
2024 		if (!attrs->vals_str) {
2025 			ret = -ENOMEM;
2026 			goto out;
2027 		}
2028 	} else if (str_has_prefix(str, "sort=")) {
2029 		attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
2030 		if (!attrs->sort_key_str) {
2031 			ret = -ENOMEM;
2032 			goto out;
2033 		}
2034 	} else if (str_has_prefix(str, "name=")) {
2035 		attrs->name = kstrdup(str, GFP_KERNEL);
2036 		if (!attrs->name) {
2037 			ret = -ENOMEM;
2038 			goto out;
2039 		}
2040 	} else if (str_has_prefix(str, "clock=")) {
2041 		strsep(&str, "=");
2042 		if (!str) {
2043 			ret = -EINVAL;
2044 			goto out;
2045 		}
2046 
2047 		str = strstrip(str);
2048 		attrs->clock = kstrdup(str, GFP_KERNEL);
2049 		if (!attrs->clock) {
2050 			ret = -ENOMEM;
2051 			goto out;
2052 		}
2053 	} else if (str_has_prefix(str, "size=")) {
2054 		int map_bits = parse_map_size(str);
2055 
2056 		if (map_bits < 0) {
2057 			ret = map_bits;
2058 			goto out;
2059 		}
2060 		attrs->map_bits = map_bits;
2061 	} else {
2062 		char *assignment;
2063 
2064 		if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
2065 			hist_err("Too many variables defined: ", str);
2066 			ret = -EINVAL;
2067 			goto out;
2068 		}
2069 
2070 		assignment = kstrdup(str, GFP_KERNEL);
2071 		if (!assignment) {
2072 			ret = -ENOMEM;
2073 			goto out;
2074 		}
2075 
2076 		attrs->assignment_str[attrs->n_assignments++] = assignment;
2077 	}
2078  out:
2079 	return ret;
2080 }
2081 
2082 static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
2083 {
2084 	struct hist_trigger_attrs *attrs;
2085 	int ret = 0;
2086 
2087 	attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
2088 	if (!attrs)
2089 		return ERR_PTR(-ENOMEM);
2090 
2091 	while (trigger_str) {
2092 		char *str = strsep(&trigger_str, ":");
2093 
2094 		if (strchr(str, '=')) {
2095 			ret = parse_assignment(str, attrs);
2096 			if (ret)
2097 				goto free;
2098 		} else if (strcmp(str, "pause") == 0)
2099 			attrs->pause = true;
2100 		else if ((strcmp(str, "cont") == 0) ||
2101 			 (strcmp(str, "continue") == 0))
2102 			attrs->cont = true;
2103 		else if (strcmp(str, "clear") == 0)
2104 			attrs->clear = true;
2105 		else {
2106 			ret = parse_action(str, attrs);
2107 			if (ret)
2108 				goto free;
2109 		}
2110 	}
2111 
2112 	if (!attrs->keys_str) {
2113 		ret = -EINVAL;
2114 		goto free;
2115 	}
2116 
2117 	if (!attrs->clock) {
2118 		attrs->clock = kstrdup("global", GFP_KERNEL);
2119 		if (!attrs->clock) {
2120 			ret = -ENOMEM;
2121 			goto free;
2122 		}
2123 	}
2124 
2125 	return attrs;
2126  free:
2127 	destroy_hist_trigger_attrs(attrs);
2128 
2129 	return ERR_PTR(ret);
2130 }
2131 
2132 static inline void save_comm(char *comm, struct task_struct *task)
2133 {
2134 	if (!task->pid) {
2135 		strcpy(comm, "<idle>");
2136 		return;
2137 	}
2138 
2139 	if (WARN_ON_ONCE(task->pid < 0)) {
2140 		strcpy(comm, "<XXX>");
2141 		return;
2142 	}
2143 
2144 	strncpy(comm, task->comm, TASK_COMM_LEN);
2145 }
2146 
2147 static void hist_elt_data_free(struct hist_elt_data *elt_data)
2148 {
2149 	unsigned int i;
2150 
2151 	for (i = 0; i < SYNTH_FIELDS_MAX; i++)
2152 		kfree(elt_data->field_var_str[i]);
2153 
2154 	kfree(elt_data->comm);
2155 	kfree(elt_data);
2156 }
2157 
2158 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
2159 {
2160 	struct hist_elt_data *elt_data = elt->private_data;
2161 
2162 	hist_elt_data_free(elt_data);
2163 }
2164 
2165 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
2166 {
2167 	struct hist_trigger_data *hist_data = elt->map->private_data;
2168 	unsigned int size = TASK_COMM_LEN;
2169 	struct hist_elt_data *elt_data;
2170 	struct hist_field *key_field;
2171 	unsigned int i, n_str;
2172 
2173 	elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
2174 	if (!elt_data)
2175 		return -ENOMEM;
2176 
2177 	for_each_hist_key_field(i, hist_data) {
2178 		key_field = hist_data->fields[i];
2179 
2180 		if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
2181 			elt_data->comm = kzalloc(size, GFP_KERNEL);
2182 			if (!elt_data->comm) {
2183 				kfree(elt_data);
2184 				return -ENOMEM;
2185 			}
2186 			break;
2187 		}
2188 	}
2189 
2190 	n_str = hist_data->n_field_var_str + hist_data->n_save_var_str;
2191 
2192 	size = STR_VAR_LEN_MAX;
2193 
2194 	for (i = 0; i < n_str; i++) {
2195 		elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
2196 		if (!elt_data->field_var_str[i]) {
2197 			hist_elt_data_free(elt_data);
2198 			return -ENOMEM;
2199 		}
2200 	}
2201 
2202 	elt->private_data = elt_data;
2203 
2204 	return 0;
2205 }
2206 
2207 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
2208 {
2209 	struct hist_elt_data *elt_data = elt->private_data;
2210 
2211 	if (elt_data->comm)
2212 		save_comm(elt_data->comm, current);
2213 }
2214 
2215 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
2216 	.elt_alloc	= hist_trigger_elt_data_alloc,
2217 	.elt_free	= hist_trigger_elt_data_free,
2218 	.elt_init	= hist_trigger_elt_data_init,
2219 };
2220 
2221 static const char *get_hist_field_flags(struct hist_field *hist_field)
2222 {
2223 	const char *flags_str = NULL;
2224 
2225 	if (hist_field->flags & HIST_FIELD_FL_HEX)
2226 		flags_str = "hex";
2227 	else if (hist_field->flags & HIST_FIELD_FL_SYM)
2228 		flags_str = "sym";
2229 	else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
2230 		flags_str = "sym-offset";
2231 	else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
2232 		flags_str = "execname";
2233 	else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
2234 		flags_str = "syscall";
2235 	else if (hist_field->flags & HIST_FIELD_FL_LOG2)
2236 		flags_str = "log2";
2237 	else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2238 		flags_str = "usecs";
2239 
2240 	return flags_str;
2241 }
2242 
2243 static void expr_field_str(struct hist_field *field, char *expr)
2244 {
2245 	if (field->flags & HIST_FIELD_FL_VAR_REF)
2246 		strcat(expr, "$");
2247 
2248 	strcat(expr, hist_field_name(field, 0));
2249 
2250 	if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
2251 		const char *flags_str = get_hist_field_flags(field);
2252 
2253 		if (flags_str) {
2254 			strcat(expr, ".");
2255 			strcat(expr, flags_str);
2256 		}
2257 	}
2258 }
2259 
2260 static char *expr_str(struct hist_field *field, unsigned int level)
2261 {
2262 	char *expr;
2263 
2264 	if (level > 1)
2265 		return NULL;
2266 
2267 	expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2268 	if (!expr)
2269 		return NULL;
2270 
2271 	if (!field->operands[0]) {
2272 		expr_field_str(field, expr);
2273 		return expr;
2274 	}
2275 
2276 	if (field->operator == FIELD_OP_UNARY_MINUS) {
2277 		char *subexpr;
2278 
2279 		strcat(expr, "-(");
2280 		subexpr = expr_str(field->operands[0], ++level);
2281 		if (!subexpr) {
2282 			kfree(expr);
2283 			return NULL;
2284 		}
2285 		strcat(expr, subexpr);
2286 		strcat(expr, ")");
2287 
2288 		kfree(subexpr);
2289 
2290 		return expr;
2291 	}
2292 
2293 	expr_field_str(field->operands[0], expr);
2294 
2295 	switch (field->operator) {
2296 	case FIELD_OP_MINUS:
2297 		strcat(expr, "-");
2298 		break;
2299 	case FIELD_OP_PLUS:
2300 		strcat(expr, "+");
2301 		break;
2302 	default:
2303 		kfree(expr);
2304 		return NULL;
2305 	}
2306 
2307 	expr_field_str(field->operands[1], expr);
2308 
2309 	return expr;
2310 }
2311 
2312 static int contains_operator(char *str)
2313 {
2314 	enum field_op_id field_op = FIELD_OP_NONE;
2315 	char *op;
2316 
2317 	op = strpbrk(str, "+-");
2318 	if (!op)
2319 		return FIELD_OP_NONE;
2320 
2321 	switch (*op) {
2322 	case '-':
2323 		if (*str == '-')
2324 			field_op = FIELD_OP_UNARY_MINUS;
2325 		else
2326 			field_op = FIELD_OP_MINUS;
2327 		break;
2328 	case '+':
2329 		field_op = FIELD_OP_PLUS;
2330 		break;
2331 	default:
2332 		break;
2333 	}
2334 
2335 	return field_op;
2336 }
2337 
2338 static void __destroy_hist_field(struct hist_field *hist_field)
2339 {
2340 	kfree(hist_field->var.name);
2341 	kfree(hist_field->name);
2342 	kfree(hist_field->type);
2343 
2344 	kfree(hist_field);
2345 }
2346 
2347 static void destroy_hist_field(struct hist_field *hist_field,
2348 			       unsigned int level)
2349 {
2350 	unsigned int i;
2351 
2352 	if (level > 3)
2353 		return;
2354 
2355 	if (!hist_field)
2356 		return;
2357 
2358 	if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
2359 		return; /* var refs will be destroyed separately */
2360 
2361 	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2362 		destroy_hist_field(hist_field->operands[i], level + 1);
2363 
2364 	__destroy_hist_field(hist_field);
2365 }
2366 
2367 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
2368 					    struct ftrace_event_field *field,
2369 					    unsigned long flags,
2370 					    char *var_name)
2371 {
2372 	struct hist_field *hist_field;
2373 
2374 	if (field && is_function_field(field))
2375 		return NULL;
2376 
2377 	hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2378 	if (!hist_field)
2379 		return NULL;
2380 
2381 	hist_field->hist_data = hist_data;
2382 
2383 	if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
2384 		goto out; /* caller will populate */
2385 
2386 	if (flags & HIST_FIELD_FL_VAR_REF) {
2387 		hist_field->fn = hist_field_var_ref;
2388 		goto out;
2389 	}
2390 
2391 	if (flags & HIST_FIELD_FL_HITCOUNT) {
2392 		hist_field->fn = hist_field_counter;
2393 		hist_field->size = sizeof(u64);
2394 		hist_field->type = kstrdup("u64", GFP_KERNEL);
2395 		if (!hist_field->type)
2396 			goto free;
2397 		goto out;
2398 	}
2399 
2400 	if (flags & HIST_FIELD_FL_STACKTRACE) {
2401 		hist_field->fn = hist_field_none;
2402 		goto out;
2403 	}
2404 
2405 	if (flags & HIST_FIELD_FL_LOG2) {
2406 		unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
2407 		hist_field->fn = hist_field_log2;
2408 		hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2409 		hist_field->size = hist_field->operands[0]->size;
2410 		hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
2411 		if (!hist_field->type)
2412 			goto free;
2413 		goto out;
2414 	}
2415 
2416 	if (flags & HIST_FIELD_FL_TIMESTAMP) {
2417 		hist_field->fn = hist_field_timestamp;
2418 		hist_field->size = sizeof(u64);
2419 		hist_field->type = kstrdup("u64", GFP_KERNEL);
2420 		if (!hist_field->type)
2421 			goto free;
2422 		goto out;
2423 	}
2424 
2425 	if (flags & HIST_FIELD_FL_CPU) {
2426 		hist_field->fn = hist_field_cpu;
2427 		hist_field->size = sizeof(int);
2428 		hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
2429 		if (!hist_field->type)
2430 			goto free;
2431 		goto out;
2432 	}
2433 
2434 	if (WARN_ON_ONCE(!field))
2435 		goto out;
2436 
2437 	if (is_string_field(field)) {
2438 		flags |= HIST_FIELD_FL_STRING;
2439 
2440 		hist_field->size = MAX_FILTER_STR_VAL;
2441 		hist_field->type = kstrdup(field->type, GFP_KERNEL);
2442 		if (!hist_field->type)
2443 			goto free;
2444 
2445 		if (field->filter_type == FILTER_STATIC_STRING)
2446 			hist_field->fn = hist_field_string;
2447 		else if (field->filter_type == FILTER_DYN_STRING)
2448 			hist_field->fn = hist_field_dynstring;
2449 		else
2450 			hist_field->fn = hist_field_pstring;
2451 	} else {
2452 		hist_field->size = field->size;
2453 		hist_field->is_signed = field->is_signed;
2454 		hist_field->type = kstrdup(field->type, GFP_KERNEL);
2455 		if (!hist_field->type)
2456 			goto free;
2457 
2458 		hist_field->fn = select_value_fn(field->size,
2459 						 field->is_signed);
2460 		if (!hist_field->fn) {
2461 			destroy_hist_field(hist_field, 0);
2462 			return NULL;
2463 		}
2464 	}
2465  out:
2466 	hist_field->field = field;
2467 	hist_field->flags = flags;
2468 
2469 	if (var_name) {
2470 		hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2471 		if (!hist_field->var.name)
2472 			goto free;
2473 	}
2474 
2475 	return hist_field;
2476  free:
2477 	destroy_hist_field(hist_field, 0);
2478 	return NULL;
2479 }
2480 
2481 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2482 {
2483 	unsigned int i;
2484 
2485 	for (i = 0; i < HIST_FIELDS_MAX; i++) {
2486 		if (hist_data->fields[i]) {
2487 			destroy_hist_field(hist_data->fields[i], 0);
2488 			hist_data->fields[i] = NULL;
2489 		}
2490 	}
2491 
2492 	for (i = 0; i < hist_data->n_var_refs; i++) {
2493 		WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2494 		__destroy_hist_field(hist_data->var_refs[i]);
2495 		hist_data->var_refs[i] = NULL;
2496 	}
2497 }
2498 
2499 static int init_var_ref(struct hist_field *ref_field,
2500 			struct hist_field *var_field,
2501 			char *system, char *event_name)
2502 {
2503 	int err = 0;
2504 
2505 	ref_field->var.idx = var_field->var.idx;
2506 	ref_field->var.hist_data = var_field->hist_data;
2507 	ref_field->size = var_field->size;
2508 	ref_field->is_signed = var_field->is_signed;
2509 	ref_field->flags |= var_field->flags &
2510 		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2511 
2512 	if (system) {
2513 		ref_field->system = kstrdup(system, GFP_KERNEL);
2514 		if (!ref_field->system)
2515 			return -ENOMEM;
2516 	}
2517 
2518 	if (event_name) {
2519 		ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2520 		if (!ref_field->event_name) {
2521 			err = -ENOMEM;
2522 			goto free;
2523 		}
2524 	}
2525 
2526 	if (var_field->var.name) {
2527 		ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2528 		if (!ref_field->name) {
2529 			err = -ENOMEM;
2530 			goto free;
2531 		}
2532 	} else if (var_field->name) {
2533 		ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2534 		if (!ref_field->name) {
2535 			err = -ENOMEM;
2536 			goto free;
2537 		}
2538 	}
2539 
2540 	ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
2541 	if (!ref_field->type) {
2542 		err = -ENOMEM;
2543 		goto free;
2544 	}
2545  out:
2546 	return err;
2547  free:
2548 	kfree(ref_field->system);
2549 	kfree(ref_field->event_name);
2550 	kfree(ref_field->name);
2551 
2552 	goto out;
2553 }
2554 
2555 /**
2556  * create_var_ref - Create a variable reference and attach it to trigger
2557  * @hist_data: The trigger that will be referencing the variable
2558  * @var_field: The VAR field to create a reference to
2559  * @system: The optional system string
2560  * @event_name: The optional event_name string
2561  *
2562  * Given a variable hist_field, create a VAR_REF hist_field that
2563  * represents a reference to it.
2564  *
2565  * This function also adds the reference to the trigger that
2566  * now references the variable.
2567  *
2568  * Return: The VAR_REF field if successful, NULL if not
2569  */
2570 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2571 					 struct hist_field *var_field,
2572 					 char *system, char *event_name)
2573 {
2574 	unsigned long flags = HIST_FIELD_FL_VAR_REF;
2575 	struct hist_field *ref_field;
2576 
2577 	ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2578 	if (ref_field) {
2579 		if (init_var_ref(ref_field, var_field, system, event_name)) {
2580 			destroy_hist_field(ref_field, 0);
2581 			return NULL;
2582 		}
2583 
2584 		hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2585 		ref_field->var_ref_idx = hist_data->n_var_refs++;
2586 	}
2587 
2588 	return ref_field;
2589 }
2590 
2591 static bool is_var_ref(char *var_name)
2592 {
2593 	if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2594 		return false;
2595 
2596 	return true;
2597 }
2598 
2599 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2600 				 char *var_name)
2601 {
2602 	char *name, *field;
2603 	unsigned int i;
2604 
2605 	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2606 		name = hist_data->attrs->var_defs.name[i];
2607 
2608 		if (strcmp(var_name, name) == 0) {
2609 			field = hist_data->attrs->var_defs.expr[i];
2610 			if (contains_operator(field) || is_var_ref(field))
2611 				continue;
2612 			return field;
2613 		}
2614 	}
2615 
2616 	return NULL;
2617 }
2618 
2619 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2620 				 char *system, char *event_name,
2621 				 char *var_name)
2622 {
2623 	struct trace_event_call *call;
2624 
2625 	if (system && event_name) {
2626 		call = hist_data->event_file->event_call;
2627 
2628 		if (strcmp(system, call->class->system) != 0)
2629 			return NULL;
2630 
2631 		if (strcmp(event_name, trace_event_name(call)) != 0)
2632 			return NULL;
2633 	}
2634 
2635 	if (!!system != !!event_name)
2636 		return NULL;
2637 
2638 	if (!is_var_ref(var_name))
2639 		return NULL;
2640 
2641 	var_name++;
2642 
2643 	return field_name_from_var(hist_data, var_name);
2644 }
2645 
2646 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2647 					char *system, char *event_name,
2648 					char *var_name)
2649 {
2650 	struct hist_field *var_field = NULL, *ref_field = NULL;
2651 
2652 	if (!is_var_ref(var_name))
2653 		return NULL;
2654 
2655 	var_name++;
2656 
2657 	var_field = find_event_var(hist_data, system, event_name, var_name);
2658 	if (var_field)
2659 		ref_field = create_var_ref(hist_data, var_field,
2660 					   system, event_name);
2661 
2662 	if (!ref_field)
2663 		hist_err_event("Couldn't find variable: $",
2664 			       system, event_name, var_name);
2665 
2666 	return ref_field;
2667 }
2668 
2669 static struct ftrace_event_field *
2670 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2671 	    char *field_str, unsigned long *flags)
2672 {
2673 	struct ftrace_event_field *field = NULL;
2674 	char *field_name, *modifier, *str;
2675 
2676 	modifier = str = kstrdup(field_str, GFP_KERNEL);
2677 	if (!modifier)
2678 		return ERR_PTR(-ENOMEM);
2679 
2680 	field_name = strsep(&modifier, ".");
2681 	if (modifier) {
2682 		if (strcmp(modifier, "hex") == 0)
2683 			*flags |= HIST_FIELD_FL_HEX;
2684 		else if (strcmp(modifier, "sym") == 0)
2685 			*flags |= HIST_FIELD_FL_SYM;
2686 		else if (strcmp(modifier, "sym-offset") == 0)
2687 			*flags |= HIST_FIELD_FL_SYM_OFFSET;
2688 		else if ((strcmp(modifier, "execname") == 0) &&
2689 			 (strcmp(field_name, "common_pid") == 0))
2690 			*flags |= HIST_FIELD_FL_EXECNAME;
2691 		else if (strcmp(modifier, "syscall") == 0)
2692 			*flags |= HIST_FIELD_FL_SYSCALL;
2693 		else if (strcmp(modifier, "log2") == 0)
2694 			*flags |= HIST_FIELD_FL_LOG2;
2695 		else if (strcmp(modifier, "usecs") == 0)
2696 			*flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2697 		else {
2698 			hist_err("Invalid field modifier: ", modifier);
2699 			field = ERR_PTR(-EINVAL);
2700 			goto out;
2701 		}
2702 	}
2703 
2704 	if (strcmp(field_name, "common_timestamp") == 0) {
2705 		*flags |= HIST_FIELD_FL_TIMESTAMP;
2706 		hist_data->enable_timestamps = true;
2707 		if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2708 			hist_data->attrs->ts_in_usecs = true;
2709 	} else if (strcmp(field_name, "cpu") == 0)
2710 		*flags |= HIST_FIELD_FL_CPU;
2711 	else {
2712 		field = trace_find_event_field(file->event_call, field_name);
2713 		if (!field || !field->size) {
2714 			hist_err("Couldn't find field: ", field_name);
2715 			field = ERR_PTR(-EINVAL);
2716 			goto out;
2717 		}
2718 	}
2719  out:
2720 	kfree(str);
2721 
2722 	return field;
2723 }
2724 
2725 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2726 				       struct hist_field *var_ref,
2727 				       char *var_name)
2728 {
2729 	struct hist_field *alias = NULL;
2730 	unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2731 
2732 	alias = create_hist_field(hist_data, NULL, flags, var_name);
2733 	if (!alias)
2734 		return NULL;
2735 
2736 	alias->fn = var_ref->fn;
2737 	alias->operands[0] = var_ref;
2738 
2739 	if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2740 		destroy_hist_field(alias, 0);
2741 		return NULL;
2742 	}
2743 
2744 	return alias;
2745 }
2746 
2747 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2748 				     struct trace_event_file *file, char *str,
2749 				     unsigned long *flags, char *var_name)
2750 {
2751 	char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2752 	struct ftrace_event_field *field = NULL;
2753 	struct hist_field *hist_field = NULL;
2754 	int ret = 0;
2755 
2756 	s = strchr(str, '.');
2757 	if (s) {
2758 		s = strchr(++s, '.');
2759 		if (s) {
2760 			ref_system = strsep(&str, ".");
2761 			if (!str) {
2762 				ret = -EINVAL;
2763 				goto out;
2764 			}
2765 			ref_event = strsep(&str, ".");
2766 			if (!str) {
2767 				ret = -EINVAL;
2768 				goto out;
2769 			}
2770 			ref_var = str;
2771 		}
2772 	}
2773 
2774 	s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2775 	if (!s) {
2776 		hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var);
2777 		if (hist_field) {
2778 			if (var_name) {
2779 				hist_field = create_alias(hist_data, hist_field, var_name);
2780 				if (!hist_field) {
2781 					ret = -ENOMEM;
2782 					goto out;
2783 				}
2784 			}
2785 			return hist_field;
2786 		}
2787 	} else
2788 		str = s;
2789 
2790 	field = parse_field(hist_data, file, str, flags);
2791 	if (IS_ERR(field)) {
2792 		ret = PTR_ERR(field);
2793 		goto out;
2794 	}
2795 
2796 	hist_field = create_hist_field(hist_data, field, *flags, var_name);
2797 	if (!hist_field) {
2798 		ret = -ENOMEM;
2799 		goto out;
2800 	}
2801 
2802 	return hist_field;
2803  out:
2804 	return ERR_PTR(ret);
2805 }
2806 
2807 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2808 				     struct trace_event_file *file,
2809 				     char *str, unsigned long flags,
2810 				     char *var_name, unsigned int level);
2811 
2812 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2813 				      struct trace_event_file *file,
2814 				      char *str, unsigned long flags,
2815 				      char *var_name, unsigned int level)
2816 {
2817 	struct hist_field *operand1, *expr = NULL;
2818 	unsigned long operand_flags;
2819 	int ret = 0;
2820 	char *s;
2821 
2822 	/* we support only -(xxx) i.e. explicit parens required */
2823 
2824 	if (level > 3) {
2825 		hist_err("Too many subexpressions (3 max): ", str);
2826 		ret = -EINVAL;
2827 		goto free;
2828 	}
2829 
2830 	str++; /* skip leading '-' */
2831 
2832 	s = strchr(str, '(');
2833 	if (s)
2834 		str++;
2835 	else {
2836 		ret = -EINVAL;
2837 		goto free;
2838 	}
2839 
2840 	s = strrchr(str, ')');
2841 	if (s)
2842 		*s = '\0';
2843 	else {
2844 		ret = -EINVAL; /* no closing ')' */
2845 		goto free;
2846 	}
2847 
2848 	flags |= HIST_FIELD_FL_EXPR;
2849 	expr = create_hist_field(hist_data, NULL, flags, var_name);
2850 	if (!expr) {
2851 		ret = -ENOMEM;
2852 		goto free;
2853 	}
2854 
2855 	operand_flags = 0;
2856 	operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2857 	if (IS_ERR(operand1)) {
2858 		ret = PTR_ERR(operand1);
2859 		goto free;
2860 	}
2861 
2862 	expr->flags |= operand1->flags &
2863 		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2864 	expr->fn = hist_field_unary_minus;
2865 	expr->operands[0] = operand1;
2866 	expr->operator = FIELD_OP_UNARY_MINUS;
2867 	expr->name = expr_str(expr, 0);
2868 	expr->type = kstrdup(operand1->type, GFP_KERNEL);
2869 	if (!expr->type) {
2870 		ret = -ENOMEM;
2871 		goto free;
2872 	}
2873 
2874 	return expr;
2875  free:
2876 	destroy_hist_field(expr, 0);
2877 	return ERR_PTR(ret);
2878 }
2879 
2880 static int check_expr_operands(struct hist_field *operand1,
2881 			       struct hist_field *operand2)
2882 {
2883 	unsigned long operand1_flags = operand1->flags;
2884 	unsigned long operand2_flags = operand2->flags;
2885 
2886 	if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2887 	    (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2888 		struct hist_field *var;
2889 
2890 		var = find_var_field(operand1->var.hist_data, operand1->name);
2891 		if (!var)
2892 			return -EINVAL;
2893 		operand1_flags = var->flags;
2894 	}
2895 
2896 	if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2897 	    (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2898 		struct hist_field *var;
2899 
2900 		var = find_var_field(operand2->var.hist_data, operand2->name);
2901 		if (!var)
2902 			return -EINVAL;
2903 		operand2_flags = var->flags;
2904 	}
2905 
2906 	if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2907 	    (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2908 		hist_err("Timestamp units in expression don't match", NULL);
2909 		return -EINVAL;
2910 	}
2911 
2912 	return 0;
2913 }
2914 
2915 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2916 				     struct trace_event_file *file,
2917 				     char *str, unsigned long flags,
2918 				     char *var_name, unsigned int level)
2919 {
2920 	struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2921 	unsigned long operand_flags;
2922 	int field_op, ret = -EINVAL;
2923 	char *sep, *operand1_str;
2924 
2925 	if (level > 3) {
2926 		hist_err("Too many subexpressions (3 max): ", str);
2927 		return ERR_PTR(-EINVAL);
2928 	}
2929 
2930 	field_op = contains_operator(str);
2931 
2932 	if (field_op == FIELD_OP_NONE)
2933 		return parse_atom(hist_data, file, str, &flags, var_name);
2934 
2935 	if (field_op == FIELD_OP_UNARY_MINUS)
2936 		return parse_unary(hist_data, file, str, flags, var_name, ++level);
2937 
2938 	switch (field_op) {
2939 	case FIELD_OP_MINUS:
2940 		sep = "-";
2941 		break;
2942 	case FIELD_OP_PLUS:
2943 		sep = "+";
2944 		break;
2945 	default:
2946 		goto free;
2947 	}
2948 
2949 	operand1_str = strsep(&str, sep);
2950 	if (!operand1_str || !str)
2951 		goto free;
2952 
2953 	operand_flags = 0;
2954 	operand1 = parse_atom(hist_data, file, operand1_str,
2955 			      &operand_flags, NULL);
2956 	if (IS_ERR(operand1)) {
2957 		ret = PTR_ERR(operand1);
2958 		operand1 = NULL;
2959 		goto free;
2960 	}
2961 
2962 	/* rest of string could be another expression e.g. b+c in a+b+c */
2963 	operand_flags = 0;
2964 	operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2965 	if (IS_ERR(operand2)) {
2966 		ret = PTR_ERR(operand2);
2967 		operand2 = NULL;
2968 		goto free;
2969 	}
2970 
2971 	ret = check_expr_operands(operand1, operand2);
2972 	if (ret)
2973 		goto free;
2974 
2975 	flags |= HIST_FIELD_FL_EXPR;
2976 
2977 	flags |= operand1->flags &
2978 		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2979 
2980 	expr = create_hist_field(hist_data, NULL, flags, var_name);
2981 	if (!expr) {
2982 		ret = -ENOMEM;
2983 		goto free;
2984 	}
2985 
2986 	operand1->read_once = true;
2987 	operand2->read_once = true;
2988 
2989 	expr->operands[0] = operand1;
2990 	expr->operands[1] = operand2;
2991 	expr->operator = field_op;
2992 	expr->name = expr_str(expr, 0);
2993 	expr->type = kstrdup(operand1->type, GFP_KERNEL);
2994 	if (!expr->type) {
2995 		ret = -ENOMEM;
2996 		goto free;
2997 	}
2998 
2999 	switch (field_op) {
3000 	case FIELD_OP_MINUS:
3001 		expr->fn = hist_field_minus;
3002 		break;
3003 	case FIELD_OP_PLUS:
3004 		expr->fn = hist_field_plus;
3005 		break;
3006 	default:
3007 		ret = -EINVAL;
3008 		goto free;
3009 	}
3010 
3011 	return expr;
3012  free:
3013 	destroy_hist_field(operand1, 0);
3014 	destroy_hist_field(operand2, 0);
3015 	destroy_hist_field(expr, 0);
3016 
3017 	return ERR_PTR(ret);
3018 }
3019 
3020 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
3021 				 struct trace_event_file *file)
3022 {
3023 	struct event_trigger_data *test;
3024 
3025 	list_for_each_entry_rcu(test, &file->triggers, list) {
3026 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3027 			if (test->private_data == hist_data)
3028 				return test->filter_str;
3029 		}
3030 	}
3031 
3032 	return NULL;
3033 }
3034 
3035 static struct event_command trigger_hist_cmd;
3036 static int event_hist_trigger_func(struct event_command *cmd_ops,
3037 				   struct trace_event_file *file,
3038 				   char *glob, char *cmd, char *param);
3039 
3040 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
3041 			    struct hist_trigger_data *hist_data,
3042 			    unsigned int n_keys)
3043 {
3044 	struct hist_field *target_hist_field, *hist_field;
3045 	unsigned int n, i, j;
3046 
3047 	if (hist_data->n_fields - hist_data->n_vals != n_keys)
3048 		return false;
3049 
3050 	i = hist_data->n_vals;
3051 	j = target_hist_data->n_vals;
3052 
3053 	for (n = 0; n < n_keys; n++) {
3054 		hist_field = hist_data->fields[i + n];
3055 		target_hist_field = target_hist_data->fields[j + n];
3056 
3057 		if (strcmp(hist_field->type, target_hist_field->type) != 0)
3058 			return false;
3059 		if (hist_field->size != target_hist_field->size)
3060 			return false;
3061 		if (hist_field->is_signed != target_hist_field->is_signed)
3062 			return false;
3063 	}
3064 
3065 	return true;
3066 }
3067 
3068 static struct hist_trigger_data *
3069 find_compatible_hist(struct hist_trigger_data *target_hist_data,
3070 		     struct trace_event_file *file)
3071 {
3072 	struct hist_trigger_data *hist_data;
3073 	struct event_trigger_data *test;
3074 	unsigned int n_keys;
3075 
3076 	n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
3077 
3078 	list_for_each_entry_rcu(test, &file->triggers, list) {
3079 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3080 			hist_data = test->private_data;
3081 
3082 			if (compatible_keys(target_hist_data, hist_data, n_keys))
3083 				return hist_data;
3084 		}
3085 	}
3086 
3087 	return NULL;
3088 }
3089 
3090 static struct trace_event_file *event_file(struct trace_array *tr,
3091 					   char *system, char *event_name)
3092 {
3093 	struct trace_event_file *file;
3094 
3095 	file = __find_event_file(tr, system, event_name);
3096 	if (!file)
3097 		return ERR_PTR(-EINVAL);
3098 
3099 	return file;
3100 }
3101 
3102 static struct hist_field *
3103 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
3104 			 char *system, char *event_name, char *field_name)
3105 {
3106 	struct hist_field *event_var;
3107 	char *synthetic_name;
3108 
3109 	synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3110 	if (!synthetic_name)
3111 		return ERR_PTR(-ENOMEM);
3112 
3113 	strcpy(synthetic_name, "synthetic_");
3114 	strcat(synthetic_name, field_name);
3115 
3116 	event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
3117 
3118 	kfree(synthetic_name);
3119 
3120 	return event_var;
3121 }
3122 
3123 /**
3124  * create_field_var_hist - Automatically create a histogram and var for a field
3125  * @target_hist_data: The target hist trigger
3126  * @subsys_name: Optional subsystem name
3127  * @event_name: Optional event name
3128  * @field_name: The name of the field (and the resulting variable)
3129  *
3130  * Hist trigger actions fetch data from variables, not directly from
3131  * events.  However, for convenience, users are allowed to directly
3132  * specify an event field in an action, which will be automatically
3133  * converted into a variable on their behalf.
3134 
3135  * If a user specifies a field on an event that isn't the event the
3136  * histogram currently being defined (the target event histogram), the
3137  * only way that can be accomplished is if a new hist trigger is
3138  * created and the field variable defined on that.
3139  *
3140  * This function creates a new histogram compatible with the target
3141  * event (meaning a histogram with the same key as the target
3142  * histogram), and creates a variable for the specified field, but
3143  * with 'synthetic_' prepended to the variable name in order to avoid
3144  * collision with normal field variables.
3145  *
3146  * Return: The variable created for the field.
3147  */
3148 static struct hist_field *
3149 create_field_var_hist(struct hist_trigger_data *target_hist_data,
3150 		      char *subsys_name, char *event_name, char *field_name)
3151 {
3152 	struct trace_array *tr = target_hist_data->event_file->tr;
3153 	struct hist_field *event_var = ERR_PTR(-EINVAL);
3154 	struct hist_trigger_data *hist_data;
3155 	unsigned int i, n, first = true;
3156 	struct field_var_hist *var_hist;
3157 	struct trace_event_file *file;
3158 	struct hist_field *key_field;
3159 	char *saved_filter;
3160 	char *cmd;
3161 	int ret;
3162 
3163 	if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
3164 		hist_err_event("trace action: Too many field variables defined: ",
3165 			       subsys_name, event_name, field_name);
3166 		return ERR_PTR(-EINVAL);
3167 	}
3168 
3169 	file = event_file(tr, subsys_name, event_name);
3170 
3171 	if (IS_ERR(file)) {
3172 		hist_err_event("trace action: Event file not found: ",
3173 			       subsys_name, event_name, field_name);
3174 		ret = PTR_ERR(file);
3175 		return ERR_PTR(ret);
3176 	}
3177 
3178 	/*
3179 	 * Look for a histogram compatible with target.  We'll use the
3180 	 * found histogram specification to create a new matching
3181 	 * histogram with our variable on it.  target_hist_data is not
3182 	 * yet a registered histogram so we can't use that.
3183 	 */
3184 	hist_data = find_compatible_hist(target_hist_data, file);
3185 	if (!hist_data) {
3186 		hist_err_event("trace action: Matching event histogram not found: ",
3187 			       subsys_name, event_name, field_name);
3188 		return ERR_PTR(-EINVAL);
3189 	}
3190 
3191 	/* See if a synthetic field variable has already been created */
3192 	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3193 					     event_name, field_name);
3194 	if (!IS_ERR_OR_NULL(event_var))
3195 		return event_var;
3196 
3197 	var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3198 	if (!var_hist)
3199 		return ERR_PTR(-ENOMEM);
3200 
3201 	cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3202 	if (!cmd) {
3203 		kfree(var_hist);
3204 		return ERR_PTR(-ENOMEM);
3205 	}
3206 
3207 	/* Use the same keys as the compatible histogram */
3208 	strcat(cmd, "keys=");
3209 
3210 	for_each_hist_key_field(i, hist_data) {
3211 		key_field = hist_data->fields[i];
3212 		if (!first)
3213 			strcat(cmd, ",");
3214 		strcat(cmd, key_field->field->name);
3215 		first = false;
3216 	}
3217 
3218 	/* Create the synthetic field variable specification */
3219 	strcat(cmd, ":synthetic_");
3220 	strcat(cmd, field_name);
3221 	strcat(cmd, "=");
3222 	strcat(cmd, field_name);
3223 
3224 	/* Use the same filter as the compatible histogram */
3225 	saved_filter = find_trigger_filter(hist_data, file);
3226 	if (saved_filter) {
3227 		strcat(cmd, " if ");
3228 		strcat(cmd, saved_filter);
3229 	}
3230 
3231 	var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3232 	if (!var_hist->cmd) {
3233 		kfree(cmd);
3234 		kfree(var_hist);
3235 		return ERR_PTR(-ENOMEM);
3236 	}
3237 
3238 	/* Save the compatible histogram information */
3239 	var_hist->hist_data = hist_data;
3240 
3241 	/* Create the new histogram with our variable */
3242 	ret = event_hist_trigger_func(&trigger_hist_cmd, file,
3243 				      "", "hist", cmd);
3244 	if (ret) {
3245 		kfree(cmd);
3246 		kfree(var_hist->cmd);
3247 		kfree(var_hist);
3248 		hist_err_event("trace action: Couldn't create histogram for field: ",
3249 			       subsys_name, event_name, field_name);
3250 		return ERR_PTR(ret);
3251 	}
3252 
3253 	kfree(cmd);
3254 
3255 	/* If we can't find the variable, something went wrong */
3256 	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3257 					     event_name, field_name);
3258 	if (IS_ERR_OR_NULL(event_var)) {
3259 		kfree(var_hist->cmd);
3260 		kfree(var_hist);
3261 		hist_err_event("trace action: Couldn't find synthetic variable: ",
3262 			       subsys_name, event_name, field_name);
3263 		return ERR_PTR(-EINVAL);
3264 	}
3265 
3266 	n = target_hist_data->n_field_var_hists;
3267 	target_hist_data->field_var_hists[n] = var_hist;
3268 	target_hist_data->n_field_var_hists++;
3269 
3270 	return event_var;
3271 }
3272 
3273 static struct hist_field *
3274 find_target_event_var(struct hist_trigger_data *hist_data,
3275 		      char *subsys_name, char *event_name, char *var_name)
3276 {
3277 	struct trace_event_file *file = hist_data->event_file;
3278 	struct hist_field *hist_field = NULL;
3279 
3280 	if (subsys_name) {
3281 		struct trace_event_call *call;
3282 
3283 		if (!event_name)
3284 			return NULL;
3285 
3286 		call = file->event_call;
3287 
3288 		if (strcmp(subsys_name, call->class->system) != 0)
3289 			return NULL;
3290 
3291 		if (strcmp(event_name, trace_event_name(call)) != 0)
3292 			return NULL;
3293 	}
3294 
3295 	hist_field = find_var_field(hist_data, var_name);
3296 
3297 	return hist_field;
3298 }
3299 
3300 static inline void __update_field_vars(struct tracing_map_elt *elt,
3301 				       struct ring_buffer_event *rbe,
3302 				       void *rec,
3303 				       struct field_var **field_vars,
3304 				       unsigned int n_field_vars,
3305 				       unsigned int field_var_str_start)
3306 {
3307 	struct hist_elt_data *elt_data = elt->private_data;
3308 	unsigned int i, j, var_idx;
3309 	u64 var_val;
3310 
3311 	for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3312 		struct field_var *field_var = field_vars[i];
3313 		struct hist_field *var = field_var->var;
3314 		struct hist_field *val = field_var->val;
3315 
3316 		var_val = val->fn(val, elt, rbe, rec);
3317 		var_idx = var->var.idx;
3318 
3319 		if (val->flags & HIST_FIELD_FL_STRING) {
3320 			char *str = elt_data->field_var_str[j++];
3321 			char *val_str = (char *)(uintptr_t)var_val;
3322 
3323 			strscpy(str, val_str, STR_VAR_LEN_MAX);
3324 			var_val = (u64)(uintptr_t)str;
3325 		}
3326 		tracing_map_set_var(elt, var_idx, var_val);
3327 	}
3328 }
3329 
3330 static void update_field_vars(struct hist_trigger_data *hist_data,
3331 			      struct tracing_map_elt *elt,
3332 			      struct ring_buffer_event *rbe,
3333 			      void *rec)
3334 {
3335 	__update_field_vars(elt, rbe, rec, hist_data->field_vars,
3336 			    hist_data->n_field_vars, 0);
3337 }
3338 
3339 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3340 				 struct tracing_map_elt *elt, void *rec,
3341 				 struct ring_buffer_event *rbe, void *key,
3342 				 struct action_data *data, u64 *var_ref_vals)
3343 {
3344 	__update_field_vars(elt, rbe, rec, hist_data->save_vars,
3345 			    hist_data->n_save_vars, hist_data->n_field_var_str);
3346 }
3347 
3348 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3349 				     struct trace_event_file *file,
3350 				     char *name, int size, const char *type)
3351 {
3352 	struct hist_field *var;
3353 	int idx;
3354 
3355 	if (find_var(hist_data, file, name) && !hist_data->remove) {
3356 		var = ERR_PTR(-EINVAL);
3357 		goto out;
3358 	}
3359 
3360 	var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3361 	if (!var) {
3362 		var = ERR_PTR(-ENOMEM);
3363 		goto out;
3364 	}
3365 
3366 	idx = tracing_map_add_var(hist_data->map);
3367 	if (idx < 0) {
3368 		kfree(var);
3369 		var = ERR_PTR(-EINVAL);
3370 		goto out;
3371 	}
3372 
3373 	var->flags = HIST_FIELD_FL_VAR;
3374 	var->var.idx = idx;
3375 	var->var.hist_data = var->hist_data = hist_data;
3376 	var->size = size;
3377 	var->var.name = kstrdup(name, GFP_KERNEL);
3378 	var->type = kstrdup(type, GFP_KERNEL);
3379 	if (!var->var.name || !var->type) {
3380 		kfree(var->var.name);
3381 		kfree(var->type);
3382 		kfree(var);
3383 		var = ERR_PTR(-ENOMEM);
3384 	}
3385  out:
3386 	return var;
3387 }
3388 
3389 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3390 					  struct trace_event_file *file,
3391 					  char *field_name)
3392 {
3393 	struct hist_field *val = NULL, *var = NULL;
3394 	unsigned long flags = HIST_FIELD_FL_VAR;
3395 	struct field_var *field_var;
3396 	int ret = 0;
3397 
3398 	if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3399 		hist_err("Too many field variables defined: ", field_name);
3400 		ret = -EINVAL;
3401 		goto err;
3402 	}
3403 
3404 	val = parse_atom(hist_data, file, field_name, &flags, NULL);
3405 	if (IS_ERR(val)) {
3406 		hist_err("Couldn't parse field variable: ", field_name);
3407 		ret = PTR_ERR(val);
3408 		goto err;
3409 	}
3410 
3411 	var = create_var(hist_data, file, field_name, val->size, val->type);
3412 	if (IS_ERR(var)) {
3413 		hist_err("Couldn't create or find variable: ", field_name);
3414 		kfree(val);
3415 		ret = PTR_ERR(var);
3416 		goto err;
3417 	}
3418 
3419 	field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3420 	if (!field_var) {
3421 		kfree(val);
3422 		kfree(var);
3423 		ret =  -ENOMEM;
3424 		goto err;
3425 	}
3426 
3427 	field_var->var = var;
3428 	field_var->val = val;
3429  out:
3430 	return field_var;
3431  err:
3432 	field_var = ERR_PTR(ret);
3433 	goto out;
3434 }
3435 
3436 /**
3437  * create_target_field_var - Automatically create a variable for a field
3438  * @target_hist_data: The target hist trigger
3439  * @subsys_name: Optional subsystem name
3440  * @event_name: Optional event name
3441  * @var_name: The name of the field (and the resulting variable)
3442  *
3443  * Hist trigger actions fetch data from variables, not directly from
3444  * events.  However, for convenience, users are allowed to directly
3445  * specify an event field in an action, which will be automatically
3446  * converted into a variable on their behalf.
3447 
3448  * This function creates a field variable with the name var_name on
3449  * the hist trigger currently being defined on the target event.  If
3450  * subsys_name and event_name are specified, this function simply
3451  * verifies that they do in fact match the target event subsystem and
3452  * event name.
3453  *
3454  * Return: The variable created for the field.
3455  */
3456 static struct field_var *
3457 create_target_field_var(struct hist_trigger_data *target_hist_data,
3458 			char *subsys_name, char *event_name, char *var_name)
3459 {
3460 	struct trace_event_file *file = target_hist_data->event_file;
3461 
3462 	if (subsys_name) {
3463 		struct trace_event_call *call;
3464 
3465 		if (!event_name)
3466 			return NULL;
3467 
3468 		call = file->event_call;
3469 
3470 		if (strcmp(subsys_name, call->class->system) != 0)
3471 			return NULL;
3472 
3473 		if (strcmp(event_name, trace_event_name(call)) != 0)
3474 			return NULL;
3475 	}
3476 
3477 	return create_field_var(target_hist_data, file, var_name);
3478 }
3479 
3480 static bool check_track_val_max(u64 track_val, u64 var_val)
3481 {
3482 	if (var_val <= track_val)
3483 		return false;
3484 
3485 	return true;
3486 }
3487 
3488 static bool check_track_val_changed(u64 track_val, u64 var_val)
3489 {
3490 	if (var_val == track_val)
3491 		return false;
3492 
3493 	return true;
3494 }
3495 
3496 static u64 get_track_val(struct hist_trigger_data *hist_data,
3497 			 struct tracing_map_elt *elt,
3498 			 struct action_data *data)
3499 {
3500 	unsigned int track_var_idx = data->track_data.track_var->var.idx;
3501 	u64 track_val;
3502 
3503 	track_val = tracing_map_read_var(elt, track_var_idx);
3504 
3505 	return track_val;
3506 }
3507 
3508 static void save_track_val(struct hist_trigger_data *hist_data,
3509 			   struct tracing_map_elt *elt,
3510 			   struct action_data *data, u64 var_val)
3511 {
3512 	unsigned int track_var_idx = data->track_data.track_var->var.idx;
3513 
3514 	tracing_map_set_var(elt, track_var_idx, var_val);
3515 }
3516 
3517 static void save_track_data(struct hist_trigger_data *hist_data,
3518 			    struct tracing_map_elt *elt, void *rec,
3519 			    struct ring_buffer_event *rbe, void *key,
3520 			    struct action_data *data, u64 *var_ref_vals)
3521 {
3522 	if (data->track_data.save_data)
3523 		data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
3524 }
3525 
3526 static bool check_track_val(struct tracing_map_elt *elt,
3527 			    struct action_data *data,
3528 			    u64 var_val)
3529 {
3530 	struct hist_trigger_data *hist_data;
3531 	u64 track_val;
3532 
3533 	hist_data = data->track_data.track_var->hist_data;
3534 	track_val = get_track_val(hist_data, elt, data);
3535 
3536 	return data->track_data.check_val(track_val, var_val);
3537 }
3538 
3539 #ifdef CONFIG_TRACER_SNAPSHOT
3540 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3541 {
3542 	/* called with tr->max_lock held */
3543 	struct track_data *track_data = tr->cond_snapshot->cond_data;
3544 	struct hist_elt_data *elt_data, *track_elt_data;
3545 	struct snapshot_context *context = cond_data;
3546 	u64 track_val;
3547 
3548 	if (!track_data)
3549 		return false;
3550 
3551 	track_val = get_track_val(track_data->hist_data, context->elt,
3552 				  track_data->action_data);
3553 
3554 	track_data->track_val = track_val;
3555 	memcpy(track_data->key, context->key, track_data->key_len);
3556 
3557 	elt_data = context->elt->private_data;
3558 	track_elt_data = track_data->elt.private_data;
3559 	if (elt_data->comm)
3560 		strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3561 
3562 	track_data->updated = true;
3563 
3564 	return true;
3565 }
3566 
3567 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3568 				     struct tracing_map_elt *elt, void *rec,
3569 				     struct ring_buffer_event *rbe, void *key,
3570 				     struct action_data *data,
3571 				     u64 *var_ref_vals)
3572 {
3573 	struct trace_event_file *file = hist_data->event_file;
3574 	struct snapshot_context context;
3575 
3576 	context.elt = elt;
3577 	context.key = key;
3578 
3579 	tracing_snapshot_cond(file->tr, &context);
3580 }
3581 
3582 static void hist_trigger_print_key(struct seq_file *m,
3583 				   struct hist_trigger_data *hist_data,
3584 				   void *key,
3585 				   struct tracing_map_elt *elt);
3586 
3587 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3588 {
3589 	unsigned int i;
3590 
3591 	if (!hist_data->n_actions)
3592 		return NULL;
3593 
3594 	for (i = 0; i < hist_data->n_actions; i++) {
3595 		struct action_data *data = hist_data->actions[i];
3596 
3597 		if (data->action == ACTION_SNAPSHOT)
3598 			return data;
3599 	}
3600 
3601 	return NULL;
3602 }
3603 
3604 static void track_data_snapshot_print(struct seq_file *m,
3605 				      struct hist_trigger_data *hist_data)
3606 {
3607 	struct trace_event_file *file = hist_data->event_file;
3608 	struct track_data *track_data;
3609 	struct action_data *action;
3610 
3611 	track_data = tracing_cond_snapshot_data(file->tr);
3612 	if (!track_data)
3613 		return;
3614 
3615 	if (!track_data->updated)
3616 		return;
3617 
3618 	action = snapshot_action(hist_data);
3619 	if (!action)
3620 		return;
3621 
3622 	seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3623 	seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3624 		   action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3625 		   action->track_data.var_str, track_data->track_val);
3626 
3627 	seq_puts(m, "\ttriggered by event with key: ");
3628 	hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3629 	seq_putc(m, '\n');
3630 }
3631 #else
3632 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3633 {
3634 	return false;
3635 }
3636 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3637 				     struct tracing_map_elt *elt, void *rec,
3638 				     struct ring_buffer_event *rbe, void *key,
3639 				     struct action_data *data,
3640 				     u64 *var_ref_vals) {}
3641 static void track_data_snapshot_print(struct seq_file *m,
3642 				      struct hist_trigger_data *hist_data) {}
3643 #endif /* CONFIG_TRACER_SNAPSHOT */
3644 
3645 static void track_data_print(struct seq_file *m,
3646 			     struct hist_trigger_data *hist_data,
3647 			     struct tracing_map_elt *elt,
3648 			     struct action_data *data)
3649 {
3650 	u64 track_val = get_track_val(hist_data, elt, data);
3651 	unsigned int i, save_var_idx;
3652 
3653 	if (data->handler == HANDLER_ONMAX)
3654 		seq_printf(m, "\n\tmax: %10llu", track_val);
3655 	else if (data->handler == HANDLER_ONCHANGE)
3656 		seq_printf(m, "\n\tchanged: %10llu", track_val);
3657 
3658 	if (data->action == ACTION_SNAPSHOT)
3659 		return;
3660 
3661 	for (i = 0; i < hist_data->n_save_vars; i++) {
3662 		struct hist_field *save_val = hist_data->save_vars[i]->val;
3663 		struct hist_field *save_var = hist_data->save_vars[i]->var;
3664 		u64 val;
3665 
3666 		save_var_idx = save_var->var.idx;
3667 
3668 		val = tracing_map_read_var(elt, save_var_idx);
3669 
3670 		if (save_val->flags & HIST_FIELD_FL_STRING) {
3671 			seq_printf(m, "  %s: %-32s", save_var->var.name,
3672 				   (char *)(uintptr_t)(val));
3673 		} else
3674 			seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3675 	}
3676 }
3677 
3678 static void ontrack_action(struct hist_trigger_data *hist_data,
3679 			   struct tracing_map_elt *elt, void *rec,
3680 			   struct ring_buffer_event *rbe, void *key,
3681 			   struct action_data *data, u64 *var_ref_vals)
3682 {
3683 	u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3684 
3685 	if (check_track_val(elt, data, var_val)) {
3686 		save_track_val(hist_data, elt, data, var_val);
3687 		save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
3688 	}
3689 }
3690 
3691 static void action_data_destroy(struct action_data *data)
3692 {
3693 	unsigned int i;
3694 
3695 	lockdep_assert_held(&event_mutex);
3696 
3697 	kfree(data->action_name);
3698 
3699 	for (i = 0; i < data->n_params; i++)
3700 		kfree(data->params[i]);
3701 
3702 	if (data->synth_event)
3703 		data->synth_event->ref--;
3704 
3705 	kfree(data->synth_event_name);
3706 
3707 	kfree(data);
3708 }
3709 
3710 static void track_data_destroy(struct hist_trigger_data *hist_data,
3711 			       struct action_data *data)
3712 {
3713 	struct trace_event_file *file = hist_data->event_file;
3714 
3715 	destroy_hist_field(data->track_data.track_var, 0);
3716 
3717 	if (data->action == ACTION_SNAPSHOT) {
3718 		struct track_data *track_data;
3719 
3720 		track_data = tracing_cond_snapshot_data(file->tr);
3721 		if (track_data && track_data->hist_data == hist_data) {
3722 			tracing_snapshot_cond_disable(file->tr);
3723 			track_data_free(track_data);
3724 		}
3725 	}
3726 
3727 	kfree(data->track_data.var_str);
3728 
3729 	action_data_destroy(data);
3730 }
3731 
3732 static int action_create(struct hist_trigger_data *hist_data,
3733 			 struct action_data *data);
3734 
3735 static int track_data_create(struct hist_trigger_data *hist_data,
3736 			     struct action_data *data)
3737 {
3738 	struct hist_field *var_field, *ref_field, *track_var = NULL;
3739 	struct trace_event_file *file = hist_data->event_file;
3740 	char *track_data_var_str;
3741 	int ret = 0;
3742 
3743 	track_data_var_str = data->track_data.var_str;
3744 	if (track_data_var_str[0] != '$') {
3745 		hist_err("For onmax(x) or onchange(x), x must be a variable: ", track_data_var_str);
3746 		return -EINVAL;
3747 	}
3748 	track_data_var_str++;
3749 
3750 	var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3751 	if (!var_field) {
3752 		hist_err("Couldn't find onmax or onchange variable: ", track_data_var_str);
3753 		return -EINVAL;
3754 	}
3755 
3756 	ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3757 	if (!ref_field)
3758 		return -ENOMEM;
3759 
3760 	data->track_data.var_ref = ref_field;
3761 
3762 	if (data->handler == HANDLER_ONMAX)
3763 		track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3764 	if (IS_ERR(track_var)) {
3765 		hist_err("Couldn't create onmax variable: ", "__max");
3766 		ret = PTR_ERR(track_var);
3767 		goto out;
3768 	}
3769 
3770 	if (data->handler == HANDLER_ONCHANGE)
3771 		track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3772 	if (IS_ERR(track_var)) {
3773 		hist_err("Couldn't create onchange variable: ", "__change");
3774 		ret = PTR_ERR(track_var);
3775 		goto out;
3776 	}
3777 	data->track_data.track_var = track_var;
3778 
3779 	ret = action_create(hist_data, data);
3780  out:
3781 	return ret;
3782 }
3783 
3784 static int parse_action_params(char *params, struct action_data *data)
3785 {
3786 	char *param, *saved_param;
3787 	bool first_param = true;
3788 	int ret = 0;
3789 
3790 	while (params) {
3791 		if (data->n_params >= SYNTH_FIELDS_MAX) {
3792 			hist_err("Too many action params", "");
3793 			goto out;
3794 		}
3795 
3796 		param = strsep(&params, ",");
3797 		if (!param) {
3798 			hist_err("No action param found", "");
3799 			ret = -EINVAL;
3800 			goto out;
3801 		}
3802 
3803 		param = strstrip(param);
3804 		if (strlen(param) < 2) {
3805 			hist_err("Invalid action param: ", param);
3806 			ret = -EINVAL;
3807 			goto out;
3808 		}
3809 
3810 		saved_param = kstrdup(param, GFP_KERNEL);
3811 		if (!saved_param) {
3812 			ret = -ENOMEM;
3813 			goto out;
3814 		}
3815 
3816 		if (first_param && data->use_trace_keyword) {
3817 			data->synth_event_name = saved_param;
3818 			first_param = false;
3819 			continue;
3820 		}
3821 		first_param = false;
3822 
3823 		data->params[data->n_params++] = saved_param;
3824 	}
3825  out:
3826 	return ret;
3827 }
3828 
3829 static int action_parse(char *str, struct action_data *data,
3830 			enum handler_id handler)
3831 {
3832 	char *action_name;
3833 	int ret = 0;
3834 
3835 	strsep(&str, ".");
3836 	if (!str) {
3837 		hist_err("action parsing: No action found", "");
3838 		ret = -EINVAL;
3839 		goto out;
3840 	}
3841 
3842 	action_name = strsep(&str, "(");
3843 	if (!action_name || !str) {
3844 		hist_err("action parsing: No action found", "");
3845 		ret = -EINVAL;
3846 		goto out;
3847 	}
3848 
3849 	if (str_has_prefix(action_name, "save")) {
3850 		char *params = strsep(&str, ")");
3851 
3852 		if (!params) {
3853 			hist_err("action parsing: No params found for %s", "save");
3854 			ret = -EINVAL;
3855 			goto out;
3856 		}
3857 
3858 		ret = parse_action_params(params, data);
3859 		if (ret)
3860 			goto out;
3861 
3862 		if (handler == HANDLER_ONMAX)
3863 			data->track_data.check_val = check_track_val_max;
3864 		else if (handler == HANDLER_ONCHANGE)
3865 			data->track_data.check_val = check_track_val_changed;
3866 		else {
3867 			hist_err("action parsing: Handler doesn't support action: ", action_name);
3868 			ret = -EINVAL;
3869 			goto out;
3870 		}
3871 
3872 		data->track_data.save_data = save_track_data_vars;
3873 		data->fn = ontrack_action;
3874 		data->action = ACTION_SAVE;
3875 	} else if (str_has_prefix(action_name, "snapshot")) {
3876 		char *params = strsep(&str, ")");
3877 
3878 		if (!str) {
3879 			hist_err("action parsing: No closing paren found: %s", params);
3880 			ret = -EINVAL;
3881 			goto out;
3882 		}
3883 
3884 		if (handler == HANDLER_ONMAX)
3885 			data->track_data.check_val = check_track_val_max;
3886 		else if (handler == HANDLER_ONCHANGE)
3887 			data->track_data.check_val = check_track_val_changed;
3888 		else {
3889 			hist_err("action parsing: Handler doesn't support action: ", action_name);
3890 			ret = -EINVAL;
3891 			goto out;
3892 		}
3893 
3894 		data->track_data.save_data = save_track_data_snapshot;
3895 		data->fn = ontrack_action;
3896 		data->action = ACTION_SNAPSHOT;
3897 	} else {
3898 		char *params = strsep(&str, ")");
3899 
3900 		if (str_has_prefix(action_name, "trace"))
3901 			data->use_trace_keyword = true;
3902 
3903 		if (params) {
3904 			ret = parse_action_params(params, data);
3905 			if (ret)
3906 				goto out;
3907 		}
3908 
3909 		if (handler == HANDLER_ONMAX)
3910 			data->track_data.check_val = check_track_val_max;
3911 		else if (handler == HANDLER_ONCHANGE)
3912 			data->track_data.check_val = check_track_val_changed;
3913 
3914 		if (handler != HANDLER_ONMATCH) {
3915 			data->track_data.save_data = action_trace;
3916 			data->fn = ontrack_action;
3917 		} else
3918 			data->fn = action_trace;
3919 
3920 		data->action = ACTION_TRACE;
3921 	}
3922 
3923 	data->action_name = kstrdup(action_name, GFP_KERNEL);
3924 	if (!data->action_name) {
3925 		ret = -ENOMEM;
3926 		goto out;
3927 	}
3928 
3929 	data->handler = handler;
3930  out:
3931 	return ret;
3932 }
3933 
3934 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3935 					    char *str, enum handler_id handler)
3936 {
3937 	struct action_data *data;
3938 	int ret = -EINVAL;
3939 	char *var_str;
3940 
3941 	data = kzalloc(sizeof(*data), GFP_KERNEL);
3942 	if (!data)
3943 		return ERR_PTR(-ENOMEM);
3944 
3945 	var_str = strsep(&str, ")");
3946 	if (!var_str || !str) {
3947 		ret = -EINVAL;
3948 		goto free;
3949 	}
3950 
3951 	data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3952 	if (!data->track_data.var_str) {
3953 		ret = -ENOMEM;
3954 		goto free;
3955 	}
3956 
3957 	ret = action_parse(str, data, handler);
3958 	if (ret)
3959 		goto free;
3960  out:
3961 	return data;
3962  free:
3963 	track_data_destroy(hist_data, data);
3964 	data = ERR_PTR(ret);
3965 	goto out;
3966 }
3967 
3968 static void onmatch_destroy(struct action_data *data)
3969 {
3970 	kfree(data->match_data.event);
3971 	kfree(data->match_data.event_system);
3972 
3973 	action_data_destroy(data);
3974 }
3975 
3976 static void destroy_field_var(struct field_var *field_var)
3977 {
3978 	if (!field_var)
3979 		return;
3980 
3981 	destroy_hist_field(field_var->var, 0);
3982 	destroy_hist_field(field_var->val, 0);
3983 
3984 	kfree(field_var);
3985 }
3986 
3987 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3988 {
3989 	unsigned int i;
3990 
3991 	for (i = 0; i < hist_data->n_field_vars; i++)
3992 		destroy_field_var(hist_data->field_vars[i]);
3993 }
3994 
3995 static void save_field_var(struct hist_trigger_data *hist_data,
3996 			   struct field_var *field_var)
3997 {
3998 	hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3999 
4000 	if (field_var->val->flags & HIST_FIELD_FL_STRING)
4001 		hist_data->n_field_var_str++;
4002 }
4003 
4004 
4005 static int check_synth_field(struct synth_event *event,
4006 			     struct hist_field *hist_field,
4007 			     unsigned int field_pos)
4008 {
4009 	struct synth_field *field;
4010 
4011 	if (field_pos >= event->n_fields)
4012 		return -EINVAL;
4013 
4014 	field = event->fields[field_pos];
4015 
4016 	if (strcmp(field->type, hist_field->type) != 0)
4017 		return -EINVAL;
4018 
4019 	return 0;
4020 }
4021 
4022 static struct hist_field *
4023 trace_action_find_var(struct hist_trigger_data *hist_data,
4024 		      struct action_data *data,
4025 		      char *system, char *event, char *var)
4026 {
4027 	struct hist_field *hist_field;
4028 
4029 	var++; /* skip '$' */
4030 
4031 	hist_field = find_target_event_var(hist_data, system, event, var);
4032 	if (!hist_field) {
4033 		if (!system && data->handler == HANDLER_ONMATCH) {
4034 			system = data->match_data.event_system;
4035 			event = data->match_data.event;
4036 		}
4037 
4038 		hist_field = find_event_var(hist_data, system, event, var);
4039 	}
4040 
4041 	if (!hist_field)
4042 		hist_err_event("trace action: Couldn't find param: $", system, event, var);
4043 
4044 	return hist_field;
4045 }
4046 
4047 static struct hist_field *
4048 trace_action_create_field_var(struct hist_trigger_data *hist_data,
4049 			      struct action_data *data, char *system,
4050 			      char *event, char *var)
4051 {
4052 	struct hist_field *hist_field = NULL;
4053 	struct field_var *field_var;
4054 
4055 	/*
4056 	 * First try to create a field var on the target event (the
4057 	 * currently being defined).  This will create a variable for
4058 	 * unqualified fields on the target event, or if qualified,
4059 	 * target fields that have qualified names matching the target.
4060 	 */
4061 	field_var = create_target_field_var(hist_data, system, event, var);
4062 
4063 	if (field_var && !IS_ERR(field_var)) {
4064 		save_field_var(hist_data, field_var);
4065 		hist_field = field_var->var;
4066 	} else {
4067 		field_var = NULL;
4068 		/*
4069 		 * If no explicit system.event is specfied, default to
4070 		 * looking for fields on the onmatch(system.event.xxx)
4071 		 * event.
4072 		 */
4073 		if (!system && data->handler == HANDLER_ONMATCH) {
4074 			system = data->match_data.event_system;
4075 			event = data->match_data.event;
4076 		}
4077 
4078 		/*
4079 		 * At this point, we're looking at a field on another
4080 		 * event.  Because we can't modify a hist trigger on
4081 		 * another event to add a variable for a field, we need
4082 		 * to create a new trigger on that event and create the
4083 		 * variable at the same time.
4084 		 */
4085 		hist_field = create_field_var_hist(hist_data, system, event, var);
4086 		if (IS_ERR(hist_field))
4087 			goto free;
4088 	}
4089  out:
4090 	return hist_field;
4091  free:
4092 	destroy_field_var(field_var);
4093 	hist_field = NULL;
4094 	goto out;
4095 }
4096 
4097 static int trace_action_create(struct hist_trigger_data *hist_data,
4098 			       struct action_data *data)
4099 {
4100 	char *event_name, *param, *system = NULL;
4101 	struct hist_field *hist_field, *var_ref;
4102 	unsigned int i, var_ref_idx;
4103 	unsigned int field_pos = 0;
4104 	struct synth_event *event;
4105 	char *synth_event_name;
4106 	int ret = 0;
4107 
4108 	lockdep_assert_held(&event_mutex);
4109 
4110 	if (data->use_trace_keyword)
4111 		synth_event_name = data->synth_event_name;
4112 	else
4113 		synth_event_name = data->action_name;
4114 
4115 	event = find_synth_event(synth_event_name);
4116 	if (!event) {
4117 		hist_err("trace action: Couldn't find synthetic event: ", synth_event_name);
4118 		return -EINVAL;
4119 	}
4120 
4121 	event->ref++;
4122 
4123 	var_ref_idx = hist_data->n_var_refs;
4124 
4125 	for (i = 0; i < data->n_params; i++) {
4126 		char *p;
4127 
4128 		p = param = kstrdup(data->params[i], GFP_KERNEL);
4129 		if (!param) {
4130 			ret = -ENOMEM;
4131 			goto err;
4132 		}
4133 
4134 		system = strsep(&param, ".");
4135 		if (!param) {
4136 			param = (char *)system;
4137 			system = event_name = NULL;
4138 		} else {
4139 			event_name = strsep(&param, ".");
4140 			if (!param) {
4141 				kfree(p);
4142 				ret = -EINVAL;
4143 				goto err;
4144 			}
4145 		}
4146 
4147 		if (param[0] == '$')
4148 			hist_field = trace_action_find_var(hist_data, data,
4149 							   system, event_name,
4150 							   param);
4151 		else
4152 			hist_field = trace_action_create_field_var(hist_data,
4153 								   data,
4154 								   system,
4155 								   event_name,
4156 								   param);
4157 
4158 		if (!hist_field) {
4159 			kfree(p);
4160 			ret = -EINVAL;
4161 			goto err;
4162 		}
4163 
4164 		if (check_synth_field(event, hist_field, field_pos) == 0) {
4165 			var_ref = create_var_ref(hist_data, hist_field,
4166 						 system, event_name);
4167 			if (!var_ref) {
4168 				kfree(p);
4169 				ret = -ENOMEM;
4170 				goto err;
4171 			}
4172 
4173 			field_pos++;
4174 			kfree(p);
4175 			continue;
4176 		}
4177 
4178 		hist_err_event("trace action: Param type doesn't match synthetic event field type: ",
4179 			       system, event_name, param);
4180 		kfree(p);
4181 		ret = -EINVAL;
4182 		goto err;
4183 	}
4184 
4185 	if (field_pos != event->n_fields) {
4186 		hist_err("trace action: Param count doesn't match synthetic event field count: ", event->name);
4187 		ret = -EINVAL;
4188 		goto err;
4189 	}
4190 
4191 	data->synth_event = event;
4192 	data->var_ref_idx = var_ref_idx;
4193  out:
4194 	return ret;
4195  err:
4196 	event->ref--;
4197 
4198 	goto out;
4199 }
4200 
4201 static int action_create(struct hist_trigger_data *hist_data,
4202 			 struct action_data *data)
4203 {
4204 	struct trace_event_file *file = hist_data->event_file;
4205 	struct track_data *track_data;
4206 	struct field_var *field_var;
4207 	unsigned int i;
4208 	char *param;
4209 	int ret = 0;
4210 
4211 	if (data->action == ACTION_TRACE)
4212 		return trace_action_create(hist_data, data);
4213 
4214 	if (data->action == ACTION_SNAPSHOT) {
4215 		track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4216 		if (IS_ERR(track_data)) {
4217 			ret = PTR_ERR(track_data);
4218 			goto out;
4219 		}
4220 
4221 		ret = tracing_snapshot_cond_enable(file->tr, track_data,
4222 						   cond_snapshot_update);
4223 		if (ret)
4224 			track_data_free(track_data);
4225 
4226 		goto out;
4227 	}
4228 
4229 	if (data->action == ACTION_SAVE) {
4230 		if (hist_data->n_save_vars) {
4231 			ret = -EEXIST;
4232 			hist_err("save action: Can't have more than one save() action per hist", "");
4233 			goto out;
4234 		}
4235 
4236 		for (i = 0; i < data->n_params; i++) {
4237 			param = kstrdup(data->params[i], GFP_KERNEL);
4238 			if (!param) {
4239 				ret = -ENOMEM;
4240 				goto out;
4241 			}
4242 
4243 			field_var = create_target_field_var(hist_data, NULL, NULL, param);
4244 			if (IS_ERR(field_var)) {
4245 				hist_err("save action: Couldn't create field variable: ", param);
4246 				ret = PTR_ERR(field_var);
4247 				kfree(param);
4248 				goto out;
4249 			}
4250 
4251 			hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4252 			if (field_var->val->flags & HIST_FIELD_FL_STRING)
4253 				hist_data->n_save_var_str++;
4254 			kfree(param);
4255 		}
4256 	}
4257  out:
4258 	return ret;
4259 }
4260 
4261 static int onmatch_create(struct hist_trigger_data *hist_data,
4262 			  struct action_data *data)
4263 {
4264 	return action_create(hist_data, data);
4265 }
4266 
4267 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4268 {
4269 	char *match_event, *match_event_system;
4270 	struct action_data *data;
4271 	int ret = -EINVAL;
4272 
4273 	data = kzalloc(sizeof(*data), GFP_KERNEL);
4274 	if (!data)
4275 		return ERR_PTR(-ENOMEM);
4276 
4277 	match_event = strsep(&str, ")");
4278 	if (!match_event || !str) {
4279 		hist_err("onmatch: Missing closing paren: ", match_event);
4280 		goto free;
4281 	}
4282 
4283 	match_event_system = strsep(&match_event, ".");
4284 	if (!match_event) {
4285 		hist_err("onmatch: Missing subsystem for match event: ", match_event_system);
4286 		goto free;
4287 	}
4288 
4289 	if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4290 		hist_err_event("onmatch: Invalid subsystem or event name: ",
4291 			       match_event_system, match_event, NULL);
4292 		goto free;
4293 	}
4294 
4295 	data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4296 	if (!data->match_data.event) {
4297 		ret = -ENOMEM;
4298 		goto free;
4299 	}
4300 
4301 	data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4302 	if (!data->match_data.event_system) {
4303 		ret = -ENOMEM;
4304 		goto free;
4305 	}
4306 
4307 	ret = action_parse(str, data, HANDLER_ONMATCH);
4308 	if (ret)
4309 		goto free;
4310  out:
4311 	return data;
4312  free:
4313 	onmatch_destroy(data);
4314 	data = ERR_PTR(ret);
4315 	goto out;
4316 }
4317 
4318 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4319 {
4320 	hist_data->fields[HITCOUNT_IDX] =
4321 		create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4322 	if (!hist_data->fields[HITCOUNT_IDX])
4323 		return -ENOMEM;
4324 
4325 	hist_data->n_vals++;
4326 	hist_data->n_fields++;
4327 
4328 	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4329 		return -EINVAL;
4330 
4331 	return 0;
4332 }
4333 
4334 static int __create_val_field(struct hist_trigger_data *hist_data,
4335 			      unsigned int val_idx,
4336 			      struct trace_event_file *file,
4337 			      char *var_name, char *field_str,
4338 			      unsigned long flags)
4339 {
4340 	struct hist_field *hist_field;
4341 	int ret = 0;
4342 
4343 	hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
4344 	if (IS_ERR(hist_field)) {
4345 		ret = PTR_ERR(hist_field);
4346 		goto out;
4347 	}
4348 
4349 	hist_data->fields[val_idx] = hist_field;
4350 
4351 	++hist_data->n_vals;
4352 	++hist_data->n_fields;
4353 
4354 	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4355 		ret = -EINVAL;
4356  out:
4357 	return ret;
4358 }
4359 
4360 static int create_val_field(struct hist_trigger_data *hist_data,
4361 			    unsigned int val_idx,
4362 			    struct trace_event_file *file,
4363 			    char *field_str)
4364 {
4365 	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4366 		return -EINVAL;
4367 
4368 	return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4369 }
4370 
4371 static int create_var_field(struct hist_trigger_data *hist_data,
4372 			    unsigned int val_idx,
4373 			    struct trace_event_file *file,
4374 			    char *var_name, char *expr_str)
4375 {
4376 	unsigned long flags = 0;
4377 
4378 	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4379 		return -EINVAL;
4380 
4381 	if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4382 		hist_err("Variable already defined: ", var_name);
4383 		return -EINVAL;
4384 	}
4385 
4386 	flags |= HIST_FIELD_FL_VAR;
4387 	hist_data->n_vars++;
4388 	if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4389 		return -EINVAL;
4390 
4391 	return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4392 }
4393 
4394 static int create_val_fields(struct hist_trigger_data *hist_data,
4395 			     struct trace_event_file *file)
4396 {
4397 	char *fields_str, *field_str;
4398 	unsigned int i, j = 1;
4399 	int ret;
4400 
4401 	ret = create_hitcount_val(hist_data);
4402 	if (ret)
4403 		goto out;
4404 
4405 	fields_str = hist_data->attrs->vals_str;
4406 	if (!fields_str)
4407 		goto out;
4408 
4409 	strsep(&fields_str, "=");
4410 	if (!fields_str)
4411 		goto out;
4412 
4413 	for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4414 		     j < TRACING_MAP_VALS_MAX; i++) {
4415 		field_str = strsep(&fields_str, ",");
4416 		if (!field_str)
4417 			break;
4418 
4419 		if (strcmp(field_str, "hitcount") == 0)
4420 			continue;
4421 
4422 		ret = create_val_field(hist_data, j++, file, field_str);
4423 		if (ret)
4424 			goto out;
4425 	}
4426 
4427 	if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4428 		ret = -EINVAL;
4429  out:
4430 	return ret;
4431 }
4432 
4433 static int create_key_field(struct hist_trigger_data *hist_data,
4434 			    unsigned int key_idx,
4435 			    unsigned int key_offset,
4436 			    struct trace_event_file *file,
4437 			    char *field_str)
4438 {
4439 	struct hist_field *hist_field = NULL;
4440 
4441 	unsigned long flags = 0;
4442 	unsigned int key_size;
4443 	int ret = 0;
4444 
4445 	if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4446 		return -EINVAL;
4447 
4448 	flags |= HIST_FIELD_FL_KEY;
4449 
4450 	if (strcmp(field_str, "stacktrace") == 0) {
4451 		flags |= HIST_FIELD_FL_STACKTRACE;
4452 		key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4453 		hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4454 	} else {
4455 		hist_field = parse_expr(hist_data, file, field_str, flags,
4456 					NULL, 0);
4457 		if (IS_ERR(hist_field)) {
4458 			ret = PTR_ERR(hist_field);
4459 			goto out;
4460 		}
4461 
4462 		if (hist_field->flags & HIST_FIELD_FL_VAR_REF) {
4463 			hist_err("Using variable references as keys not supported: ", field_str);
4464 			destroy_hist_field(hist_field, 0);
4465 			ret = -EINVAL;
4466 			goto out;
4467 		}
4468 
4469 		key_size = hist_field->size;
4470 	}
4471 
4472 	hist_data->fields[key_idx] = hist_field;
4473 
4474 	key_size = ALIGN(key_size, sizeof(u64));
4475 	hist_data->fields[key_idx]->size = key_size;
4476 	hist_data->fields[key_idx]->offset = key_offset;
4477 
4478 	hist_data->key_size += key_size;
4479 
4480 	if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4481 		ret = -EINVAL;
4482 		goto out;
4483 	}
4484 
4485 	hist_data->n_keys++;
4486 	hist_data->n_fields++;
4487 
4488 	if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4489 		return -EINVAL;
4490 
4491 	ret = key_size;
4492  out:
4493 	return ret;
4494 }
4495 
4496 static int create_key_fields(struct hist_trigger_data *hist_data,
4497 			     struct trace_event_file *file)
4498 {
4499 	unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4500 	char *fields_str, *field_str;
4501 	int ret = -EINVAL;
4502 
4503 	fields_str = hist_data->attrs->keys_str;
4504 	if (!fields_str)
4505 		goto out;
4506 
4507 	strsep(&fields_str, "=");
4508 	if (!fields_str)
4509 		goto out;
4510 
4511 	for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4512 		field_str = strsep(&fields_str, ",");
4513 		if (!field_str)
4514 			break;
4515 		ret = create_key_field(hist_data, i, key_offset,
4516 				       file, field_str);
4517 		if (ret < 0)
4518 			goto out;
4519 		key_offset += ret;
4520 	}
4521 	if (fields_str) {
4522 		ret = -EINVAL;
4523 		goto out;
4524 	}
4525 	ret = 0;
4526  out:
4527 	return ret;
4528 }
4529 
4530 static int create_var_fields(struct hist_trigger_data *hist_data,
4531 			     struct trace_event_file *file)
4532 {
4533 	unsigned int i, j = hist_data->n_vals;
4534 	int ret = 0;
4535 
4536 	unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4537 
4538 	for (i = 0; i < n_vars; i++) {
4539 		char *var_name = hist_data->attrs->var_defs.name[i];
4540 		char *expr = hist_data->attrs->var_defs.expr[i];
4541 
4542 		ret = create_var_field(hist_data, j++, file, var_name, expr);
4543 		if (ret)
4544 			goto out;
4545 	}
4546  out:
4547 	return ret;
4548 }
4549 
4550 static void free_var_defs(struct hist_trigger_data *hist_data)
4551 {
4552 	unsigned int i;
4553 
4554 	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4555 		kfree(hist_data->attrs->var_defs.name[i]);
4556 		kfree(hist_data->attrs->var_defs.expr[i]);
4557 	}
4558 
4559 	hist_data->attrs->var_defs.n_vars = 0;
4560 }
4561 
4562 static int parse_var_defs(struct hist_trigger_data *hist_data)
4563 {
4564 	char *s, *str, *var_name, *field_str;
4565 	unsigned int i, j, n_vars = 0;
4566 	int ret = 0;
4567 
4568 	for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4569 		str = hist_data->attrs->assignment_str[i];
4570 		for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4571 			field_str = strsep(&str, ",");
4572 			if (!field_str)
4573 				break;
4574 
4575 			var_name = strsep(&field_str, "=");
4576 			if (!var_name || !field_str) {
4577 				hist_err("Malformed assignment: ", var_name);
4578 				ret = -EINVAL;
4579 				goto free;
4580 			}
4581 
4582 			if (n_vars == TRACING_MAP_VARS_MAX) {
4583 				hist_err("Too many variables defined: ", var_name);
4584 				ret = -EINVAL;
4585 				goto free;
4586 			}
4587 
4588 			s = kstrdup(var_name, GFP_KERNEL);
4589 			if (!s) {
4590 				ret = -ENOMEM;
4591 				goto free;
4592 			}
4593 			hist_data->attrs->var_defs.name[n_vars] = s;
4594 
4595 			s = kstrdup(field_str, GFP_KERNEL);
4596 			if (!s) {
4597 				kfree(hist_data->attrs->var_defs.name[n_vars]);
4598 				ret = -ENOMEM;
4599 				goto free;
4600 			}
4601 			hist_data->attrs->var_defs.expr[n_vars++] = s;
4602 
4603 			hist_data->attrs->var_defs.n_vars = n_vars;
4604 		}
4605 	}
4606 
4607 	return ret;
4608  free:
4609 	free_var_defs(hist_data);
4610 
4611 	return ret;
4612 }
4613 
4614 static int create_hist_fields(struct hist_trigger_data *hist_data,
4615 			      struct trace_event_file *file)
4616 {
4617 	int ret;
4618 
4619 	ret = parse_var_defs(hist_data);
4620 	if (ret)
4621 		goto out;
4622 
4623 	ret = create_val_fields(hist_data, file);
4624 	if (ret)
4625 		goto out;
4626 
4627 	ret = create_var_fields(hist_data, file);
4628 	if (ret)
4629 		goto out;
4630 
4631 	ret = create_key_fields(hist_data, file);
4632 	if (ret)
4633 		goto out;
4634  out:
4635 	free_var_defs(hist_data);
4636 
4637 	return ret;
4638 }
4639 
4640 static int is_descending(const char *str)
4641 {
4642 	if (!str)
4643 		return 0;
4644 
4645 	if (strcmp(str, "descending") == 0)
4646 		return 1;
4647 
4648 	if (strcmp(str, "ascending") == 0)
4649 		return 0;
4650 
4651 	return -EINVAL;
4652 }
4653 
4654 static int create_sort_keys(struct hist_trigger_data *hist_data)
4655 {
4656 	char *fields_str = hist_data->attrs->sort_key_str;
4657 	struct tracing_map_sort_key *sort_key;
4658 	int descending, ret = 0;
4659 	unsigned int i, j, k;
4660 
4661 	hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4662 
4663 	if (!fields_str)
4664 		goto out;
4665 
4666 	strsep(&fields_str, "=");
4667 	if (!fields_str) {
4668 		ret = -EINVAL;
4669 		goto out;
4670 	}
4671 
4672 	for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4673 		struct hist_field *hist_field;
4674 		char *field_str, *field_name;
4675 		const char *test_name;
4676 
4677 		sort_key = &hist_data->sort_keys[i];
4678 
4679 		field_str = strsep(&fields_str, ",");
4680 		if (!field_str) {
4681 			if (i == 0)
4682 				ret = -EINVAL;
4683 			break;
4684 		}
4685 
4686 		if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4687 			ret = -EINVAL;
4688 			break;
4689 		}
4690 
4691 		field_name = strsep(&field_str, ".");
4692 		if (!field_name) {
4693 			ret = -EINVAL;
4694 			break;
4695 		}
4696 
4697 		if (strcmp(field_name, "hitcount") == 0) {
4698 			descending = is_descending(field_str);
4699 			if (descending < 0) {
4700 				ret = descending;
4701 				break;
4702 			}
4703 			sort_key->descending = descending;
4704 			continue;
4705 		}
4706 
4707 		for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4708 			unsigned int idx;
4709 
4710 			hist_field = hist_data->fields[j];
4711 			if (hist_field->flags & HIST_FIELD_FL_VAR)
4712 				continue;
4713 
4714 			idx = k++;
4715 
4716 			test_name = hist_field_name(hist_field, 0);
4717 
4718 			if (strcmp(field_name, test_name) == 0) {
4719 				sort_key->field_idx = idx;
4720 				descending = is_descending(field_str);
4721 				if (descending < 0) {
4722 					ret = descending;
4723 					goto out;
4724 				}
4725 				sort_key->descending = descending;
4726 				break;
4727 			}
4728 		}
4729 		if (j == hist_data->n_fields) {
4730 			ret = -EINVAL;
4731 			break;
4732 		}
4733 	}
4734 
4735 	hist_data->n_sort_keys = i;
4736  out:
4737 	return ret;
4738 }
4739 
4740 static void destroy_actions(struct hist_trigger_data *hist_data)
4741 {
4742 	unsigned int i;
4743 
4744 	for (i = 0; i < hist_data->n_actions; i++) {
4745 		struct action_data *data = hist_data->actions[i];
4746 
4747 		if (data->handler == HANDLER_ONMATCH)
4748 			onmatch_destroy(data);
4749 		else if (data->handler == HANDLER_ONMAX ||
4750 			 data->handler == HANDLER_ONCHANGE)
4751 			track_data_destroy(hist_data, data);
4752 		else
4753 			kfree(data);
4754 	}
4755 }
4756 
4757 static int parse_actions(struct hist_trigger_data *hist_data)
4758 {
4759 	struct trace_array *tr = hist_data->event_file->tr;
4760 	struct action_data *data;
4761 	unsigned int i;
4762 	int ret = 0;
4763 	char *str;
4764 	int len;
4765 
4766 	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4767 		str = hist_data->attrs->action_str[i];
4768 
4769 		if ((len = str_has_prefix(str, "onmatch("))) {
4770 			char *action_str = str + len;
4771 
4772 			data = onmatch_parse(tr, action_str);
4773 			if (IS_ERR(data)) {
4774 				ret = PTR_ERR(data);
4775 				break;
4776 			}
4777 		} else if ((len = str_has_prefix(str, "onmax("))) {
4778 			char *action_str = str + len;
4779 
4780 			data = track_data_parse(hist_data, action_str,
4781 						HANDLER_ONMAX);
4782 			if (IS_ERR(data)) {
4783 				ret = PTR_ERR(data);
4784 				break;
4785 			}
4786 		} else if ((len = str_has_prefix(str, "onchange("))) {
4787 			char *action_str = str + len;
4788 
4789 			data = track_data_parse(hist_data, action_str,
4790 						HANDLER_ONCHANGE);
4791 			if (IS_ERR(data)) {
4792 				ret = PTR_ERR(data);
4793 				break;
4794 			}
4795 		} else {
4796 			ret = -EINVAL;
4797 			break;
4798 		}
4799 
4800 		hist_data->actions[hist_data->n_actions++] = data;
4801 	}
4802 
4803 	return ret;
4804 }
4805 
4806 static int create_actions(struct hist_trigger_data *hist_data)
4807 {
4808 	struct action_data *data;
4809 	unsigned int i;
4810 	int ret = 0;
4811 
4812 	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4813 		data = hist_data->actions[i];
4814 
4815 		if (data->handler == HANDLER_ONMATCH) {
4816 			ret = onmatch_create(hist_data, data);
4817 			if (ret)
4818 				break;
4819 		} else if (data->handler == HANDLER_ONMAX ||
4820 			   data->handler == HANDLER_ONCHANGE) {
4821 			ret = track_data_create(hist_data, data);
4822 			if (ret)
4823 				break;
4824 		} else {
4825 			ret = -EINVAL;
4826 			break;
4827 		}
4828 	}
4829 
4830 	return ret;
4831 }
4832 
4833 static void print_actions(struct seq_file *m,
4834 			  struct hist_trigger_data *hist_data,
4835 			  struct tracing_map_elt *elt)
4836 {
4837 	unsigned int i;
4838 
4839 	for (i = 0; i < hist_data->n_actions; i++) {
4840 		struct action_data *data = hist_data->actions[i];
4841 
4842 		if (data->action == ACTION_SNAPSHOT)
4843 			continue;
4844 
4845 		if (data->handler == HANDLER_ONMAX ||
4846 		    data->handler == HANDLER_ONCHANGE)
4847 			track_data_print(m, hist_data, elt, data);
4848 	}
4849 }
4850 
4851 static void print_action_spec(struct seq_file *m,
4852 			      struct hist_trigger_data *hist_data,
4853 			      struct action_data *data)
4854 {
4855 	unsigned int i;
4856 
4857 	if (data->action == ACTION_SAVE) {
4858 		for (i = 0; i < hist_data->n_save_vars; i++) {
4859 			seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4860 			if (i < hist_data->n_save_vars - 1)
4861 				seq_puts(m, ",");
4862 		}
4863 	} else if (data->action == ACTION_TRACE) {
4864 		if (data->use_trace_keyword)
4865 			seq_printf(m, "%s", data->synth_event_name);
4866 		for (i = 0; i < data->n_params; i++) {
4867 			if (i || data->use_trace_keyword)
4868 				seq_puts(m, ",");
4869 			seq_printf(m, "%s", data->params[i]);
4870 		}
4871 	}
4872 }
4873 
4874 static void print_track_data_spec(struct seq_file *m,
4875 				  struct hist_trigger_data *hist_data,
4876 				  struct action_data *data)
4877 {
4878 	if (data->handler == HANDLER_ONMAX)
4879 		seq_puts(m, ":onmax(");
4880 	else if (data->handler == HANDLER_ONCHANGE)
4881 		seq_puts(m, ":onchange(");
4882 	seq_printf(m, "%s", data->track_data.var_str);
4883 	seq_printf(m, ").%s(", data->action_name);
4884 
4885 	print_action_spec(m, hist_data, data);
4886 
4887 	seq_puts(m, ")");
4888 }
4889 
4890 static void print_onmatch_spec(struct seq_file *m,
4891 			       struct hist_trigger_data *hist_data,
4892 			       struct action_data *data)
4893 {
4894 	seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4895 		   data->match_data.event);
4896 
4897 	seq_printf(m, "%s(", data->action_name);
4898 
4899 	print_action_spec(m, hist_data, data);
4900 
4901 	seq_puts(m, ")");
4902 }
4903 
4904 static bool actions_match(struct hist_trigger_data *hist_data,
4905 			  struct hist_trigger_data *hist_data_test)
4906 {
4907 	unsigned int i, j;
4908 
4909 	if (hist_data->n_actions != hist_data_test->n_actions)
4910 		return false;
4911 
4912 	for (i = 0; i < hist_data->n_actions; i++) {
4913 		struct action_data *data = hist_data->actions[i];
4914 		struct action_data *data_test = hist_data_test->actions[i];
4915 		char *action_name, *action_name_test;
4916 
4917 		if (data->handler != data_test->handler)
4918 			return false;
4919 		if (data->action != data_test->action)
4920 			return false;
4921 
4922 		if (data->n_params != data_test->n_params)
4923 			return false;
4924 
4925 		for (j = 0; j < data->n_params; j++) {
4926 			if (strcmp(data->params[j], data_test->params[j]) != 0)
4927 				return false;
4928 		}
4929 
4930 		if (data->use_trace_keyword)
4931 			action_name = data->synth_event_name;
4932 		else
4933 			action_name = data->action_name;
4934 
4935 		if (data_test->use_trace_keyword)
4936 			action_name_test = data_test->synth_event_name;
4937 		else
4938 			action_name_test = data_test->action_name;
4939 
4940 		if (strcmp(action_name, action_name_test) != 0)
4941 			return false;
4942 
4943 		if (data->handler == HANDLER_ONMATCH) {
4944 			if (strcmp(data->match_data.event_system,
4945 				   data_test->match_data.event_system) != 0)
4946 				return false;
4947 			if (strcmp(data->match_data.event,
4948 				   data_test->match_data.event) != 0)
4949 				return false;
4950 		} else if (data->handler == HANDLER_ONMAX ||
4951 			   data->handler == HANDLER_ONCHANGE) {
4952 			if (strcmp(data->track_data.var_str,
4953 				   data_test->track_data.var_str) != 0)
4954 				return false;
4955 		}
4956 	}
4957 
4958 	return true;
4959 }
4960 
4961 
4962 static void print_actions_spec(struct seq_file *m,
4963 			       struct hist_trigger_data *hist_data)
4964 {
4965 	unsigned int i;
4966 
4967 	for (i = 0; i < hist_data->n_actions; i++) {
4968 		struct action_data *data = hist_data->actions[i];
4969 
4970 		if (data->handler == HANDLER_ONMATCH)
4971 			print_onmatch_spec(m, hist_data, data);
4972 		else if (data->handler == HANDLER_ONMAX ||
4973 			 data->handler == HANDLER_ONCHANGE)
4974 			print_track_data_spec(m, hist_data, data);
4975 	}
4976 }
4977 
4978 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4979 {
4980 	unsigned int i;
4981 
4982 	for (i = 0; i < hist_data->n_field_var_hists; i++) {
4983 		kfree(hist_data->field_var_hists[i]->cmd);
4984 		kfree(hist_data->field_var_hists[i]);
4985 	}
4986 }
4987 
4988 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4989 {
4990 	if (!hist_data)
4991 		return;
4992 
4993 	destroy_hist_trigger_attrs(hist_data->attrs);
4994 	destroy_hist_fields(hist_data);
4995 	tracing_map_destroy(hist_data->map);
4996 
4997 	destroy_actions(hist_data);
4998 	destroy_field_vars(hist_data);
4999 	destroy_field_var_hists(hist_data);
5000 
5001 	kfree(hist_data);
5002 }
5003 
5004 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
5005 {
5006 	struct tracing_map *map = hist_data->map;
5007 	struct ftrace_event_field *field;
5008 	struct hist_field *hist_field;
5009 	int i, idx = 0;
5010 
5011 	for_each_hist_field(i, hist_data) {
5012 		hist_field = hist_data->fields[i];
5013 		if (hist_field->flags & HIST_FIELD_FL_KEY) {
5014 			tracing_map_cmp_fn_t cmp_fn;
5015 
5016 			field = hist_field->field;
5017 
5018 			if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
5019 				cmp_fn = tracing_map_cmp_none;
5020 			else if (!field)
5021 				cmp_fn = tracing_map_cmp_num(hist_field->size,
5022 							     hist_field->is_signed);
5023 			else if (is_string_field(field))
5024 				cmp_fn = tracing_map_cmp_string;
5025 			else
5026 				cmp_fn = tracing_map_cmp_num(field->size,
5027 							     field->is_signed);
5028 			idx = tracing_map_add_key_field(map,
5029 							hist_field->offset,
5030 							cmp_fn);
5031 		} else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5032 			idx = tracing_map_add_sum_field(map);
5033 
5034 		if (idx < 0)
5035 			return idx;
5036 
5037 		if (hist_field->flags & HIST_FIELD_FL_VAR) {
5038 			idx = tracing_map_add_var(map);
5039 			if (idx < 0)
5040 				return idx;
5041 			hist_field->var.idx = idx;
5042 			hist_field->var.hist_data = hist_data;
5043 		}
5044 	}
5045 
5046 	return 0;
5047 }
5048 
5049 static struct hist_trigger_data *
5050 create_hist_data(unsigned int map_bits,
5051 		 struct hist_trigger_attrs *attrs,
5052 		 struct trace_event_file *file,
5053 		 bool remove)
5054 {
5055 	const struct tracing_map_ops *map_ops = NULL;
5056 	struct hist_trigger_data *hist_data;
5057 	int ret = 0;
5058 
5059 	hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5060 	if (!hist_data)
5061 		return ERR_PTR(-ENOMEM);
5062 
5063 	hist_data->attrs = attrs;
5064 	hist_data->remove = remove;
5065 	hist_data->event_file = file;
5066 
5067 	ret = parse_actions(hist_data);
5068 	if (ret)
5069 		goto free;
5070 
5071 	ret = create_hist_fields(hist_data, file);
5072 	if (ret)
5073 		goto free;
5074 
5075 	ret = create_sort_keys(hist_data);
5076 	if (ret)
5077 		goto free;
5078 
5079 	map_ops = &hist_trigger_elt_data_ops;
5080 
5081 	hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5082 					    map_ops, hist_data);
5083 	if (IS_ERR(hist_data->map)) {
5084 		ret = PTR_ERR(hist_data->map);
5085 		hist_data->map = NULL;
5086 		goto free;
5087 	}
5088 
5089 	ret = create_tracing_map_fields(hist_data);
5090 	if (ret)
5091 		goto free;
5092  out:
5093 	return hist_data;
5094  free:
5095 	hist_data->attrs = NULL;
5096 
5097 	destroy_hist_data(hist_data);
5098 
5099 	hist_data = ERR_PTR(ret);
5100 
5101 	goto out;
5102 }
5103 
5104 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5105 				    struct tracing_map_elt *elt, void *rec,
5106 				    struct ring_buffer_event *rbe,
5107 				    u64 *var_ref_vals)
5108 {
5109 	struct hist_elt_data *elt_data;
5110 	struct hist_field *hist_field;
5111 	unsigned int i, var_idx;
5112 	u64 hist_val;
5113 
5114 	elt_data = elt->private_data;
5115 	elt_data->var_ref_vals = var_ref_vals;
5116 
5117 	for_each_hist_val_field(i, hist_data) {
5118 		hist_field = hist_data->fields[i];
5119 		hist_val = hist_field->fn(hist_field, elt, rbe, rec);
5120 		if (hist_field->flags & HIST_FIELD_FL_VAR) {
5121 			var_idx = hist_field->var.idx;
5122 			tracing_map_set_var(elt, var_idx, hist_val);
5123 			continue;
5124 		}
5125 		tracing_map_update_sum(elt, i, hist_val);
5126 	}
5127 
5128 	for_each_hist_key_field(i, hist_data) {
5129 		hist_field = hist_data->fields[i];
5130 		if (hist_field->flags & HIST_FIELD_FL_VAR) {
5131 			hist_val = hist_field->fn(hist_field, elt, rbe, rec);
5132 			var_idx = hist_field->var.idx;
5133 			tracing_map_set_var(elt, var_idx, hist_val);
5134 		}
5135 	}
5136 
5137 	update_field_vars(hist_data, elt, rbe, rec);
5138 }
5139 
5140 static inline void add_to_key(char *compound_key, void *key,
5141 			      struct hist_field *key_field, void *rec)
5142 {
5143 	size_t size = key_field->size;
5144 
5145 	if (key_field->flags & HIST_FIELD_FL_STRING) {
5146 		struct ftrace_event_field *field;
5147 
5148 		field = key_field->field;
5149 		if (field->filter_type == FILTER_DYN_STRING)
5150 			size = *(u32 *)(rec + field->offset) >> 16;
5151 		else if (field->filter_type == FILTER_PTR_STRING)
5152 			size = strlen(key);
5153 		else if (field->filter_type == FILTER_STATIC_STRING)
5154 			size = field->size;
5155 
5156 		/* ensure NULL-termination */
5157 		if (size > key_field->size - 1)
5158 			size = key_field->size - 1;
5159 
5160 		strncpy(compound_key + key_field->offset, (char *)key, size);
5161 	} else
5162 		memcpy(compound_key + key_field->offset, key, size);
5163 }
5164 
5165 static void
5166 hist_trigger_actions(struct hist_trigger_data *hist_data,
5167 		     struct tracing_map_elt *elt, void *rec,
5168 		     struct ring_buffer_event *rbe, void *key,
5169 		     u64 *var_ref_vals)
5170 {
5171 	struct action_data *data;
5172 	unsigned int i;
5173 
5174 	for (i = 0; i < hist_data->n_actions; i++) {
5175 		data = hist_data->actions[i];
5176 		data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals);
5177 	}
5178 }
5179 
5180 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
5181 			       struct ring_buffer_event *rbe)
5182 {
5183 	struct hist_trigger_data *hist_data = data->private_data;
5184 	bool use_compound_key = (hist_data->n_keys > 1);
5185 	unsigned long entries[HIST_STACKTRACE_DEPTH];
5186 	u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5187 	char compound_key[HIST_KEY_SIZE_MAX];
5188 	struct tracing_map_elt *elt = NULL;
5189 	struct stack_trace stacktrace;
5190 	struct hist_field *key_field;
5191 	u64 field_contents;
5192 	void *key = NULL;
5193 	unsigned int i;
5194 
5195 	memset(compound_key, 0, hist_data->key_size);
5196 
5197 	for_each_hist_key_field(i, hist_data) {
5198 		key_field = hist_data->fields[i];
5199 
5200 		if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5201 			stacktrace.max_entries = HIST_STACKTRACE_DEPTH;
5202 			stacktrace.entries = entries;
5203 			stacktrace.nr_entries = 0;
5204 			stacktrace.skip = HIST_STACKTRACE_SKIP;
5205 
5206 			memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE);
5207 			save_stack_trace(&stacktrace);
5208 
5209 			key = entries;
5210 		} else {
5211 			field_contents = key_field->fn(key_field, elt, rbe, rec);
5212 			if (key_field->flags & HIST_FIELD_FL_STRING) {
5213 				key = (void *)(unsigned long)field_contents;
5214 				use_compound_key = true;
5215 			} else
5216 				key = (void *)&field_contents;
5217 		}
5218 
5219 		if (use_compound_key)
5220 			add_to_key(compound_key, key, key_field, rec);
5221 	}
5222 
5223 	if (use_compound_key)
5224 		key = compound_key;
5225 
5226 	if (hist_data->n_var_refs &&
5227 	    !resolve_var_refs(hist_data, key, var_ref_vals, false))
5228 		return;
5229 
5230 	elt = tracing_map_insert(hist_data->map, key);
5231 	if (!elt)
5232 		return;
5233 
5234 	hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
5235 
5236 	if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5237 		hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals);
5238 }
5239 
5240 static void hist_trigger_stacktrace_print(struct seq_file *m,
5241 					  unsigned long *stacktrace_entries,
5242 					  unsigned int max_entries)
5243 {
5244 	char str[KSYM_SYMBOL_LEN];
5245 	unsigned int spaces = 8;
5246 	unsigned int i;
5247 
5248 	for (i = 0; i < max_entries; i++) {
5249 		if (stacktrace_entries[i] == ULONG_MAX)
5250 			return;
5251 
5252 		seq_printf(m, "%*c", 1 + spaces, ' ');
5253 		sprint_symbol(str, stacktrace_entries[i]);
5254 		seq_printf(m, "%s\n", str);
5255 	}
5256 }
5257 
5258 static void hist_trigger_print_key(struct seq_file *m,
5259 				   struct hist_trigger_data *hist_data,
5260 				   void *key,
5261 				   struct tracing_map_elt *elt)
5262 {
5263 	struct hist_field *key_field;
5264 	char str[KSYM_SYMBOL_LEN];
5265 	bool multiline = false;
5266 	const char *field_name;
5267 	unsigned int i;
5268 	u64 uval;
5269 
5270 	seq_puts(m, "{ ");
5271 
5272 	for_each_hist_key_field(i, hist_data) {
5273 		key_field = hist_data->fields[i];
5274 
5275 		if (i > hist_data->n_vals)
5276 			seq_puts(m, ", ");
5277 
5278 		field_name = hist_field_name(key_field, 0);
5279 
5280 		if (key_field->flags & HIST_FIELD_FL_HEX) {
5281 			uval = *(u64 *)(key + key_field->offset);
5282 			seq_printf(m, "%s: %llx", field_name, uval);
5283 		} else if (key_field->flags & HIST_FIELD_FL_SYM) {
5284 			uval = *(u64 *)(key + key_field->offset);
5285 			sprint_symbol_no_offset(str, uval);
5286 			seq_printf(m, "%s: [%llx] %-45s", field_name,
5287 				   uval, str);
5288 		} else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5289 			uval = *(u64 *)(key + key_field->offset);
5290 			sprint_symbol(str, uval);
5291 			seq_printf(m, "%s: [%llx] %-55s", field_name,
5292 				   uval, str);
5293 		} else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5294 			struct hist_elt_data *elt_data = elt->private_data;
5295 			char *comm;
5296 
5297 			if (WARN_ON_ONCE(!elt_data))
5298 				return;
5299 
5300 			comm = elt_data->comm;
5301 
5302 			uval = *(u64 *)(key + key_field->offset);
5303 			seq_printf(m, "%s: %-16s[%10llu]", field_name,
5304 				   comm, uval);
5305 		} else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5306 			const char *syscall_name;
5307 
5308 			uval = *(u64 *)(key + key_field->offset);
5309 			syscall_name = get_syscall_name(uval);
5310 			if (!syscall_name)
5311 				syscall_name = "unknown_syscall";
5312 
5313 			seq_printf(m, "%s: %-30s[%3llu]", field_name,
5314 				   syscall_name, uval);
5315 		} else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5316 			seq_puts(m, "stacktrace:\n");
5317 			hist_trigger_stacktrace_print(m,
5318 						      key + key_field->offset,
5319 						      HIST_STACKTRACE_DEPTH);
5320 			multiline = true;
5321 		} else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5322 			seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5323 				   *(u64 *)(key + key_field->offset));
5324 		} else if (key_field->flags & HIST_FIELD_FL_STRING) {
5325 			seq_printf(m, "%s: %-50s", field_name,
5326 				   (char *)(key + key_field->offset));
5327 		} else {
5328 			uval = *(u64 *)(key + key_field->offset);
5329 			seq_printf(m, "%s: %10llu", field_name, uval);
5330 		}
5331 	}
5332 
5333 	if (!multiline)
5334 		seq_puts(m, " ");
5335 
5336 	seq_puts(m, "}");
5337 }
5338 
5339 static void hist_trigger_entry_print(struct seq_file *m,
5340 				     struct hist_trigger_data *hist_data,
5341 				     void *key,
5342 				     struct tracing_map_elt *elt)
5343 {
5344 	const char *field_name;
5345 	unsigned int i;
5346 
5347 	hist_trigger_print_key(m, hist_data, key, elt);
5348 
5349 	seq_printf(m, " hitcount: %10llu",
5350 		   tracing_map_read_sum(elt, HITCOUNT_IDX));
5351 
5352 	for (i = 1; i < hist_data->n_vals; i++) {
5353 		field_name = hist_field_name(hist_data->fields[i], 0);
5354 
5355 		if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
5356 		    hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
5357 			continue;
5358 
5359 		if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
5360 			seq_printf(m, "  %s: %10llx", field_name,
5361 				   tracing_map_read_sum(elt, i));
5362 		} else {
5363 			seq_printf(m, "  %s: %10llu", field_name,
5364 				   tracing_map_read_sum(elt, i));
5365 		}
5366 	}
5367 
5368 	print_actions(m, hist_data, elt);
5369 
5370 	seq_puts(m, "\n");
5371 }
5372 
5373 static int print_entries(struct seq_file *m,
5374 			 struct hist_trigger_data *hist_data)
5375 {
5376 	struct tracing_map_sort_entry **sort_entries = NULL;
5377 	struct tracing_map *map = hist_data->map;
5378 	int i, n_entries;
5379 
5380 	n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5381 					     hist_data->n_sort_keys,
5382 					     &sort_entries);
5383 	if (n_entries < 0)
5384 		return n_entries;
5385 
5386 	for (i = 0; i < n_entries; i++)
5387 		hist_trigger_entry_print(m, hist_data,
5388 					 sort_entries[i]->key,
5389 					 sort_entries[i]->elt);
5390 
5391 	tracing_map_destroy_sort_entries(sort_entries, n_entries);
5392 
5393 	return n_entries;
5394 }
5395 
5396 static void hist_trigger_show(struct seq_file *m,
5397 			      struct event_trigger_data *data, int n)
5398 {
5399 	struct hist_trigger_data *hist_data;
5400 	int n_entries;
5401 
5402 	if (n > 0)
5403 		seq_puts(m, "\n\n");
5404 
5405 	seq_puts(m, "# event histogram\n#\n# trigger info: ");
5406 	data->ops->print(m, data->ops, data);
5407 	seq_puts(m, "#\n\n");
5408 
5409 	hist_data = data->private_data;
5410 	n_entries = print_entries(m, hist_data);
5411 	if (n_entries < 0)
5412 		n_entries = 0;
5413 
5414 	track_data_snapshot_print(m, hist_data);
5415 
5416 	seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5417 		   (u64)atomic64_read(&hist_data->map->hits),
5418 		   n_entries, (u64)atomic64_read(&hist_data->map->drops));
5419 }
5420 
5421 static int hist_show(struct seq_file *m, void *v)
5422 {
5423 	struct event_trigger_data *data;
5424 	struct trace_event_file *event_file;
5425 	int n = 0, ret = 0;
5426 
5427 	mutex_lock(&event_mutex);
5428 
5429 	event_file = event_file_data(m->private);
5430 	if (unlikely(!event_file)) {
5431 		ret = -ENODEV;
5432 		goto out_unlock;
5433 	}
5434 
5435 	list_for_each_entry_rcu(data, &event_file->triggers, list) {
5436 		if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5437 			hist_trigger_show(m, data, n++);
5438 	}
5439 
5440 	if (have_hist_err()) {
5441 		seq_printf(m, "\nERROR: %s\n", hist_err_str);
5442 		seq_printf(m, "  Last command: %s\n", last_hist_cmd);
5443 	}
5444 
5445  out_unlock:
5446 	mutex_unlock(&event_mutex);
5447 
5448 	return ret;
5449 }
5450 
5451 static int event_hist_open(struct inode *inode, struct file *file)
5452 {
5453 	return single_open(file, hist_show, file);
5454 }
5455 
5456 const struct file_operations event_hist_fops = {
5457 	.open = event_hist_open,
5458 	.read = seq_read,
5459 	.llseek = seq_lseek,
5460 	.release = single_release,
5461 };
5462 
5463 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5464 {
5465 	const char *field_name = hist_field_name(hist_field, 0);
5466 
5467 	if (hist_field->var.name)
5468 		seq_printf(m, "%s=", hist_field->var.name);
5469 
5470 	if (hist_field->flags & HIST_FIELD_FL_CPU)
5471 		seq_puts(m, "cpu");
5472 	else if (field_name) {
5473 		if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5474 		    hist_field->flags & HIST_FIELD_FL_ALIAS)
5475 			seq_putc(m, '$');
5476 		seq_printf(m, "%s", field_name);
5477 	} else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5478 		seq_puts(m, "common_timestamp");
5479 
5480 	if (hist_field->flags) {
5481 		if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5482 		    !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5483 			const char *flags = get_hist_field_flags(hist_field);
5484 
5485 			if (flags)
5486 				seq_printf(m, ".%s", flags);
5487 		}
5488 	}
5489 }
5490 
5491 static int event_hist_trigger_print(struct seq_file *m,
5492 				    struct event_trigger_ops *ops,
5493 				    struct event_trigger_data *data)
5494 {
5495 	struct hist_trigger_data *hist_data = data->private_data;
5496 	struct hist_field *field;
5497 	bool have_var = false;
5498 	unsigned int i;
5499 
5500 	seq_puts(m, "hist:");
5501 
5502 	if (data->name)
5503 		seq_printf(m, "%s:", data->name);
5504 
5505 	seq_puts(m, "keys=");
5506 
5507 	for_each_hist_key_field(i, hist_data) {
5508 		field = hist_data->fields[i];
5509 
5510 		if (i > hist_data->n_vals)
5511 			seq_puts(m, ",");
5512 
5513 		if (field->flags & HIST_FIELD_FL_STACKTRACE)
5514 			seq_puts(m, "stacktrace");
5515 		else
5516 			hist_field_print(m, field);
5517 	}
5518 
5519 	seq_puts(m, ":vals=");
5520 
5521 	for_each_hist_val_field(i, hist_data) {
5522 		field = hist_data->fields[i];
5523 		if (field->flags & HIST_FIELD_FL_VAR) {
5524 			have_var = true;
5525 			continue;
5526 		}
5527 
5528 		if (i == HITCOUNT_IDX)
5529 			seq_puts(m, "hitcount");
5530 		else {
5531 			seq_puts(m, ",");
5532 			hist_field_print(m, field);
5533 		}
5534 	}
5535 
5536 	if (have_var) {
5537 		unsigned int n = 0;
5538 
5539 		seq_puts(m, ":");
5540 
5541 		for_each_hist_val_field(i, hist_data) {
5542 			field = hist_data->fields[i];
5543 
5544 			if (field->flags & HIST_FIELD_FL_VAR) {
5545 				if (n++)
5546 					seq_puts(m, ",");
5547 				hist_field_print(m, field);
5548 			}
5549 		}
5550 	}
5551 
5552 	seq_puts(m, ":sort=");
5553 
5554 	for (i = 0; i < hist_data->n_sort_keys; i++) {
5555 		struct tracing_map_sort_key *sort_key;
5556 		unsigned int idx, first_key_idx;
5557 
5558 		/* skip VAR vals */
5559 		first_key_idx = hist_data->n_vals - hist_data->n_vars;
5560 
5561 		sort_key = &hist_data->sort_keys[i];
5562 		idx = sort_key->field_idx;
5563 
5564 		if (WARN_ON(idx >= HIST_FIELDS_MAX))
5565 			return -EINVAL;
5566 
5567 		if (i > 0)
5568 			seq_puts(m, ",");
5569 
5570 		if (idx == HITCOUNT_IDX)
5571 			seq_puts(m, "hitcount");
5572 		else {
5573 			if (idx >= first_key_idx)
5574 				idx += hist_data->n_vars;
5575 			hist_field_print(m, hist_data->fields[idx]);
5576 		}
5577 
5578 		if (sort_key->descending)
5579 			seq_puts(m, ".descending");
5580 	}
5581 	seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5582 	if (hist_data->enable_timestamps)
5583 		seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5584 
5585 	print_actions_spec(m, hist_data);
5586 
5587 	if (data->filter_str)
5588 		seq_printf(m, " if %s", data->filter_str);
5589 
5590 	if (data->paused)
5591 		seq_puts(m, " [paused]");
5592 	else
5593 		seq_puts(m, " [active]");
5594 
5595 	seq_putc(m, '\n');
5596 
5597 	return 0;
5598 }
5599 
5600 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5601 				   struct event_trigger_data *data)
5602 {
5603 	struct hist_trigger_data *hist_data = data->private_data;
5604 
5605 	if (!data->ref && hist_data->attrs->name)
5606 		save_named_trigger(hist_data->attrs->name, data);
5607 
5608 	data->ref++;
5609 
5610 	return 0;
5611 }
5612 
5613 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5614 {
5615 	struct trace_event_file *file;
5616 	unsigned int i;
5617 	char *cmd;
5618 	int ret;
5619 
5620 	for (i = 0; i < hist_data->n_field_var_hists; i++) {
5621 		file = hist_data->field_var_hists[i]->hist_data->event_file;
5622 		cmd = hist_data->field_var_hists[i]->cmd;
5623 		ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5624 					      "!hist", "hist", cmd);
5625 	}
5626 }
5627 
5628 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5629 				    struct event_trigger_data *data)
5630 {
5631 	struct hist_trigger_data *hist_data = data->private_data;
5632 
5633 	if (WARN_ON_ONCE(data->ref <= 0))
5634 		return;
5635 
5636 	data->ref--;
5637 	if (!data->ref) {
5638 		if (data->name)
5639 			del_named_trigger(data);
5640 
5641 		trigger_data_free(data);
5642 
5643 		remove_hist_vars(hist_data);
5644 
5645 		unregister_field_var_hists(hist_data);
5646 
5647 		destroy_hist_data(hist_data);
5648 	}
5649 }
5650 
5651 static struct event_trigger_ops event_hist_trigger_ops = {
5652 	.func			= event_hist_trigger,
5653 	.print			= event_hist_trigger_print,
5654 	.init			= event_hist_trigger_init,
5655 	.free			= event_hist_trigger_free,
5656 };
5657 
5658 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5659 					 struct event_trigger_data *data)
5660 {
5661 	data->ref++;
5662 
5663 	save_named_trigger(data->named_data->name, data);
5664 
5665 	event_hist_trigger_init(ops, data->named_data);
5666 
5667 	return 0;
5668 }
5669 
5670 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5671 					  struct event_trigger_data *data)
5672 {
5673 	if (WARN_ON_ONCE(data->ref <= 0))
5674 		return;
5675 
5676 	event_hist_trigger_free(ops, data->named_data);
5677 
5678 	data->ref--;
5679 	if (!data->ref) {
5680 		del_named_trigger(data);
5681 		trigger_data_free(data);
5682 	}
5683 }
5684 
5685 static struct event_trigger_ops event_hist_trigger_named_ops = {
5686 	.func			= event_hist_trigger,
5687 	.print			= event_hist_trigger_print,
5688 	.init			= event_hist_trigger_named_init,
5689 	.free			= event_hist_trigger_named_free,
5690 };
5691 
5692 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5693 							    char *param)
5694 {
5695 	return &event_hist_trigger_ops;
5696 }
5697 
5698 static void hist_clear(struct event_trigger_data *data)
5699 {
5700 	struct hist_trigger_data *hist_data = data->private_data;
5701 
5702 	if (data->name)
5703 		pause_named_trigger(data);
5704 
5705 	tracepoint_synchronize_unregister();
5706 
5707 	tracing_map_clear(hist_data->map);
5708 
5709 	if (data->name)
5710 		unpause_named_trigger(data);
5711 }
5712 
5713 static bool compatible_field(struct ftrace_event_field *field,
5714 			     struct ftrace_event_field *test_field)
5715 {
5716 	if (field == test_field)
5717 		return true;
5718 	if (field == NULL || test_field == NULL)
5719 		return false;
5720 	if (strcmp(field->name, test_field->name) != 0)
5721 		return false;
5722 	if (strcmp(field->type, test_field->type) != 0)
5723 		return false;
5724 	if (field->size != test_field->size)
5725 		return false;
5726 	if (field->is_signed != test_field->is_signed)
5727 		return false;
5728 
5729 	return true;
5730 }
5731 
5732 static bool hist_trigger_match(struct event_trigger_data *data,
5733 			       struct event_trigger_data *data_test,
5734 			       struct event_trigger_data *named_data,
5735 			       bool ignore_filter)
5736 {
5737 	struct tracing_map_sort_key *sort_key, *sort_key_test;
5738 	struct hist_trigger_data *hist_data, *hist_data_test;
5739 	struct hist_field *key_field, *key_field_test;
5740 	unsigned int i;
5741 
5742 	if (named_data && (named_data != data_test) &&
5743 	    (named_data != data_test->named_data))
5744 		return false;
5745 
5746 	if (!named_data && is_named_trigger(data_test))
5747 		return false;
5748 
5749 	hist_data = data->private_data;
5750 	hist_data_test = data_test->private_data;
5751 
5752 	if (hist_data->n_vals != hist_data_test->n_vals ||
5753 	    hist_data->n_fields != hist_data_test->n_fields ||
5754 	    hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5755 		return false;
5756 
5757 	if (!ignore_filter) {
5758 		if ((data->filter_str && !data_test->filter_str) ||
5759 		   (!data->filter_str && data_test->filter_str))
5760 			return false;
5761 	}
5762 
5763 	for_each_hist_field(i, hist_data) {
5764 		key_field = hist_data->fields[i];
5765 		key_field_test = hist_data_test->fields[i];
5766 
5767 		if (key_field->flags != key_field_test->flags)
5768 			return false;
5769 		if (!compatible_field(key_field->field, key_field_test->field))
5770 			return false;
5771 		if (key_field->offset != key_field_test->offset)
5772 			return false;
5773 		if (key_field->size != key_field_test->size)
5774 			return false;
5775 		if (key_field->is_signed != key_field_test->is_signed)
5776 			return false;
5777 		if (!!key_field->var.name != !!key_field_test->var.name)
5778 			return false;
5779 		if (key_field->var.name &&
5780 		    strcmp(key_field->var.name, key_field_test->var.name) != 0)
5781 			return false;
5782 	}
5783 
5784 	for (i = 0; i < hist_data->n_sort_keys; i++) {
5785 		sort_key = &hist_data->sort_keys[i];
5786 		sort_key_test = &hist_data_test->sort_keys[i];
5787 
5788 		if (sort_key->field_idx != sort_key_test->field_idx ||
5789 		    sort_key->descending != sort_key_test->descending)
5790 			return false;
5791 	}
5792 
5793 	if (!ignore_filter && data->filter_str &&
5794 	    (strcmp(data->filter_str, data_test->filter_str) != 0))
5795 		return false;
5796 
5797 	if (!actions_match(hist_data, hist_data_test))
5798 		return false;
5799 
5800 	return true;
5801 }
5802 
5803 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5804 				 struct event_trigger_data *data,
5805 				 struct trace_event_file *file)
5806 {
5807 	struct hist_trigger_data *hist_data = data->private_data;
5808 	struct event_trigger_data *test, *named_data = NULL;
5809 	int ret = 0;
5810 
5811 	if (hist_data->attrs->name) {
5812 		named_data = find_named_trigger(hist_data->attrs->name);
5813 		if (named_data) {
5814 			if (!hist_trigger_match(data, named_data, named_data,
5815 						true)) {
5816 				hist_err("Named hist trigger doesn't match existing named trigger (includes variables): ", hist_data->attrs->name);
5817 				ret = -EINVAL;
5818 				goto out;
5819 			}
5820 		}
5821 	}
5822 
5823 	if (hist_data->attrs->name && !named_data)
5824 		goto new;
5825 
5826 	list_for_each_entry_rcu(test, &file->triggers, list) {
5827 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5828 			if (!hist_trigger_match(data, test, named_data, false))
5829 				continue;
5830 			if (hist_data->attrs->pause)
5831 				test->paused = true;
5832 			else if (hist_data->attrs->cont)
5833 				test->paused = false;
5834 			else if (hist_data->attrs->clear)
5835 				hist_clear(test);
5836 			else {
5837 				hist_err("Hist trigger already exists", NULL);
5838 				ret = -EEXIST;
5839 			}
5840 			goto out;
5841 		}
5842 	}
5843  new:
5844 	if (hist_data->attrs->cont || hist_data->attrs->clear) {
5845 		hist_err("Can't clear or continue a nonexistent hist trigger", NULL);
5846 		ret = -ENOENT;
5847 		goto out;
5848 	}
5849 
5850 	if (hist_data->attrs->pause)
5851 		data->paused = true;
5852 
5853 	if (named_data) {
5854 		data->private_data = named_data->private_data;
5855 		set_named_trigger_data(data, named_data);
5856 		data->ops = &event_hist_trigger_named_ops;
5857 	}
5858 
5859 	if (data->ops->init) {
5860 		ret = data->ops->init(data->ops, data);
5861 		if (ret < 0)
5862 			goto out;
5863 	}
5864 
5865 	if (hist_data->enable_timestamps) {
5866 		char *clock = hist_data->attrs->clock;
5867 
5868 		ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5869 		if (ret) {
5870 			hist_err("Couldn't set trace_clock: ", clock);
5871 			goto out;
5872 		}
5873 
5874 		tracing_set_time_stamp_abs(file->tr, true);
5875 	}
5876 
5877 	if (named_data)
5878 		destroy_hist_data(hist_data);
5879 
5880 	ret++;
5881  out:
5882 	return ret;
5883 }
5884 
5885 static int hist_trigger_enable(struct event_trigger_data *data,
5886 			       struct trace_event_file *file)
5887 {
5888 	int ret = 0;
5889 
5890 	list_add_tail_rcu(&data->list, &file->triggers);
5891 
5892 	update_cond_flag(file);
5893 
5894 	if (trace_event_trigger_enable_disable(file, 1) < 0) {
5895 		list_del_rcu(&data->list);
5896 		update_cond_flag(file);
5897 		ret--;
5898 	}
5899 
5900 	return ret;
5901 }
5902 
5903 static bool have_hist_trigger_match(struct event_trigger_data *data,
5904 				    struct trace_event_file *file)
5905 {
5906 	struct hist_trigger_data *hist_data = data->private_data;
5907 	struct event_trigger_data *test, *named_data = NULL;
5908 	bool match = false;
5909 
5910 	if (hist_data->attrs->name)
5911 		named_data = find_named_trigger(hist_data->attrs->name);
5912 
5913 	list_for_each_entry_rcu(test, &file->triggers, list) {
5914 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5915 			if (hist_trigger_match(data, test, named_data, false)) {
5916 				match = true;
5917 				break;
5918 			}
5919 		}
5920 	}
5921 
5922 	return match;
5923 }
5924 
5925 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5926 				    struct trace_event_file *file)
5927 {
5928 	struct hist_trigger_data *hist_data = data->private_data;
5929 	struct event_trigger_data *test, *named_data = NULL;
5930 
5931 	if (hist_data->attrs->name)
5932 		named_data = find_named_trigger(hist_data->attrs->name);
5933 
5934 	list_for_each_entry_rcu(test, &file->triggers, list) {
5935 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5936 			if (!hist_trigger_match(data, test, named_data, false))
5937 				continue;
5938 			hist_data = test->private_data;
5939 			if (check_var_refs(hist_data))
5940 				return true;
5941 			break;
5942 		}
5943 	}
5944 
5945 	return false;
5946 }
5947 
5948 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5949 				    struct event_trigger_data *data,
5950 				    struct trace_event_file *file)
5951 {
5952 	struct hist_trigger_data *hist_data = data->private_data;
5953 	struct event_trigger_data *test, *named_data = NULL;
5954 	bool unregistered = false;
5955 
5956 	if (hist_data->attrs->name)
5957 		named_data = find_named_trigger(hist_data->attrs->name);
5958 
5959 	list_for_each_entry_rcu(test, &file->triggers, list) {
5960 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5961 			if (!hist_trigger_match(data, test, named_data, false))
5962 				continue;
5963 			unregistered = true;
5964 			list_del_rcu(&test->list);
5965 			trace_event_trigger_enable_disable(file, 0);
5966 			update_cond_flag(file);
5967 			break;
5968 		}
5969 	}
5970 
5971 	if (unregistered && test->ops->free)
5972 		test->ops->free(test->ops, test);
5973 
5974 	if (hist_data->enable_timestamps) {
5975 		if (!hist_data->remove || unregistered)
5976 			tracing_set_time_stamp_abs(file->tr, false);
5977 	}
5978 }
5979 
5980 static bool hist_file_check_refs(struct trace_event_file *file)
5981 {
5982 	struct hist_trigger_data *hist_data;
5983 	struct event_trigger_data *test;
5984 
5985 	list_for_each_entry_rcu(test, &file->triggers, list) {
5986 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5987 			hist_data = test->private_data;
5988 			if (check_var_refs(hist_data))
5989 				return true;
5990 		}
5991 	}
5992 
5993 	return false;
5994 }
5995 
5996 static void hist_unreg_all(struct trace_event_file *file)
5997 {
5998 	struct event_trigger_data *test, *n;
5999 	struct hist_trigger_data *hist_data;
6000 	struct synth_event *se;
6001 	const char *se_name;
6002 
6003 	lockdep_assert_held(&event_mutex);
6004 
6005 	if (hist_file_check_refs(file))
6006 		return;
6007 
6008 	list_for_each_entry_safe(test, n, &file->triggers, list) {
6009 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6010 			hist_data = test->private_data;
6011 			list_del_rcu(&test->list);
6012 			trace_event_trigger_enable_disable(file, 0);
6013 
6014 			se_name = trace_event_name(file->event_call);
6015 			se = find_synth_event(se_name);
6016 			if (se)
6017 				se->ref--;
6018 
6019 			update_cond_flag(file);
6020 			if (hist_data->enable_timestamps)
6021 				tracing_set_time_stamp_abs(file->tr, false);
6022 			if (test->ops->free)
6023 				test->ops->free(test->ops, test);
6024 		}
6025 	}
6026 }
6027 
6028 static int event_hist_trigger_func(struct event_command *cmd_ops,
6029 				   struct trace_event_file *file,
6030 				   char *glob, char *cmd, char *param)
6031 {
6032 	unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6033 	struct event_trigger_data *trigger_data;
6034 	struct hist_trigger_attrs *attrs;
6035 	struct event_trigger_ops *trigger_ops;
6036 	struct hist_trigger_data *hist_data;
6037 	struct synth_event *se;
6038 	const char *se_name;
6039 	bool remove = false;
6040 	char *trigger, *p;
6041 	int ret = 0;
6042 
6043 	lockdep_assert_held(&event_mutex);
6044 
6045 	if (glob && strlen(glob)) {
6046 		last_cmd_set(param);
6047 		hist_err_clear();
6048 	}
6049 
6050 	if (!param)
6051 		return -EINVAL;
6052 
6053 	if (glob[0] == '!')
6054 		remove = true;
6055 
6056 	/*
6057 	 * separate the trigger from the filter (k:v [if filter])
6058 	 * allowing for whitespace in the trigger
6059 	 */
6060 	p = trigger = param;
6061 	do {
6062 		p = strstr(p, "if");
6063 		if (!p)
6064 			break;
6065 		if (p == param)
6066 			return -EINVAL;
6067 		if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6068 			p++;
6069 			continue;
6070 		}
6071 		if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
6072 			return -EINVAL;
6073 		if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6074 			p++;
6075 			continue;
6076 		}
6077 		break;
6078 	} while (p);
6079 
6080 	if (!p)
6081 		param = NULL;
6082 	else {
6083 		*(p - 1) = '\0';
6084 		param = strstrip(p);
6085 		trigger = strstrip(trigger);
6086 	}
6087 
6088 	attrs = parse_hist_trigger_attrs(trigger);
6089 	if (IS_ERR(attrs))
6090 		return PTR_ERR(attrs);
6091 
6092 	if (attrs->map_bits)
6093 		hist_trigger_bits = attrs->map_bits;
6094 
6095 	hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6096 	if (IS_ERR(hist_data)) {
6097 		destroy_hist_trigger_attrs(attrs);
6098 		return PTR_ERR(hist_data);
6099 	}
6100 
6101 	trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
6102 
6103 	trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
6104 	if (!trigger_data) {
6105 		ret = -ENOMEM;
6106 		goto out_free;
6107 	}
6108 
6109 	trigger_data->count = -1;
6110 	trigger_data->ops = trigger_ops;
6111 	trigger_data->cmd_ops = cmd_ops;
6112 
6113 	INIT_LIST_HEAD(&trigger_data->list);
6114 	RCU_INIT_POINTER(trigger_data->filter, NULL);
6115 
6116 	trigger_data->private_data = hist_data;
6117 
6118 	/* if param is non-empty, it's supposed to be a filter */
6119 	if (param && cmd_ops->set_filter) {
6120 		ret = cmd_ops->set_filter(param, trigger_data, file);
6121 		if (ret < 0)
6122 			goto out_free;
6123 	}
6124 
6125 	if (remove) {
6126 		if (!have_hist_trigger_match(trigger_data, file))
6127 			goto out_free;
6128 
6129 		if (hist_trigger_check_refs(trigger_data, file)) {
6130 			ret = -EBUSY;
6131 			goto out_free;
6132 		}
6133 
6134 		cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6135 		se_name = trace_event_name(file->event_call);
6136 		se = find_synth_event(se_name);
6137 		if (se)
6138 			se->ref--;
6139 		ret = 0;
6140 		goto out_free;
6141 	}
6142 
6143 	ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
6144 	/*
6145 	 * The above returns on success the # of triggers registered,
6146 	 * but if it didn't register any it returns zero.  Consider no
6147 	 * triggers registered a failure too.
6148 	 */
6149 	if (!ret) {
6150 		if (!(attrs->pause || attrs->cont || attrs->clear))
6151 			ret = -ENOENT;
6152 		goto out_free;
6153 	} else if (ret < 0)
6154 		goto out_free;
6155 
6156 	if (get_named_trigger_data(trigger_data))
6157 		goto enable;
6158 
6159 	if (has_hist_vars(hist_data))
6160 		save_hist_vars(hist_data);
6161 
6162 	ret = create_actions(hist_data);
6163 	if (ret)
6164 		goto out_unreg;
6165 
6166 	ret = tracing_map_init(hist_data->map);
6167 	if (ret)
6168 		goto out_unreg;
6169 enable:
6170 	ret = hist_trigger_enable(trigger_data, file);
6171 	if (ret)
6172 		goto out_unreg;
6173 
6174 	se_name = trace_event_name(file->event_call);
6175 	se = find_synth_event(se_name);
6176 	if (se)
6177 		se->ref++;
6178 	/* Just return zero, not the number of registered triggers */
6179 	ret = 0;
6180  out:
6181 	if (ret == 0)
6182 		hist_err_clear();
6183 
6184 	return ret;
6185  out_unreg:
6186 	cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
6187  out_free:
6188 	if (cmd_ops->set_filter)
6189 		cmd_ops->set_filter(NULL, trigger_data, NULL);
6190 
6191 	remove_hist_vars(hist_data);
6192 
6193 	kfree(trigger_data);
6194 
6195 	destroy_hist_data(hist_data);
6196 	goto out;
6197 }
6198 
6199 static struct event_command trigger_hist_cmd = {
6200 	.name			= "hist",
6201 	.trigger_type		= ETT_EVENT_HIST,
6202 	.flags			= EVENT_CMD_FL_NEEDS_REC,
6203 	.func			= event_hist_trigger_func,
6204 	.reg			= hist_register_trigger,
6205 	.unreg			= hist_unregister_trigger,
6206 	.unreg_all		= hist_unreg_all,
6207 	.get_trigger_ops	= event_hist_get_trigger_ops,
6208 	.set_filter		= set_trigger_filter,
6209 };
6210 
6211 __init int register_trigger_hist_cmd(void)
6212 {
6213 	int ret;
6214 
6215 	ret = register_event_command(&trigger_hist_cmd);
6216 	WARN_ON(ret < 0);
6217 
6218 	return ret;
6219 }
6220 
6221 static void
6222 hist_enable_trigger(struct event_trigger_data *data, void *rec,
6223 		    struct ring_buffer_event *event)
6224 {
6225 	struct enable_trigger_data *enable_data = data->private_data;
6226 	struct event_trigger_data *test;
6227 
6228 	list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
6229 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6230 			if (enable_data->enable)
6231 				test->paused = false;
6232 			else
6233 				test->paused = true;
6234 		}
6235 	}
6236 }
6237 
6238 static void
6239 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
6240 			  struct ring_buffer_event *event)
6241 {
6242 	if (!data->count)
6243 		return;
6244 
6245 	if (data->count != -1)
6246 		(data->count)--;
6247 
6248 	hist_enable_trigger(data, rec, event);
6249 }
6250 
6251 static struct event_trigger_ops hist_enable_trigger_ops = {
6252 	.func			= hist_enable_trigger,
6253 	.print			= event_enable_trigger_print,
6254 	.init			= event_trigger_init,
6255 	.free			= event_enable_trigger_free,
6256 };
6257 
6258 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6259 	.func			= hist_enable_count_trigger,
6260 	.print			= event_enable_trigger_print,
6261 	.init			= event_trigger_init,
6262 	.free			= event_enable_trigger_free,
6263 };
6264 
6265 static struct event_trigger_ops hist_disable_trigger_ops = {
6266 	.func			= hist_enable_trigger,
6267 	.print			= event_enable_trigger_print,
6268 	.init			= event_trigger_init,
6269 	.free			= event_enable_trigger_free,
6270 };
6271 
6272 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6273 	.func			= hist_enable_count_trigger,
6274 	.print			= event_enable_trigger_print,
6275 	.init			= event_trigger_init,
6276 	.free			= event_enable_trigger_free,
6277 };
6278 
6279 static struct event_trigger_ops *
6280 hist_enable_get_trigger_ops(char *cmd, char *param)
6281 {
6282 	struct event_trigger_ops *ops;
6283 	bool enable;
6284 
6285 	enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6286 
6287 	if (enable)
6288 		ops = param ? &hist_enable_count_trigger_ops :
6289 			&hist_enable_trigger_ops;
6290 	else
6291 		ops = param ? &hist_disable_count_trigger_ops :
6292 			&hist_disable_trigger_ops;
6293 
6294 	return ops;
6295 }
6296 
6297 static void hist_enable_unreg_all(struct trace_event_file *file)
6298 {
6299 	struct event_trigger_data *test, *n;
6300 
6301 	list_for_each_entry_safe(test, n, &file->triggers, list) {
6302 		if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6303 			list_del_rcu(&test->list);
6304 			update_cond_flag(file);
6305 			trace_event_trigger_enable_disable(file, 0);
6306 			if (test->ops->free)
6307 				test->ops->free(test->ops, test);
6308 		}
6309 	}
6310 }
6311 
6312 static struct event_command trigger_hist_enable_cmd = {
6313 	.name			= ENABLE_HIST_STR,
6314 	.trigger_type		= ETT_HIST_ENABLE,
6315 	.func			= event_enable_trigger_func,
6316 	.reg			= event_enable_register_trigger,
6317 	.unreg			= event_enable_unregister_trigger,
6318 	.unreg_all		= hist_enable_unreg_all,
6319 	.get_trigger_ops	= hist_enable_get_trigger_ops,
6320 	.set_filter		= set_trigger_filter,
6321 };
6322 
6323 static struct event_command trigger_hist_disable_cmd = {
6324 	.name			= DISABLE_HIST_STR,
6325 	.trigger_type		= ETT_HIST_ENABLE,
6326 	.func			= event_enable_trigger_func,
6327 	.reg			= event_enable_register_trigger,
6328 	.unreg			= event_enable_unregister_trigger,
6329 	.unreg_all		= hist_enable_unreg_all,
6330 	.get_trigger_ops	= hist_enable_get_trigger_ops,
6331 	.set_filter		= set_trigger_filter,
6332 };
6333 
6334 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6335 {
6336 	unregister_event_command(&trigger_hist_enable_cmd);
6337 	unregister_event_command(&trigger_hist_disable_cmd);
6338 }
6339 
6340 __init int register_trigger_hist_enable_disable_cmds(void)
6341 {
6342 	int ret;
6343 
6344 	ret = register_event_command(&trigger_hist_enable_cmd);
6345 	if (WARN_ON(ret < 0))
6346 		return ret;
6347 	ret = register_event_command(&trigger_hist_disable_cmd);
6348 	if (WARN_ON(ret < 0))
6349 		unregister_trigger_hist_enable_disable_cmds();
6350 
6351 	return ret;
6352 }
6353 
6354 static __init int trace_events_hist_init(void)
6355 {
6356 	struct dentry *entry = NULL;
6357 	struct dentry *d_tracer;
6358 	int err = 0;
6359 
6360 	err = dyn_event_register(&synth_event_ops);
6361 	if (err) {
6362 		pr_warn("Could not register synth_event_ops\n");
6363 		return err;
6364 	}
6365 
6366 	d_tracer = tracing_init_dentry();
6367 	if (IS_ERR(d_tracer)) {
6368 		err = PTR_ERR(d_tracer);
6369 		goto err;
6370 	}
6371 
6372 	entry = tracefs_create_file("synthetic_events", 0644, d_tracer,
6373 				    NULL, &synth_events_fops);
6374 	if (!entry) {
6375 		err = -ENODEV;
6376 		goto err;
6377 	}
6378 
6379 	return err;
6380  err:
6381 	pr_warn("Could not create tracefs 'synthetic_events' entry\n");
6382 
6383 	return err;
6384 }
6385 
6386 fs_initcall(trace_events_hist_init);
6387