1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * trace_events_hist - trace event hist triggers
4  *
5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
6  */
7 
8 #include <linux/module.h>
9 #include <linux/kallsyms.h>
10 #include <linux/security.h>
11 #include <linux/mutex.h>
12 #include <linux/slab.h>
13 #include <linux/stacktrace.h>
14 #include <linux/rculist.h>
15 #include <linux/tracefs.h>
16 
17 /* for gfp flag names */
18 #include <linux/trace_events.h>
19 #include <trace/events/mmflags.h>
20 
21 #include "tracing_map.h"
22 #include "trace_synth.h"
23 
24 #define ERRORS								\
25 	C(NONE,			"No error"),				\
26 	C(DUPLICATE_VAR,	"Variable already defined"),		\
27 	C(VAR_NOT_UNIQUE,	"Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
28 	C(TOO_MANY_VARS,	"Too many variables defined"),		\
29 	C(MALFORMED_ASSIGNMENT,	"Malformed assignment"),		\
30 	C(NAMED_MISMATCH,	"Named hist trigger doesn't match existing named trigger (includes variables)"), \
31 	C(TRIGGER_EEXIST,	"Hist trigger already exists"),		\
32 	C(TRIGGER_ENOENT_CLEAR,	"Can't clear or continue a nonexistent hist trigger"), \
33 	C(SET_CLOCK_FAIL,	"Couldn't set trace_clock"),		\
34 	C(BAD_FIELD_MODIFIER,	"Invalid field modifier"),		\
35 	C(TOO_MANY_SUBEXPR,	"Too many subexpressions (3 max)"),	\
36 	C(TIMESTAMP_MISMATCH,	"Timestamp units in expression don't match"), \
37 	C(TOO_MANY_FIELD_VARS,	"Too many field variables defined"),	\
38 	C(EVENT_FILE_NOT_FOUND,	"Event file not found"),		\
39 	C(HIST_NOT_FOUND,	"Matching event histogram not found"),	\
40 	C(HIST_CREATE_FAIL,	"Couldn't create histogram for field"),	\
41 	C(SYNTH_VAR_NOT_FOUND,	"Couldn't find synthetic variable"),	\
42 	C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),	\
43 	C(SYNTH_TYPE_MISMATCH,	"Param type doesn't match synthetic event field type"), \
44 	C(SYNTH_COUNT_MISMATCH,	"Param count doesn't match synthetic event field count"), \
45 	C(FIELD_VAR_PARSE_FAIL,	"Couldn't parse field variable"),	\
46 	C(VAR_CREATE_FIND_FAIL,	"Couldn't create or find variable"),	\
47 	C(ONX_NOT_VAR,		"For onmax(x) or onchange(x), x must be a variable"), \
48 	C(ONX_VAR_NOT_FOUND,	"Couldn't find onmax or onchange variable"), \
49 	C(ONX_VAR_CREATE_FAIL,	"Couldn't create onmax or onchange variable"), \
50 	C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),	\
51 	C(TOO_MANY_PARAMS,	"Too many action params"),		\
52 	C(PARAM_NOT_FOUND,	"Couldn't find param"),			\
53 	C(INVALID_PARAM,	"Invalid action param"),		\
54 	C(ACTION_NOT_FOUND,	"No action found"),			\
55 	C(NO_SAVE_PARAMS,	"No params found for save()"),		\
56 	C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
57 	C(ACTION_MISMATCH,	"Handler doesn't support action"),	\
58 	C(NO_CLOSING_PAREN,	"No closing paren found"),		\
59 	C(SUBSYS_NOT_FOUND,	"Missing subsystem"),			\
60 	C(INVALID_SUBSYS_EVENT,	"Invalid subsystem or event name"),	\
61 	C(INVALID_REF_KEY,	"Using variable references in keys not supported"), \
62 	C(VAR_NOT_FOUND,	"Couldn't find variable"),		\
63 	C(FIELD_NOT_FOUND,	"Couldn't find field"),			\
64 	C(EMPTY_ASSIGNMENT,	"Empty assignment"),			\
65 	C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),		\
66 	C(EMPTY_SORT_FIELD,	"Empty sort field"),			\
67 	C(TOO_MANY_SORT_FIELDS,	"Too many sort fields (Max = 2)"),	\
68 	C(INVALID_SORT_FIELD,	"Sort field must be a key or a val"),
69 
70 #undef C
71 #define C(a, b)		HIST_ERR_##a
72 
73 enum { ERRORS };
74 
75 #undef C
76 #define C(a, b)		b
77 
78 static const char *err_text[] = { ERRORS };
79 
80 struct hist_field;
81 
82 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
83 				struct tracing_map_elt *elt,
84 				struct ring_buffer_event *rbe,
85 				void *event);
86 
87 #define HIST_FIELD_OPERANDS_MAX	2
88 #define HIST_FIELDS_MAX		(TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
89 #define HIST_ACTIONS_MAX	8
90 
91 enum field_op_id {
92 	FIELD_OP_NONE,
93 	FIELD_OP_PLUS,
94 	FIELD_OP_MINUS,
95 	FIELD_OP_UNARY_MINUS,
96 };
97 
98 /*
99  * A hist_var (histogram variable) contains variable information for
100  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
101  * flag set.  A hist_var has a variable name e.g. ts0, and is
102  * associated with a given histogram trigger, as specified by
103  * hist_data.  The hist_var idx is the unique index assigned to the
104  * variable by the hist trigger's tracing_map.  The idx is what is
105  * used to set a variable's value and, by a variable reference, to
106  * retrieve it.
107  */
108 struct hist_var {
109 	char				*name;
110 	struct hist_trigger_data	*hist_data;
111 	unsigned int			idx;
112 };
113 
114 struct hist_field {
115 	struct ftrace_event_field	*field;
116 	unsigned long			flags;
117 	hist_field_fn_t			fn;
118 	unsigned int			ref;
119 	unsigned int			size;
120 	unsigned int			offset;
121 	unsigned int                    is_signed;
122 	const char			*type;
123 	struct hist_field		*operands[HIST_FIELD_OPERANDS_MAX];
124 	struct hist_trigger_data	*hist_data;
125 
126 	/*
127 	 * Variable fields contain variable-specific info in var.
128 	 */
129 	struct hist_var			var;
130 	enum field_op_id		operator;
131 	char				*system;
132 	char				*event_name;
133 
134 	/*
135 	 * The name field is used for EXPR and VAR_REF fields.  VAR
136 	 * fields contain the variable name in var.name.
137 	 */
138 	char				*name;
139 
140 	/*
141 	 * When a histogram trigger is hit, if it has any references
142 	 * to variables, the values of those variables are collected
143 	 * into a var_ref_vals array by resolve_var_refs().  The
144 	 * current value of each variable is read from the tracing_map
145 	 * using the hist field's hist_var.idx and entered into the
146 	 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
147 	 */
148 	unsigned int			var_ref_idx;
149 	bool                            read_once;
150 
151 	unsigned int			var_str_idx;
152 };
153 
154 static u64 hist_field_none(struct hist_field *field,
155 			   struct tracing_map_elt *elt,
156 			   struct ring_buffer_event *rbe,
157 			   void *event)
158 {
159 	return 0;
160 }
161 
162 static u64 hist_field_counter(struct hist_field *field,
163 			      struct tracing_map_elt *elt,
164 			      struct ring_buffer_event *rbe,
165 			      void *event)
166 {
167 	return 1;
168 }
169 
170 static u64 hist_field_string(struct hist_field *hist_field,
171 			     struct tracing_map_elt *elt,
172 			     struct ring_buffer_event *rbe,
173 			     void *event)
174 {
175 	char *addr = (char *)(event + hist_field->field->offset);
176 
177 	return (u64)(unsigned long)addr;
178 }
179 
180 static u64 hist_field_dynstring(struct hist_field *hist_field,
181 				struct tracing_map_elt *elt,
182 				struct ring_buffer_event *rbe,
183 				void *event)
184 {
185 	u32 str_item = *(u32 *)(event + hist_field->field->offset);
186 	int str_loc = str_item & 0xffff;
187 	char *addr = (char *)(event + str_loc);
188 
189 	return (u64)(unsigned long)addr;
190 }
191 
192 static u64 hist_field_pstring(struct hist_field *hist_field,
193 			      struct tracing_map_elt *elt,
194 			      struct ring_buffer_event *rbe,
195 			      void *event)
196 {
197 	char **addr = (char **)(event + hist_field->field->offset);
198 
199 	return (u64)(unsigned long)*addr;
200 }
201 
202 static u64 hist_field_log2(struct hist_field *hist_field,
203 			   struct tracing_map_elt *elt,
204 			   struct ring_buffer_event *rbe,
205 			   void *event)
206 {
207 	struct hist_field *operand = hist_field->operands[0];
208 
209 	u64 val = operand->fn(operand, elt, rbe, event);
210 
211 	return (u64) ilog2(roundup_pow_of_two(val));
212 }
213 
214 static u64 hist_field_plus(struct hist_field *hist_field,
215 			   struct tracing_map_elt *elt,
216 			   struct ring_buffer_event *rbe,
217 			   void *event)
218 {
219 	struct hist_field *operand1 = hist_field->operands[0];
220 	struct hist_field *operand2 = hist_field->operands[1];
221 
222 	u64 val1 = operand1->fn(operand1, elt, rbe, event);
223 	u64 val2 = operand2->fn(operand2, elt, rbe, event);
224 
225 	return val1 + val2;
226 }
227 
228 static u64 hist_field_minus(struct hist_field *hist_field,
229 			    struct tracing_map_elt *elt,
230 			    struct ring_buffer_event *rbe,
231 			    void *event)
232 {
233 	struct hist_field *operand1 = hist_field->operands[0];
234 	struct hist_field *operand2 = hist_field->operands[1];
235 
236 	u64 val1 = operand1->fn(operand1, elt, rbe, event);
237 	u64 val2 = operand2->fn(operand2, elt, rbe, event);
238 
239 	return val1 - val2;
240 }
241 
242 static u64 hist_field_unary_minus(struct hist_field *hist_field,
243 				  struct tracing_map_elt *elt,
244 				  struct ring_buffer_event *rbe,
245 				  void *event)
246 {
247 	struct hist_field *operand = hist_field->operands[0];
248 
249 	s64 sval = (s64)operand->fn(operand, elt, rbe, event);
250 	u64 val = (u64)-sval;
251 
252 	return val;
253 }
254 
255 #define DEFINE_HIST_FIELD_FN(type)					\
256 	static u64 hist_field_##type(struct hist_field *hist_field,	\
257 				     struct tracing_map_elt *elt,	\
258 				     struct ring_buffer_event *rbe,	\
259 				     void *event)			\
260 {									\
261 	type *addr = (type *)(event + hist_field->field->offset);	\
262 									\
263 	return (u64)(unsigned long)*addr;				\
264 }
265 
266 DEFINE_HIST_FIELD_FN(s64);
267 DEFINE_HIST_FIELD_FN(u64);
268 DEFINE_HIST_FIELD_FN(s32);
269 DEFINE_HIST_FIELD_FN(u32);
270 DEFINE_HIST_FIELD_FN(s16);
271 DEFINE_HIST_FIELD_FN(u16);
272 DEFINE_HIST_FIELD_FN(s8);
273 DEFINE_HIST_FIELD_FN(u8);
274 
275 #define for_each_hist_field(i, hist_data)	\
276 	for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
277 
278 #define for_each_hist_val_field(i, hist_data)	\
279 	for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
280 
281 #define for_each_hist_key_field(i, hist_data)	\
282 	for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
283 
284 #define HIST_STACKTRACE_DEPTH	16
285 #define HIST_STACKTRACE_SIZE	(HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
286 #define HIST_STACKTRACE_SKIP	5
287 
288 #define HITCOUNT_IDX		0
289 #define HIST_KEY_SIZE_MAX	(MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
290 
291 enum hist_field_flags {
292 	HIST_FIELD_FL_HITCOUNT		= 1 << 0,
293 	HIST_FIELD_FL_KEY		= 1 << 1,
294 	HIST_FIELD_FL_STRING		= 1 << 2,
295 	HIST_FIELD_FL_HEX		= 1 << 3,
296 	HIST_FIELD_FL_SYM		= 1 << 4,
297 	HIST_FIELD_FL_SYM_OFFSET	= 1 << 5,
298 	HIST_FIELD_FL_EXECNAME		= 1 << 6,
299 	HIST_FIELD_FL_SYSCALL		= 1 << 7,
300 	HIST_FIELD_FL_STACKTRACE	= 1 << 8,
301 	HIST_FIELD_FL_LOG2		= 1 << 9,
302 	HIST_FIELD_FL_TIMESTAMP		= 1 << 10,
303 	HIST_FIELD_FL_TIMESTAMP_USECS	= 1 << 11,
304 	HIST_FIELD_FL_VAR		= 1 << 12,
305 	HIST_FIELD_FL_EXPR		= 1 << 13,
306 	HIST_FIELD_FL_VAR_REF		= 1 << 14,
307 	HIST_FIELD_FL_CPU		= 1 << 15,
308 	HIST_FIELD_FL_ALIAS		= 1 << 16,
309 };
310 
311 struct var_defs {
312 	unsigned int	n_vars;
313 	char		*name[TRACING_MAP_VARS_MAX];
314 	char		*expr[TRACING_MAP_VARS_MAX];
315 };
316 
317 struct hist_trigger_attrs {
318 	char		*keys_str;
319 	char		*vals_str;
320 	char		*sort_key_str;
321 	char		*name;
322 	char		*clock;
323 	bool		pause;
324 	bool		cont;
325 	bool		clear;
326 	bool		ts_in_usecs;
327 	unsigned int	map_bits;
328 
329 	char		*assignment_str[TRACING_MAP_VARS_MAX];
330 	unsigned int	n_assignments;
331 
332 	char		*action_str[HIST_ACTIONS_MAX];
333 	unsigned int	n_actions;
334 
335 	struct var_defs	var_defs;
336 };
337 
338 struct field_var {
339 	struct hist_field	*var;
340 	struct hist_field	*val;
341 };
342 
343 struct field_var_hist {
344 	struct hist_trigger_data	*hist_data;
345 	char				*cmd;
346 };
347 
348 struct hist_trigger_data {
349 	struct hist_field               *fields[HIST_FIELDS_MAX];
350 	unsigned int			n_vals;
351 	unsigned int			n_keys;
352 	unsigned int			n_fields;
353 	unsigned int			n_vars;
354 	unsigned int			n_var_str;
355 	unsigned int			key_size;
356 	struct tracing_map_sort_key	sort_keys[TRACING_MAP_SORT_KEYS_MAX];
357 	unsigned int			n_sort_keys;
358 	struct trace_event_file		*event_file;
359 	struct hist_trigger_attrs	*attrs;
360 	struct tracing_map		*map;
361 	bool				enable_timestamps;
362 	bool				remove;
363 	struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
364 	unsigned int			n_var_refs;
365 
366 	struct action_data		*actions[HIST_ACTIONS_MAX];
367 	unsigned int			n_actions;
368 
369 	struct field_var		*field_vars[SYNTH_FIELDS_MAX];
370 	unsigned int			n_field_vars;
371 	unsigned int			n_field_var_str;
372 	struct field_var_hist		*field_var_hists[SYNTH_FIELDS_MAX];
373 	unsigned int			n_field_var_hists;
374 
375 	struct field_var		*save_vars[SYNTH_FIELDS_MAX];
376 	unsigned int			n_save_vars;
377 	unsigned int			n_save_var_str;
378 };
379 
380 struct action_data;
381 
382 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
383 			     struct tracing_map_elt *elt, void *rec,
384 			     struct ring_buffer_event *rbe, void *key,
385 			     struct action_data *data, u64 *var_ref_vals);
386 
387 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
388 
389 enum handler_id {
390 	HANDLER_ONMATCH = 1,
391 	HANDLER_ONMAX,
392 	HANDLER_ONCHANGE,
393 };
394 
395 enum action_id {
396 	ACTION_SAVE = 1,
397 	ACTION_TRACE,
398 	ACTION_SNAPSHOT,
399 };
400 
401 struct action_data {
402 	enum handler_id		handler;
403 	enum action_id		action;
404 	char			*action_name;
405 	action_fn_t		fn;
406 
407 	unsigned int		n_params;
408 	char			*params[SYNTH_FIELDS_MAX];
409 
410 	/*
411 	 * When a histogram trigger is hit, the values of any
412 	 * references to variables, including variables being passed
413 	 * as parameters to synthetic events, are collected into a
414 	 * var_ref_vals array.  This var_ref_idx array is an array of
415 	 * indices into the var_ref_vals array, one for each synthetic
416 	 * event param, and is passed to the synthetic event
417 	 * invocation.
418 	 */
419 	unsigned int		var_ref_idx[TRACING_MAP_VARS_MAX];
420 	struct synth_event	*synth_event;
421 	bool			use_trace_keyword;
422 	char			*synth_event_name;
423 
424 	union {
425 		struct {
426 			char			*event;
427 			char			*event_system;
428 		} match_data;
429 
430 		struct {
431 			/*
432 			 * var_str contains the $-unstripped variable
433 			 * name referenced by var_ref, and used when
434 			 * printing the action.  Because var_ref
435 			 * creation is deferred to create_actions(),
436 			 * we need a per-action way to save it until
437 			 * then, thus var_str.
438 			 */
439 			char			*var_str;
440 
441 			/*
442 			 * var_ref refers to the variable being
443 			 * tracked e.g onmax($var).
444 			 */
445 			struct hist_field	*var_ref;
446 
447 			/*
448 			 * track_var contains the 'invisible' tracking
449 			 * variable created to keep the current
450 			 * e.g. max value.
451 			 */
452 			struct hist_field	*track_var;
453 
454 			check_track_val_fn_t	check_val;
455 			action_fn_t		save_data;
456 		} track_data;
457 	};
458 };
459 
460 struct track_data {
461 	u64				track_val;
462 	bool				updated;
463 
464 	unsigned int			key_len;
465 	void				*key;
466 	struct tracing_map_elt		elt;
467 
468 	struct action_data		*action_data;
469 	struct hist_trigger_data	*hist_data;
470 };
471 
472 struct hist_elt_data {
473 	char *comm;
474 	u64 *var_ref_vals;
475 	char *field_var_str[SYNTH_FIELDS_MAX];
476 };
477 
478 struct snapshot_context {
479 	struct tracing_map_elt	*elt;
480 	void			*key;
481 };
482 
483 static void track_data_free(struct track_data *track_data)
484 {
485 	struct hist_elt_data *elt_data;
486 
487 	if (!track_data)
488 		return;
489 
490 	kfree(track_data->key);
491 
492 	elt_data = track_data->elt.private_data;
493 	if (elt_data) {
494 		kfree(elt_data->comm);
495 		kfree(elt_data);
496 	}
497 
498 	kfree(track_data);
499 }
500 
501 static struct track_data *track_data_alloc(unsigned int key_len,
502 					   struct action_data *action_data,
503 					   struct hist_trigger_data *hist_data)
504 {
505 	struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
506 	struct hist_elt_data *elt_data;
507 
508 	if (!data)
509 		return ERR_PTR(-ENOMEM);
510 
511 	data->key = kzalloc(key_len, GFP_KERNEL);
512 	if (!data->key) {
513 		track_data_free(data);
514 		return ERR_PTR(-ENOMEM);
515 	}
516 
517 	data->key_len = key_len;
518 	data->action_data = action_data;
519 	data->hist_data = hist_data;
520 
521 	elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
522 	if (!elt_data) {
523 		track_data_free(data);
524 		return ERR_PTR(-ENOMEM);
525 	}
526 
527 	data->elt.private_data = elt_data;
528 
529 	elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
530 	if (!elt_data->comm) {
531 		track_data_free(data);
532 		return ERR_PTR(-ENOMEM);
533 	}
534 
535 	return data;
536 }
537 
538 static char last_cmd[MAX_FILTER_STR_VAL];
539 static char last_cmd_loc[MAX_FILTER_STR_VAL];
540 
541 static int errpos(char *str)
542 {
543 	return err_pos(last_cmd, str);
544 }
545 
546 static void last_cmd_set(struct trace_event_file *file, char *str)
547 {
548 	const char *system = NULL, *name = NULL;
549 	struct trace_event_call *call;
550 
551 	if (!str)
552 		return;
553 
554 	strcpy(last_cmd, "hist:");
555 	strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
556 
557 	if (file) {
558 		call = file->event_call;
559 		system = call->class->system;
560 		if (system) {
561 			name = trace_event_name(call);
562 			if (!name)
563 				system = NULL;
564 		}
565 	}
566 
567 	if (system)
568 		snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
569 }
570 
571 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
572 {
573 	tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
574 			err_type, err_pos);
575 }
576 
577 static void hist_err_clear(void)
578 {
579 	last_cmd[0] = '\0';
580 	last_cmd_loc[0] = '\0';
581 }
582 
583 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
584 				    unsigned int *var_ref_idx);
585 
586 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
587 			       unsigned int *var_ref_idx)
588 {
589 	struct tracepoint *tp = event->tp;
590 
591 	if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
592 		struct tracepoint_func *probe_func_ptr;
593 		synth_probe_func_t probe_func;
594 		void *__data;
595 
596 		if (!(cpu_online(raw_smp_processor_id())))
597 			return;
598 
599 		probe_func_ptr = rcu_dereference_sched((tp)->funcs);
600 		if (probe_func_ptr) {
601 			do {
602 				probe_func = probe_func_ptr->func;
603 				__data = probe_func_ptr->data;
604 				probe_func(__data, var_ref_vals, var_ref_idx);
605 			} while ((++probe_func_ptr)->func);
606 		}
607 	}
608 }
609 
610 static void action_trace(struct hist_trigger_data *hist_data,
611 			 struct tracing_map_elt *elt, void *rec,
612 			 struct ring_buffer_event *rbe, void *key,
613 			 struct action_data *data, u64 *var_ref_vals)
614 {
615 	struct synth_event *event = data->synth_event;
616 
617 	trace_synth(event, var_ref_vals, data->var_ref_idx);
618 }
619 
620 struct hist_var_data {
621 	struct list_head list;
622 	struct hist_trigger_data *hist_data;
623 };
624 
625 static u64 hist_field_timestamp(struct hist_field *hist_field,
626 				struct tracing_map_elt *elt,
627 				struct ring_buffer_event *rbe,
628 				void *event)
629 {
630 	struct hist_trigger_data *hist_data = hist_field->hist_data;
631 	struct trace_array *tr = hist_data->event_file->tr;
632 
633 	u64 ts = ring_buffer_event_time_stamp(rbe);
634 
635 	if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
636 		ts = ns2usecs(ts);
637 
638 	return ts;
639 }
640 
641 static u64 hist_field_cpu(struct hist_field *hist_field,
642 			  struct tracing_map_elt *elt,
643 			  struct ring_buffer_event *rbe,
644 			  void *event)
645 {
646 	int cpu = smp_processor_id();
647 
648 	return cpu;
649 }
650 
651 /**
652  * check_field_for_var_ref - Check if a VAR_REF field references a variable
653  * @hist_field: The VAR_REF field to check
654  * @var_data: The hist trigger that owns the variable
655  * @var_idx: The trigger variable identifier
656  *
657  * Check the given VAR_REF field to see whether or not it references
658  * the given variable associated with the given trigger.
659  *
660  * Return: The VAR_REF field if it does reference the variable, NULL if not
661  */
662 static struct hist_field *
663 check_field_for_var_ref(struct hist_field *hist_field,
664 			struct hist_trigger_data *var_data,
665 			unsigned int var_idx)
666 {
667 	WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
668 
669 	if (hist_field && hist_field->var.idx == var_idx &&
670 	    hist_field->var.hist_data == var_data)
671 		return hist_field;
672 
673 	return NULL;
674 }
675 
676 /**
677  * find_var_ref - Check if a trigger has a reference to a trigger variable
678  * @hist_data: The hist trigger that might have a reference to the variable
679  * @var_data: The hist trigger that owns the variable
680  * @var_idx: The trigger variable identifier
681  *
682  * Check the list of var_refs[] on the first hist trigger to see
683  * whether any of them are references to the variable on the second
684  * trigger.
685  *
686  * Return: The VAR_REF field referencing the variable if so, NULL if not
687  */
688 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
689 				       struct hist_trigger_data *var_data,
690 				       unsigned int var_idx)
691 {
692 	struct hist_field *hist_field;
693 	unsigned int i;
694 
695 	for (i = 0; i < hist_data->n_var_refs; i++) {
696 		hist_field = hist_data->var_refs[i];
697 		if (check_field_for_var_ref(hist_field, var_data, var_idx))
698 			return hist_field;
699 	}
700 
701 	return NULL;
702 }
703 
704 /**
705  * find_any_var_ref - Check if there is a reference to a given trigger variable
706  * @hist_data: The hist trigger
707  * @var_idx: The trigger variable identifier
708  *
709  * Check to see whether the given variable is currently referenced by
710  * any other trigger.
711  *
712  * The trigger the variable is defined on is explicitly excluded - the
713  * assumption being that a self-reference doesn't prevent a trigger
714  * from being removed.
715  *
716  * Return: The VAR_REF field referencing the variable if so, NULL if not
717  */
718 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
719 					   unsigned int var_idx)
720 {
721 	struct trace_array *tr = hist_data->event_file->tr;
722 	struct hist_field *found = NULL;
723 	struct hist_var_data *var_data;
724 
725 	list_for_each_entry(var_data, &tr->hist_vars, list) {
726 		if (var_data->hist_data == hist_data)
727 			continue;
728 		found = find_var_ref(var_data->hist_data, hist_data, var_idx);
729 		if (found)
730 			break;
731 	}
732 
733 	return found;
734 }
735 
736 /**
737  * check_var_refs - Check if there is a reference to any of trigger's variables
738  * @hist_data: The hist trigger
739  *
740  * A trigger can define one or more variables.  If any one of them is
741  * currently referenced by any other trigger, this function will
742  * determine that.
743 
744  * Typically used to determine whether or not a trigger can be removed
745  * - if there are any references to a trigger's variables, it cannot.
746  *
747  * Return: True if there is a reference to any of trigger's variables
748  */
749 static bool check_var_refs(struct hist_trigger_data *hist_data)
750 {
751 	struct hist_field *field;
752 	bool found = false;
753 	int i;
754 
755 	for_each_hist_field(i, hist_data) {
756 		field = hist_data->fields[i];
757 		if (field && field->flags & HIST_FIELD_FL_VAR) {
758 			if (find_any_var_ref(hist_data, field->var.idx)) {
759 				found = true;
760 				break;
761 			}
762 		}
763 	}
764 
765 	return found;
766 }
767 
768 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
769 {
770 	struct trace_array *tr = hist_data->event_file->tr;
771 	struct hist_var_data *var_data, *found = NULL;
772 
773 	list_for_each_entry(var_data, &tr->hist_vars, list) {
774 		if (var_data->hist_data == hist_data) {
775 			found = var_data;
776 			break;
777 		}
778 	}
779 
780 	return found;
781 }
782 
783 static bool field_has_hist_vars(struct hist_field *hist_field,
784 				unsigned int level)
785 {
786 	int i;
787 
788 	if (level > 3)
789 		return false;
790 
791 	if (!hist_field)
792 		return false;
793 
794 	if (hist_field->flags & HIST_FIELD_FL_VAR ||
795 	    hist_field->flags & HIST_FIELD_FL_VAR_REF)
796 		return true;
797 
798 	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
799 		struct hist_field *operand;
800 
801 		operand = hist_field->operands[i];
802 		if (field_has_hist_vars(operand, level + 1))
803 			return true;
804 	}
805 
806 	return false;
807 }
808 
809 static bool has_hist_vars(struct hist_trigger_data *hist_data)
810 {
811 	struct hist_field *hist_field;
812 	int i;
813 
814 	for_each_hist_field(i, hist_data) {
815 		hist_field = hist_data->fields[i];
816 		if (field_has_hist_vars(hist_field, 0))
817 			return true;
818 	}
819 
820 	return false;
821 }
822 
823 static int save_hist_vars(struct hist_trigger_data *hist_data)
824 {
825 	struct trace_array *tr = hist_data->event_file->tr;
826 	struct hist_var_data *var_data;
827 
828 	var_data = find_hist_vars(hist_data);
829 	if (var_data)
830 		return 0;
831 
832 	if (tracing_check_open_get_tr(tr))
833 		return -ENODEV;
834 
835 	var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
836 	if (!var_data) {
837 		trace_array_put(tr);
838 		return -ENOMEM;
839 	}
840 
841 	var_data->hist_data = hist_data;
842 	list_add(&var_data->list, &tr->hist_vars);
843 
844 	return 0;
845 }
846 
847 static void remove_hist_vars(struct hist_trigger_data *hist_data)
848 {
849 	struct trace_array *tr = hist_data->event_file->tr;
850 	struct hist_var_data *var_data;
851 
852 	var_data = find_hist_vars(hist_data);
853 	if (!var_data)
854 		return;
855 
856 	if (WARN_ON(check_var_refs(hist_data)))
857 		return;
858 
859 	list_del(&var_data->list);
860 
861 	kfree(var_data);
862 
863 	trace_array_put(tr);
864 }
865 
866 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
867 					 const char *var_name)
868 {
869 	struct hist_field *hist_field, *found = NULL;
870 	int i;
871 
872 	for_each_hist_field(i, hist_data) {
873 		hist_field = hist_data->fields[i];
874 		if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
875 		    strcmp(hist_field->var.name, var_name) == 0) {
876 			found = hist_field;
877 			break;
878 		}
879 	}
880 
881 	return found;
882 }
883 
884 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
885 				   struct trace_event_file *file,
886 				   const char *var_name)
887 {
888 	struct hist_trigger_data *test_data;
889 	struct event_trigger_data *test;
890 	struct hist_field *hist_field;
891 
892 	lockdep_assert_held(&event_mutex);
893 
894 	hist_field = find_var_field(hist_data, var_name);
895 	if (hist_field)
896 		return hist_field;
897 
898 	list_for_each_entry(test, &file->triggers, list) {
899 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
900 			test_data = test->private_data;
901 			hist_field = find_var_field(test_data, var_name);
902 			if (hist_field)
903 				return hist_field;
904 		}
905 	}
906 
907 	return NULL;
908 }
909 
910 static struct trace_event_file *find_var_file(struct trace_array *tr,
911 					      char *system,
912 					      char *event_name,
913 					      char *var_name)
914 {
915 	struct hist_trigger_data *var_hist_data;
916 	struct hist_var_data *var_data;
917 	struct trace_event_file *file, *found = NULL;
918 
919 	if (system)
920 		return find_event_file(tr, system, event_name);
921 
922 	list_for_each_entry(var_data, &tr->hist_vars, list) {
923 		var_hist_data = var_data->hist_data;
924 		file = var_hist_data->event_file;
925 		if (file == found)
926 			continue;
927 
928 		if (find_var_field(var_hist_data, var_name)) {
929 			if (found) {
930 				hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
931 				return NULL;
932 			}
933 
934 			found = file;
935 		}
936 	}
937 
938 	return found;
939 }
940 
941 static struct hist_field *find_file_var(struct trace_event_file *file,
942 					const char *var_name)
943 {
944 	struct hist_trigger_data *test_data;
945 	struct event_trigger_data *test;
946 	struct hist_field *hist_field;
947 
948 	lockdep_assert_held(&event_mutex);
949 
950 	list_for_each_entry(test, &file->triggers, list) {
951 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
952 			test_data = test->private_data;
953 			hist_field = find_var_field(test_data, var_name);
954 			if (hist_field)
955 				return hist_field;
956 		}
957 	}
958 
959 	return NULL;
960 }
961 
962 static struct hist_field *
963 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
964 {
965 	struct trace_array *tr = hist_data->event_file->tr;
966 	struct hist_field *hist_field, *found = NULL;
967 	struct trace_event_file *file;
968 	unsigned int i;
969 
970 	for (i = 0; i < hist_data->n_actions; i++) {
971 		struct action_data *data = hist_data->actions[i];
972 
973 		if (data->handler == HANDLER_ONMATCH) {
974 			char *system = data->match_data.event_system;
975 			char *event_name = data->match_data.event;
976 
977 			file = find_var_file(tr, system, event_name, var_name);
978 			if (!file)
979 				continue;
980 			hist_field = find_file_var(file, var_name);
981 			if (hist_field) {
982 				if (found) {
983 					hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
984 						 errpos(var_name));
985 					return ERR_PTR(-EINVAL);
986 				}
987 
988 				found = hist_field;
989 			}
990 		}
991 	}
992 	return found;
993 }
994 
995 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
996 					 char *system,
997 					 char *event_name,
998 					 char *var_name)
999 {
1000 	struct trace_array *tr = hist_data->event_file->tr;
1001 	struct hist_field *hist_field = NULL;
1002 	struct trace_event_file *file;
1003 
1004 	if (!system || !event_name) {
1005 		hist_field = find_match_var(hist_data, var_name);
1006 		if (IS_ERR(hist_field))
1007 			return NULL;
1008 		if (hist_field)
1009 			return hist_field;
1010 	}
1011 
1012 	file = find_var_file(tr, system, event_name, var_name);
1013 	if (!file)
1014 		return NULL;
1015 
1016 	hist_field = find_file_var(file, var_name);
1017 
1018 	return hist_field;
1019 }
1020 
1021 static u64 hist_field_var_ref(struct hist_field *hist_field,
1022 			      struct tracing_map_elt *elt,
1023 			      struct ring_buffer_event *rbe,
1024 			      void *event)
1025 {
1026 	struct hist_elt_data *elt_data;
1027 	u64 var_val = 0;
1028 
1029 	if (WARN_ON_ONCE(!elt))
1030 		return var_val;
1031 
1032 	elt_data = elt->private_data;
1033 	var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1034 
1035 	return var_val;
1036 }
1037 
1038 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1039 			     u64 *var_ref_vals, bool self)
1040 {
1041 	struct hist_trigger_data *var_data;
1042 	struct tracing_map_elt *var_elt;
1043 	struct hist_field *hist_field;
1044 	unsigned int i, var_idx;
1045 	bool resolved = true;
1046 	u64 var_val = 0;
1047 
1048 	for (i = 0; i < hist_data->n_var_refs; i++) {
1049 		hist_field = hist_data->var_refs[i];
1050 		var_idx = hist_field->var.idx;
1051 		var_data = hist_field->var.hist_data;
1052 
1053 		if (var_data == NULL) {
1054 			resolved = false;
1055 			break;
1056 		}
1057 
1058 		if ((self && var_data != hist_data) ||
1059 		    (!self && var_data == hist_data))
1060 			continue;
1061 
1062 		var_elt = tracing_map_lookup(var_data->map, key);
1063 		if (!var_elt) {
1064 			resolved = false;
1065 			break;
1066 		}
1067 
1068 		if (!tracing_map_var_set(var_elt, var_idx)) {
1069 			resolved = false;
1070 			break;
1071 		}
1072 
1073 		if (self || !hist_field->read_once)
1074 			var_val = tracing_map_read_var(var_elt, var_idx);
1075 		else
1076 			var_val = tracing_map_read_var_once(var_elt, var_idx);
1077 
1078 		var_ref_vals[i] = var_val;
1079 	}
1080 
1081 	return resolved;
1082 }
1083 
1084 static const char *hist_field_name(struct hist_field *field,
1085 				   unsigned int level)
1086 {
1087 	const char *field_name = "";
1088 
1089 	if (level > 1)
1090 		return field_name;
1091 
1092 	if (field->field)
1093 		field_name = field->field->name;
1094 	else if (field->flags & HIST_FIELD_FL_LOG2 ||
1095 		 field->flags & HIST_FIELD_FL_ALIAS)
1096 		field_name = hist_field_name(field->operands[0], ++level);
1097 	else if (field->flags & HIST_FIELD_FL_CPU)
1098 		field_name = "cpu";
1099 	else if (field->flags & HIST_FIELD_FL_EXPR ||
1100 		 field->flags & HIST_FIELD_FL_VAR_REF) {
1101 		if (field->system) {
1102 			static char full_name[MAX_FILTER_STR_VAL];
1103 
1104 			strcat(full_name, field->system);
1105 			strcat(full_name, ".");
1106 			strcat(full_name, field->event_name);
1107 			strcat(full_name, ".");
1108 			strcat(full_name, field->name);
1109 			field_name = full_name;
1110 		} else
1111 			field_name = field->name;
1112 	} else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1113 		field_name = "common_timestamp";
1114 
1115 	if (field_name == NULL)
1116 		field_name = "";
1117 
1118 	return field_name;
1119 }
1120 
1121 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1122 {
1123 	hist_field_fn_t fn = NULL;
1124 
1125 	switch (field_size) {
1126 	case 8:
1127 		if (field_is_signed)
1128 			fn = hist_field_s64;
1129 		else
1130 			fn = hist_field_u64;
1131 		break;
1132 	case 4:
1133 		if (field_is_signed)
1134 			fn = hist_field_s32;
1135 		else
1136 			fn = hist_field_u32;
1137 		break;
1138 	case 2:
1139 		if (field_is_signed)
1140 			fn = hist_field_s16;
1141 		else
1142 			fn = hist_field_u16;
1143 		break;
1144 	case 1:
1145 		if (field_is_signed)
1146 			fn = hist_field_s8;
1147 		else
1148 			fn = hist_field_u8;
1149 		break;
1150 	}
1151 
1152 	return fn;
1153 }
1154 
1155 static int parse_map_size(char *str)
1156 {
1157 	unsigned long size, map_bits;
1158 	int ret;
1159 
1160 	ret = kstrtoul(str, 0, &size);
1161 	if (ret)
1162 		goto out;
1163 
1164 	map_bits = ilog2(roundup_pow_of_two(size));
1165 	if (map_bits < TRACING_MAP_BITS_MIN ||
1166 	    map_bits > TRACING_MAP_BITS_MAX)
1167 		ret = -EINVAL;
1168 	else
1169 		ret = map_bits;
1170  out:
1171 	return ret;
1172 }
1173 
1174 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1175 {
1176 	unsigned int i;
1177 
1178 	if (!attrs)
1179 		return;
1180 
1181 	for (i = 0; i < attrs->n_assignments; i++)
1182 		kfree(attrs->assignment_str[i]);
1183 
1184 	for (i = 0; i < attrs->n_actions; i++)
1185 		kfree(attrs->action_str[i]);
1186 
1187 	kfree(attrs->name);
1188 	kfree(attrs->sort_key_str);
1189 	kfree(attrs->keys_str);
1190 	kfree(attrs->vals_str);
1191 	kfree(attrs->clock);
1192 	kfree(attrs);
1193 }
1194 
1195 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1196 {
1197 	int ret = -EINVAL;
1198 
1199 	if (attrs->n_actions >= HIST_ACTIONS_MAX)
1200 		return ret;
1201 
1202 	if ((str_has_prefix(str, "onmatch(")) ||
1203 	    (str_has_prefix(str, "onmax(")) ||
1204 	    (str_has_prefix(str, "onchange("))) {
1205 		attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1206 		if (!attrs->action_str[attrs->n_actions]) {
1207 			ret = -ENOMEM;
1208 			return ret;
1209 		}
1210 		attrs->n_actions++;
1211 		ret = 0;
1212 	}
1213 	return ret;
1214 }
1215 
1216 static int parse_assignment(struct trace_array *tr,
1217 			    char *str, struct hist_trigger_attrs *attrs)
1218 {
1219 	int len, ret = 0;
1220 
1221 	if ((len = str_has_prefix(str, "key=")) ||
1222 	    (len = str_has_prefix(str, "keys="))) {
1223 		attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1224 		if (!attrs->keys_str) {
1225 			ret = -ENOMEM;
1226 			goto out;
1227 		}
1228 	} else if ((len = str_has_prefix(str, "val=")) ||
1229 		   (len = str_has_prefix(str, "vals=")) ||
1230 		   (len = str_has_prefix(str, "values="))) {
1231 		attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1232 		if (!attrs->vals_str) {
1233 			ret = -ENOMEM;
1234 			goto out;
1235 		}
1236 	} else if ((len = str_has_prefix(str, "sort="))) {
1237 		attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1238 		if (!attrs->sort_key_str) {
1239 			ret = -ENOMEM;
1240 			goto out;
1241 		}
1242 	} else if (str_has_prefix(str, "name=")) {
1243 		attrs->name = kstrdup(str, GFP_KERNEL);
1244 		if (!attrs->name) {
1245 			ret = -ENOMEM;
1246 			goto out;
1247 		}
1248 	} else if ((len = str_has_prefix(str, "clock="))) {
1249 		str += len;
1250 
1251 		str = strstrip(str);
1252 		attrs->clock = kstrdup(str, GFP_KERNEL);
1253 		if (!attrs->clock) {
1254 			ret = -ENOMEM;
1255 			goto out;
1256 		}
1257 	} else if ((len = str_has_prefix(str, "size="))) {
1258 		int map_bits = parse_map_size(str + len);
1259 
1260 		if (map_bits < 0) {
1261 			ret = map_bits;
1262 			goto out;
1263 		}
1264 		attrs->map_bits = map_bits;
1265 	} else {
1266 		char *assignment;
1267 
1268 		if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1269 			hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1270 			ret = -EINVAL;
1271 			goto out;
1272 		}
1273 
1274 		assignment = kstrdup(str, GFP_KERNEL);
1275 		if (!assignment) {
1276 			ret = -ENOMEM;
1277 			goto out;
1278 		}
1279 
1280 		attrs->assignment_str[attrs->n_assignments++] = assignment;
1281 	}
1282  out:
1283 	return ret;
1284 }
1285 
1286 static struct hist_trigger_attrs *
1287 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1288 {
1289 	struct hist_trigger_attrs *attrs;
1290 	int ret = 0;
1291 
1292 	attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1293 	if (!attrs)
1294 		return ERR_PTR(-ENOMEM);
1295 
1296 	while (trigger_str) {
1297 		char *str = strsep(&trigger_str, ":");
1298 		char *rhs;
1299 
1300 		rhs = strchr(str, '=');
1301 		if (rhs) {
1302 			if (!strlen(++rhs)) {
1303 				ret = -EINVAL;
1304 				hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1305 				goto free;
1306 			}
1307 			ret = parse_assignment(tr, str, attrs);
1308 			if (ret)
1309 				goto free;
1310 		} else if (strcmp(str, "pause") == 0)
1311 			attrs->pause = true;
1312 		else if ((strcmp(str, "cont") == 0) ||
1313 			 (strcmp(str, "continue") == 0))
1314 			attrs->cont = true;
1315 		else if (strcmp(str, "clear") == 0)
1316 			attrs->clear = true;
1317 		else {
1318 			ret = parse_action(str, attrs);
1319 			if (ret)
1320 				goto free;
1321 		}
1322 	}
1323 
1324 	if (!attrs->keys_str) {
1325 		ret = -EINVAL;
1326 		goto free;
1327 	}
1328 
1329 	if (!attrs->clock) {
1330 		attrs->clock = kstrdup("global", GFP_KERNEL);
1331 		if (!attrs->clock) {
1332 			ret = -ENOMEM;
1333 			goto free;
1334 		}
1335 	}
1336 
1337 	return attrs;
1338  free:
1339 	destroy_hist_trigger_attrs(attrs);
1340 
1341 	return ERR_PTR(ret);
1342 }
1343 
1344 static inline void save_comm(char *comm, struct task_struct *task)
1345 {
1346 	if (!task->pid) {
1347 		strcpy(comm, "<idle>");
1348 		return;
1349 	}
1350 
1351 	if (WARN_ON_ONCE(task->pid < 0)) {
1352 		strcpy(comm, "<XXX>");
1353 		return;
1354 	}
1355 
1356 	strncpy(comm, task->comm, TASK_COMM_LEN);
1357 }
1358 
1359 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1360 {
1361 	unsigned int i;
1362 
1363 	for (i = 0; i < SYNTH_FIELDS_MAX; i++)
1364 		kfree(elt_data->field_var_str[i]);
1365 
1366 	kfree(elt_data->comm);
1367 	kfree(elt_data);
1368 }
1369 
1370 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1371 {
1372 	struct hist_elt_data *elt_data = elt->private_data;
1373 
1374 	hist_elt_data_free(elt_data);
1375 }
1376 
1377 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1378 {
1379 	struct hist_trigger_data *hist_data = elt->map->private_data;
1380 	unsigned int size = TASK_COMM_LEN;
1381 	struct hist_elt_data *elt_data;
1382 	struct hist_field *key_field;
1383 	unsigned int i, n_str;
1384 
1385 	elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1386 	if (!elt_data)
1387 		return -ENOMEM;
1388 
1389 	for_each_hist_key_field(i, hist_data) {
1390 		key_field = hist_data->fields[i];
1391 
1392 		if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
1393 			elt_data->comm = kzalloc(size, GFP_KERNEL);
1394 			if (!elt_data->comm) {
1395 				kfree(elt_data);
1396 				return -ENOMEM;
1397 			}
1398 			break;
1399 		}
1400 	}
1401 
1402 	n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1403 		hist_data->n_var_str;
1404 	if (n_str > SYNTH_FIELDS_MAX) {
1405 		hist_elt_data_free(elt_data);
1406 		return -EINVAL;
1407 	}
1408 
1409 	BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1410 
1411 	size = STR_VAR_LEN_MAX;
1412 
1413 	for (i = 0; i < n_str; i++) {
1414 		elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1415 		if (!elt_data->field_var_str[i]) {
1416 			hist_elt_data_free(elt_data);
1417 			return -ENOMEM;
1418 		}
1419 	}
1420 
1421 	elt->private_data = elt_data;
1422 
1423 	return 0;
1424 }
1425 
1426 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1427 {
1428 	struct hist_elt_data *elt_data = elt->private_data;
1429 
1430 	if (elt_data->comm)
1431 		save_comm(elt_data->comm, current);
1432 }
1433 
1434 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1435 	.elt_alloc	= hist_trigger_elt_data_alloc,
1436 	.elt_free	= hist_trigger_elt_data_free,
1437 	.elt_init	= hist_trigger_elt_data_init,
1438 };
1439 
1440 static const char *get_hist_field_flags(struct hist_field *hist_field)
1441 {
1442 	const char *flags_str = NULL;
1443 
1444 	if (hist_field->flags & HIST_FIELD_FL_HEX)
1445 		flags_str = "hex";
1446 	else if (hist_field->flags & HIST_FIELD_FL_SYM)
1447 		flags_str = "sym";
1448 	else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1449 		flags_str = "sym-offset";
1450 	else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1451 		flags_str = "execname";
1452 	else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1453 		flags_str = "syscall";
1454 	else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1455 		flags_str = "log2";
1456 	else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1457 		flags_str = "usecs";
1458 
1459 	return flags_str;
1460 }
1461 
1462 static void expr_field_str(struct hist_field *field, char *expr)
1463 {
1464 	if (field->flags & HIST_FIELD_FL_VAR_REF)
1465 		strcat(expr, "$");
1466 
1467 	strcat(expr, hist_field_name(field, 0));
1468 
1469 	if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1470 		const char *flags_str = get_hist_field_flags(field);
1471 
1472 		if (flags_str) {
1473 			strcat(expr, ".");
1474 			strcat(expr, flags_str);
1475 		}
1476 	}
1477 }
1478 
1479 static char *expr_str(struct hist_field *field, unsigned int level)
1480 {
1481 	char *expr;
1482 
1483 	if (level > 1)
1484 		return NULL;
1485 
1486 	expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1487 	if (!expr)
1488 		return NULL;
1489 
1490 	if (!field->operands[0]) {
1491 		expr_field_str(field, expr);
1492 		return expr;
1493 	}
1494 
1495 	if (field->operator == FIELD_OP_UNARY_MINUS) {
1496 		char *subexpr;
1497 
1498 		strcat(expr, "-(");
1499 		subexpr = expr_str(field->operands[0], ++level);
1500 		if (!subexpr) {
1501 			kfree(expr);
1502 			return NULL;
1503 		}
1504 		strcat(expr, subexpr);
1505 		strcat(expr, ")");
1506 
1507 		kfree(subexpr);
1508 
1509 		return expr;
1510 	}
1511 
1512 	expr_field_str(field->operands[0], expr);
1513 
1514 	switch (field->operator) {
1515 	case FIELD_OP_MINUS:
1516 		strcat(expr, "-");
1517 		break;
1518 	case FIELD_OP_PLUS:
1519 		strcat(expr, "+");
1520 		break;
1521 	default:
1522 		kfree(expr);
1523 		return NULL;
1524 	}
1525 
1526 	expr_field_str(field->operands[1], expr);
1527 
1528 	return expr;
1529 }
1530 
1531 static int contains_operator(char *str)
1532 {
1533 	enum field_op_id field_op = FIELD_OP_NONE;
1534 	char *op;
1535 
1536 	op = strpbrk(str, "+-");
1537 	if (!op)
1538 		return FIELD_OP_NONE;
1539 
1540 	switch (*op) {
1541 	case '-':
1542 		if (*str == '-')
1543 			field_op = FIELD_OP_UNARY_MINUS;
1544 		else
1545 			field_op = FIELD_OP_MINUS;
1546 		break;
1547 	case '+':
1548 		field_op = FIELD_OP_PLUS;
1549 		break;
1550 	default:
1551 		break;
1552 	}
1553 
1554 	return field_op;
1555 }
1556 
1557 static void get_hist_field(struct hist_field *hist_field)
1558 {
1559 	hist_field->ref++;
1560 }
1561 
1562 static void __destroy_hist_field(struct hist_field *hist_field)
1563 {
1564 	if (--hist_field->ref > 1)
1565 		return;
1566 
1567 	kfree(hist_field->var.name);
1568 	kfree(hist_field->name);
1569 	kfree(hist_field->type);
1570 
1571 	kfree(hist_field->system);
1572 	kfree(hist_field->event_name);
1573 
1574 	kfree(hist_field);
1575 }
1576 
1577 static void destroy_hist_field(struct hist_field *hist_field,
1578 			       unsigned int level)
1579 {
1580 	unsigned int i;
1581 
1582 	if (level > 3)
1583 		return;
1584 
1585 	if (!hist_field)
1586 		return;
1587 
1588 	if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1589 		return; /* var refs will be destroyed separately */
1590 
1591 	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1592 		destroy_hist_field(hist_field->operands[i], level + 1);
1593 
1594 	__destroy_hist_field(hist_field);
1595 }
1596 
1597 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1598 					    struct ftrace_event_field *field,
1599 					    unsigned long flags,
1600 					    char *var_name)
1601 {
1602 	struct hist_field *hist_field;
1603 
1604 	if (field && is_function_field(field))
1605 		return NULL;
1606 
1607 	hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1608 	if (!hist_field)
1609 		return NULL;
1610 
1611 	hist_field->ref = 1;
1612 
1613 	hist_field->hist_data = hist_data;
1614 
1615 	if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1616 		goto out; /* caller will populate */
1617 
1618 	if (flags & HIST_FIELD_FL_VAR_REF) {
1619 		hist_field->fn = hist_field_var_ref;
1620 		goto out;
1621 	}
1622 
1623 	if (flags & HIST_FIELD_FL_HITCOUNT) {
1624 		hist_field->fn = hist_field_counter;
1625 		hist_field->size = sizeof(u64);
1626 		hist_field->type = kstrdup("u64", GFP_KERNEL);
1627 		if (!hist_field->type)
1628 			goto free;
1629 		goto out;
1630 	}
1631 
1632 	if (flags & HIST_FIELD_FL_STACKTRACE) {
1633 		hist_field->fn = hist_field_none;
1634 		goto out;
1635 	}
1636 
1637 	if (flags & HIST_FIELD_FL_LOG2) {
1638 		unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
1639 		hist_field->fn = hist_field_log2;
1640 		hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1641 		hist_field->size = hist_field->operands[0]->size;
1642 		hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
1643 		if (!hist_field->type)
1644 			goto free;
1645 		goto out;
1646 	}
1647 
1648 	if (flags & HIST_FIELD_FL_TIMESTAMP) {
1649 		hist_field->fn = hist_field_timestamp;
1650 		hist_field->size = sizeof(u64);
1651 		hist_field->type = kstrdup("u64", GFP_KERNEL);
1652 		if (!hist_field->type)
1653 			goto free;
1654 		goto out;
1655 	}
1656 
1657 	if (flags & HIST_FIELD_FL_CPU) {
1658 		hist_field->fn = hist_field_cpu;
1659 		hist_field->size = sizeof(int);
1660 		hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
1661 		if (!hist_field->type)
1662 			goto free;
1663 		goto out;
1664 	}
1665 
1666 	if (WARN_ON_ONCE(!field))
1667 		goto out;
1668 
1669 	if (is_string_field(field)) {
1670 		flags |= HIST_FIELD_FL_STRING;
1671 
1672 		hist_field->size = MAX_FILTER_STR_VAL;
1673 		hist_field->type = kstrdup(field->type, GFP_KERNEL);
1674 		if (!hist_field->type)
1675 			goto free;
1676 
1677 		if (field->filter_type == FILTER_STATIC_STRING)
1678 			hist_field->fn = hist_field_string;
1679 		else if (field->filter_type == FILTER_DYN_STRING)
1680 			hist_field->fn = hist_field_dynstring;
1681 		else
1682 			hist_field->fn = hist_field_pstring;
1683 	} else {
1684 		hist_field->size = field->size;
1685 		hist_field->is_signed = field->is_signed;
1686 		hist_field->type = kstrdup(field->type, GFP_KERNEL);
1687 		if (!hist_field->type)
1688 			goto free;
1689 
1690 		hist_field->fn = select_value_fn(field->size,
1691 						 field->is_signed);
1692 		if (!hist_field->fn) {
1693 			destroy_hist_field(hist_field, 0);
1694 			return NULL;
1695 		}
1696 	}
1697  out:
1698 	hist_field->field = field;
1699 	hist_field->flags = flags;
1700 
1701 	if (var_name) {
1702 		hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1703 		if (!hist_field->var.name)
1704 			goto free;
1705 	}
1706 
1707 	return hist_field;
1708  free:
1709 	destroy_hist_field(hist_field, 0);
1710 	return NULL;
1711 }
1712 
1713 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
1714 {
1715 	unsigned int i;
1716 
1717 	for (i = 0; i < HIST_FIELDS_MAX; i++) {
1718 		if (hist_data->fields[i]) {
1719 			destroy_hist_field(hist_data->fields[i], 0);
1720 			hist_data->fields[i] = NULL;
1721 		}
1722 	}
1723 
1724 	for (i = 0; i < hist_data->n_var_refs; i++) {
1725 		WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
1726 		__destroy_hist_field(hist_data->var_refs[i]);
1727 		hist_data->var_refs[i] = NULL;
1728 	}
1729 }
1730 
1731 static int init_var_ref(struct hist_field *ref_field,
1732 			struct hist_field *var_field,
1733 			char *system, char *event_name)
1734 {
1735 	int err = 0;
1736 
1737 	ref_field->var.idx = var_field->var.idx;
1738 	ref_field->var.hist_data = var_field->hist_data;
1739 	ref_field->size = var_field->size;
1740 	ref_field->is_signed = var_field->is_signed;
1741 	ref_field->flags |= var_field->flags &
1742 		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
1743 
1744 	if (system) {
1745 		ref_field->system = kstrdup(system, GFP_KERNEL);
1746 		if (!ref_field->system)
1747 			return -ENOMEM;
1748 	}
1749 
1750 	if (event_name) {
1751 		ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
1752 		if (!ref_field->event_name) {
1753 			err = -ENOMEM;
1754 			goto free;
1755 		}
1756 	}
1757 
1758 	if (var_field->var.name) {
1759 		ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
1760 		if (!ref_field->name) {
1761 			err = -ENOMEM;
1762 			goto free;
1763 		}
1764 	} else if (var_field->name) {
1765 		ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
1766 		if (!ref_field->name) {
1767 			err = -ENOMEM;
1768 			goto free;
1769 		}
1770 	}
1771 
1772 	ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
1773 	if (!ref_field->type) {
1774 		err = -ENOMEM;
1775 		goto free;
1776 	}
1777  out:
1778 	return err;
1779  free:
1780 	kfree(ref_field->system);
1781 	kfree(ref_field->event_name);
1782 	kfree(ref_field->name);
1783 
1784 	goto out;
1785 }
1786 
1787 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
1788 			    struct hist_field *var_field)
1789 {
1790 	struct hist_field *ref_field;
1791 	int i;
1792 
1793 	for (i = 0; i < hist_data->n_var_refs; i++) {
1794 		ref_field = hist_data->var_refs[i];
1795 		if (ref_field->var.idx == var_field->var.idx &&
1796 		    ref_field->var.hist_data == var_field->hist_data)
1797 			return i;
1798 	}
1799 
1800 	return -ENOENT;
1801 }
1802 
1803 /**
1804  * create_var_ref - Create a variable reference and attach it to trigger
1805  * @hist_data: The trigger that will be referencing the variable
1806  * @var_field: The VAR field to create a reference to
1807  * @system: The optional system string
1808  * @event_name: The optional event_name string
1809  *
1810  * Given a variable hist_field, create a VAR_REF hist_field that
1811  * represents a reference to it.
1812  *
1813  * This function also adds the reference to the trigger that
1814  * now references the variable.
1815  *
1816  * Return: The VAR_REF field if successful, NULL if not
1817  */
1818 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
1819 					 struct hist_field *var_field,
1820 					 char *system, char *event_name)
1821 {
1822 	unsigned long flags = HIST_FIELD_FL_VAR_REF;
1823 	struct hist_field *ref_field;
1824 	int i;
1825 
1826 	/* Check if the variable already exists */
1827 	for (i = 0; i < hist_data->n_var_refs; i++) {
1828 		ref_field = hist_data->var_refs[i];
1829 		if (ref_field->var.idx == var_field->var.idx &&
1830 		    ref_field->var.hist_data == var_field->hist_data) {
1831 			get_hist_field(ref_field);
1832 			return ref_field;
1833 		}
1834 	}
1835 
1836 	ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
1837 	if (ref_field) {
1838 		if (init_var_ref(ref_field, var_field, system, event_name)) {
1839 			destroy_hist_field(ref_field, 0);
1840 			return NULL;
1841 		}
1842 
1843 		hist_data->var_refs[hist_data->n_var_refs] = ref_field;
1844 		ref_field->var_ref_idx = hist_data->n_var_refs++;
1845 	}
1846 
1847 	return ref_field;
1848 }
1849 
1850 static bool is_var_ref(char *var_name)
1851 {
1852 	if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
1853 		return false;
1854 
1855 	return true;
1856 }
1857 
1858 static char *field_name_from_var(struct hist_trigger_data *hist_data,
1859 				 char *var_name)
1860 {
1861 	char *name, *field;
1862 	unsigned int i;
1863 
1864 	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
1865 		name = hist_data->attrs->var_defs.name[i];
1866 
1867 		if (strcmp(var_name, name) == 0) {
1868 			field = hist_data->attrs->var_defs.expr[i];
1869 			if (contains_operator(field) || is_var_ref(field))
1870 				continue;
1871 			return field;
1872 		}
1873 	}
1874 
1875 	return NULL;
1876 }
1877 
1878 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
1879 				 char *system, char *event_name,
1880 				 char *var_name)
1881 {
1882 	struct trace_event_call *call;
1883 
1884 	if (system && event_name) {
1885 		call = hist_data->event_file->event_call;
1886 
1887 		if (strcmp(system, call->class->system) != 0)
1888 			return NULL;
1889 
1890 		if (strcmp(event_name, trace_event_name(call)) != 0)
1891 			return NULL;
1892 	}
1893 
1894 	if (!!system != !!event_name)
1895 		return NULL;
1896 
1897 	if (!is_var_ref(var_name))
1898 		return NULL;
1899 
1900 	var_name++;
1901 
1902 	return field_name_from_var(hist_data, var_name);
1903 }
1904 
1905 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
1906 					char *system, char *event_name,
1907 					char *var_name)
1908 {
1909 	struct hist_field *var_field = NULL, *ref_field = NULL;
1910 	struct trace_array *tr = hist_data->event_file->tr;
1911 
1912 	if (!is_var_ref(var_name))
1913 		return NULL;
1914 
1915 	var_name++;
1916 
1917 	var_field = find_event_var(hist_data, system, event_name, var_name);
1918 	if (var_field)
1919 		ref_field = create_var_ref(hist_data, var_field,
1920 					   system, event_name);
1921 
1922 	if (!ref_field)
1923 		hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
1924 
1925 	return ref_field;
1926 }
1927 
1928 static struct ftrace_event_field *
1929 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
1930 	    char *field_str, unsigned long *flags)
1931 {
1932 	struct ftrace_event_field *field = NULL;
1933 	char *field_name, *modifier, *str;
1934 	struct trace_array *tr = file->tr;
1935 
1936 	modifier = str = kstrdup(field_str, GFP_KERNEL);
1937 	if (!modifier)
1938 		return ERR_PTR(-ENOMEM);
1939 
1940 	field_name = strsep(&modifier, ".");
1941 	if (modifier) {
1942 		if (strcmp(modifier, "hex") == 0)
1943 			*flags |= HIST_FIELD_FL_HEX;
1944 		else if (strcmp(modifier, "sym") == 0)
1945 			*flags |= HIST_FIELD_FL_SYM;
1946 		else if (strcmp(modifier, "sym-offset") == 0)
1947 			*flags |= HIST_FIELD_FL_SYM_OFFSET;
1948 		else if ((strcmp(modifier, "execname") == 0) &&
1949 			 (strcmp(field_name, "common_pid") == 0))
1950 			*flags |= HIST_FIELD_FL_EXECNAME;
1951 		else if (strcmp(modifier, "syscall") == 0)
1952 			*flags |= HIST_FIELD_FL_SYSCALL;
1953 		else if (strcmp(modifier, "log2") == 0)
1954 			*flags |= HIST_FIELD_FL_LOG2;
1955 		else if (strcmp(modifier, "usecs") == 0)
1956 			*flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
1957 		else {
1958 			hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
1959 			field = ERR_PTR(-EINVAL);
1960 			goto out;
1961 		}
1962 	}
1963 
1964 	if (strcmp(field_name, "common_timestamp") == 0) {
1965 		*flags |= HIST_FIELD_FL_TIMESTAMP;
1966 		hist_data->enable_timestamps = true;
1967 		if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1968 			hist_data->attrs->ts_in_usecs = true;
1969 	} else if (strcmp(field_name, "cpu") == 0)
1970 		*flags |= HIST_FIELD_FL_CPU;
1971 	else {
1972 		field = trace_find_event_field(file->event_call, field_name);
1973 		if (!field || !field->size) {
1974 			hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name));
1975 			field = ERR_PTR(-EINVAL);
1976 			goto out;
1977 		}
1978 	}
1979  out:
1980 	kfree(str);
1981 
1982 	return field;
1983 }
1984 
1985 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
1986 				       struct hist_field *var_ref,
1987 				       char *var_name)
1988 {
1989 	struct hist_field *alias = NULL;
1990 	unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
1991 
1992 	alias = create_hist_field(hist_data, NULL, flags, var_name);
1993 	if (!alias)
1994 		return NULL;
1995 
1996 	alias->fn = var_ref->fn;
1997 	alias->operands[0] = var_ref;
1998 
1999 	if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2000 		destroy_hist_field(alias, 0);
2001 		return NULL;
2002 	}
2003 
2004 	alias->var_ref_idx = var_ref->var_ref_idx;
2005 
2006 	return alias;
2007 }
2008 
2009 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2010 				     struct trace_event_file *file, char *str,
2011 				     unsigned long *flags, char *var_name)
2012 {
2013 	char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2014 	struct ftrace_event_field *field = NULL;
2015 	struct hist_field *hist_field = NULL;
2016 	int ret = 0;
2017 
2018 	s = strchr(str, '.');
2019 	if (s) {
2020 		s = strchr(++s, '.');
2021 		if (s) {
2022 			ref_system = strsep(&str, ".");
2023 			if (!str) {
2024 				ret = -EINVAL;
2025 				goto out;
2026 			}
2027 			ref_event = strsep(&str, ".");
2028 			if (!str) {
2029 				ret = -EINVAL;
2030 				goto out;
2031 			}
2032 			ref_var = str;
2033 		}
2034 	}
2035 
2036 	s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2037 	if (!s) {
2038 		hist_field = parse_var_ref(hist_data, ref_system,
2039 					   ref_event, ref_var);
2040 		if (hist_field) {
2041 			if (var_name) {
2042 				hist_field = create_alias(hist_data, hist_field, var_name);
2043 				if (!hist_field) {
2044 					ret = -ENOMEM;
2045 					goto out;
2046 				}
2047 			}
2048 			return hist_field;
2049 		}
2050 	} else
2051 		str = s;
2052 
2053 	field = parse_field(hist_data, file, str, flags);
2054 	if (IS_ERR(field)) {
2055 		ret = PTR_ERR(field);
2056 		goto out;
2057 	}
2058 
2059 	hist_field = create_hist_field(hist_data, field, *flags, var_name);
2060 	if (!hist_field) {
2061 		ret = -ENOMEM;
2062 		goto out;
2063 	}
2064 
2065 	return hist_field;
2066  out:
2067 	return ERR_PTR(ret);
2068 }
2069 
2070 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2071 				     struct trace_event_file *file,
2072 				     char *str, unsigned long flags,
2073 				     char *var_name, unsigned int level);
2074 
2075 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2076 				      struct trace_event_file *file,
2077 				      char *str, unsigned long flags,
2078 				      char *var_name, unsigned int level)
2079 {
2080 	struct hist_field *operand1, *expr = NULL;
2081 	unsigned long operand_flags;
2082 	int ret = 0;
2083 	char *s;
2084 
2085 	/* we support only -(xxx) i.e. explicit parens required */
2086 
2087 	if (level > 3) {
2088 		hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2089 		ret = -EINVAL;
2090 		goto free;
2091 	}
2092 
2093 	str++; /* skip leading '-' */
2094 
2095 	s = strchr(str, '(');
2096 	if (s)
2097 		str++;
2098 	else {
2099 		ret = -EINVAL;
2100 		goto free;
2101 	}
2102 
2103 	s = strrchr(str, ')');
2104 	if (s)
2105 		*s = '\0';
2106 	else {
2107 		ret = -EINVAL; /* no closing ')' */
2108 		goto free;
2109 	}
2110 
2111 	flags |= HIST_FIELD_FL_EXPR;
2112 	expr = create_hist_field(hist_data, NULL, flags, var_name);
2113 	if (!expr) {
2114 		ret = -ENOMEM;
2115 		goto free;
2116 	}
2117 
2118 	operand_flags = 0;
2119 	operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2120 	if (IS_ERR(operand1)) {
2121 		ret = PTR_ERR(operand1);
2122 		goto free;
2123 	}
2124 
2125 	expr->flags |= operand1->flags &
2126 		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2127 	expr->fn = hist_field_unary_minus;
2128 	expr->operands[0] = operand1;
2129 	expr->operator = FIELD_OP_UNARY_MINUS;
2130 	expr->name = expr_str(expr, 0);
2131 	expr->type = kstrdup(operand1->type, GFP_KERNEL);
2132 	if (!expr->type) {
2133 		ret = -ENOMEM;
2134 		goto free;
2135 	}
2136 
2137 	return expr;
2138  free:
2139 	destroy_hist_field(expr, 0);
2140 	return ERR_PTR(ret);
2141 }
2142 
2143 static int check_expr_operands(struct trace_array *tr,
2144 			       struct hist_field *operand1,
2145 			       struct hist_field *operand2)
2146 {
2147 	unsigned long operand1_flags = operand1->flags;
2148 	unsigned long operand2_flags = operand2->flags;
2149 
2150 	if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2151 	    (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2152 		struct hist_field *var;
2153 
2154 		var = find_var_field(operand1->var.hist_data, operand1->name);
2155 		if (!var)
2156 			return -EINVAL;
2157 		operand1_flags = var->flags;
2158 	}
2159 
2160 	if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2161 	    (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2162 		struct hist_field *var;
2163 
2164 		var = find_var_field(operand2->var.hist_data, operand2->name);
2165 		if (!var)
2166 			return -EINVAL;
2167 		operand2_flags = var->flags;
2168 	}
2169 
2170 	if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2171 	    (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2172 		hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2173 		return -EINVAL;
2174 	}
2175 
2176 	return 0;
2177 }
2178 
2179 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2180 				     struct trace_event_file *file,
2181 				     char *str, unsigned long flags,
2182 				     char *var_name, unsigned int level)
2183 {
2184 	struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2185 	unsigned long operand_flags;
2186 	int field_op, ret = -EINVAL;
2187 	char *sep, *operand1_str;
2188 
2189 	if (level > 3) {
2190 		hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2191 		return ERR_PTR(-EINVAL);
2192 	}
2193 
2194 	field_op = contains_operator(str);
2195 
2196 	if (field_op == FIELD_OP_NONE)
2197 		return parse_atom(hist_data, file, str, &flags, var_name);
2198 
2199 	if (field_op == FIELD_OP_UNARY_MINUS)
2200 		return parse_unary(hist_data, file, str, flags, var_name, ++level);
2201 
2202 	switch (field_op) {
2203 	case FIELD_OP_MINUS:
2204 		sep = "-";
2205 		break;
2206 	case FIELD_OP_PLUS:
2207 		sep = "+";
2208 		break;
2209 	default:
2210 		goto free;
2211 	}
2212 
2213 	operand1_str = strsep(&str, sep);
2214 	if (!operand1_str || !str)
2215 		goto free;
2216 
2217 	operand_flags = 0;
2218 	operand1 = parse_atom(hist_data, file, operand1_str,
2219 			      &operand_flags, NULL);
2220 	if (IS_ERR(operand1)) {
2221 		ret = PTR_ERR(operand1);
2222 		operand1 = NULL;
2223 		goto free;
2224 	}
2225 
2226 	/* rest of string could be another expression e.g. b+c in a+b+c */
2227 	operand_flags = 0;
2228 	operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2229 	if (IS_ERR(operand2)) {
2230 		ret = PTR_ERR(operand2);
2231 		operand2 = NULL;
2232 		goto free;
2233 	}
2234 
2235 	ret = check_expr_operands(file->tr, operand1, operand2);
2236 	if (ret)
2237 		goto free;
2238 
2239 	flags |= HIST_FIELD_FL_EXPR;
2240 
2241 	flags |= operand1->flags &
2242 		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2243 
2244 	expr = create_hist_field(hist_data, NULL, flags, var_name);
2245 	if (!expr) {
2246 		ret = -ENOMEM;
2247 		goto free;
2248 	}
2249 
2250 	operand1->read_once = true;
2251 	operand2->read_once = true;
2252 
2253 	expr->operands[0] = operand1;
2254 	expr->operands[1] = operand2;
2255 	expr->operator = field_op;
2256 	expr->name = expr_str(expr, 0);
2257 	expr->type = kstrdup(operand1->type, GFP_KERNEL);
2258 	if (!expr->type) {
2259 		ret = -ENOMEM;
2260 		goto free;
2261 	}
2262 
2263 	switch (field_op) {
2264 	case FIELD_OP_MINUS:
2265 		expr->fn = hist_field_minus;
2266 		break;
2267 	case FIELD_OP_PLUS:
2268 		expr->fn = hist_field_plus;
2269 		break;
2270 	default:
2271 		ret = -EINVAL;
2272 		goto free;
2273 	}
2274 
2275 	return expr;
2276  free:
2277 	destroy_hist_field(operand1, 0);
2278 	destroy_hist_field(operand2, 0);
2279 	destroy_hist_field(expr, 0);
2280 
2281 	return ERR_PTR(ret);
2282 }
2283 
2284 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2285 				 struct trace_event_file *file)
2286 {
2287 	struct event_trigger_data *test;
2288 
2289 	lockdep_assert_held(&event_mutex);
2290 
2291 	list_for_each_entry(test, &file->triggers, list) {
2292 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2293 			if (test->private_data == hist_data)
2294 				return test->filter_str;
2295 		}
2296 	}
2297 
2298 	return NULL;
2299 }
2300 
2301 static struct event_command trigger_hist_cmd;
2302 static int event_hist_trigger_func(struct event_command *cmd_ops,
2303 				   struct trace_event_file *file,
2304 				   char *glob, char *cmd, char *param);
2305 
2306 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2307 			    struct hist_trigger_data *hist_data,
2308 			    unsigned int n_keys)
2309 {
2310 	struct hist_field *target_hist_field, *hist_field;
2311 	unsigned int n, i, j;
2312 
2313 	if (hist_data->n_fields - hist_data->n_vals != n_keys)
2314 		return false;
2315 
2316 	i = hist_data->n_vals;
2317 	j = target_hist_data->n_vals;
2318 
2319 	for (n = 0; n < n_keys; n++) {
2320 		hist_field = hist_data->fields[i + n];
2321 		target_hist_field = target_hist_data->fields[j + n];
2322 
2323 		if (strcmp(hist_field->type, target_hist_field->type) != 0)
2324 			return false;
2325 		if (hist_field->size != target_hist_field->size)
2326 			return false;
2327 		if (hist_field->is_signed != target_hist_field->is_signed)
2328 			return false;
2329 	}
2330 
2331 	return true;
2332 }
2333 
2334 static struct hist_trigger_data *
2335 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2336 		     struct trace_event_file *file)
2337 {
2338 	struct hist_trigger_data *hist_data;
2339 	struct event_trigger_data *test;
2340 	unsigned int n_keys;
2341 
2342 	lockdep_assert_held(&event_mutex);
2343 
2344 	n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2345 
2346 	list_for_each_entry(test, &file->triggers, list) {
2347 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2348 			hist_data = test->private_data;
2349 
2350 			if (compatible_keys(target_hist_data, hist_data, n_keys))
2351 				return hist_data;
2352 		}
2353 	}
2354 
2355 	return NULL;
2356 }
2357 
2358 static struct trace_event_file *event_file(struct trace_array *tr,
2359 					   char *system, char *event_name)
2360 {
2361 	struct trace_event_file *file;
2362 
2363 	file = __find_event_file(tr, system, event_name);
2364 	if (!file)
2365 		return ERR_PTR(-EINVAL);
2366 
2367 	return file;
2368 }
2369 
2370 static struct hist_field *
2371 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2372 			 char *system, char *event_name, char *field_name)
2373 {
2374 	struct hist_field *event_var;
2375 	char *synthetic_name;
2376 
2377 	synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2378 	if (!synthetic_name)
2379 		return ERR_PTR(-ENOMEM);
2380 
2381 	strcpy(synthetic_name, "synthetic_");
2382 	strcat(synthetic_name, field_name);
2383 
2384 	event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2385 
2386 	kfree(synthetic_name);
2387 
2388 	return event_var;
2389 }
2390 
2391 /**
2392  * create_field_var_hist - Automatically create a histogram and var for a field
2393  * @target_hist_data: The target hist trigger
2394  * @subsys_name: Optional subsystem name
2395  * @event_name: Optional event name
2396  * @field_name: The name of the field (and the resulting variable)
2397  *
2398  * Hist trigger actions fetch data from variables, not directly from
2399  * events.  However, for convenience, users are allowed to directly
2400  * specify an event field in an action, which will be automatically
2401  * converted into a variable on their behalf.
2402 
2403  * If a user specifies a field on an event that isn't the event the
2404  * histogram currently being defined (the target event histogram), the
2405  * only way that can be accomplished is if a new hist trigger is
2406  * created and the field variable defined on that.
2407  *
2408  * This function creates a new histogram compatible with the target
2409  * event (meaning a histogram with the same key as the target
2410  * histogram), and creates a variable for the specified field, but
2411  * with 'synthetic_' prepended to the variable name in order to avoid
2412  * collision with normal field variables.
2413  *
2414  * Return: The variable created for the field.
2415  */
2416 static struct hist_field *
2417 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2418 		      char *subsys_name, char *event_name, char *field_name)
2419 {
2420 	struct trace_array *tr = target_hist_data->event_file->tr;
2421 	struct hist_field *event_var = ERR_PTR(-EINVAL);
2422 	struct hist_trigger_data *hist_data;
2423 	unsigned int i, n, first = true;
2424 	struct field_var_hist *var_hist;
2425 	struct trace_event_file *file;
2426 	struct hist_field *key_field;
2427 	char *saved_filter;
2428 	char *cmd;
2429 	int ret;
2430 
2431 	if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2432 		hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2433 		return ERR_PTR(-EINVAL);
2434 	}
2435 
2436 	file = event_file(tr, subsys_name, event_name);
2437 
2438 	if (IS_ERR(file)) {
2439 		hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2440 		ret = PTR_ERR(file);
2441 		return ERR_PTR(ret);
2442 	}
2443 
2444 	/*
2445 	 * Look for a histogram compatible with target.  We'll use the
2446 	 * found histogram specification to create a new matching
2447 	 * histogram with our variable on it.  target_hist_data is not
2448 	 * yet a registered histogram so we can't use that.
2449 	 */
2450 	hist_data = find_compatible_hist(target_hist_data, file);
2451 	if (!hist_data) {
2452 		hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2453 		return ERR_PTR(-EINVAL);
2454 	}
2455 
2456 	/* See if a synthetic field variable has already been created */
2457 	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2458 					     event_name, field_name);
2459 	if (!IS_ERR_OR_NULL(event_var))
2460 		return event_var;
2461 
2462 	var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2463 	if (!var_hist)
2464 		return ERR_PTR(-ENOMEM);
2465 
2466 	cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2467 	if (!cmd) {
2468 		kfree(var_hist);
2469 		return ERR_PTR(-ENOMEM);
2470 	}
2471 
2472 	/* Use the same keys as the compatible histogram */
2473 	strcat(cmd, "keys=");
2474 
2475 	for_each_hist_key_field(i, hist_data) {
2476 		key_field = hist_data->fields[i];
2477 		if (!first)
2478 			strcat(cmd, ",");
2479 		strcat(cmd, key_field->field->name);
2480 		first = false;
2481 	}
2482 
2483 	/* Create the synthetic field variable specification */
2484 	strcat(cmd, ":synthetic_");
2485 	strcat(cmd, field_name);
2486 	strcat(cmd, "=");
2487 	strcat(cmd, field_name);
2488 
2489 	/* Use the same filter as the compatible histogram */
2490 	saved_filter = find_trigger_filter(hist_data, file);
2491 	if (saved_filter) {
2492 		strcat(cmd, " if ");
2493 		strcat(cmd, saved_filter);
2494 	}
2495 
2496 	var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2497 	if (!var_hist->cmd) {
2498 		kfree(cmd);
2499 		kfree(var_hist);
2500 		return ERR_PTR(-ENOMEM);
2501 	}
2502 
2503 	/* Save the compatible histogram information */
2504 	var_hist->hist_data = hist_data;
2505 
2506 	/* Create the new histogram with our variable */
2507 	ret = event_hist_trigger_func(&trigger_hist_cmd, file,
2508 				      "", "hist", cmd);
2509 	if (ret) {
2510 		kfree(cmd);
2511 		kfree(var_hist->cmd);
2512 		kfree(var_hist);
2513 		hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2514 		return ERR_PTR(ret);
2515 	}
2516 
2517 	kfree(cmd);
2518 
2519 	/* If we can't find the variable, something went wrong */
2520 	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2521 					     event_name, field_name);
2522 	if (IS_ERR_OR_NULL(event_var)) {
2523 		kfree(var_hist->cmd);
2524 		kfree(var_hist);
2525 		hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
2526 		return ERR_PTR(-EINVAL);
2527 	}
2528 
2529 	n = target_hist_data->n_field_var_hists;
2530 	target_hist_data->field_var_hists[n] = var_hist;
2531 	target_hist_data->n_field_var_hists++;
2532 
2533 	return event_var;
2534 }
2535 
2536 static struct hist_field *
2537 find_target_event_var(struct hist_trigger_data *hist_data,
2538 		      char *subsys_name, char *event_name, char *var_name)
2539 {
2540 	struct trace_event_file *file = hist_data->event_file;
2541 	struct hist_field *hist_field = NULL;
2542 
2543 	if (subsys_name) {
2544 		struct trace_event_call *call;
2545 
2546 		if (!event_name)
2547 			return NULL;
2548 
2549 		call = file->event_call;
2550 
2551 		if (strcmp(subsys_name, call->class->system) != 0)
2552 			return NULL;
2553 
2554 		if (strcmp(event_name, trace_event_name(call)) != 0)
2555 			return NULL;
2556 	}
2557 
2558 	hist_field = find_var_field(hist_data, var_name);
2559 
2560 	return hist_field;
2561 }
2562 
2563 static inline void __update_field_vars(struct tracing_map_elt *elt,
2564 				       struct ring_buffer_event *rbe,
2565 				       void *rec,
2566 				       struct field_var **field_vars,
2567 				       unsigned int n_field_vars,
2568 				       unsigned int field_var_str_start)
2569 {
2570 	struct hist_elt_data *elt_data = elt->private_data;
2571 	unsigned int i, j, var_idx;
2572 	u64 var_val;
2573 
2574 	for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
2575 		struct field_var *field_var = field_vars[i];
2576 		struct hist_field *var = field_var->var;
2577 		struct hist_field *val = field_var->val;
2578 
2579 		var_val = val->fn(val, elt, rbe, rec);
2580 		var_idx = var->var.idx;
2581 
2582 		if (val->flags & HIST_FIELD_FL_STRING) {
2583 			char *str = elt_data->field_var_str[j++];
2584 			char *val_str = (char *)(uintptr_t)var_val;
2585 
2586 			strscpy(str, val_str, STR_VAR_LEN_MAX);
2587 			var_val = (u64)(uintptr_t)str;
2588 		}
2589 		tracing_map_set_var(elt, var_idx, var_val);
2590 	}
2591 }
2592 
2593 static void update_field_vars(struct hist_trigger_data *hist_data,
2594 			      struct tracing_map_elt *elt,
2595 			      struct ring_buffer_event *rbe,
2596 			      void *rec)
2597 {
2598 	__update_field_vars(elt, rbe, rec, hist_data->field_vars,
2599 			    hist_data->n_field_vars, 0);
2600 }
2601 
2602 static void save_track_data_vars(struct hist_trigger_data *hist_data,
2603 				 struct tracing_map_elt *elt, void *rec,
2604 				 struct ring_buffer_event *rbe, void *key,
2605 				 struct action_data *data, u64 *var_ref_vals)
2606 {
2607 	__update_field_vars(elt, rbe, rec, hist_data->save_vars,
2608 			    hist_data->n_save_vars, hist_data->n_field_var_str);
2609 }
2610 
2611 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
2612 				     struct trace_event_file *file,
2613 				     char *name, int size, const char *type)
2614 {
2615 	struct hist_field *var;
2616 	int idx;
2617 
2618 	if (find_var(hist_data, file, name) && !hist_data->remove) {
2619 		var = ERR_PTR(-EINVAL);
2620 		goto out;
2621 	}
2622 
2623 	var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2624 	if (!var) {
2625 		var = ERR_PTR(-ENOMEM);
2626 		goto out;
2627 	}
2628 
2629 	idx = tracing_map_add_var(hist_data->map);
2630 	if (idx < 0) {
2631 		kfree(var);
2632 		var = ERR_PTR(-EINVAL);
2633 		goto out;
2634 	}
2635 
2636 	var->ref = 1;
2637 	var->flags = HIST_FIELD_FL_VAR;
2638 	var->var.idx = idx;
2639 	var->var.hist_data = var->hist_data = hist_data;
2640 	var->size = size;
2641 	var->var.name = kstrdup(name, GFP_KERNEL);
2642 	var->type = kstrdup(type, GFP_KERNEL);
2643 	if (!var->var.name || !var->type) {
2644 		kfree(var->var.name);
2645 		kfree(var->type);
2646 		kfree(var);
2647 		var = ERR_PTR(-ENOMEM);
2648 	}
2649  out:
2650 	return var;
2651 }
2652 
2653 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
2654 					  struct trace_event_file *file,
2655 					  char *field_name)
2656 {
2657 	struct hist_field *val = NULL, *var = NULL;
2658 	unsigned long flags = HIST_FIELD_FL_VAR;
2659 	struct trace_array *tr = file->tr;
2660 	struct field_var *field_var;
2661 	int ret = 0;
2662 
2663 	if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
2664 		hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2665 		ret = -EINVAL;
2666 		goto err;
2667 	}
2668 
2669 	val = parse_atom(hist_data, file, field_name, &flags, NULL);
2670 	if (IS_ERR(val)) {
2671 		hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
2672 		ret = PTR_ERR(val);
2673 		goto err;
2674 	}
2675 
2676 	var = create_var(hist_data, file, field_name, val->size, val->type);
2677 	if (IS_ERR(var)) {
2678 		hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
2679 		kfree(val);
2680 		ret = PTR_ERR(var);
2681 		goto err;
2682 	}
2683 
2684 	field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
2685 	if (!field_var) {
2686 		kfree(val);
2687 		kfree(var);
2688 		ret =  -ENOMEM;
2689 		goto err;
2690 	}
2691 
2692 	field_var->var = var;
2693 	field_var->val = val;
2694  out:
2695 	return field_var;
2696  err:
2697 	field_var = ERR_PTR(ret);
2698 	goto out;
2699 }
2700 
2701 /**
2702  * create_target_field_var - Automatically create a variable for a field
2703  * @target_hist_data: The target hist trigger
2704  * @subsys_name: Optional subsystem name
2705  * @event_name: Optional event name
2706  * @var_name: The name of the field (and the resulting variable)
2707  *
2708  * Hist trigger actions fetch data from variables, not directly from
2709  * events.  However, for convenience, users are allowed to directly
2710  * specify an event field in an action, which will be automatically
2711  * converted into a variable on their behalf.
2712 
2713  * This function creates a field variable with the name var_name on
2714  * the hist trigger currently being defined on the target event.  If
2715  * subsys_name and event_name are specified, this function simply
2716  * verifies that they do in fact match the target event subsystem and
2717  * event name.
2718  *
2719  * Return: The variable created for the field.
2720  */
2721 static struct field_var *
2722 create_target_field_var(struct hist_trigger_data *target_hist_data,
2723 			char *subsys_name, char *event_name, char *var_name)
2724 {
2725 	struct trace_event_file *file = target_hist_data->event_file;
2726 
2727 	if (subsys_name) {
2728 		struct trace_event_call *call;
2729 
2730 		if (!event_name)
2731 			return NULL;
2732 
2733 		call = file->event_call;
2734 
2735 		if (strcmp(subsys_name, call->class->system) != 0)
2736 			return NULL;
2737 
2738 		if (strcmp(event_name, trace_event_name(call)) != 0)
2739 			return NULL;
2740 	}
2741 
2742 	return create_field_var(target_hist_data, file, var_name);
2743 }
2744 
2745 static bool check_track_val_max(u64 track_val, u64 var_val)
2746 {
2747 	if (var_val <= track_val)
2748 		return false;
2749 
2750 	return true;
2751 }
2752 
2753 static bool check_track_val_changed(u64 track_val, u64 var_val)
2754 {
2755 	if (var_val == track_val)
2756 		return false;
2757 
2758 	return true;
2759 }
2760 
2761 static u64 get_track_val(struct hist_trigger_data *hist_data,
2762 			 struct tracing_map_elt *elt,
2763 			 struct action_data *data)
2764 {
2765 	unsigned int track_var_idx = data->track_data.track_var->var.idx;
2766 	u64 track_val;
2767 
2768 	track_val = tracing_map_read_var(elt, track_var_idx);
2769 
2770 	return track_val;
2771 }
2772 
2773 static void save_track_val(struct hist_trigger_data *hist_data,
2774 			   struct tracing_map_elt *elt,
2775 			   struct action_data *data, u64 var_val)
2776 {
2777 	unsigned int track_var_idx = data->track_data.track_var->var.idx;
2778 
2779 	tracing_map_set_var(elt, track_var_idx, var_val);
2780 }
2781 
2782 static void save_track_data(struct hist_trigger_data *hist_data,
2783 			    struct tracing_map_elt *elt, void *rec,
2784 			    struct ring_buffer_event *rbe, void *key,
2785 			    struct action_data *data, u64 *var_ref_vals)
2786 {
2787 	if (data->track_data.save_data)
2788 		data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
2789 }
2790 
2791 static bool check_track_val(struct tracing_map_elt *elt,
2792 			    struct action_data *data,
2793 			    u64 var_val)
2794 {
2795 	struct hist_trigger_data *hist_data;
2796 	u64 track_val;
2797 
2798 	hist_data = data->track_data.track_var->hist_data;
2799 	track_val = get_track_val(hist_data, elt, data);
2800 
2801 	return data->track_data.check_val(track_val, var_val);
2802 }
2803 
2804 #ifdef CONFIG_TRACER_SNAPSHOT
2805 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2806 {
2807 	/* called with tr->max_lock held */
2808 	struct track_data *track_data = tr->cond_snapshot->cond_data;
2809 	struct hist_elt_data *elt_data, *track_elt_data;
2810 	struct snapshot_context *context = cond_data;
2811 	struct action_data *action;
2812 	u64 track_val;
2813 
2814 	if (!track_data)
2815 		return false;
2816 
2817 	action = track_data->action_data;
2818 
2819 	track_val = get_track_val(track_data->hist_data, context->elt,
2820 				  track_data->action_data);
2821 
2822 	if (!action->track_data.check_val(track_data->track_val, track_val))
2823 		return false;
2824 
2825 	track_data->track_val = track_val;
2826 	memcpy(track_data->key, context->key, track_data->key_len);
2827 
2828 	elt_data = context->elt->private_data;
2829 	track_elt_data = track_data->elt.private_data;
2830 	if (elt_data->comm)
2831 		strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
2832 
2833 	track_data->updated = true;
2834 
2835 	return true;
2836 }
2837 
2838 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2839 				     struct tracing_map_elt *elt, void *rec,
2840 				     struct ring_buffer_event *rbe, void *key,
2841 				     struct action_data *data,
2842 				     u64 *var_ref_vals)
2843 {
2844 	struct trace_event_file *file = hist_data->event_file;
2845 	struct snapshot_context context;
2846 
2847 	context.elt = elt;
2848 	context.key = key;
2849 
2850 	tracing_snapshot_cond(file->tr, &context);
2851 }
2852 
2853 static void hist_trigger_print_key(struct seq_file *m,
2854 				   struct hist_trigger_data *hist_data,
2855 				   void *key,
2856 				   struct tracing_map_elt *elt);
2857 
2858 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
2859 {
2860 	unsigned int i;
2861 
2862 	if (!hist_data->n_actions)
2863 		return NULL;
2864 
2865 	for (i = 0; i < hist_data->n_actions; i++) {
2866 		struct action_data *data = hist_data->actions[i];
2867 
2868 		if (data->action == ACTION_SNAPSHOT)
2869 			return data;
2870 	}
2871 
2872 	return NULL;
2873 }
2874 
2875 static void track_data_snapshot_print(struct seq_file *m,
2876 				      struct hist_trigger_data *hist_data)
2877 {
2878 	struct trace_event_file *file = hist_data->event_file;
2879 	struct track_data *track_data;
2880 	struct action_data *action;
2881 
2882 	track_data = tracing_cond_snapshot_data(file->tr);
2883 	if (!track_data)
2884 		return;
2885 
2886 	if (!track_data->updated)
2887 		return;
2888 
2889 	action = snapshot_action(hist_data);
2890 	if (!action)
2891 		return;
2892 
2893 	seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
2894 	seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
2895 		   action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
2896 		   action->track_data.var_str, track_data->track_val);
2897 
2898 	seq_puts(m, "\ttriggered by event with key: ");
2899 	hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
2900 	seq_putc(m, '\n');
2901 }
2902 #else
2903 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2904 {
2905 	return false;
2906 }
2907 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2908 				     struct tracing_map_elt *elt, void *rec,
2909 				     struct ring_buffer_event *rbe, void *key,
2910 				     struct action_data *data,
2911 				     u64 *var_ref_vals) {}
2912 static void track_data_snapshot_print(struct seq_file *m,
2913 				      struct hist_trigger_data *hist_data) {}
2914 #endif /* CONFIG_TRACER_SNAPSHOT */
2915 
2916 static void track_data_print(struct seq_file *m,
2917 			     struct hist_trigger_data *hist_data,
2918 			     struct tracing_map_elt *elt,
2919 			     struct action_data *data)
2920 {
2921 	u64 track_val = get_track_val(hist_data, elt, data);
2922 	unsigned int i, save_var_idx;
2923 
2924 	if (data->handler == HANDLER_ONMAX)
2925 		seq_printf(m, "\n\tmax: %10llu", track_val);
2926 	else if (data->handler == HANDLER_ONCHANGE)
2927 		seq_printf(m, "\n\tchanged: %10llu", track_val);
2928 
2929 	if (data->action == ACTION_SNAPSHOT)
2930 		return;
2931 
2932 	for (i = 0; i < hist_data->n_save_vars; i++) {
2933 		struct hist_field *save_val = hist_data->save_vars[i]->val;
2934 		struct hist_field *save_var = hist_data->save_vars[i]->var;
2935 		u64 val;
2936 
2937 		save_var_idx = save_var->var.idx;
2938 
2939 		val = tracing_map_read_var(elt, save_var_idx);
2940 
2941 		if (save_val->flags & HIST_FIELD_FL_STRING) {
2942 			seq_printf(m, "  %s: %-32s", save_var->var.name,
2943 				   (char *)(uintptr_t)(val));
2944 		} else
2945 			seq_printf(m, "  %s: %10llu", save_var->var.name, val);
2946 	}
2947 }
2948 
2949 static void ontrack_action(struct hist_trigger_data *hist_data,
2950 			   struct tracing_map_elt *elt, void *rec,
2951 			   struct ring_buffer_event *rbe, void *key,
2952 			   struct action_data *data, u64 *var_ref_vals)
2953 {
2954 	u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
2955 
2956 	if (check_track_val(elt, data, var_val)) {
2957 		save_track_val(hist_data, elt, data, var_val);
2958 		save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
2959 	}
2960 }
2961 
2962 static void action_data_destroy(struct action_data *data)
2963 {
2964 	unsigned int i;
2965 
2966 	lockdep_assert_held(&event_mutex);
2967 
2968 	kfree(data->action_name);
2969 
2970 	for (i = 0; i < data->n_params; i++)
2971 		kfree(data->params[i]);
2972 
2973 	if (data->synth_event)
2974 		data->synth_event->ref--;
2975 
2976 	kfree(data->synth_event_name);
2977 
2978 	kfree(data);
2979 }
2980 
2981 static void track_data_destroy(struct hist_trigger_data *hist_data,
2982 			       struct action_data *data)
2983 {
2984 	struct trace_event_file *file = hist_data->event_file;
2985 
2986 	destroy_hist_field(data->track_data.track_var, 0);
2987 
2988 	if (data->action == ACTION_SNAPSHOT) {
2989 		struct track_data *track_data;
2990 
2991 		track_data = tracing_cond_snapshot_data(file->tr);
2992 		if (track_data && track_data->hist_data == hist_data) {
2993 			tracing_snapshot_cond_disable(file->tr);
2994 			track_data_free(track_data);
2995 		}
2996 	}
2997 
2998 	kfree(data->track_data.var_str);
2999 
3000 	action_data_destroy(data);
3001 }
3002 
3003 static int action_create(struct hist_trigger_data *hist_data,
3004 			 struct action_data *data);
3005 
3006 static int track_data_create(struct hist_trigger_data *hist_data,
3007 			     struct action_data *data)
3008 {
3009 	struct hist_field *var_field, *ref_field, *track_var = NULL;
3010 	struct trace_event_file *file = hist_data->event_file;
3011 	struct trace_array *tr = file->tr;
3012 	char *track_data_var_str;
3013 	int ret = 0;
3014 
3015 	track_data_var_str = data->track_data.var_str;
3016 	if (track_data_var_str[0] != '$') {
3017 		hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3018 		return -EINVAL;
3019 	}
3020 	track_data_var_str++;
3021 
3022 	var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3023 	if (!var_field) {
3024 		hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3025 		return -EINVAL;
3026 	}
3027 
3028 	ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3029 	if (!ref_field)
3030 		return -ENOMEM;
3031 
3032 	data->track_data.var_ref = ref_field;
3033 
3034 	if (data->handler == HANDLER_ONMAX)
3035 		track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3036 	if (IS_ERR(track_var)) {
3037 		hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3038 		ret = PTR_ERR(track_var);
3039 		goto out;
3040 	}
3041 
3042 	if (data->handler == HANDLER_ONCHANGE)
3043 		track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3044 	if (IS_ERR(track_var)) {
3045 		hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3046 		ret = PTR_ERR(track_var);
3047 		goto out;
3048 	}
3049 	data->track_data.track_var = track_var;
3050 
3051 	ret = action_create(hist_data, data);
3052  out:
3053 	return ret;
3054 }
3055 
3056 static int parse_action_params(struct trace_array *tr, char *params,
3057 			       struct action_data *data)
3058 {
3059 	char *param, *saved_param;
3060 	bool first_param = true;
3061 	int ret = 0;
3062 
3063 	while (params) {
3064 		if (data->n_params >= SYNTH_FIELDS_MAX) {
3065 			hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3066 			goto out;
3067 		}
3068 
3069 		param = strsep(&params, ",");
3070 		if (!param) {
3071 			hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3072 			ret = -EINVAL;
3073 			goto out;
3074 		}
3075 
3076 		param = strstrip(param);
3077 		if (strlen(param) < 2) {
3078 			hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3079 			ret = -EINVAL;
3080 			goto out;
3081 		}
3082 
3083 		saved_param = kstrdup(param, GFP_KERNEL);
3084 		if (!saved_param) {
3085 			ret = -ENOMEM;
3086 			goto out;
3087 		}
3088 
3089 		if (first_param && data->use_trace_keyword) {
3090 			data->synth_event_name = saved_param;
3091 			first_param = false;
3092 			continue;
3093 		}
3094 		first_param = false;
3095 
3096 		data->params[data->n_params++] = saved_param;
3097 	}
3098  out:
3099 	return ret;
3100 }
3101 
3102 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3103 			enum handler_id handler)
3104 {
3105 	char *action_name;
3106 	int ret = 0;
3107 
3108 	strsep(&str, ".");
3109 	if (!str) {
3110 		hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3111 		ret = -EINVAL;
3112 		goto out;
3113 	}
3114 
3115 	action_name = strsep(&str, "(");
3116 	if (!action_name || !str) {
3117 		hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3118 		ret = -EINVAL;
3119 		goto out;
3120 	}
3121 
3122 	if (str_has_prefix(action_name, "save")) {
3123 		char *params = strsep(&str, ")");
3124 
3125 		if (!params) {
3126 			hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3127 			ret = -EINVAL;
3128 			goto out;
3129 		}
3130 
3131 		ret = parse_action_params(tr, params, data);
3132 		if (ret)
3133 			goto out;
3134 
3135 		if (handler == HANDLER_ONMAX)
3136 			data->track_data.check_val = check_track_val_max;
3137 		else if (handler == HANDLER_ONCHANGE)
3138 			data->track_data.check_val = check_track_val_changed;
3139 		else {
3140 			hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3141 			ret = -EINVAL;
3142 			goto out;
3143 		}
3144 
3145 		data->track_data.save_data = save_track_data_vars;
3146 		data->fn = ontrack_action;
3147 		data->action = ACTION_SAVE;
3148 	} else if (str_has_prefix(action_name, "snapshot")) {
3149 		char *params = strsep(&str, ")");
3150 
3151 		if (!str) {
3152 			hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3153 			ret = -EINVAL;
3154 			goto out;
3155 		}
3156 
3157 		if (handler == HANDLER_ONMAX)
3158 			data->track_data.check_val = check_track_val_max;
3159 		else if (handler == HANDLER_ONCHANGE)
3160 			data->track_data.check_val = check_track_val_changed;
3161 		else {
3162 			hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3163 			ret = -EINVAL;
3164 			goto out;
3165 		}
3166 
3167 		data->track_data.save_data = save_track_data_snapshot;
3168 		data->fn = ontrack_action;
3169 		data->action = ACTION_SNAPSHOT;
3170 	} else {
3171 		char *params = strsep(&str, ")");
3172 
3173 		if (str_has_prefix(action_name, "trace"))
3174 			data->use_trace_keyword = true;
3175 
3176 		if (params) {
3177 			ret = parse_action_params(tr, params, data);
3178 			if (ret)
3179 				goto out;
3180 		}
3181 
3182 		if (handler == HANDLER_ONMAX)
3183 			data->track_data.check_val = check_track_val_max;
3184 		else if (handler == HANDLER_ONCHANGE)
3185 			data->track_data.check_val = check_track_val_changed;
3186 
3187 		if (handler != HANDLER_ONMATCH) {
3188 			data->track_data.save_data = action_trace;
3189 			data->fn = ontrack_action;
3190 		} else
3191 			data->fn = action_trace;
3192 
3193 		data->action = ACTION_TRACE;
3194 	}
3195 
3196 	data->action_name = kstrdup(action_name, GFP_KERNEL);
3197 	if (!data->action_name) {
3198 		ret = -ENOMEM;
3199 		goto out;
3200 	}
3201 
3202 	data->handler = handler;
3203  out:
3204 	return ret;
3205 }
3206 
3207 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3208 					    char *str, enum handler_id handler)
3209 {
3210 	struct action_data *data;
3211 	int ret = -EINVAL;
3212 	char *var_str;
3213 
3214 	data = kzalloc(sizeof(*data), GFP_KERNEL);
3215 	if (!data)
3216 		return ERR_PTR(-ENOMEM);
3217 
3218 	var_str = strsep(&str, ")");
3219 	if (!var_str || !str) {
3220 		ret = -EINVAL;
3221 		goto free;
3222 	}
3223 
3224 	data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3225 	if (!data->track_data.var_str) {
3226 		ret = -ENOMEM;
3227 		goto free;
3228 	}
3229 
3230 	ret = action_parse(hist_data->event_file->tr, str, data, handler);
3231 	if (ret)
3232 		goto free;
3233  out:
3234 	return data;
3235  free:
3236 	track_data_destroy(hist_data, data);
3237 	data = ERR_PTR(ret);
3238 	goto out;
3239 }
3240 
3241 static void onmatch_destroy(struct action_data *data)
3242 {
3243 	kfree(data->match_data.event);
3244 	kfree(data->match_data.event_system);
3245 
3246 	action_data_destroy(data);
3247 }
3248 
3249 static void destroy_field_var(struct field_var *field_var)
3250 {
3251 	if (!field_var)
3252 		return;
3253 
3254 	destroy_hist_field(field_var->var, 0);
3255 	destroy_hist_field(field_var->val, 0);
3256 
3257 	kfree(field_var);
3258 }
3259 
3260 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3261 {
3262 	unsigned int i;
3263 
3264 	for (i = 0; i < hist_data->n_field_vars; i++)
3265 		destroy_field_var(hist_data->field_vars[i]);
3266 
3267 	for (i = 0; i < hist_data->n_save_vars; i++)
3268 		destroy_field_var(hist_data->save_vars[i]);
3269 }
3270 
3271 static void save_field_var(struct hist_trigger_data *hist_data,
3272 			   struct field_var *field_var)
3273 {
3274 	hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3275 
3276 	if (field_var->val->flags & HIST_FIELD_FL_STRING)
3277 		hist_data->n_field_var_str++;
3278 }
3279 
3280 
3281 static int check_synth_field(struct synth_event *event,
3282 			     struct hist_field *hist_field,
3283 			     unsigned int field_pos)
3284 {
3285 	struct synth_field *field;
3286 
3287 	if (field_pos >= event->n_fields)
3288 		return -EINVAL;
3289 
3290 	field = event->fields[field_pos];
3291 
3292 	/*
3293 	 * A dynamic string synth field can accept static or
3294 	 * dynamic. A static string synth field can only accept a
3295 	 * same-sized static string, which is checked for later.
3296 	 */
3297 	if (strstr(hist_field->type, "char[") && field->is_string
3298 	    && field->is_dynamic)
3299 		return 0;
3300 
3301 	if (strcmp(field->type, hist_field->type) != 0) {
3302 		if (field->size != hist_field->size ||
3303 		    field->is_signed != hist_field->is_signed)
3304 			return -EINVAL;
3305 	}
3306 
3307 	return 0;
3308 }
3309 
3310 static struct hist_field *
3311 trace_action_find_var(struct hist_trigger_data *hist_data,
3312 		      struct action_data *data,
3313 		      char *system, char *event, char *var)
3314 {
3315 	struct trace_array *tr = hist_data->event_file->tr;
3316 	struct hist_field *hist_field;
3317 
3318 	var++; /* skip '$' */
3319 
3320 	hist_field = find_target_event_var(hist_data, system, event, var);
3321 	if (!hist_field) {
3322 		if (!system && data->handler == HANDLER_ONMATCH) {
3323 			system = data->match_data.event_system;
3324 			event = data->match_data.event;
3325 		}
3326 
3327 		hist_field = find_event_var(hist_data, system, event, var);
3328 	}
3329 
3330 	if (!hist_field)
3331 		hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3332 
3333 	return hist_field;
3334 }
3335 
3336 static struct hist_field *
3337 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3338 			      struct action_data *data, char *system,
3339 			      char *event, char *var)
3340 {
3341 	struct hist_field *hist_field = NULL;
3342 	struct field_var *field_var;
3343 
3344 	/*
3345 	 * First try to create a field var on the target event (the
3346 	 * currently being defined).  This will create a variable for
3347 	 * unqualified fields on the target event, or if qualified,
3348 	 * target fields that have qualified names matching the target.
3349 	 */
3350 	field_var = create_target_field_var(hist_data, system, event, var);
3351 
3352 	if (field_var && !IS_ERR(field_var)) {
3353 		save_field_var(hist_data, field_var);
3354 		hist_field = field_var->var;
3355 	} else {
3356 		field_var = NULL;
3357 		/*
3358 		 * If no explicit system.event is specfied, default to
3359 		 * looking for fields on the onmatch(system.event.xxx)
3360 		 * event.
3361 		 */
3362 		if (!system && data->handler == HANDLER_ONMATCH) {
3363 			system = data->match_data.event_system;
3364 			event = data->match_data.event;
3365 		}
3366 
3367 		/*
3368 		 * At this point, we're looking at a field on another
3369 		 * event.  Because we can't modify a hist trigger on
3370 		 * another event to add a variable for a field, we need
3371 		 * to create a new trigger on that event and create the
3372 		 * variable at the same time.
3373 		 */
3374 		hist_field = create_field_var_hist(hist_data, system, event, var);
3375 		if (IS_ERR(hist_field))
3376 			goto free;
3377 	}
3378  out:
3379 	return hist_field;
3380  free:
3381 	destroy_field_var(field_var);
3382 	hist_field = NULL;
3383 	goto out;
3384 }
3385 
3386 static int trace_action_create(struct hist_trigger_data *hist_data,
3387 			       struct action_data *data)
3388 {
3389 	struct trace_array *tr = hist_data->event_file->tr;
3390 	char *event_name, *param, *system = NULL;
3391 	struct hist_field *hist_field, *var_ref;
3392 	unsigned int i;
3393 	unsigned int field_pos = 0;
3394 	struct synth_event *event;
3395 	char *synth_event_name;
3396 	int var_ref_idx, ret = 0;
3397 
3398 	lockdep_assert_held(&event_mutex);
3399 
3400 	if (data->use_trace_keyword)
3401 		synth_event_name = data->synth_event_name;
3402 	else
3403 		synth_event_name = data->action_name;
3404 
3405 	event = find_synth_event(synth_event_name);
3406 	if (!event) {
3407 		hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3408 		return -EINVAL;
3409 	}
3410 
3411 	event->ref++;
3412 
3413 	for (i = 0; i < data->n_params; i++) {
3414 		char *p;
3415 
3416 		p = param = kstrdup(data->params[i], GFP_KERNEL);
3417 		if (!param) {
3418 			ret = -ENOMEM;
3419 			goto err;
3420 		}
3421 
3422 		system = strsep(&param, ".");
3423 		if (!param) {
3424 			param = (char *)system;
3425 			system = event_name = NULL;
3426 		} else {
3427 			event_name = strsep(&param, ".");
3428 			if (!param) {
3429 				kfree(p);
3430 				ret = -EINVAL;
3431 				goto err;
3432 			}
3433 		}
3434 
3435 		if (param[0] == '$')
3436 			hist_field = trace_action_find_var(hist_data, data,
3437 							   system, event_name,
3438 							   param);
3439 		else
3440 			hist_field = trace_action_create_field_var(hist_data,
3441 								   data,
3442 								   system,
3443 								   event_name,
3444 								   param);
3445 
3446 		if (!hist_field) {
3447 			kfree(p);
3448 			ret = -EINVAL;
3449 			goto err;
3450 		}
3451 
3452 		if (check_synth_field(event, hist_field, field_pos) == 0) {
3453 			var_ref = create_var_ref(hist_data, hist_field,
3454 						 system, event_name);
3455 			if (!var_ref) {
3456 				kfree(p);
3457 				ret = -ENOMEM;
3458 				goto err;
3459 			}
3460 
3461 			var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3462 			if (WARN_ON(var_ref_idx < 0)) {
3463 				ret = var_ref_idx;
3464 				goto err;
3465 			}
3466 
3467 			data->var_ref_idx[i] = var_ref_idx;
3468 
3469 			field_pos++;
3470 			kfree(p);
3471 			continue;
3472 		}
3473 
3474 		hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3475 		kfree(p);
3476 		ret = -EINVAL;
3477 		goto err;
3478 	}
3479 
3480 	if (field_pos != event->n_fields) {
3481 		hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3482 		ret = -EINVAL;
3483 		goto err;
3484 	}
3485 
3486 	data->synth_event = event;
3487  out:
3488 	return ret;
3489  err:
3490 	event->ref--;
3491 
3492 	goto out;
3493 }
3494 
3495 static int action_create(struct hist_trigger_data *hist_data,
3496 			 struct action_data *data)
3497 {
3498 	struct trace_event_file *file = hist_data->event_file;
3499 	struct trace_array *tr = file->tr;
3500 	struct track_data *track_data;
3501 	struct field_var *field_var;
3502 	unsigned int i;
3503 	char *param;
3504 	int ret = 0;
3505 
3506 	if (data->action == ACTION_TRACE)
3507 		return trace_action_create(hist_data, data);
3508 
3509 	if (data->action == ACTION_SNAPSHOT) {
3510 		track_data = track_data_alloc(hist_data->key_size, data, hist_data);
3511 		if (IS_ERR(track_data)) {
3512 			ret = PTR_ERR(track_data);
3513 			goto out;
3514 		}
3515 
3516 		ret = tracing_snapshot_cond_enable(file->tr, track_data,
3517 						   cond_snapshot_update);
3518 		if (ret)
3519 			track_data_free(track_data);
3520 
3521 		goto out;
3522 	}
3523 
3524 	if (data->action == ACTION_SAVE) {
3525 		if (hist_data->n_save_vars) {
3526 			ret = -EEXIST;
3527 			hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
3528 			goto out;
3529 		}
3530 
3531 		for (i = 0; i < data->n_params; i++) {
3532 			param = kstrdup(data->params[i], GFP_KERNEL);
3533 			if (!param) {
3534 				ret = -ENOMEM;
3535 				goto out;
3536 			}
3537 
3538 			field_var = create_target_field_var(hist_data, NULL, NULL, param);
3539 			if (IS_ERR(field_var)) {
3540 				hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
3541 					 errpos(param));
3542 				ret = PTR_ERR(field_var);
3543 				kfree(param);
3544 				goto out;
3545 			}
3546 
3547 			hist_data->save_vars[hist_data->n_save_vars++] = field_var;
3548 			if (field_var->val->flags & HIST_FIELD_FL_STRING)
3549 				hist_data->n_save_var_str++;
3550 			kfree(param);
3551 		}
3552 	}
3553  out:
3554 	return ret;
3555 }
3556 
3557 static int onmatch_create(struct hist_trigger_data *hist_data,
3558 			  struct action_data *data)
3559 {
3560 	return action_create(hist_data, data);
3561 }
3562 
3563 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3564 {
3565 	char *match_event, *match_event_system;
3566 	struct action_data *data;
3567 	int ret = -EINVAL;
3568 
3569 	data = kzalloc(sizeof(*data), GFP_KERNEL);
3570 	if (!data)
3571 		return ERR_PTR(-ENOMEM);
3572 
3573 	match_event = strsep(&str, ")");
3574 	if (!match_event || !str) {
3575 		hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
3576 		goto free;
3577 	}
3578 
3579 	match_event_system = strsep(&match_event, ".");
3580 	if (!match_event) {
3581 		hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
3582 		goto free;
3583 	}
3584 
3585 	if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3586 		hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
3587 		goto free;
3588 	}
3589 
3590 	data->match_data.event = kstrdup(match_event, GFP_KERNEL);
3591 	if (!data->match_data.event) {
3592 		ret = -ENOMEM;
3593 		goto free;
3594 	}
3595 
3596 	data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
3597 	if (!data->match_data.event_system) {
3598 		ret = -ENOMEM;
3599 		goto free;
3600 	}
3601 
3602 	ret = action_parse(tr, str, data, HANDLER_ONMATCH);
3603 	if (ret)
3604 		goto free;
3605  out:
3606 	return data;
3607  free:
3608 	onmatch_destroy(data);
3609 	data = ERR_PTR(ret);
3610 	goto out;
3611 }
3612 
3613 static int create_hitcount_val(struct hist_trigger_data *hist_data)
3614 {
3615 	hist_data->fields[HITCOUNT_IDX] =
3616 		create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
3617 	if (!hist_data->fields[HITCOUNT_IDX])
3618 		return -ENOMEM;
3619 
3620 	hist_data->n_vals++;
3621 	hist_data->n_fields++;
3622 
3623 	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3624 		return -EINVAL;
3625 
3626 	return 0;
3627 }
3628 
3629 static int __create_val_field(struct hist_trigger_data *hist_data,
3630 			      unsigned int val_idx,
3631 			      struct trace_event_file *file,
3632 			      char *var_name, char *field_str,
3633 			      unsigned long flags)
3634 {
3635 	struct hist_field *hist_field;
3636 	int ret = 0;
3637 
3638 	hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3639 	if (IS_ERR(hist_field)) {
3640 		ret = PTR_ERR(hist_field);
3641 		goto out;
3642 	}
3643 
3644 	hist_data->fields[val_idx] = hist_field;
3645 
3646 	++hist_data->n_vals;
3647 	++hist_data->n_fields;
3648 
3649 	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3650 		ret = -EINVAL;
3651  out:
3652 	return ret;
3653 }
3654 
3655 static int create_val_field(struct hist_trigger_data *hist_data,
3656 			    unsigned int val_idx,
3657 			    struct trace_event_file *file,
3658 			    char *field_str)
3659 {
3660 	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3661 		return -EINVAL;
3662 
3663 	return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3664 }
3665 
3666 static int create_var_field(struct hist_trigger_data *hist_data,
3667 			    unsigned int val_idx,
3668 			    struct trace_event_file *file,
3669 			    char *var_name, char *expr_str)
3670 {
3671 	struct trace_array *tr = hist_data->event_file->tr;
3672 	unsigned long flags = 0;
3673 	int ret;
3674 
3675 	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3676 		return -EINVAL;
3677 
3678 	if (find_var(hist_data, file, var_name) && !hist_data->remove) {
3679 		hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
3680 		return -EINVAL;
3681 	}
3682 
3683 	flags |= HIST_FIELD_FL_VAR;
3684 	hist_data->n_vars++;
3685 	if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3686 		return -EINVAL;
3687 
3688 	ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3689 
3690 	if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_STRING)
3691 		hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
3692 
3693 	return ret;
3694 }
3695 
3696 static int create_val_fields(struct hist_trigger_data *hist_data,
3697 			     struct trace_event_file *file)
3698 {
3699 	char *fields_str, *field_str;
3700 	unsigned int i, j = 1;
3701 	int ret;
3702 
3703 	ret = create_hitcount_val(hist_data);
3704 	if (ret)
3705 		goto out;
3706 
3707 	fields_str = hist_data->attrs->vals_str;
3708 	if (!fields_str)
3709 		goto out;
3710 
3711 	for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3712 		     j < TRACING_MAP_VALS_MAX; i++) {
3713 		field_str = strsep(&fields_str, ",");
3714 		if (!field_str)
3715 			break;
3716 
3717 		if (strcmp(field_str, "hitcount") == 0)
3718 			continue;
3719 
3720 		ret = create_val_field(hist_data, j++, file, field_str);
3721 		if (ret)
3722 			goto out;
3723 	}
3724 
3725 	if (fields_str && (strcmp(fields_str, "hitcount") != 0))
3726 		ret = -EINVAL;
3727  out:
3728 	return ret;
3729 }
3730 
3731 static int create_key_field(struct hist_trigger_data *hist_data,
3732 			    unsigned int key_idx,
3733 			    unsigned int key_offset,
3734 			    struct trace_event_file *file,
3735 			    char *field_str)
3736 {
3737 	struct trace_array *tr = hist_data->event_file->tr;
3738 	struct hist_field *hist_field = NULL;
3739 	unsigned long flags = 0;
3740 	unsigned int key_size;
3741 	int ret = 0;
3742 
3743 	if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
3744 		return -EINVAL;
3745 
3746 	flags |= HIST_FIELD_FL_KEY;
3747 
3748 	if (strcmp(field_str, "stacktrace") == 0) {
3749 		flags |= HIST_FIELD_FL_STACKTRACE;
3750 		key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
3751 		hist_field = create_hist_field(hist_data, NULL, flags, NULL);
3752 	} else {
3753 		hist_field = parse_expr(hist_data, file, field_str, flags,
3754 					NULL, 0);
3755 		if (IS_ERR(hist_field)) {
3756 			ret = PTR_ERR(hist_field);
3757 			goto out;
3758 		}
3759 
3760 		if (field_has_hist_vars(hist_field, 0))	{
3761 			hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
3762 			destroy_hist_field(hist_field, 0);
3763 			ret = -EINVAL;
3764 			goto out;
3765 		}
3766 
3767 		key_size = hist_field->size;
3768 	}
3769 
3770 	hist_data->fields[key_idx] = hist_field;
3771 
3772 	key_size = ALIGN(key_size, sizeof(u64));
3773 	hist_data->fields[key_idx]->size = key_size;
3774 	hist_data->fields[key_idx]->offset = key_offset;
3775 
3776 	hist_data->key_size += key_size;
3777 
3778 	if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
3779 		ret = -EINVAL;
3780 		goto out;
3781 	}
3782 
3783 	hist_data->n_keys++;
3784 	hist_data->n_fields++;
3785 
3786 	if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
3787 		return -EINVAL;
3788 
3789 	ret = key_size;
3790  out:
3791 	return ret;
3792 }
3793 
3794 static int create_key_fields(struct hist_trigger_data *hist_data,
3795 			     struct trace_event_file *file)
3796 {
3797 	unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
3798 	char *fields_str, *field_str;
3799 	int ret = -EINVAL;
3800 
3801 	fields_str = hist_data->attrs->keys_str;
3802 	if (!fields_str)
3803 		goto out;
3804 
3805 	for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
3806 		field_str = strsep(&fields_str, ",");
3807 		if (!field_str)
3808 			break;
3809 		ret = create_key_field(hist_data, i, key_offset,
3810 				       file, field_str);
3811 		if (ret < 0)
3812 			goto out;
3813 		key_offset += ret;
3814 	}
3815 	if (fields_str) {
3816 		ret = -EINVAL;
3817 		goto out;
3818 	}
3819 	ret = 0;
3820  out:
3821 	return ret;
3822 }
3823 
3824 static int create_var_fields(struct hist_trigger_data *hist_data,
3825 			     struct trace_event_file *file)
3826 {
3827 	unsigned int i, j = hist_data->n_vals;
3828 	int ret = 0;
3829 
3830 	unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
3831 
3832 	for (i = 0; i < n_vars; i++) {
3833 		char *var_name = hist_data->attrs->var_defs.name[i];
3834 		char *expr = hist_data->attrs->var_defs.expr[i];
3835 
3836 		ret = create_var_field(hist_data, j++, file, var_name, expr);
3837 		if (ret)
3838 			goto out;
3839 	}
3840  out:
3841 	return ret;
3842 }
3843 
3844 static void free_var_defs(struct hist_trigger_data *hist_data)
3845 {
3846 	unsigned int i;
3847 
3848 	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
3849 		kfree(hist_data->attrs->var_defs.name[i]);
3850 		kfree(hist_data->attrs->var_defs.expr[i]);
3851 	}
3852 
3853 	hist_data->attrs->var_defs.n_vars = 0;
3854 }
3855 
3856 static int parse_var_defs(struct hist_trigger_data *hist_data)
3857 {
3858 	struct trace_array *tr = hist_data->event_file->tr;
3859 	char *s, *str, *var_name, *field_str;
3860 	unsigned int i, j, n_vars = 0;
3861 	int ret = 0;
3862 
3863 	for (i = 0; i < hist_data->attrs->n_assignments; i++) {
3864 		str = hist_data->attrs->assignment_str[i];
3865 		for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
3866 			field_str = strsep(&str, ",");
3867 			if (!field_str)
3868 				break;
3869 
3870 			var_name = strsep(&field_str, "=");
3871 			if (!var_name || !field_str) {
3872 				hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
3873 					 errpos(var_name));
3874 				ret = -EINVAL;
3875 				goto free;
3876 			}
3877 
3878 			if (n_vars == TRACING_MAP_VARS_MAX) {
3879 				hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
3880 				ret = -EINVAL;
3881 				goto free;
3882 			}
3883 
3884 			s = kstrdup(var_name, GFP_KERNEL);
3885 			if (!s) {
3886 				ret = -ENOMEM;
3887 				goto free;
3888 			}
3889 			hist_data->attrs->var_defs.name[n_vars] = s;
3890 
3891 			s = kstrdup(field_str, GFP_KERNEL);
3892 			if (!s) {
3893 				ret = -ENOMEM;
3894 				goto free;
3895 			}
3896 			hist_data->attrs->var_defs.expr[n_vars++] = s;
3897 
3898 			hist_data->attrs->var_defs.n_vars = n_vars;
3899 		}
3900 	}
3901 
3902 	return ret;
3903  free:
3904 	free_var_defs(hist_data);
3905 
3906 	return ret;
3907 }
3908 
3909 static int create_hist_fields(struct hist_trigger_data *hist_data,
3910 			      struct trace_event_file *file)
3911 {
3912 	int ret;
3913 
3914 	ret = parse_var_defs(hist_data);
3915 	if (ret)
3916 		goto out;
3917 
3918 	ret = create_val_fields(hist_data, file);
3919 	if (ret)
3920 		goto out;
3921 
3922 	ret = create_var_fields(hist_data, file);
3923 	if (ret)
3924 		goto out;
3925 
3926 	ret = create_key_fields(hist_data, file);
3927 	if (ret)
3928 		goto out;
3929  out:
3930 	free_var_defs(hist_data);
3931 
3932 	return ret;
3933 }
3934 
3935 static int is_descending(struct trace_array *tr, const char *str)
3936 {
3937 	if (!str)
3938 		return 0;
3939 
3940 	if (strcmp(str, "descending") == 0)
3941 		return 1;
3942 
3943 	if (strcmp(str, "ascending") == 0)
3944 		return 0;
3945 
3946 	hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
3947 
3948 	return -EINVAL;
3949 }
3950 
3951 static int create_sort_keys(struct hist_trigger_data *hist_data)
3952 {
3953 	struct trace_array *tr = hist_data->event_file->tr;
3954 	char *fields_str = hist_data->attrs->sort_key_str;
3955 	struct tracing_map_sort_key *sort_key;
3956 	int descending, ret = 0;
3957 	unsigned int i, j, k;
3958 
3959 	hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
3960 
3961 	if (!fields_str)
3962 		goto out;
3963 
3964 	for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
3965 		struct hist_field *hist_field;
3966 		char *field_str, *field_name;
3967 		const char *test_name;
3968 
3969 		sort_key = &hist_data->sort_keys[i];
3970 
3971 		field_str = strsep(&fields_str, ",");
3972 		if (!field_str)
3973 			break;
3974 
3975 		if (!*field_str) {
3976 			ret = -EINVAL;
3977 			hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
3978 			break;
3979 		}
3980 
3981 		if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
3982 			hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
3983 			ret = -EINVAL;
3984 			break;
3985 		}
3986 
3987 		field_name = strsep(&field_str, ".");
3988 		if (!field_name || !*field_name) {
3989 			ret = -EINVAL;
3990 			hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
3991 			break;
3992 		}
3993 
3994 		if (strcmp(field_name, "hitcount") == 0) {
3995 			descending = is_descending(tr, field_str);
3996 			if (descending < 0) {
3997 				ret = descending;
3998 				break;
3999 			}
4000 			sort_key->descending = descending;
4001 			continue;
4002 		}
4003 
4004 		for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4005 			unsigned int idx;
4006 
4007 			hist_field = hist_data->fields[j];
4008 			if (hist_field->flags & HIST_FIELD_FL_VAR)
4009 				continue;
4010 
4011 			idx = k++;
4012 
4013 			test_name = hist_field_name(hist_field, 0);
4014 
4015 			if (strcmp(field_name, test_name) == 0) {
4016 				sort_key->field_idx = idx;
4017 				descending = is_descending(tr, field_str);
4018 				if (descending < 0) {
4019 					ret = descending;
4020 					goto out;
4021 				}
4022 				sort_key->descending = descending;
4023 				break;
4024 			}
4025 		}
4026 		if (j == hist_data->n_fields) {
4027 			ret = -EINVAL;
4028 			hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4029 			break;
4030 		}
4031 	}
4032 
4033 	hist_data->n_sort_keys = i;
4034  out:
4035 	return ret;
4036 }
4037 
4038 static void destroy_actions(struct hist_trigger_data *hist_data)
4039 {
4040 	unsigned int i;
4041 
4042 	for (i = 0; i < hist_data->n_actions; i++) {
4043 		struct action_data *data = hist_data->actions[i];
4044 
4045 		if (data->handler == HANDLER_ONMATCH)
4046 			onmatch_destroy(data);
4047 		else if (data->handler == HANDLER_ONMAX ||
4048 			 data->handler == HANDLER_ONCHANGE)
4049 			track_data_destroy(hist_data, data);
4050 		else
4051 			kfree(data);
4052 	}
4053 }
4054 
4055 static int parse_actions(struct hist_trigger_data *hist_data)
4056 {
4057 	struct trace_array *tr = hist_data->event_file->tr;
4058 	struct action_data *data;
4059 	unsigned int i;
4060 	int ret = 0;
4061 	char *str;
4062 	int len;
4063 
4064 	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4065 		str = hist_data->attrs->action_str[i];
4066 
4067 		if ((len = str_has_prefix(str, "onmatch("))) {
4068 			char *action_str = str + len;
4069 
4070 			data = onmatch_parse(tr, action_str);
4071 			if (IS_ERR(data)) {
4072 				ret = PTR_ERR(data);
4073 				break;
4074 			}
4075 		} else if ((len = str_has_prefix(str, "onmax("))) {
4076 			char *action_str = str + len;
4077 
4078 			data = track_data_parse(hist_data, action_str,
4079 						HANDLER_ONMAX);
4080 			if (IS_ERR(data)) {
4081 				ret = PTR_ERR(data);
4082 				break;
4083 			}
4084 		} else if ((len = str_has_prefix(str, "onchange("))) {
4085 			char *action_str = str + len;
4086 
4087 			data = track_data_parse(hist_data, action_str,
4088 						HANDLER_ONCHANGE);
4089 			if (IS_ERR(data)) {
4090 				ret = PTR_ERR(data);
4091 				break;
4092 			}
4093 		} else {
4094 			ret = -EINVAL;
4095 			break;
4096 		}
4097 
4098 		hist_data->actions[hist_data->n_actions++] = data;
4099 	}
4100 
4101 	return ret;
4102 }
4103 
4104 static int create_actions(struct hist_trigger_data *hist_data)
4105 {
4106 	struct action_data *data;
4107 	unsigned int i;
4108 	int ret = 0;
4109 
4110 	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4111 		data = hist_data->actions[i];
4112 
4113 		if (data->handler == HANDLER_ONMATCH) {
4114 			ret = onmatch_create(hist_data, data);
4115 			if (ret)
4116 				break;
4117 		} else if (data->handler == HANDLER_ONMAX ||
4118 			   data->handler == HANDLER_ONCHANGE) {
4119 			ret = track_data_create(hist_data, data);
4120 			if (ret)
4121 				break;
4122 		} else {
4123 			ret = -EINVAL;
4124 			break;
4125 		}
4126 	}
4127 
4128 	return ret;
4129 }
4130 
4131 static void print_actions(struct seq_file *m,
4132 			  struct hist_trigger_data *hist_data,
4133 			  struct tracing_map_elt *elt)
4134 {
4135 	unsigned int i;
4136 
4137 	for (i = 0; i < hist_data->n_actions; i++) {
4138 		struct action_data *data = hist_data->actions[i];
4139 
4140 		if (data->action == ACTION_SNAPSHOT)
4141 			continue;
4142 
4143 		if (data->handler == HANDLER_ONMAX ||
4144 		    data->handler == HANDLER_ONCHANGE)
4145 			track_data_print(m, hist_data, elt, data);
4146 	}
4147 }
4148 
4149 static void print_action_spec(struct seq_file *m,
4150 			      struct hist_trigger_data *hist_data,
4151 			      struct action_data *data)
4152 {
4153 	unsigned int i;
4154 
4155 	if (data->action == ACTION_SAVE) {
4156 		for (i = 0; i < hist_data->n_save_vars; i++) {
4157 			seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4158 			if (i < hist_data->n_save_vars - 1)
4159 				seq_puts(m, ",");
4160 		}
4161 	} else if (data->action == ACTION_TRACE) {
4162 		if (data->use_trace_keyword)
4163 			seq_printf(m, "%s", data->synth_event_name);
4164 		for (i = 0; i < data->n_params; i++) {
4165 			if (i || data->use_trace_keyword)
4166 				seq_puts(m, ",");
4167 			seq_printf(m, "%s", data->params[i]);
4168 		}
4169 	}
4170 }
4171 
4172 static void print_track_data_spec(struct seq_file *m,
4173 				  struct hist_trigger_data *hist_data,
4174 				  struct action_data *data)
4175 {
4176 	if (data->handler == HANDLER_ONMAX)
4177 		seq_puts(m, ":onmax(");
4178 	else if (data->handler == HANDLER_ONCHANGE)
4179 		seq_puts(m, ":onchange(");
4180 	seq_printf(m, "%s", data->track_data.var_str);
4181 	seq_printf(m, ").%s(", data->action_name);
4182 
4183 	print_action_spec(m, hist_data, data);
4184 
4185 	seq_puts(m, ")");
4186 }
4187 
4188 static void print_onmatch_spec(struct seq_file *m,
4189 			       struct hist_trigger_data *hist_data,
4190 			       struct action_data *data)
4191 {
4192 	seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4193 		   data->match_data.event);
4194 
4195 	seq_printf(m, "%s(", data->action_name);
4196 
4197 	print_action_spec(m, hist_data, data);
4198 
4199 	seq_puts(m, ")");
4200 }
4201 
4202 static bool actions_match(struct hist_trigger_data *hist_data,
4203 			  struct hist_trigger_data *hist_data_test)
4204 {
4205 	unsigned int i, j;
4206 
4207 	if (hist_data->n_actions != hist_data_test->n_actions)
4208 		return false;
4209 
4210 	for (i = 0; i < hist_data->n_actions; i++) {
4211 		struct action_data *data = hist_data->actions[i];
4212 		struct action_data *data_test = hist_data_test->actions[i];
4213 		char *action_name, *action_name_test;
4214 
4215 		if (data->handler != data_test->handler)
4216 			return false;
4217 		if (data->action != data_test->action)
4218 			return false;
4219 
4220 		if (data->n_params != data_test->n_params)
4221 			return false;
4222 
4223 		for (j = 0; j < data->n_params; j++) {
4224 			if (strcmp(data->params[j], data_test->params[j]) != 0)
4225 				return false;
4226 		}
4227 
4228 		if (data->use_trace_keyword)
4229 			action_name = data->synth_event_name;
4230 		else
4231 			action_name = data->action_name;
4232 
4233 		if (data_test->use_trace_keyword)
4234 			action_name_test = data_test->synth_event_name;
4235 		else
4236 			action_name_test = data_test->action_name;
4237 
4238 		if (strcmp(action_name, action_name_test) != 0)
4239 			return false;
4240 
4241 		if (data->handler == HANDLER_ONMATCH) {
4242 			if (strcmp(data->match_data.event_system,
4243 				   data_test->match_data.event_system) != 0)
4244 				return false;
4245 			if (strcmp(data->match_data.event,
4246 				   data_test->match_data.event) != 0)
4247 				return false;
4248 		} else if (data->handler == HANDLER_ONMAX ||
4249 			   data->handler == HANDLER_ONCHANGE) {
4250 			if (strcmp(data->track_data.var_str,
4251 				   data_test->track_data.var_str) != 0)
4252 				return false;
4253 		}
4254 	}
4255 
4256 	return true;
4257 }
4258 
4259 
4260 static void print_actions_spec(struct seq_file *m,
4261 			       struct hist_trigger_data *hist_data)
4262 {
4263 	unsigned int i;
4264 
4265 	for (i = 0; i < hist_data->n_actions; i++) {
4266 		struct action_data *data = hist_data->actions[i];
4267 
4268 		if (data->handler == HANDLER_ONMATCH)
4269 			print_onmatch_spec(m, hist_data, data);
4270 		else if (data->handler == HANDLER_ONMAX ||
4271 			 data->handler == HANDLER_ONCHANGE)
4272 			print_track_data_spec(m, hist_data, data);
4273 	}
4274 }
4275 
4276 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4277 {
4278 	unsigned int i;
4279 
4280 	for (i = 0; i < hist_data->n_field_var_hists; i++) {
4281 		kfree(hist_data->field_var_hists[i]->cmd);
4282 		kfree(hist_data->field_var_hists[i]);
4283 	}
4284 }
4285 
4286 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4287 {
4288 	if (!hist_data)
4289 		return;
4290 
4291 	destroy_hist_trigger_attrs(hist_data->attrs);
4292 	destroy_hist_fields(hist_data);
4293 	tracing_map_destroy(hist_data->map);
4294 
4295 	destroy_actions(hist_data);
4296 	destroy_field_vars(hist_data);
4297 	destroy_field_var_hists(hist_data);
4298 
4299 	kfree(hist_data);
4300 }
4301 
4302 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4303 {
4304 	struct tracing_map *map = hist_data->map;
4305 	struct ftrace_event_field *field;
4306 	struct hist_field *hist_field;
4307 	int i, idx = 0;
4308 
4309 	for_each_hist_field(i, hist_data) {
4310 		hist_field = hist_data->fields[i];
4311 		if (hist_field->flags & HIST_FIELD_FL_KEY) {
4312 			tracing_map_cmp_fn_t cmp_fn;
4313 
4314 			field = hist_field->field;
4315 
4316 			if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4317 				cmp_fn = tracing_map_cmp_none;
4318 			else if (!field)
4319 				cmp_fn = tracing_map_cmp_num(hist_field->size,
4320 							     hist_field->is_signed);
4321 			else if (is_string_field(field))
4322 				cmp_fn = tracing_map_cmp_string;
4323 			else
4324 				cmp_fn = tracing_map_cmp_num(field->size,
4325 							     field->is_signed);
4326 			idx = tracing_map_add_key_field(map,
4327 							hist_field->offset,
4328 							cmp_fn);
4329 		} else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4330 			idx = tracing_map_add_sum_field(map);
4331 
4332 		if (idx < 0)
4333 			return idx;
4334 
4335 		if (hist_field->flags & HIST_FIELD_FL_VAR) {
4336 			idx = tracing_map_add_var(map);
4337 			if (idx < 0)
4338 				return idx;
4339 			hist_field->var.idx = idx;
4340 			hist_field->var.hist_data = hist_data;
4341 		}
4342 	}
4343 
4344 	return 0;
4345 }
4346 
4347 static struct hist_trigger_data *
4348 create_hist_data(unsigned int map_bits,
4349 		 struct hist_trigger_attrs *attrs,
4350 		 struct trace_event_file *file,
4351 		 bool remove)
4352 {
4353 	const struct tracing_map_ops *map_ops = NULL;
4354 	struct hist_trigger_data *hist_data;
4355 	int ret = 0;
4356 
4357 	hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4358 	if (!hist_data)
4359 		return ERR_PTR(-ENOMEM);
4360 
4361 	hist_data->attrs = attrs;
4362 	hist_data->remove = remove;
4363 	hist_data->event_file = file;
4364 
4365 	ret = parse_actions(hist_data);
4366 	if (ret)
4367 		goto free;
4368 
4369 	ret = create_hist_fields(hist_data, file);
4370 	if (ret)
4371 		goto free;
4372 
4373 	ret = create_sort_keys(hist_data);
4374 	if (ret)
4375 		goto free;
4376 
4377 	map_ops = &hist_trigger_elt_data_ops;
4378 
4379 	hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4380 					    map_ops, hist_data);
4381 	if (IS_ERR(hist_data->map)) {
4382 		ret = PTR_ERR(hist_data->map);
4383 		hist_data->map = NULL;
4384 		goto free;
4385 	}
4386 
4387 	ret = create_tracing_map_fields(hist_data);
4388 	if (ret)
4389 		goto free;
4390  out:
4391 	return hist_data;
4392  free:
4393 	hist_data->attrs = NULL;
4394 
4395 	destroy_hist_data(hist_data);
4396 
4397 	hist_data = ERR_PTR(ret);
4398 
4399 	goto out;
4400 }
4401 
4402 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4403 				    struct tracing_map_elt *elt, void *rec,
4404 				    struct ring_buffer_event *rbe,
4405 				    u64 *var_ref_vals)
4406 {
4407 	struct hist_elt_data *elt_data;
4408 	struct hist_field *hist_field;
4409 	unsigned int i, var_idx;
4410 	u64 hist_val;
4411 
4412 	elt_data = elt->private_data;
4413 	elt_data->var_ref_vals = var_ref_vals;
4414 
4415 	for_each_hist_val_field(i, hist_data) {
4416 		hist_field = hist_data->fields[i];
4417 		hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4418 		if (hist_field->flags & HIST_FIELD_FL_VAR) {
4419 			var_idx = hist_field->var.idx;
4420 
4421 			if (hist_field->flags & HIST_FIELD_FL_STRING) {
4422 				unsigned int str_start, var_str_idx, idx;
4423 				char *str, *val_str;
4424 
4425 				str_start = hist_data->n_field_var_str +
4426 					hist_data->n_save_var_str;
4427 				var_str_idx = hist_field->var_str_idx;
4428 				idx = str_start + var_str_idx;
4429 
4430 				str = elt_data->field_var_str[idx];
4431 				val_str = (char *)(uintptr_t)hist_val;
4432 				strscpy(str, val_str, STR_VAR_LEN_MAX);
4433 
4434 				hist_val = (u64)(uintptr_t)str;
4435 			}
4436 			tracing_map_set_var(elt, var_idx, hist_val);
4437 			continue;
4438 		}
4439 		tracing_map_update_sum(elt, i, hist_val);
4440 	}
4441 
4442 	for_each_hist_key_field(i, hist_data) {
4443 		hist_field = hist_data->fields[i];
4444 		if (hist_field->flags & HIST_FIELD_FL_VAR) {
4445 			hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4446 			var_idx = hist_field->var.idx;
4447 			tracing_map_set_var(elt, var_idx, hist_val);
4448 		}
4449 	}
4450 
4451 	update_field_vars(hist_data, elt, rbe, rec);
4452 }
4453 
4454 static inline void add_to_key(char *compound_key, void *key,
4455 			      struct hist_field *key_field, void *rec)
4456 {
4457 	size_t size = key_field->size;
4458 
4459 	if (key_field->flags & HIST_FIELD_FL_STRING) {
4460 		struct ftrace_event_field *field;
4461 
4462 		field = key_field->field;
4463 		if (field->filter_type == FILTER_DYN_STRING)
4464 			size = *(u32 *)(rec + field->offset) >> 16;
4465 		else if (field->filter_type == FILTER_PTR_STRING)
4466 			size = strlen(key);
4467 		else if (field->filter_type == FILTER_STATIC_STRING)
4468 			size = field->size;
4469 
4470 		/* ensure NULL-termination */
4471 		if (size > key_field->size - 1)
4472 			size = key_field->size - 1;
4473 
4474 		strncpy(compound_key + key_field->offset, (char *)key, size);
4475 	} else
4476 		memcpy(compound_key + key_field->offset, key, size);
4477 }
4478 
4479 static void
4480 hist_trigger_actions(struct hist_trigger_data *hist_data,
4481 		     struct tracing_map_elt *elt, void *rec,
4482 		     struct ring_buffer_event *rbe, void *key,
4483 		     u64 *var_ref_vals)
4484 {
4485 	struct action_data *data;
4486 	unsigned int i;
4487 
4488 	for (i = 0; i < hist_data->n_actions; i++) {
4489 		data = hist_data->actions[i];
4490 		data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals);
4491 	}
4492 }
4493 
4494 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
4495 			       struct ring_buffer_event *rbe)
4496 {
4497 	struct hist_trigger_data *hist_data = data->private_data;
4498 	bool use_compound_key = (hist_data->n_keys > 1);
4499 	unsigned long entries[HIST_STACKTRACE_DEPTH];
4500 	u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4501 	char compound_key[HIST_KEY_SIZE_MAX];
4502 	struct tracing_map_elt *elt = NULL;
4503 	struct hist_field *key_field;
4504 	u64 field_contents;
4505 	void *key = NULL;
4506 	unsigned int i;
4507 
4508 	memset(compound_key, 0, hist_data->key_size);
4509 
4510 	for_each_hist_key_field(i, hist_data) {
4511 		key_field = hist_data->fields[i];
4512 
4513 		if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4514 			memset(entries, 0, HIST_STACKTRACE_SIZE);
4515 			stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
4516 					 HIST_STACKTRACE_SKIP);
4517 			key = entries;
4518 		} else {
4519 			field_contents = key_field->fn(key_field, elt, rbe, rec);
4520 			if (key_field->flags & HIST_FIELD_FL_STRING) {
4521 				key = (void *)(unsigned long)field_contents;
4522 				use_compound_key = true;
4523 			} else
4524 				key = (void *)&field_contents;
4525 		}
4526 
4527 		if (use_compound_key)
4528 			add_to_key(compound_key, key, key_field, rec);
4529 	}
4530 
4531 	if (use_compound_key)
4532 		key = compound_key;
4533 
4534 	if (hist_data->n_var_refs &&
4535 	    !resolve_var_refs(hist_data, key, var_ref_vals, false))
4536 		return;
4537 
4538 	elt = tracing_map_insert(hist_data->map, key);
4539 	if (!elt)
4540 		return;
4541 
4542 	hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
4543 
4544 	if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4545 		hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals);
4546 }
4547 
4548 static void hist_trigger_stacktrace_print(struct seq_file *m,
4549 					  unsigned long *stacktrace_entries,
4550 					  unsigned int max_entries)
4551 {
4552 	char str[KSYM_SYMBOL_LEN];
4553 	unsigned int spaces = 8;
4554 	unsigned int i;
4555 
4556 	for (i = 0; i < max_entries; i++) {
4557 		if (!stacktrace_entries[i])
4558 			return;
4559 
4560 		seq_printf(m, "%*c", 1 + spaces, ' ');
4561 		sprint_symbol(str, stacktrace_entries[i]);
4562 		seq_printf(m, "%s\n", str);
4563 	}
4564 }
4565 
4566 static void hist_trigger_print_key(struct seq_file *m,
4567 				   struct hist_trigger_data *hist_data,
4568 				   void *key,
4569 				   struct tracing_map_elt *elt)
4570 {
4571 	struct hist_field *key_field;
4572 	char str[KSYM_SYMBOL_LEN];
4573 	bool multiline = false;
4574 	const char *field_name;
4575 	unsigned int i;
4576 	u64 uval;
4577 
4578 	seq_puts(m, "{ ");
4579 
4580 	for_each_hist_key_field(i, hist_data) {
4581 		key_field = hist_data->fields[i];
4582 
4583 		if (i > hist_data->n_vals)
4584 			seq_puts(m, ", ");
4585 
4586 		field_name = hist_field_name(key_field, 0);
4587 
4588 		if (key_field->flags & HIST_FIELD_FL_HEX) {
4589 			uval = *(u64 *)(key + key_field->offset);
4590 			seq_printf(m, "%s: %llx", field_name, uval);
4591 		} else if (key_field->flags & HIST_FIELD_FL_SYM) {
4592 			uval = *(u64 *)(key + key_field->offset);
4593 			sprint_symbol_no_offset(str, uval);
4594 			seq_printf(m, "%s: [%llx] %-45s", field_name,
4595 				   uval, str);
4596 		} else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4597 			uval = *(u64 *)(key + key_field->offset);
4598 			sprint_symbol(str, uval);
4599 			seq_printf(m, "%s: [%llx] %-55s", field_name,
4600 				   uval, str);
4601 		} else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
4602 			struct hist_elt_data *elt_data = elt->private_data;
4603 			char *comm;
4604 
4605 			if (WARN_ON_ONCE(!elt_data))
4606 				return;
4607 
4608 			comm = elt_data->comm;
4609 
4610 			uval = *(u64 *)(key + key_field->offset);
4611 			seq_printf(m, "%s: %-16s[%10llu]", field_name,
4612 				   comm, uval);
4613 		} else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4614 			const char *syscall_name;
4615 
4616 			uval = *(u64 *)(key + key_field->offset);
4617 			syscall_name = get_syscall_name(uval);
4618 			if (!syscall_name)
4619 				syscall_name = "unknown_syscall";
4620 
4621 			seq_printf(m, "%s: %-30s[%3llu]", field_name,
4622 				   syscall_name, uval);
4623 		} else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4624 			seq_puts(m, "stacktrace:\n");
4625 			hist_trigger_stacktrace_print(m,
4626 						      key + key_field->offset,
4627 						      HIST_STACKTRACE_DEPTH);
4628 			multiline = true;
4629 		} else if (key_field->flags & HIST_FIELD_FL_LOG2) {
4630 			seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4631 				   *(u64 *)(key + key_field->offset));
4632 		} else if (key_field->flags & HIST_FIELD_FL_STRING) {
4633 			seq_printf(m, "%s: %-50s", field_name,
4634 				   (char *)(key + key_field->offset));
4635 		} else {
4636 			uval = *(u64 *)(key + key_field->offset);
4637 			seq_printf(m, "%s: %10llu", field_name, uval);
4638 		}
4639 	}
4640 
4641 	if (!multiline)
4642 		seq_puts(m, " ");
4643 
4644 	seq_puts(m, "}");
4645 }
4646 
4647 static void hist_trigger_entry_print(struct seq_file *m,
4648 				     struct hist_trigger_data *hist_data,
4649 				     void *key,
4650 				     struct tracing_map_elt *elt)
4651 {
4652 	const char *field_name;
4653 	unsigned int i;
4654 
4655 	hist_trigger_print_key(m, hist_data, key, elt);
4656 
4657 	seq_printf(m, " hitcount: %10llu",
4658 		   tracing_map_read_sum(elt, HITCOUNT_IDX));
4659 
4660 	for (i = 1; i < hist_data->n_vals; i++) {
4661 		field_name = hist_field_name(hist_data->fields[i], 0);
4662 
4663 		if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4664 		    hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
4665 			continue;
4666 
4667 		if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
4668 			seq_printf(m, "  %s: %10llx", field_name,
4669 				   tracing_map_read_sum(elt, i));
4670 		} else {
4671 			seq_printf(m, "  %s: %10llu", field_name,
4672 				   tracing_map_read_sum(elt, i));
4673 		}
4674 	}
4675 
4676 	print_actions(m, hist_data, elt);
4677 
4678 	seq_puts(m, "\n");
4679 }
4680 
4681 static int print_entries(struct seq_file *m,
4682 			 struct hist_trigger_data *hist_data)
4683 {
4684 	struct tracing_map_sort_entry **sort_entries = NULL;
4685 	struct tracing_map *map = hist_data->map;
4686 	int i, n_entries;
4687 
4688 	n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4689 					     hist_data->n_sort_keys,
4690 					     &sort_entries);
4691 	if (n_entries < 0)
4692 		return n_entries;
4693 
4694 	for (i = 0; i < n_entries; i++)
4695 		hist_trigger_entry_print(m, hist_data,
4696 					 sort_entries[i]->key,
4697 					 sort_entries[i]->elt);
4698 
4699 	tracing_map_destroy_sort_entries(sort_entries, n_entries);
4700 
4701 	return n_entries;
4702 }
4703 
4704 static void hist_trigger_show(struct seq_file *m,
4705 			      struct event_trigger_data *data, int n)
4706 {
4707 	struct hist_trigger_data *hist_data;
4708 	int n_entries;
4709 
4710 	if (n > 0)
4711 		seq_puts(m, "\n\n");
4712 
4713 	seq_puts(m, "# event histogram\n#\n# trigger info: ");
4714 	data->ops->print(m, data->ops, data);
4715 	seq_puts(m, "#\n\n");
4716 
4717 	hist_data = data->private_data;
4718 	n_entries = print_entries(m, hist_data);
4719 	if (n_entries < 0)
4720 		n_entries = 0;
4721 
4722 	track_data_snapshot_print(m, hist_data);
4723 
4724 	seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
4725 		   (u64)atomic64_read(&hist_data->map->hits),
4726 		   n_entries, (u64)atomic64_read(&hist_data->map->drops));
4727 }
4728 
4729 static int hist_show(struct seq_file *m, void *v)
4730 {
4731 	struct event_trigger_data *data;
4732 	struct trace_event_file *event_file;
4733 	int n = 0, ret = 0;
4734 
4735 	mutex_lock(&event_mutex);
4736 
4737 	event_file = event_file_data(m->private);
4738 	if (unlikely(!event_file)) {
4739 		ret = -ENODEV;
4740 		goto out_unlock;
4741 	}
4742 
4743 	list_for_each_entry(data, &event_file->triggers, list) {
4744 		if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4745 			hist_trigger_show(m, data, n++);
4746 	}
4747 
4748  out_unlock:
4749 	mutex_unlock(&event_mutex);
4750 
4751 	return ret;
4752 }
4753 
4754 static int event_hist_open(struct inode *inode, struct file *file)
4755 {
4756 	int ret;
4757 
4758 	ret = security_locked_down(LOCKDOWN_TRACEFS);
4759 	if (ret)
4760 		return ret;
4761 
4762 	return single_open(file, hist_show, file);
4763 }
4764 
4765 const struct file_operations event_hist_fops = {
4766 	.open = event_hist_open,
4767 	.read = seq_read,
4768 	.llseek = seq_lseek,
4769 	.release = single_release,
4770 };
4771 
4772 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
4773 static void hist_field_debug_show_flags(struct seq_file *m,
4774 					unsigned long flags)
4775 {
4776 	seq_puts(m, "      flags:\n");
4777 
4778 	if (flags & HIST_FIELD_FL_KEY)
4779 		seq_puts(m, "        HIST_FIELD_FL_KEY\n");
4780 	else if (flags & HIST_FIELD_FL_HITCOUNT)
4781 		seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
4782 	else if (flags & HIST_FIELD_FL_VAR)
4783 		seq_puts(m, "        HIST_FIELD_FL_VAR\n");
4784 	else if (flags & HIST_FIELD_FL_VAR_REF)
4785 		seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
4786 	else
4787 		seq_puts(m, "        VAL: normal u64 value\n");
4788 
4789 	if (flags & HIST_FIELD_FL_ALIAS)
4790 		seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
4791 }
4792 
4793 static int hist_field_debug_show(struct seq_file *m,
4794 				 struct hist_field *field, unsigned long flags)
4795 {
4796 	if ((field->flags & flags) != flags) {
4797 		seq_printf(m, "ERROR: bad flags - %lx\n", flags);
4798 		return -EINVAL;
4799 	}
4800 
4801 	hist_field_debug_show_flags(m, field->flags);
4802 	if (field->field)
4803 		seq_printf(m, "      ftrace_event_field name: %s\n",
4804 			   field->field->name);
4805 
4806 	if (field->flags & HIST_FIELD_FL_VAR) {
4807 		seq_printf(m, "      var.name: %s\n", field->var.name);
4808 		seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4809 			   field->var.idx);
4810 	}
4811 
4812 	if (field->flags & HIST_FIELD_FL_ALIAS)
4813 		seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4814 			   field->var_ref_idx);
4815 
4816 	if (field->flags & HIST_FIELD_FL_VAR_REF) {
4817 		seq_printf(m, "      name: %s\n", field->name);
4818 		seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4819 			   field->var.idx);
4820 		seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
4821 		seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4822 			   field->var_ref_idx);
4823 		if (field->system)
4824 			seq_printf(m, "      system: %s\n", field->system);
4825 		if (field->event_name)
4826 			seq_printf(m, "      event_name: %s\n", field->event_name);
4827 	}
4828 
4829 	seq_printf(m, "      type: %s\n", field->type);
4830 	seq_printf(m, "      size: %u\n", field->size);
4831 	seq_printf(m, "      is_signed: %u\n", field->is_signed);
4832 
4833 	return 0;
4834 }
4835 
4836 static int field_var_debug_show(struct seq_file *m,
4837 				struct field_var *field_var, unsigned int i,
4838 				bool save_vars)
4839 {
4840 	const char *vars_name = save_vars ? "save_vars" : "field_vars";
4841 	struct hist_field *field;
4842 	int ret = 0;
4843 
4844 	seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
4845 
4846 	field = field_var->var;
4847 
4848 	seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
4849 
4850 	hist_field_debug_show_flags(m, field->flags);
4851 	seq_printf(m, "      var.name: %s\n", field->var.name);
4852 	seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4853 		   field->var.idx);
4854 
4855 	field = field_var->val;
4856 
4857 	seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
4858 	if (field->field)
4859 		seq_printf(m, "      ftrace_event_field name: %s\n",
4860 			   field->field->name);
4861 	else {
4862 		ret = -EINVAL;
4863 		goto out;
4864 	}
4865 
4866 	seq_printf(m, "      type: %s\n", field->type);
4867 	seq_printf(m, "      size: %u\n", field->size);
4868 	seq_printf(m, "      is_signed: %u\n", field->is_signed);
4869 out:
4870 	return ret;
4871 }
4872 
4873 static int hist_action_debug_show(struct seq_file *m,
4874 				  struct action_data *data, int i)
4875 {
4876 	int ret = 0;
4877 
4878 	if (data->handler == HANDLER_ONMAX ||
4879 	    data->handler == HANDLER_ONCHANGE) {
4880 		seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
4881 		ret = hist_field_debug_show(m, data->track_data.var_ref,
4882 					    HIST_FIELD_FL_VAR_REF);
4883 		if (ret)
4884 			goto out;
4885 
4886 		seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
4887 		ret = hist_field_debug_show(m, data->track_data.track_var,
4888 					    HIST_FIELD_FL_VAR);
4889 		if (ret)
4890 			goto out;
4891 	}
4892 
4893 	if (data->handler == HANDLER_ONMATCH) {
4894 		seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
4895 			   i, data->match_data.event_system);
4896 		seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
4897 			   i, data->match_data.event);
4898 	}
4899 out:
4900 	return ret;
4901 }
4902 
4903 static int hist_actions_debug_show(struct seq_file *m,
4904 				   struct hist_trigger_data *hist_data)
4905 {
4906 	int i, ret = 0;
4907 
4908 	if (hist_data->n_actions)
4909 		seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
4910 
4911 	for (i = 0; i < hist_data->n_actions; i++) {
4912 		struct action_data *action = hist_data->actions[i];
4913 
4914 		ret = hist_action_debug_show(m, action, i);
4915 		if (ret)
4916 			goto out;
4917 	}
4918 
4919 	if (hist_data->n_save_vars)
4920 		seq_puts(m, "\n  save action variables (save() params):\n");
4921 
4922 	for (i = 0; i < hist_data->n_save_vars; i++) {
4923 		ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
4924 		if (ret)
4925 			goto out;
4926 	}
4927 out:
4928 	return ret;
4929 }
4930 
4931 static void hist_trigger_debug_show(struct seq_file *m,
4932 				    struct event_trigger_data *data, int n)
4933 {
4934 	struct hist_trigger_data *hist_data;
4935 	int i, ret;
4936 
4937 	if (n > 0)
4938 		seq_puts(m, "\n\n");
4939 
4940 	seq_puts(m, "# event histogram\n#\n# trigger info: ");
4941 	data->ops->print(m, data->ops, data);
4942 	seq_puts(m, "#\n\n");
4943 
4944 	hist_data = data->private_data;
4945 
4946 	seq_printf(m, "hist_data: %p\n\n", hist_data);
4947 	seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
4948 	seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
4949 	seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
4950 
4951 	seq_puts(m, "\n  val fields:\n\n");
4952 
4953 	seq_puts(m, "    hist_data->fields[0]:\n");
4954 	ret = hist_field_debug_show(m, hist_data->fields[0],
4955 				    HIST_FIELD_FL_HITCOUNT);
4956 	if (ret)
4957 		return;
4958 
4959 	for (i = 1; i < hist_data->n_vals; i++) {
4960 		seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
4961 		ret = hist_field_debug_show(m, hist_data->fields[i], 0);
4962 		if (ret)
4963 			return;
4964 	}
4965 
4966 	seq_puts(m, "\n  key fields:\n");
4967 
4968 	for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
4969 		seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
4970 		ret = hist_field_debug_show(m, hist_data->fields[i],
4971 					    HIST_FIELD_FL_KEY);
4972 		if (ret)
4973 			return;
4974 	}
4975 
4976 	if (hist_data->n_var_refs)
4977 		seq_puts(m, "\n  variable reference fields:\n");
4978 
4979 	for (i = 0; i < hist_data->n_var_refs; i++) {
4980 		seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
4981 		ret = hist_field_debug_show(m, hist_data->var_refs[i],
4982 					    HIST_FIELD_FL_VAR_REF);
4983 		if (ret)
4984 			return;
4985 	}
4986 
4987 	if (hist_data->n_field_vars)
4988 		seq_puts(m, "\n  field variables:\n");
4989 
4990 	for (i = 0; i < hist_data->n_field_vars; i++) {
4991 		ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
4992 		if (ret)
4993 			return;
4994 	}
4995 
4996 	ret = hist_actions_debug_show(m, hist_data);
4997 	if (ret)
4998 		return;
4999 }
5000 
5001 static int hist_debug_show(struct seq_file *m, void *v)
5002 {
5003 	struct event_trigger_data *data;
5004 	struct trace_event_file *event_file;
5005 	int n = 0, ret = 0;
5006 
5007 	mutex_lock(&event_mutex);
5008 
5009 	event_file = event_file_data(m->private);
5010 	if (unlikely(!event_file)) {
5011 		ret = -ENODEV;
5012 		goto out_unlock;
5013 	}
5014 
5015 	list_for_each_entry(data, &event_file->triggers, list) {
5016 		if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5017 			hist_trigger_debug_show(m, data, n++);
5018 	}
5019 
5020  out_unlock:
5021 	mutex_unlock(&event_mutex);
5022 
5023 	return ret;
5024 }
5025 
5026 static int event_hist_debug_open(struct inode *inode, struct file *file)
5027 {
5028 	int ret;
5029 
5030 	ret = security_locked_down(LOCKDOWN_TRACEFS);
5031 	if (ret)
5032 		return ret;
5033 
5034 	return single_open(file, hist_debug_show, file);
5035 }
5036 
5037 const struct file_operations event_hist_debug_fops = {
5038 	.open = event_hist_debug_open,
5039 	.read = seq_read,
5040 	.llseek = seq_lseek,
5041 	.release = single_release,
5042 };
5043 #endif
5044 
5045 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5046 {
5047 	const char *field_name = hist_field_name(hist_field, 0);
5048 
5049 	if (hist_field->var.name)
5050 		seq_printf(m, "%s=", hist_field->var.name);
5051 
5052 	if (hist_field->flags & HIST_FIELD_FL_CPU)
5053 		seq_puts(m, "cpu");
5054 	else if (field_name) {
5055 		if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5056 		    hist_field->flags & HIST_FIELD_FL_ALIAS)
5057 			seq_putc(m, '$');
5058 		seq_printf(m, "%s", field_name);
5059 	} else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5060 		seq_puts(m, "common_timestamp");
5061 
5062 	if (hist_field->flags) {
5063 		if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5064 		    !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5065 			const char *flags = get_hist_field_flags(hist_field);
5066 
5067 			if (flags)
5068 				seq_printf(m, ".%s", flags);
5069 		}
5070 	}
5071 }
5072 
5073 static int event_hist_trigger_print(struct seq_file *m,
5074 				    struct event_trigger_ops *ops,
5075 				    struct event_trigger_data *data)
5076 {
5077 	struct hist_trigger_data *hist_data = data->private_data;
5078 	struct hist_field *field;
5079 	bool have_var = false;
5080 	unsigned int i;
5081 
5082 	seq_puts(m, "hist:");
5083 
5084 	if (data->name)
5085 		seq_printf(m, "%s:", data->name);
5086 
5087 	seq_puts(m, "keys=");
5088 
5089 	for_each_hist_key_field(i, hist_data) {
5090 		field = hist_data->fields[i];
5091 
5092 		if (i > hist_data->n_vals)
5093 			seq_puts(m, ",");
5094 
5095 		if (field->flags & HIST_FIELD_FL_STACKTRACE)
5096 			seq_puts(m, "stacktrace");
5097 		else
5098 			hist_field_print(m, field);
5099 	}
5100 
5101 	seq_puts(m, ":vals=");
5102 
5103 	for_each_hist_val_field(i, hist_data) {
5104 		field = hist_data->fields[i];
5105 		if (field->flags & HIST_FIELD_FL_VAR) {
5106 			have_var = true;
5107 			continue;
5108 		}
5109 
5110 		if (i == HITCOUNT_IDX)
5111 			seq_puts(m, "hitcount");
5112 		else {
5113 			seq_puts(m, ",");
5114 			hist_field_print(m, field);
5115 		}
5116 	}
5117 
5118 	if (have_var) {
5119 		unsigned int n = 0;
5120 
5121 		seq_puts(m, ":");
5122 
5123 		for_each_hist_val_field(i, hist_data) {
5124 			field = hist_data->fields[i];
5125 
5126 			if (field->flags & HIST_FIELD_FL_VAR) {
5127 				if (n++)
5128 					seq_puts(m, ",");
5129 				hist_field_print(m, field);
5130 			}
5131 		}
5132 	}
5133 
5134 	seq_puts(m, ":sort=");
5135 
5136 	for (i = 0; i < hist_data->n_sort_keys; i++) {
5137 		struct tracing_map_sort_key *sort_key;
5138 		unsigned int idx, first_key_idx;
5139 
5140 		/* skip VAR vals */
5141 		first_key_idx = hist_data->n_vals - hist_data->n_vars;
5142 
5143 		sort_key = &hist_data->sort_keys[i];
5144 		idx = sort_key->field_idx;
5145 
5146 		if (WARN_ON(idx >= HIST_FIELDS_MAX))
5147 			return -EINVAL;
5148 
5149 		if (i > 0)
5150 			seq_puts(m, ",");
5151 
5152 		if (idx == HITCOUNT_IDX)
5153 			seq_puts(m, "hitcount");
5154 		else {
5155 			if (idx >= first_key_idx)
5156 				idx += hist_data->n_vars;
5157 			hist_field_print(m, hist_data->fields[idx]);
5158 		}
5159 
5160 		if (sort_key->descending)
5161 			seq_puts(m, ".descending");
5162 	}
5163 	seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5164 	if (hist_data->enable_timestamps)
5165 		seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5166 
5167 	print_actions_spec(m, hist_data);
5168 
5169 	if (data->filter_str)
5170 		seq_printf(m, " if %s", data->filter_str);
5171 
5172 	if (data->paused)
5173 		seq_puts(m, " [paused]");
5174 	else
5175 		seq_puts(m, " [active]");
5176 
5177 	seq_putc(m, '\n');
5178 
5179 	return 0;
5180 }
5181 
5182 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5183 				   struct event_trigger_data *data)
5184 {
5185 	struct hist_trigger_data *hist_data = data->private_data;
5186 
5187 	if (!data->ref && hist_data->attrs->name)
5188 		save_named_trigger(hist_data->attrs->name, data);
5189 
5190 	data->ref++;
5191 
5192 	return 0;
5193 }
5194 
5195 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5196 {
5197 	struct trace_event_file *file;
5198 	unsigned int i;
5199 	char *cmd;
5200 	int ret;
5201 
5202 	for (i = 0; i < hist_data->n_field_var_hists; i++) {
5203 		file = hist_data->field_var_hists[i]->hist_data->event_file;
5204 		cmd = hist_data->field_var_hists[i]->cmd;
5205 		ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5206 					      "!hist", "hist", cmd);
5207 	}
5208 }
5209 
5210 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5211 				    struct event_trigger_data *data)
5212 {
5213 	struct hist_trigger_data *hist_data = data->private_data;
5214 
5215 	if (WARN_ON_ONCE(data->ref <= 0))
5216 		return;
5217 
5218 	data->ref--;
5219 	if (!data->ref) {
5220 		if (data->name)
5221 			del_named_trigger(data);
5222 
5223 		trigger_data_free(data);
5224 
5225 		remove_hist_vars(hist_data);
5226 
5227 		unregister_field_var_hists(hist_data);
5228 
5229 		destroy_hist_data(hist_data);
5230 	}
5231 }
5232 
5233 static struct event_trigger_ops event_hist_trigger_ops = {
5234 	.func			= event_hist_trigger,
5235 	.print			= event_hist_trigger_print,
5236 	.init			= event_hist_trigger_init,
5237 	.free			= event_hist_trigger_free,
5238 };
5239 
5240 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5241 					 struct event_trigger_data *data)
5242 {
5243 	data->ref++;
5244 
5245 	save_named_trigger(data->named_data->name, data);
5246 
5247 	event_hist_trigger_init(ops, data->named_data);
5248 
5249 	return 0;
5250 }
5251 
5252 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5253 					  struct event_trigger_data *data)
5254 {
5255 	if (WARN_ON_ONCE(data->ref <= 0))
5256 		return;
5257 
5258 	event_hist_trigger_free(ops, data->named_data);
5259 
5260 	data->ref--;
5261 	if (!data->ref) {
5262 		del_named_trigger(data);
5263 		trigger_data_free(data);
5264 	}
5265 }
5266 
5267 static struct event_trigger_ops event_hist_trigger_named_ops = {
5268 	.func			= event_hist_trigger,
5269 	.print			= event_hist_trigger_print,
5270 	.init			= event_hist_trigger_named_init,
5271 	.free			= event_hist_trigger_named_free,
5272 };
5273 
5274 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5275 							    char *param)
5276 {
5277 	return &event_hist_trigger_ops;
5278 }
5279 
5280 static void hist_clear(struct event_trigger_data *data)
5281 {
5282 	struct hist_trigger_data *hist_data = data->private_data;
5283 
5284 	if (data->name)
5285 		pause_named_trigger(data);
5286 
5287 	tracepoint_synchronize_unregister();
5288 
5289 	tracing_map_clear(hist_data->map);
5290 
5291 	if (data->name)
5292 		unpause_named_trigger(data);
5293 }
5294 
5295 static bool compatible_field(struct ftrace_event_field *field,
5296 			     struct ftrace_event_field *test_field)
5297 {
5298 	if (field == test_field)
5299 		return true;
5300 	if (field == NULL || test_field == NULL)
5301 		return false;
5302 	if (strcmp(field->name, test_field->name) != 0)
5303 		return false;
5304 	if (strcmp(field->type, test_field->type) != 0)
5305 		return false;
5306 	if (field->size != test_field->size)
5307 		return false;
5308 	if (field->is_signed != test_field->is_signed)
5309 		return false;
5310 
5311 	return true;
5312 }
5313 
5314 static bool hist_trigger_match(struct event_trigger_data *data,
5315 			       struct event_trigger_data *data_test,
5316 			       struct event_trigger_data *named_data,
5317 			       bool ignore_filter)
5318 {
5319 	struct tracing_map_sort_key *sort_key, *sort_key_test;
5320 	struct hist_trigger_data *hist_data, *hist_data_test;
5321 	struct hist_field *key_field, *key_field_test;
5322 	unsigned int i;
5323 
5324 	if (named_data && (named_data != data_test) &&
5325 	    (named_data != data_test->named_data))
5326 		return false;
5327 
5328 	if (!named_data && is_named_trigger(data_test))
5329 		return false;
5330 
5331 	hist_data = data->private_data;
5332 	hist_data_test = data_test->private_data;
5333 
5334 	if (hist_data->n_vals != hist_data_test->n_vals ||
5335 	    hist_data->n_fields != hist_data_test->n_fields ||
5336 	    hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5337 		return false;
5338 
5339 	if (!ignore_filter) {
5340 		if ((data->filter_str && !data_test->filter_str) ||
5341 		   (!data->filter_str && data_test->filter_str))
5342 			return false;
5343 	}
5344 
5345 	for_each_hist_field(i, hist_data) {
5346 		key_field = hist_data->fields[i];
5347 		key_field_test = hist_data_test->fields[i];
5348 
5349 		if (key_field->flags != key_field_test->flags)
5350 			return false;
5351 		if (!compatible_field(key_field->field, key_field_test->field))
5352 			return false;
5353 		if (key_field->offset != key_field_test->offset)
5354 			return false;
5355 		if (key_field->size != key_field_test->size)
5356 			return false;
5357 		if (key_field->is_signed != key_field_test->is_signed)
5358 			return false;
5359 		if (!!key_field->var.name != !!key_field_test->var.name)
5360 			return false;
5361 		if (key_field->var.name &&
5362 		    strcmp(key_field->var.name, key_field_test->var.name) != 0)
5363 			return false;
5364 	}
5365 
5366 	for (i = 0; i < hist_data->n_sort_keys; i++) {
5367 		sort_key = &hist_data->sort_keys[i];
5368 		sort_key_test = &hist_data_test->sort_keys[i];
5369 
5370 		if (sort_key->field_idx != sort_key_test->field_idx ||
5371 		    sort_key->descending != sort_key_test->descending)
5372 			return false;
5373 	}
5374 
5375 	if (!ignore_filter && data->filter_str &&
5376 	    (strcmp(data->filter_str, data_test->filter_str) != 0))
5377 		return false;
5378 
5379 	if (!actions_match(hist_data, hist_data_test))
5380 		return false;
5381 
5382 	return true;
5383 }
5384 
5385 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5386 				 struct event_trigger_data *data,
5387 				 struct trace_event_file *file)
5388 {
5389 	struct hist_trigger_data *hist_data = data->private_data;
5390 	struct event_trigger_data *test, *named_data = NULL;
5391 	struct trace_array *tr = file->tr;
5392 	int ret = 0;
5393 
5394 	if (hist_data->attrs->name) {
5395 		named_data = find_named_trigger(hist_data->attrs->name);
5396 		if (named_data) {
5397 			if (!hist_trigger_match(data, named_data, named_data,
5398 						true)) {
5399 				hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5400 				ret = -EINVAL;
5401 				goto out;
5402 			}
5403 		}
5404 	}
5405 
5406 	if (hist_data->attrs->name && !named_data)
5407 		goto new;
5408 
5409 	lockdep_assert_held(&event_mutex);
5410 
5411 	list_for_each_entry(test, &file->triggers, list) {
5412 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5413 			if (!hist_trigger_match(data, test, named_data, false))
5414 				continue;
5415 			if (hist_data->attrs->pause)
5416 				test->paused = true;
5417 			else if (hist_data->attrs->cont)
5418 				test->paused = false;
5419 			else if (hist_data->attrs->clear)
5420 				hist_clear(test);
5421 			else {
5422 				hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5423 				ret = -EEXIST;
5424 			}
5425 			goto out;
5426 		}
5427 	}
5428  new:
5429 	if (hist_data->attrs->cont || hist_data->attrs->clear) {
5430 		hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5431 		ret = -ENOENT;
5432 		goto out;
5433 	}
5434 
5435 	if (hist_data->attrs->pause)
5436 		data->paused = true;
5437 
5438 	if (named_data) {
5439 		data->private_data = named_data->private_data;
5440 		set_named_trigger_data(data, named_data);
5441 		data->ops = &event_hist_trigger_named_ops;
5442 	}
5443 
5444 	if (data->ops->init) {
5445 		ret = data->ops->init(data->ops, data);
5446 		if (ret < 0)
5447 			goto out;
5448 	}
5449 
5450 	if (hist_data->enable_timestamps) {
5451 		char *clock = hist_data->attrs->clock;
5452 
5453 		ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5454 		if (ret) {
5455 			hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5456 			goto out;
5457 		}
5458 
5459 		tracing_set_time_stamp_abs(file->tr, true);
5460 	}
5461 
5462 	if (named_data)
5463 		destroy_hist_data(hist_data);
5464 
5465 	ret++;
5466  out:
5467 	return ret;
5468 }
5469 
5470 static int hist_trigger_enable(struct event_trigger_data *data,
5471 			       struct trace_event_file *file)
5472 {
5473 	int ret = 0;
5474 
5475 	list_add_tail_rcu(&data->list, &file->triggers);
5476 
5477 	update_cond_flag(file);
5478 
5479 	if (trace_event_trigger_enable_disable(file, 1) < 0) {
5480 		list_del_rcu(&data->list);
5481 		update_cond_flag(file);
5482 		ret--;
5483 	}
5484 
5485 	return ret;
5486 }
5487 
5488 static bool have_hist_trigger_match(struct event_trigger_data *data,
5489 				    struct trace_event_file *file)
5490 {
5491 	struct hist_trigger_data *hist_data = data->private_data;
5492 	struct event_trigger_data *test, *named_data = NULL;
5493 	bool match = false;
5494 
5495 	lockdep_assert_held(&event_mutex);
5496 
5497 	if (hist_data->attrs->name)
5498 		named_data = find_named_trigger(hist_data->attrs->name);
5499 
5500 	list_for_each_entry(test, &file->triggers, list) {
5501 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5502 			if (hist_trigger_match(data, test, named_data, false)) {
5503 				match = true;
5504 				break;
5505 			}
5506 		}
5507 	}
5508 
5509 	return match;
5510 }
5511 
5512 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5513 				    struct trace_event_file *file)
5514 {
5515 	struct hist_trigger_data *hist_data = data->private_data;
5516 	struct event_trigger_data *test, *named_data = NULL;
5517 
5518 	lockdep_assert_held(&event_mutex);
5519 
5520 	if (hist_data->attrs->name)
5521 		named_data = find_named_trigger(hist_data->attrs->name);
5522 
5523 	list_for_each_entry(test, &file->triggers, list) {
5524 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5525 			if (!hist_trigger_match(data, test, named_data, false))
5526 				continue;
5527 			hist_data = test->private_data;
5528 			if (check_var_refs(hist_data))
5529 				return true;
5530 			break;
5531 		}
5532 	}
5533 
5534 	return false;
5535 }
5536 
5537 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5538 				    struct event_trigger_data *data,
5539 				    struct trace_event_file *file)
5540 {
5541 	struct hist_trigger_data *hist_data = data->private_data;
5542 	struct event_trigger_data *test, *named_data = NULL;
5543 	bool unregistered = false;
5544 
5545 	lockdep_assert_held(&event_mutex);
5546 
5547 	if (hist_data->attrs->name)
5548 		named_data = find_named_trigger(hist_data->attrs->name);
5549 
5550 	list_for_each_entry(test, &file->triggers, list) {
5551 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5552 			if (!hist_trigger_match(data, test, named_data, false))
5553 				continue;
5554 			unregistered = true;
5555 			list_del_rcu(&test->list);
5556 			trace_event_trigger_enable_disable(file, 0);
5557 			update_cond_flag(file);
5558 			break;
5559 		}
5560 	}
5561 
5562 	if (unregistered && test->ops->free)
5563 		test->ops->free(test->ops, test);
5564 
5565 	if (hist_data->enable_timestamps) {
5566 		if (!hist_data->remove || unregistered)
5567 			tracing_set_time_stamp_abs(file->tr, false);
5568 	}
5569 }
5570 
5571 static bool hist_file_check_refs(struct trace_event_file *file)
5572 {
5573 	struct hist_trigger_data *hist_data;
5574 	struct event_trigger_data *test;
5575 
5576 	lockdep_assert_held(&event_mutex);
5577 
5578 	list_for_each_entry(test, &file->triggers, list) {
5579 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5580 			hist_data = test->private_data;
5581 			if (check_var_refs(hist_data))
5582 				return true;
5583 		}
5584 	}
5585 
5586 	return false;
5587 }
5588 
5589 static void hist_unreg_all(struct trace_event_file *file)
5590 {
5591 	struct event_trigger_data *test, *n;
5592 	struct hist_trigger_data *hist_data;
5593 	struct synth_event *se;
5594 	const char *se_name;
5595 
5596 	lockdep_assert_held(&event_mutex);
5597 
5598 	if (hist_file_check_refs(file))
5599 		return;
5600 
5601 	list_for_each_entry_safe(test, n, &file->triggers, list) {
5602 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5603 			hist_data = test->private_data;
5604 			list_del_rcu(&test->list);
5605 			trace_event_trigger_enable_disable(file, 0);
5606 
5607 			se_name = trace_event_name(file->event_call);
5608 			se = find_synth_event(se_name);
5609 			if (se)
5610 				se->ref--;
5611 
5612 			update_cond_flag(file);
5613 			if (hist_data->enable_timestamps)
5614 				tracing_set_time_stamp_abs(file->tr, false);
5615 			if (test->ops->free)
5616 				test->ops->free(test->ops, test);
5617 		}
5618 	}
5619 }
5620 
5621 static int event_hist_trigger_func(struct event_command *cmd_ops,
5622 				   struct trace_event_file *file,
5623 				   char *glob, char *cmd, char *param)
5624 {
5625 	unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5626 	struct event_trigger_data *trigger_data;
5627 	struct hist_trigger_attrs *attrs;
5628 	struct event_trigger_ops *trigger_ops;
5629 	struct hist_trigger_data *hist_data;
5630 	struct synth_event *se;
5631 	const char *se_name;
5632 	bool remove = false;
5633 	char *trigger, *p;
5634 	int ret = 0;
5635 
5636 	lockdep_assert_held(&event_mutex);
5637 
5638 	if (glob && strlen(glob)) {
5639 		hist_err_clear();
5640 		last_cmd_set(file, param);
5641 	}
5642 
5643 	if (!param)
5644 		return -EINVAL;
5645 
5646 	if (glob[0] == '!')
5647 		remove = true;
5648 
5649 	/*
5650 	 * separate the trigger from the filter (k:v [if filter])
5651 	 * allowing for whitespace in the trigger
5652 	 */
5653 	p = trigger = param;
5654 	do {
5655 		p = strstr(p, "if");
5656 		if (!p)
5657 			break;
5658 		if (p == param)
5659 			return -EINVAL;
5660 		if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5661 			p++;
5662 			continue;
5663 		}
5664 		if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
5665 			return -EINVAL;
5666 		if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
5667 			p++;
5668 			continue;
5669 		}
5670 		break;
5671 	} while (p);
5672 
5673 	if (!p)
5674 		param = NULL;
5675 	else {
5676 		*(p - 1) = '\0';
5677 		param = strstrip(p);
5678 		trigger = strstrip(trigger);
5679 	}
5680 
5681 	attrs = parse_hist_trigger_attrs(file->tr, trigger);
5682 	if (IS_ERR(attrs))
5683 		return PTR_ERR(attrs);
5684 
5685 	if (attrs->map_bits)
5686 		hist_trigger_bits = attrs->map_bits;
5687 
5688 	hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
5689 	if (IS_ERR(hist_data)) {
5690 		destroy_hist_trigger_attrs(attrs);
5691 		return PTR_ERR(hist_data);
5692 	}
5693 
5694 	trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5695 
5696 	trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
5697 	if (!trigger_data) {
5698 		ret = -ENOMEM;
5699 		goto out_free;
5700 	}
5701 
5702 	trigger_data->count = -1;
5703 	trigger_data->ops = trigger_ops;
5704 	trigger_data->cmd_ops = cmd_ops;
5705 
5706 	INIT_LIST_HEAD(&trigger_data->list);
5707 	RCU_INIT_POINTER(trigger_data->filter, NULL);
5708 
5709 	trigger_data->private_data = hist_data;
5710 
5711 	/* if param is non-empty, it's supposed to be a filter */
5712 	if (param && cmd_ops->set_filter) {
5713 		ret = cmd_ops->set_filter(param, trigger_data, file);
5714 		if (ret < 0)
5715 			goto out_free;
5716 	}
5717 
5718 	if (remove) {
5719 		if (!have_hist_trigger_match(trigger_data, file))
5720 			goto out_free;
5721 
5722 		if (hist_trigger_check_refs(trigger_data, file)) {
5723 			ret = -EBUSY;
5724 			goto out_free;
5725 		}
5726 
5727 		cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5728 		se_name = trace_event_name(file->event_call);
5729 		se = find_synth_event(se_name);
5730 		if (se)
5731 			se->ref--;
5732 		ret = 0;
5733 		goto out_free;
5734 	}
5735 
5736 	ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5737 	/*
5738 	 * The above returns on success the # of triggers registered,
5739 	 * but if it didn't register any it returns zero.  Consider no
5740 	 * triggers registered a failure too.
5741 	 */
5742 	if (!ret) {
5743 		if (!(attrs->pause || attrs->cont || attrs->clear))
5744 			ret = -ENOENT;
5745 		goto out_free;
5746 	} else if (ret < 0)
5747 		goto out_free;
5748 
5749 	if (get_named_trigger_data(trigger_data))
5750 		goto enable;
5751 
5752 	if (has_hist_vars(hist_data))
5753 		save_hist_vars(hist_data);
5754 
5755 	ret = create_actions(hist_data);
5756 	if (ret)
5757 		goto out_unreg;
5758 
5759 	ret = tracing_map_init(hist_data->map);
5760 	if (ret)
5761 		goto out_unreg;
5762 enable:
5763 	ret = hist_trigger_enable(trigger_data, file);
5764 	if (ret)
5765 		goto out_unreg;
5766 
5767 	se_name = trace_event_name(file->event_call);
5768 	se = find_synth_event(se_name);
5769 	if (se)
5770 		se->ref++;
5771 	/* Just return zero, not the number of registered triggers */
5772 	ret = 0;
5773  out:
5774 	if (ret == 0)
5775 		hist_err_clear();
5776 
5777 	return ret;
5778  out_unreg:
5779 	cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5780  out_free:
5781 	if (cmd_ops->set_filter)
5782 		cmd_ops->set_filter(NULL, trigger_data, NULL);
5783 
5784 	remove_hist_vars(hist_data);
5785 
5786 	kfree(trigger_data);
5787 
5788 	destroy_hist_data(hist_data);
5789 	goto out;
5790 }
5791 
5792 static struct event_command trigger_hist_cmd = {
5793 	.name			= "hist",
5794 	.trigger_type		= ETT_EVENT_HIST,
5795 	.flags			= EVENT_CMD_FL_NEEDS_REC,
5796 	.func			= event_hist_trigger_func,
5797 	.reg			= hist_register_trigger,
5798 	.unreg			= hist_unregister_trigger,
5799 	.unreg_all		= hist_unreg_all,
5800 	.get_trigger_ops	= event_hist_get_trigger_ops,
5801 	.set_filter		= set_trigger_filter,
5802 };
5803 
5804 __init int register_trigger_hist_cmd(void)
5805 {
5806 	int ret;
5807 
5808 	ret = register_event_command(&trigger_hist_cmd);
5809 	WARN_ON(ret < 0);
5810 
5811 	return ret;
5812 }
5813 
5814 static void
5815 hist_enable_trigger(struct event_trigger_data *data, void *rec,
5816 		    struct ring_buffer_event *event)
5817 {
5818 	struct enable_trigger_data *enable_data = data->private_data;
5819 	struct event_trigger_data *test;
5820 
5821 	list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
5822 				lockdep_is_held(&event_mutex)) {
5823 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5824 			if (enable_data->enable)
5825 				test->paused = false;
5826 			else
5827 				test->paused = true;
5828 		}
5829 	}
5830 }
5831 
5832 static void
5833 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
5834 			  struct ring_buffer_event *event)
5835 {
5836 	if (!data->count)
5837 		return;
5838 
5839 	if (data->count != -1)
5840 		(data->count)--;
5841 
5842 	hist_enable_trigger(data, rec, event);
5843 }
5844 
5845 static struct event_trigger_ops hist_enable_trigger_ops = {
5846 	.func			= hist_enable_trigger,
5847 	.print			= event_enable_trigger_print,
5848 	.init			= event_trigger_init,
5849 	.free			= event_enable_trigger_free,
5850 };
5851 
5852 static struct event_trigger_ops hist_enable_count_trigger_ops = {
5853 	.func			= hist_enable_count_trigger,
5854 	.print			= event_enable_trigger_print,
5855 	.init			= event_trigger_init,
5856 	.free			= event_enable_trigger_free,
5857 };
5858 
5859 static struct event_trigger_ops hist_disable_trigger_ops = {
5860 	.func			= hist_enable_trigger,
5861 	.print			= event_enable_trigger_print,
5862 	.init			= event_trigger_init,
5863 	.free			= event_enable_trigger_free,
5864 };
5865 
5866 static struct event_trigger_ops hist_disable_count_trigger_ops = {
5867 	.func			= hist_enable_count_trigger,
5868 	.print			= event_enable_trigger_print,
5869 	.init			= event_trigger_init,
5870 	.free			= event_enable_trigger_free,
5871 };
5872 
5873 static struct event_trigger_ops *
5874 hist_enable_get_trigger_ops(char *cmd, char *param)
5875 {
5876 	struct event_trigger_ops *ops;
5877 	bool enable;
5878 
5879 	enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5880 
5881 	if (enable)
5882 		ops = param ? &hist_enable_count_trigger_ops :
5883 			&hist_enable_trigger_ops;
5884 	else
5885 		ops = param ? &hist_disable_count_trigger_ops :
5886 			&hist_disable_trigger_ops;
5887 
5888 	return ops;
5889 }
5890 
5891 static void hist_enable_unreg_all(struct trace_event_file *file)
5892 {
5893 	struct event_trigger_data *test, *n;
5894 
5895 	list_for_each_entry_safe(test, n, &file->triggers, list) {
5896 		if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5897 			list_del_rcu(&test->list);
5898 			update_cond_flag(file);
5899 			trace_event_trigger_enable_disable(file, 0);
5900 			if (test->ops->free)
5901 				test->ops->free(test->ops, test);
5902 		}
5903 	}
5904 }
5905 
5906 static struct event_command trigger_hist_enable_cmd = {
5907 	.name			= ENABLE_HIST_STR,
5908 	.trigger_type		= ETT_HIST_ENABLE,
5909 	.func			= event_enable_trigger_func,
5910 	.reg			= event_enable_register_trigger,
5911 	.unreg			= event_enable_unregister_trigger,
5912 	.unreg_all		= hist_enable_unreg_all,
5913 	.get_trigger_ops	= hist_enable_get_trigger_ops,
5914 	.set_filter		= set_trigger_filter,
5915 };
5916 
5917 static struct event_command trigger_hist_disable_cmd = {
5918 	.name			= DISABLE_HIST_STR,
5919 	.trigger_type		= ETT_HIST_ENABLE,
5920 	.func			= event_enable_trigger_func,
5921 	.reg			= event_enable_register_trigger,
5922 	.unreg			= event_enable_unregister_trigger,
5923 	.unreg_all		= hist_enable_unreg_all,
5924 	.get_trigger_ops	= hist_enable_get_trigger_ops,
5925 	.set_filter		= set_trigger_filter,
5926 };
5927 
5928 static __init void unregister_trigger_hist_enable_disable_cmds(void)
5929 {
5930 	unregister_event_command(&trigger_hist_enable_cmd);
5931 	unregister_event_command(&trigger_hist_disable_cmd);
5932 }
5933 
5934 __init int register_trigger_hist_enable_disable_cmds(void)
5935 {
5936 	int ret;
5937 
5938 	ret = register_event_command(&trigger_hist_enable_cmd);
5939 	if (WARN_ON(ret < 0))
5940 		return ret;
5941 	ret = register_event_command(&trigger_hist_disable_cmd);
5942 	if (WARN_ON(ret < 0))
5943 		unregister_trigger_hist_enable_disable_cmds();
5944 
5945 	return ret;
5946 }
5947