1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * trace_events_hist - trace event hist triggers
4  *
5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
6  */
7 
8 #include <linux/module.h>
9 #include <linux/kallsyms.h>
10 #include <linux/security.h>
11 #include <linux/mutex.h>
12 #include <linux/slab.h>
13 #include <linux/stacktrace.h>
14 #include <linux/rculist.h>
15 #include <linux/tracefs.h>
16 
17 /* for gfp flag names */
18 #include <linux/trace_events.h>
19 #include <trace/events/mmflags.h>
20 
21 #include "tracing_map.h"
22 #include "trace_synth.h"
23 
24 #define ERRORS								\
25 	C(NONE,			"No error"),				\
26 	C(DUPLICATE_VAR,	"Variable already defined"),		\
27 	C(VAR_NOT_UNIQUE,	"Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
28 	C(TOO_MANY_VARS,	"Too many variables defined"),		\
29 	C(MALFORMED_ASSIGNMENT,	"Malformed assignment"),		\
30 	C(NAMED_MISMATCH,	"Named hist trigger doesn't match existing named trigger (includes variables)"), \
31 	C(TRIGGER_EEXIST,	"Hist trigger already exists"),		\
32 	C(TRIGGER_ENOENT_CLEAR,	"Can't clear or continue a nonexistent hist trigger"), \
33 	C(SET_CLOCK_FAIL,	"Couldn't set trace_clock"),		\
34 	C(BAD_FIELD_MODIFIER,	"Invalid field modifier"),		\
35 	C(TOO_MANY_SUBEXPR,	"Too many subexpressions (3 max)"),	\
36 	C(TIMESTAMP_MISMATCH,	"Timestamp units in expression don't match"), \
37 	C(TOO_MANY_FIELD_VARS,	"Too many field variables defined"),	\
38 	C(EVENT_FILE_NOT_FOUND,	"Event file not found"),		\
39 	C(HIST_NOT_FOUND,	"Matching event histogram not found"),	\
40 	C(HIST_CREATE_FAIL,	"Couldn't create histogram for field"),	\
41 	C(SYNTH_VAR_NOT_FOUND,	"Couldn't find synthetic variable"),	\
42 	C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),	\
43 	C(SYNTH_TYPE_MISMATCH,	"Param type doesn't match synthetic event field type"), \
44 	C(SYNTH_COUNT_MISMATCH,	"Param count doesn't match synthetic event field count"), \
45 	C(FIELD_VAR_PARSE_FAIL,	"Couldn't parse field variable"),	\
46 	C(VAR_CREATE_FIND_FAIL,	"Couldn't create or find variable"),	\
47 	C(ONX_NOT_VAR,		"For onmax(x) or onchange(x), x must be a variable"), \
48 	C(ONX_VAR_NOT_FOUND,	"Couldn't find onmax or onchange variable"), \
49 	C(ONX_VAR_CREATE_FAIL,	"Couldn't create onmax or onchange variable"), \
50 	C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),	\
51 	C(TOO_MANY_PARAMS,	"Too many action params"),		\
52 	C(PARAM_NOT_FOUND,	"Couldn't find param"),			\
53 	C(INVALID_PARAM,	"Invalid action param"),		\
54 	C(ACTION_NOT_FOUND,	"No action found"),			\
55 	C(NO_SAVE_PARAMS,	"No params found for save()"),		\
56 	C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
57 	C(ACTION_MISMATCH,	"Handler doesn't support action"),	\
58 	C(NO_CLOSING_PAREN,	"No closing paren found"),		\
59 	C(SUBSYS_NOT_FOUND,	"Missing subsystem"),			\
60 	C(INVALID_SUBSYS_EVENT,	"Invalid subsystem or event name"),	\
61 	C(INVALID_REF_KEY,	"Using variable references in keys not supported"), \
62 	C(VAR_NOT_FOUND,	"Couldn't find variable"),		\
63 	C(FIELD_NOT_FOUND,	"Couldn't find field"),			\
64 	C(EMPTY_ASSIGNMENT,	"Empty assignment"),			\
65 	C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),		\
66 	C(EMPTY_SORT_FIELD,	"Empty sort field"),			\
67 	C(TOO_MANY_SORT_FIELDS,	"Too many sort fields (Max = 2)"),	\
68 	C(INVALID_SORT_FIELD,	"Sort field must be a key or a val"),	\
69 	C(INVALID_STR_OPERAND,	"String type can not be an operand in expression"), \
70 	C(EXPECT_NUMBER,	"Expecting numeric literal"),		\
71 	C(UNARY_MINUS_SUBEXPR,	"Unary minus not supported in sub-expressions"), \
72 	C(DIVISION_BY_ZERO,	"Division by zero"),
73 
74 #undef C
75 #define C(a, b)		HIST_ERR_##a
76 
77 enum { ERRORS };
78 
79 #undef C
80 #define C(a, b)		b
81 
82 static const char *err_text[] = { ERRORS };
83 
84 struct hist_field;
85 
86 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
87 				struct tracing_map_elt *elt,
88 				struct trace_buffer *buffer,
89 				struct ring_buffer_event *rbe,
90 				void *event);
91 
92 #define HIST_FIELD_OPERANDS_MAX	2
93 #define HIST_FIELDS_MAX		(TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
94 #define HIST_ACTIONS_MAX	8
95 #define HIST_CONST_DIGITS_MAX	21
96 #define HIST_DIV_SHIFT		20  /* For optimizing division by constants */
97 
98 enum field_op_id {
99 	FIELD_OP_NONE,
100 	FIELD_OP_PLUS,
101 	FIELD_OP_MINUS,
102 	FIELD_OP_UNARY_MINUS,
103 	FIELD_OP_DIV,
104 	FIELD_OP_MULT,
105 };
106 
107 /*
108  * A hist_var (histogram variable) contains variable information for
109  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
110  * flag set.  A hist_var has a variable name e.g. ts0, and is
111  * associated with a given histogram trigger, as specified by
112  * hist_data.  The hist_var idx is the unique index assigned to the
113  * variable by the hist trigger's tracing_map.  The idx is what is
114  * used to set a variable's value and, by a variable reference, to
115  * retrieve it.
116  */
117 struct hist_var {
118 	char				*name;
119 	struct hist_trigger_data	*hist_data;
120 	unsigned int			idx;
121 };
122 
123 struct hist_field {
124 	struct ftrace_event_field	*field;
125 	unsigned long			flags;
126 	hist_field_fn_t			fn;
127 	unsigned int			ref;
128 	unsigned int			size;
129 	unsigned int			offset;
130 	unsigned int                    is_signed;
131 	unsigned long			buckets;
132 	const char			*type;
133 	struct hist_field		*operands[HIST_FIELD_OPERANDS_MAX];
134 	struct hist_trigger_data	*hist_data;
135 
136 	/*
137 	 * Variable fields contain variable-specific info in var.
138 	 */
139 	struct hist_var			var;
140 	enum field_op_id		operator;
141 	char				*system;
142 	char				*event_name;
143 
144 	/*
145 	 * The name field is used for EXPR and VAR_REF fields.  VAR
146 	 * fields contain the variable name in var.name.
147 	 */
148 	char				*name;
149 
150 	/*
151 	 * When a histogram trigger is hit, if it has any references
152 	 * to variables, the values of those variables are collected
153 	 * into a var_ref_vals array by resolve_var_refs().  The
154 	 * current value of each variable is read from the tracing_map
155 	 * using the hist field's hist_var.idx and entered into the
156 	 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
157 	 */
158 	unsigned int			var_ref_idx;
159 	bool                            read_once;
160 
161 	unsigned int			var_str_idx;
162 
163 	/* Numeric literals are represented as u64 */
164 	u64				constant;
165 	/* Used to optimize division by constants */
166 	u64				div_multiplier;
167 };
168 
169 static u64 hist_field_none(struct hist_field *field,
170 			   struct tracing_map_elt *elt,
171 			   struct trace_buffer *buffer,
172 			   struct ring_buffer_event *rbe,
173 			   void *event)
174 {
175 	return 0;
176 }
177 
178 static u64 hist_field_const(struct hist_field *field,
179 			   struct tracing_map_elt *elt,
180 			   struct trace_buffer *buffer,
181 			   struct ring_buffer_event *rbe,
182 			   void *event)
183 {
184 	return field->constant;
185 }
186 
187 static u64 hist_field_counter(struct hist_field *field,
188 			      struct tracing_map_elt *elt,
189 			      struct trace_buffer *buffer,
190 			      struct ring_buffer_event *rbe,
191 			      void *event)
192 {
193 	return 1;
194 }
195 
196 static u64 hist_field_string(struct hist_field *hist_field,
197 			     struct tracing_map_elt *elt,
198 			     struct trace_buffer *buffer,
199 			     struct ring_buffer_event *rbe,
200 			     void *event)
201 {
202 	char *addr = (char *)(event + hist_field->field->offset);
203 
204 	return (u64)(unsigned long)addr;
205 }
206 
207 static u64 hist_field_dynstring(struct hist_field *hist_field,
208 				struct tracing_map_elt *elt,
209 				struct trace_buffer *buffer,
210 				struct ring_buffer_event *rbe,
211 				void *event)
212 {
213 	u32 str_item = *(u32 *)(event + hist_field->field->offset);
214 	int str_loc = str_item & 0xffff;
215 	char *addr = (char *)(event + str_loc);
216 
217 	return (u64)(unsigned long)addr;
218 }
219 
220 static u64 hist_field_reldynstring(struct hist_field *hist_field,
221 				   struct tracing_map_elt *elt,
222 				   struct trace_buffer *buffer,
223 				   struct ring_buffer_event *rbe,
224 				   void *event)
225 {
226 	u32 *item = event + hist_field->field->offset;
227 	u32 str_item = *item;
228 	int str_loc = str_item & 0xffff;
229 	char *addr = (char *)&item[1] + str_loc;
230 
231 	return (u64)(unsigned long)addr;
232 }
233 
234 static u64 hist_field_pstring(struct hist_field *hist_field,
235 			      struct tracing_map_elt *elt,
236 			      struct trace_buffer *buffer,
237 			      struct ring_buffer_event *rbe,
238 			      void *event)
239 {
240 	char **addr = (char **)(event + hist_field->field->offset);
241 
242 	return (u64)(unsigned long)*addr;
243 }
244 
245 static u64 hist_field_log2(struct hist_field *hist_field,
246 			   struct tracing_map_elt *elt,
247 			   struct trace_buffer *buffer,
248 			   struct ring_buffer_event *rbe,
249 			   void *event)
250 {
251 	struct hist_field *operand = hist_field->operands[0];
252 
253 	u64 val = operand->fn(operand, elt, buffer, rbe, event);
254 
255 	return (u64) ilog2(roundup_pow_of_two(val));
256 }
257 
258 static u64 hist_field_bucket(struct hist_field *hist_field,
259 			     struct tracing_map_elt *elt,
260 			     struct trace_buffer *buffer,
261 			     struct ring_buffer_event *rbe,
262 			     void *event)
263 {
264 	struct hist_field *operand = hist_field->operands[0];
265 	unsigned long buckets = hist_field->buckets;
266 
267 	u64 val = operand->fn(operand, elt, buffer, rbe, event);
268 
269 	if (WARN_ON_ONCE(!buckets))
270 		return val;
271 
272 	if (val >= LONG_MAX)
273 		val = div64_ul(val, buckets);
274 	else
275 		val = (u64)((unsigned long)val / buckets);
276 	return val * buckets;
277 }
278 
279 static u64 hist_field_plus(struct hist_field *hist_field,
280 			   struct tracing_map_elt *elt,
281 			   struct trace_buffer *buffer,
282 			   struct ring_buffer_event *rbe,
283 			   void *event)
284 {
285 	struct hist_field *operand1 = hist_field->operands[0];
286 	struct hist_field *operand2 = hist_field->operands[1];
287 
288 	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
289 	u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
290 
291 	return val1 + val2;
292 }
293 
294 static u64 hist_field_minus(struct hist_field *hist_field,
295 			    struct tracing_map_elt *elt,
296 			    struct trace_buffer *buffer,
297 			    struct ring_buffer_event *rbe,
298 			    void *event)
299 {
300 	struct hist_field *operand1 = hist_field->operands[0];
301 	struct hist_field *operand2 = hist_field->operands[1];
302 
303 	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
304 	u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
305 
306 	return val1 - val2;
307 }
308 
309 static u64 hist_field_div(struct hist_field *hist_field,
310 			   struct tracing_map_elt *elt,
311 			   struct trace_buffer *buffer,
312 			   struct ring_buffer_event *rbe,
313 			   void *event)
314 {
315 	struct hist_field *operand1 = hist_field->operands[0];
316 	struct hist_field *operand2 = hist_field->operands[1];
317 
318 	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
319 	u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
320 
321 	/* Return -1 for the undefined case */
322 	if (!val2)
323 		return -1;
324 
325 	/* Use shift if the divisor is a power of 2 */
326 	if (!(val2 & (val2 - 1)))
327 		return val1 >> __ffs64(val2);
328 
329 	return div64_u64(val1, val2);
330 }
331 
332 static u64 div_by_power_of_two(struct hist_field *hist_field,
333 				struct tracing_map_elt *elt,
334 				struct trace_buffer *buffer,
335 				struct ring_buffer_event *rbe,
336 				void *event)
337 {
338 	struct hist_field *operand1 = hist_field->operands[0];
339 	struct hist_field *operand2 = hist_field->operands[1];
340 
341 	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
342 
343 	return val1 >> __ffs64(operand2->constant);
344 }
345 
346 static u64 div_by_not_power_of_two(struct hist_field *hist_field,
347 				struct tracing_map_elt *elt,
348 				struct trace_buffer *buffer,
349 				struct ring_buffer_event *rbe,
350 				void *event)
351 {
352 	struct hist_field *operand1 = hist_field->operands[0];
353 	struct hist_field *operand2 = hist_field->operands[1];
354 
355 	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
356 
357 	return div64_u64(val1, operand2->constant);
358 }
359 
360 static u64 div_by_mult_and_shift(struct hist_field *hist_field,
361 				struct tracing_map_elt *elt,
362 				struct trace_buffer *buffer,
363 				struct ring_buffer_event *rbe,
364 				void *event)
365 {
366 	struct hist_field *operand1 = hist_field->operands[0];
367 	struct hist_field *operand2 = hist_field->operands[1];
368 
369 	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
370 
371 	/*
372 	 * If the divisor is a constant, do a multiplication and shift instead.
373 	 *
374 	 * Choose Z = some power of 2. If Y <= Z, then:
375 	 *     X / Y = (X * (Z / Y)) / Z
376 	 *
377 	 * (Z / Y) is a constant (mult) which is calculated at parse time, so:
378 	 *     X / Y = (X * mult) / Z
379 	 *
380 	 * The division by Z can be replaced by a shift since Z is a power of 2:
381 	 *     X / Y = (X * mult) >> HIST_DIV_SHIFT
382 	 *
383 	 * As long, as X < Z the results will not be off by more than 1.
384 	 */
385 	if (val1 < (1 << HIST_DIV_SHIFT)) {
386 		u64 mult = operand2->div_multiplier;
387 
388 		return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT;
389 	}
390 
391 	return div64_u64(val1, operand2->constant);
392 }
393 
394 static u64 hist_field_mult(struct hist_field *hist_field,
395 			   struct tracing_map_elt *elt,
396 			   struct trace_buffer *buffer,
397 			   struct ring_buffer_event *rbe,
398 			   void *event)
399 {
400 	struct hist_field *operand1 = hist_field->operands[0];
401 	struct hist_field *operand2 = hist_field->operands[1];
402 
403 	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
404 	u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
405 
406 	return val1 * val2;
407 }
408 
409 static u64 hist_field_unary_minus(struct hist_field *hist_field,
410 				  struct tracing_map_elt *elt,
411 				  struct trace_buffer *buffer,
412 				  struct ring_buffer_event *rbe,
413 				  void *event)
414 {
415 	struct hist_field *operand = hist_field->operands[0];
416 
417 	s64 sval = (s64)operand->fn(operand, elt, buffer, rbe, event);
418 	u64 val = (u64)-sval;
419 
420 	return val;
421 }
422 
423 #define DEFINE_HIST_FIELD_FN(type)					\
424 	static u64 hist_field_##type(struct hist_field *hist_field,	\
425 				     struct tracing_map_elt *elt,	\
426 				     struct trace_buffer *buffer,	\
427 				     struct ring_buffer_event *rbe,	\
428 				     void *event)			\
429 {									\
430 	type *addr = (type *)(event + hist_field->field->offset);	\
431 									\
432 	return (u64)(unsigned long)*addr;				\
433 }
434 
435 DEFINE_HIST_FIELD_FN(s64);
436 DEFINE_HIST_FIELD_FN(u64);
437 DEFINE_HIST_FIELD_FN(s32);
438 DEFINE_HIST_FIELD_FN(u32);
439 DEFINE_HIST_FIELD_FN(s16);
440 DEFINE_HIST_FIELD_FN(u16);
441 DEFINE_HIST_FIELD_FN(s8);
442 DEFINE_HIST_FIELD_FN(u8);
443 
444 #define for_each_hist_field(i, hist_data)	\
445 	for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
446 
447 #define for_each_hist_val_field(i, hist_data)	\
448 	for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
449 
450 #define for_each_hist_key_field(i, hist_data)	\
451 	for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
452 
453 #define HIST_STACKTRACE_DEPTH	16
454 #define HIST_STACKTRACE_SIZE	(HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
455 #define HIST_STACKTRACE_SKIP	5
456 
457 #define HITCOUNT_IDX		0
458 #define HIST_KEY_SIZE_MAX	(MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
459 
460 enum hist_field_flags {
461 	HIST_FIELD_FL_HITCOUNT		= 1 << 0,
462 	HIST_FIELD_FL_KEY		= 1 << 1,
463 	HIST_FIELD_FL_STRING		= 1 << 2,
464 	HIST_FIELD_FL_HEX		= 1 << 3,
465 	HIST_FIELD_FL_SYM		= 1 << 4,
466 	HIST_FIELD_FL_SYM_OFFSET	= 1 << 5,
467 	HIST_FIELD_FL_EXECNAME		= 1 << 6,
468 	HIST_FIELD_FL_SYSCALL		= 1 << 7,
469 	HIST_FIELD_FL_STACKTRACE	= 1 << 8,
470 	HIST_FIELD_FL_LOG2		= 1 << 9,
471 	HIST_FIELD_FL_TIMESTAMP		= 1 << 10,
472 	HIST_FIELD_FL_TIMESTAMP_USECS	= 1 << 11,
473 	HIST_FIELD_FL_VAR		= 1 << 12,
474 	HIST_FIELD_FL_EXPR		= 1 << 13,
475 	HIST_FIELD_FL_VAR_REF		= 1 << 14,
476 	HIST_FIELD_FL_CPU		= 1 << 15,
477 	HIST_FIELD_FL_ALIAS		= 1 << 16,
478 	HIST_FIELD_FL_BUCKET		= 1 << 17,
479 	HIST_FIELD_FL_CONST		= 1 << 18,
480 };
481 
482 struct var_defs {
483 	unsigned int	n_vars;
484 	char		*name[TRACING_MAP_VARS_MAX];
485 	char		*expr[TRACING_MAP_VARS_MAX];
486 };
487 
488 struct hist_trigger_attrs {
489 	char		*keys_str;
490 	char		*vals_str;
491 	char		*sort_key_str;
492 	char		*name;
493 	char		*clock;
494 	bool		pause;
495 	bool		cont;
496 	bool		clear;
497 	bool		ts_in_usecs;
498 	unsigned int	map_bits;
499 
500 	char		*assignment_str[TRACING_MAP_VARS_MAX];
501 	unsigned int	n_assignments;
502 
503 	char		*action_str[HIST_ACTIONS_MAX];
504 	unsigned int	n_actions;
505 
506 	struct var_defs	var_defs;
507 };
508 
509 struct field_var {
510 	struct hist_field	*var;
511 	struct hist_field	*val;
512 };
513 
514 struct field_var_hist {
515 	struct hist_trigger_data	*hist_data;
516 	char				*cmd;
517 };
518 
519 struct hist_trigger_data {
520 	struct hist_field               *fields[HIST_FIELDS_MAX];
521 	unsigned int			n_vals;
522 	unsigned int			n_keys;
523 	unsigned int			n_fields;
524 	unsigned int			n_vars;
525 	unsigned int			n_var_str;
526 	unsigned int			key_size;
527 	struct tracing_map_sort_key	sort_keys[TRACING_MAP_SORT_KEYS_MAX];
528 	unsigned int			n_sort_keys;
529 	struct trace_event_file		*event_file;
530 	struct hist_trigger_attrs	*attrs;
531 	struct tracing_map		*map;
532 	bool				enable_timestamps;
533 	bool				remove;
534 	struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
535 	unsigned int			n_var_refs;
536 
537 	struct action_data		*actions[HIST_ACTIONS_MAX];
538 	unsigned int			n_actions;
539 
540 	struct field_var		*field_vars[SYNTH_FIELDS_MAX];
541 	unsigned int			n_field_vars;
542 	unsigned int			n_field_var_str;
543 	struct field_var_hist		*field_var_hists[SYNTH_FIELDS_MAX];
544 	unsigned int			n_field_var_hists;
545 
546 	struct field_var		*save_vars[SYNTH_FIELDS_MAX];
547 	unsigned int			n_save_vars;
548 	unsigned int			n_save_var_str;
549 };
550 
551 struct action_data;
552 
553 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
554 			     struct tracing_map_elt *elt,
555 			     struct trace_buffer *buffer, void *rec,
556 			     struct ring_buffer_event *rbe, void *key,
557 			     struct action_data *data, u64 *var_ref_vals);
558 
559 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
560 
561 enum handler_id {
562 	HANDLER_ONMATCH = 1,
563 	HANDLER_ONMAX,
564 	HANDLER_ONCHANGE,
565 };
566 
567 enum action_id {
568 	ACTION_SAVE = 1,
569 	ACTION_TRACE,
570 	ACTION_SNAPSHOT,
571 };
572 
573 struct action_data {
574 	enum handler_id		handler;
575 	enum action_id		action;
576 	char			*action_name;
577 	action_fn_t		fn;
578 
579 	unsigned int		n_params;
580 	char			*params[SYNTH_FIELDS_MAX];
581 
582 	/*
583 	 * When a histogram trigger is hit, the values of any
584 	 * references to variables, including variables being passed
585 	 * as parameters to synthetic events, are collected into a
586 	 * var_ref_vals array.  This var_ref_idx array is an array of
587 	 * indices into the var_ref_vals array, one for each synthetic
588 	 * event param, and is passed to the synthetic event
589 	 * invocation.
590 	 */
591 	unsigned int		var_ref_idx[TRACING_MAP_VARS_MAX];
592 	struct synth_event	*synth_event;
593 	bool			use_trace_keyword;
594 	char			*synth_event_name;
595 
596 	union {
597 		struct {
598 			char			*event;
599 			char			*event_system;
600 		} match_data;
601 
602 		struct {
603 			/*
604 			 * var_str contains the $-unstripped variable
605 			 * name referenced by var_ref, and used when
606 			 * printing the action.  Because var_ref
607 			 * creation is deferred to create_actions(),
608 			 * we need a per-action way to save it until
609 			 * then, thus var_str.
610 			 */
611 			char			*var_str;
612 
613 			/*
614 			 * var_ref refers to the variable being
615 			 * tracked e.g onmax($var).
616 			 */
617 			struct hist_field	*var_ref;
618 
619 			/*
620 			 * track_var contains the 'invisible' tracking
621 			 * variable created to keep the current
622 			 * e.g. max value.
623 			 */
624 			struct hist_field	*track_var;
625 
626 			check_track_val_fn_t	check_val;
627 			action_fn_t		save_data;
628 		} track_data;
629 	};
630 };
631 
632 struct track_data {
633 	u64				track_val;
634 	bool				updated;
635 
636 	unsigned int			key_len;
637 	void				*key;
638 	struct tracing_map_elt		elt;
639 
640 	struct action_data		*action_data;
641 	struct hist_trigger_data	*hist_data;
642 };
643 
644 struct hist_elt_data {
645 	char *comm;
646 	u64 *var_ref_vals;
647 	char **field_var_str;
648 	int n_field_var_str;
649 };
650 
651 struct snapshot_context {
652 	struct tracing_map_elt	*elt;
653 	void			*key;
654 };
655 
656 /*
657  * Returns the specific division function to use if the divisor
658  * is constant. This avoids extra branches when the trigger is hit.
659  */
660 static hist_field_fn_t hist_field_get_div_fn(struct hist_field *divisor)
661 {
662 	u64 div = divisor->constant;
663 
664 	if (!(div & (div - 1)))
665 		return div_by_power_of_two;
666 
667 	/* If the divisor is too large, do a regular division */
668 	if (div > (1 << HIST_DIV_SHIFT))
669 		return div_by_not_power_of_two;
670 
671 	divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
672 	return div_by_mult_and_shift;
673 }
674 
675 static void track_data_free(struct track_data *track_data)
676 {
677 	struct hist_elt_data *elt_data;
678 
679 	if (!track_data)
680 		return;
681 
682 	kfree(track_data->key);
683 
684 	elt_data = track_data->elt.private_data;
685 	if (elt_data) {
686 		kfree(elt_data->comm);
687 		kfree(elt_data);
688 	}
689 
690 	kfree(track_data);
691 }
692 
693 static struct track_data *track_data_alloc(unsigned int key_len,
694 					   struct action_data *action_data,
695 					   struct hist_trigger_data *hist_data)
696 {
697 	struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
698 	struct hist_elt_data *elt_data;
699 
700 	if (!data)
701 		return ERR_PTR(-ENOMEM);
702 
703 	data->key = kzalloc(key_len, GFP_KERNEL);
704 	if (!data->key) {
705 		track_data_free(data);
706 		return ERR_PTR(-ENOMEM);
707 	}
708 
709 	data->key_len = key_len;
710 	data->action_data = action_data;
711 	data->hist_data = hist_data;
712 
713 	elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
714 	if (!elt_data) {
715 		track_data_free(data);
716 		return ERR_PTR(-ENOMEM);
717 	}
718 
719 	data->elt.private_data = elt_data;
720 
721 	elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
722 	if (!elt_data->comm) {
723 		track_data_free(data);
724 		return ERR_PTR(-ENOMEM);
725 	}
726 
727 	return data;
728 }
729 
730 #define HIST_PREFIX "hist:"
731 
732 static char *last_cmd;
733 static char last_cmd_loc[MAX_FILTER_STR_VAL];
734 
735 static int errpos(char *str)
736 {
737 	if (!str || !last_cmd)
738 		return 0;
739 
740 	return err_pos(last_cmd, str);
741 }
742 
743 static void last_cmd_set(struct trace_event_file *file, char *str)
744 {
745 	const char *system = NULL, *name = NULL;
746 	struct trace_event_call *call;
747 	int len;
748 
749 	if (!str)
750 		return;
751 
752 	/* sizeof() contains the nul byte */
753 	len = sizeof(HIST_PREFIX) + strlen(str);
754 	kfree(last_cmd);
755 	last_cmd = kzalloc(len, GFP_KERNEL);
756 	if (!last_cmd)
757 		return;
758 
759 	strcpy(last_cmd, HIST_PREFIX);
760 	/* Again, sizeof() contains the nul byte */
761 	len -= sizeof(HIST_PREFIX);
762 	strncat(last_cmd, str, len);
763 
764 	if (file) {
765 		call = file->event_call;
766 		system = call->class->system;
767 		if (system) {
768 			name = trace_event_name(call);
769 			if (!name)
770 				system = NULL;
771 		}
772 	}
773 
774 	if (system)
775 		snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, HIST_PREFIX "%s:%s", system, name);
776 }
777 
778 static void hist_err(struct trace_array *tr, u8 err_type, u16 err_pos)
779 {
780 	if (!last_cmd)
781 		return;
782 
783 	tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
784 			err_type, err_pos);
785 }
786 
787 static void hist_err_clear(void)
788 {
789 	if (last_cmd)
790 		last_cmd[0] = '\0';
791 	last_cmd_loc[0] = '\0';
792 }
793 
794 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
795 				    unsigned int *var_ref_idx);
796 
797 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
798 			       unsigned int *var_ref_idx)
799 {
800 	struct tracepoint *tp = event->tp;
801 
802 	if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
803 		struct tracepoint_func *probe_func_ptr;
804 		synth_probe_func_t probe_func;
805 		void *__data;
806 
807 		if (!(cpu_online(raw_smp_processor_id())))
808 			return;
809 
810 		probe_func_ptr = rcu_dereference_sched((tp)->funcs);
811 		if (probe_func_ptr) {
812 			do {
813 				probe_func = probe_func_ptr->func;
814 				__data = probe_func_ptr->data;
815 				probe_func(__data, var_ref_vals, var_ref_idx);
816 			} while ((++probe_func_ptr)->func);
817 		}
818 	}
819 }
820 
821 static void action_trace(struct hist_trigger_data *hist_data,
822 			 struct tracing_map_elt *elt,
823 			 struct trace_buffer *buffer, void *rec,
824 			 struct ring_buffer_event *rbe, void *key,
825 			 struct action_data *data, u64 *var_ref_vals)
826 {
827 	struct synth_event *event = data->synth_event;
828 
829 	trace_synth(event, var_ref_vals, data->var_ref_idx);
830 }
831 
832 struct hist_var_data {
833 	struct list_head list;
834 	struct hist_trigger_data *hist_data;
835 };
836 
837 static u64 hist_field_timestamp(struct hist_field *hist_field,
838 				struct tracing_map_elt *elt,
839 				struct trace_buffer *buffer,
840 				struct ring_buffer_event *rbe,
841 				void *event)
842 {
843 	struct hist_trigger_data *hist_data = hist_field->hist_data;
844 	struct trace_array *tr = hist_data->event_file->tr;
845 
846 	u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
847 
848 	if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
849 		ts = ns2usecs(ts);
850 
851 	return ts;
852 }
853 
854 static u64 hist_field_cpu(struct hist_field *hist_field,
855 			  struct tracing_map_elt *elt,
856 			  struct trace_buffer *buffer,
857 			  struct ring_buffer_event *rbe,
858 			  void *event)
859 {
860 	int cpu = smp_processor_id();
861 
862 	return cpu;
863 }
864 
865 /**
866  * check_field_for_var_ref - Check if a VAR_REF field references a variable
867  * @hist_field: The VAR_REF field to check
868  * @var_data: The hist trigger that owns the variable
869  * @var_idx: The trigger variable identifier
870  *
871  * Check the given VAR_REF field to see whether or not it references
872  * the given variable associated with the given trigger.
873  *
874  * Return: The VAR_REF field if it does reference the variable, NULL if not
875  */
876 static struct hist_field *
877 check_field_for_var_ref(struct hist_field *hist_field,
878 			struct hist_trigger_data *var_data,
879 			unsigned int var_idx)
880 {
881 	WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
882 
883 	if (hist_field && hist_field->var.idx == var_idx &&
884 	    hist_field->var.hist_data == var_data)
885 		return hist_field;
886 
887 	return NULL;
888 }
889 
890 /**
891  * find_var_ref - Check if a trigger has a reference to a trigger variable
892  * @hist_data: The hist trigger that might have a reference to the variable
893  * @var_data: The hist trigger that owns the variable
894  * @var_idx: The trigger variable identifier
895  *
896  * Check the list of var_refs[] on the first hist trigger to see
897  * whether any of them are references to the variable on the second
898  * trigger.
899  *
900  * Return: The VAR_REF field referencing the variable if so, NULL if not
901  */
902 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
903 				       struct hist_trigger_data *var_data,
904 				       unsigned int var_idx)
905 {
906 	struct hist_field *hist_field;
907 	unsigned int i;
908 
909 	for (i = 0; i < hist_data->n_var_refs; i++) {
910 		hist_field = hist_data->var_refs[i];
911 		if (check_field_for_var_ref(hist_field, var_data, var_idx))
912 			return hist_field;
913 	}
914 
915 	return NULL;
916 }
917 
918 /**
919  * find_any_var_ref - Check if there is a reference to a given trigger variable
920  * @hist_data: The hist trigger
921  * @var_idx: The trigger variable identifier
922  *
923  * Check to see whether the given variable is currently referenced by
924  * any other trigger.
925  *
926  * The trigger the variable is defined on is explicitly excluded - the
927  * assumption being that a self-reference doesn't prevent a trigger
928  * from being removed.
929  *
930  * Return: The VAR_REF field referencing the variable if so, NULL if not
931  */
932 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
933 					   unsigned int var_idx)
934 {
935 	struct trace_array *tr = hist_data->event_file->tr;
936 	struct hist_field *found = NULL;
937 	struct hist_var_data *var_data;
938 
939 	list_for_each_entry(var_data, &tr->hist_vars, list) {
940 		if (var_data->hist_data == hist_data)
941 			continue;
942 		found = find_var_ref(var_data->hist_data, hist_data, var_idx);
943 		if (found)
944 			break;
945 	}
946 
947 	return found;
948 }
949 
950 /**
951  * check_var_refs - Check if there is a reference to any of trigger's variables
952  * @hist_data: The hist trigger
953  *
954  * A trigger can define one or more variables.  If any one of them is
955  * currently referenced by any other trigger, this function will
956  * determine that.
957 
958  * Typically used to determine whether or not a trigger can be removed
959  * - if there are any references to a trigger's variables, it cannot.
960  *
961  * Return: True if there is a reference to any of trigger's variables
962  */
963 static bool check_var_refs(struct hist_trigger_data *hist_data)
964 {
965 	struct hist_field *field;
966 	bool found = false;
967 	int i;
968 
969 	for_each_hist_field(i, hist_data) {
970 		field = hist_data->fields[i];
971 		if (field && field->flags & HIST_FIELD_FL_VAR) {
972 			if (find_any_var_ref(hist_data, field->var.idx)) {
973 				found = true;
974 				break;
975 			}
976 		}
977 	}
978 
979 	return found;
980 }
981 
982 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
983 {
984 	struct trace_array *tr = hist_data->event_file->tr;
985 	struct hist_var_data *var_data, *found = NULL;
986 
987 	list_for_each_entry(var_data, &tr->hist_vars, list) {
988 		if (var_data->hist_data == hist_data) {
989 			found = var_data;
990 			break;
991 		}
992 	}
993 
994 	return found;
995 }
996 
997 static bool field_has_hist_vars(struct hist_field *hist_field,
998 				unsigned int level)
999 {
1000 	int i;
1001 
1002 	if (level > 3)
1003 		return false;
1004 
1005 	if (!hist_field)
1006 		return false;
1007 
1008 	if (hist_field->flags & HIST_FIELD_FL_VAR ||
1009 	    hist_field->flags & HIST_FIELD_FL_VAR_REF)
1010 		return true;
1011 
1012 	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1013 		struct hist_field *operand;
1014 
1015 		operand = hist_field->operands[i];
1016 		if (field_has_hist_vars(operand, level + 1))
1017 			return true;
1018 	}
1019 
1020 	return false;
1021 }
1022 
1023 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1024 {
1025 	struct hist_field *hist_field;
1026 	int i;
1027 
1028 	for_each_hist_field(i, hist_data) {
1029 		hist_field = hist_data->fields[i];
1030 		if (field_has_hist_vars(hist_field, 0))
1031 			return true;
1032 	}
1033 
1034 	return false;
1035 }
1036 
1037 static int save_hist_vars(struct hist_trigger_data *hist_data)
1038 {
1039 	struct trace_array *tr = hist_data->event_file->tr;
1040 	struct hist_var_data *var_data;
1041 
1042 	var_data = find_hist_vars(hist_data);
1043 	if (var_data)
1044 		return 0;
1045 
1046 	if (tracing_check_open_get_tr(tr))
1047 		return -ENODEV;
1048 
1049 	var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1050 	if (!var_data) {
1051 		trace_array_put(tr);
1052 		return -ENOMEM;
1053 	}
1054 
1055 	var_data->hist_data = hist_data;
1056 	list_add(&var_data->list, &tr->hist_vars);
1057 
1058 	return 0;
1059 }
1060 
1061 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1062 {
1063 	struct trace_array *tr = hist_data->event_file->tr;
1064 	struct hist_var_data *var_data;
1065 
1066 	var_data = find_hist_vars(hist_data);
1067 	if (!var_data)
1068 		return;
1069 
1070 	if (WARN_ON(check_var_refs(hist_data)))
1071 		return;
1072 
1073 	list_del(&var_data->list);
1074 
1075 	kfree(var_data);
1076 
1077 	trace_array_put(tr);
1078 }
1079 
1080 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1081 					 const char *var_name)
1082 {
1083 	struct hist_field *hist_field, *found = NULL;
1084 	int i;
1085 
1086 	for_each_hist_field(i, hist_data) {
1087 		hist_field = hist_data->fields[i];
1088 		if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1089 		    strcmp(hist_field->var.name, var_name) == 0) {
1090 			found = hist_field;
1091 			break;
1092 		}
1093 	}
1094 
1095 	return found;
1096 }
1097 
1098 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1099 				   struct trace_event_file *file,
1100 				   const char *var_name)
1101 {
1102 	struct hist_trigger_data *test_data;
1103 	struct event_trigger_data *test;
1104 	struct hist_field *hist_field;
1105 
1106 	lockdep_assert_held(&event_mutex);
1107 
1108 	hist_field = find_var_field(hist_data, var_name);
1109 	if (hist_field)
1110 		return hist_field;
1111 
1112 	list_for_each_entry(test, &file->triggers, list) {
1113 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1114 			test_data = test->private_data;
1115 			hist_field = find_var_field(test_data, var_name);
1116 			if (hist_field)
1117 				return hist_field;
1118 		}
1119 	}
1120 
1121 	return NULL;
1122 }
1123 
1124 static struct trace_event_file *find_var_file(struct trace_array *tr,
1125 					      char *system,
1126 					      char *event_name,
1127 					      char *var_name)
1128 {
1129 	struct hist_trigger_data *var_hist_data;
1130 	struct hist_var_data *var_data;
1131 	struct trace_event_file *file, *found = NULL;
1132 
1133 	if (system)
1134 		return find_event_file(tr, system, event_name);
1135 
1136 	list_for_each_entry(var_data, &tr->hist_vars, list) {
1137 		var_hist_data = var_data->hist_data;
1138 		file = var_hist_data->event_file;
1139 		if (file == found)
1140 			continue;
1141 
1142 		if (find_var_field(var_hist_data, var_name)) {
1143 			if (found) {
1144 				hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1145 				return NULL;
1146 			}
1147 
1148 			found = file;
1149 		}
1150 	}
1151 
1152 	return found;
1153 }
1154 
1155 static struct hist_field *find_file_var(struct trace_event_file *file,
1156 					const char *var_name)
1157 {
1158 	struct hist_trigger_data *test_data;
1159 	struct event_trigger_data *test;
1160 	struct hist_field *hist_field;
1161 
1162 	lockdep_assert_held(&event_mutex);
1163 
1164 	list_for_each_entry(test, &file->triggers, list) {
1165 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1166 			test_data = test->private_data;
1167 			hist_field = find_var_field(test_data, var_name);
1168 			if (hist_field)
1169 				return hist_field;
1170 		}
1171 	}
1172 
1173 	return NULL;
1174 }
1175 
1176 static struct hist_field *
1177 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1178 {
1179 	struct trace_array *tr = hist_data->event_file->tr;
1180 	struct hist_field *hist_field, *found = NULL;
1181 	struct trace_event_file *file;
1182 	unsigned int i;
1183 
1184 	for (i = 0; i < hist_data->n_actions; i++) {
1185 		struct action_data *data = hist_data->actions[i];
1186 
1187 		if (data->handler == HANDLER_ONMATCH) {
1188 			char *system = data->match_data.event_system;
1189 			char *event_name = data->match_data.event;
1190 
1191 			file = find_var_file(tr, system, event_name, var_name);
1192 			if (!file)
1193 				continue;
1194 			hist_field = find_file_var(file, var_name);
1195 			if (hist_field) {
1196 				if (found) {
1197 					hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1198 						 errpos(var_name));
1199 					return ERR_PTR(-EINVAL);
1200 				}
1201 
1202 				found = hist_field;
1203 			}
1204 		}
1205 	}
1206 	return found;
1207 }
1208 
1209 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1210 					 char *system,
1211 					 char *event_name,
1212 					 char *var_name)
1213 {
1214 	struct trace_array *tr = hist_data->event_file->tr;
1215 	struct hist_field *hist_field = NULL;
1216 	struct trace_event_file *file;
1217 
1218 	if (!system || !event_name) {
1219 		hist_field = find_match_var(hist_data, var_name);
1220 		if (IS_ERR(hist_field))
1221 			return NULL;
1222 		if (hist_field)
1223 			return hist_field;
1224 	}
1225 
1226 	file = find_var_file(tr, system, event_name, var_name);
1227 	if (!file)
1228 		return NULL;
1229 
1230 	hist_field = find_file_var(file, var_name);
1231 
1232 	return hist_field;
1233 }
1234 
1235 static u64 hist_field_var_ref(struct hist_field *hist_field,
1236 			      struct tracing_map_elt *elt,
1237 			      struct trace_buffer *buffer,
1238 			      struct ring_buffer_event *rbe,
1239 			      void *event)
1240 {
1241 	struct hist_elt_data *elt_data;
1242 	u64 var_val = 0;
1243 
1244 	if (WARN_ON_ONCE(!elt))
1245 		return var_val;
1246 
1247 	elt_data = elt->private_data;
1248 	var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1249 
1250 	return var_val;
1251 }
1252 
1253 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1254 			     u64 *var_ref_vals, bool self)
1255 {
1256 	struct hist_trigger_data *var_data;
1257 	struct tracing_map_elt *var_elt;
1258 	struct hist_field *hist_field;
1259 	unsigned int i, var_idx;
1260 	bool resolved = true;
1261 	u64 var_val = 0;
1262 
1263 	for (i = 0; i < hist_data->n_var_refs; i++) {
1264 		hist_field = hist_data->var_refs[i];
1265 		var_idx = hist_field->var.idx;
1266 		var_data = hist_field->var.hist_data;
1267 
1268 		if (var_data == NULL) {
1269 			resolved = false;
1270 			break;
1271 		}
1272 
1273 		if ((self && var_data != hist_data) ||
1274 		    (!self && var_data == hist_data))
1275 			continue;
1276 
1277 		var_elt = tracing_map_lookup(var_data->map, key);
1278 		if (!var_elt) {
1279 			resolved = false;
1280 			break;
1281 		}
1282 
1283 		if (!tracing_map_var_set(var_elt, var_idx)) {
1284 			resolved = false;
1285 			break;
1286 		}
1287 
1288 		if (self || !hist_field->read_once)
1289 			var_val = tracing_map_read_var(var_elt, var_idx);
1290 		else
1291 			var_val = tracing_map_read_var_once(var_elt, var_idx);
1292 
1293 		var_ref_vals[i] = var_val;
1294 	}
1295 
1296 	return resolved;
1297 }
1298 
1299 static const char *hist_field_name(struct hist_field *field,
1300 				   unsigned int level)
1301 {
1302 	const char *field_name = "";
1303 
1304 	if (level > 1)
1305 		return field_name;
1306 
1307 	if (field->field)
1308 		field_name = field->field->name;
1309 	else if (field->flags & HIST_FIELD_FL_LOG2 ||
1310 		 field->flags & HIST_FIELD_FL_ALIAS ||
1311 		 field->flags & HIST_FIELD_FL_BUCKET)
1312 		field_name = hist_field_name(field->operands[0], ++level);
1313 	else if (field->flags & HIST_FIELD_FL_CPU)
1314 		field_name = "common_cpu";
1315 	else if (field->flags & HIST_FIELD_FL_EXPR ||
1316 		 field->flags & HIST_FIELD_FL_VAR_REF) {
1317 		if (field->system) {
1318 			static char full_name[MAX_FILTER_STR_VAL];
1319 
1320 			strcat(full_name, field->system);
1321 			strcat(full_name, ".");
1322 			strcat(full_name, field->event_name);
1323 			strcat(full_name, ".");
1324 			strcat(full_name, field->name);
1325 			field_name = full_name;
1326 		} else
1327 			field_name = field->name;
1328 	} else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1329 		field_name = "common_timestamp";
1330 
1331 	if (field_name == NULL)
1332 		field_name = "";
1333 
1334 	return field_name;
1335 }
1336 
1337 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1338 {
1339 	hist_field_fn_t fn = NULL;
1340 
1341 	switch (field_size) {
1342 	case 8:
1343 		if (field_is_signed)
1344 			fn = hist_field_s64;
1345 		else
1346 			fn = hist_field_u64;
1347 		break;
1348 	case 4:
1349 		if (field_is_signed)
1350 			fn = hist_field_s32;
1351 		else
1352 			fn = hist_field_u32;
1353 		break;
1354 	case 2:
1355 		if (field_is_signed)
1356 			fn = hist_field_s16;
1357 		else
1358 			fn = hist_field_u16;
1359 		break;
1360 	case 1:
1361 		if (field_is_signed)
1362 			fn = hist_field_s8;
1363 		else
1364 			fn = hist_field_u8;
1365 		break;
1366 	}
1367 
1368 	return fn;
1369 }
1370 
1371 static int parse_map_size(char *str)
1372 {
1373 	unsigned long size, map_bits;
1374 	int ret;
1375 
1376 	ret = kstrtoul(str, 0, &size);
1377 	if (ret)
1378 		goto out;
1379 
1380 	map_bits = ilog2(roundup_pow_of_two(size));
1381 	if (map_bits < TRACING_MAP_BITS_MIN ||
1382 	    map_bits > TRACING_MAP_BITS_MAX)
1383 		ret = -EINVAL;
1384 	else
1385 		ret = map_bits;
1386  out:
1387 	return ret;
1388 }
1389 
1390 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1391 {
1392 	unsigned int i;
1393 
1394 	if (!attrs)
1395 		return;
1396 
1397 	for (i = 0; i < attrs->n_assignments; i++)
1398 		kfree(attrs->assignment_str[i]);
1399 
1400 	for (i = 0; i < attrs->n_actions; i++)
1401 		kfree(attrs->action_str[i]);
1402 
1403 	kfree(attrs->name);
1404 	kfree(attrs->sort_key_str);
1405 	kfree(attrs->keys_str);
1406 	kfree(attrs->vals_str);
1407 	kfree(attrs->clock);
1408 	kfree(attrs);
1409 }
1410 
1411 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1412 {
1413 	int ret = -EINVAL;
1414 
1415 	if (attrs->n_actions >= HIST_ACTIONS_MAX)
1416 		return ret;
1417 
1418 	if ((str_has_prefix(str, "onmatch(")) ||
1419 	    (str_has_prefix(str, "onmax(")) ||
1420 	    (str_has_prefix(str, "onchange("))) {
1421 		attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1422 		if (!attrs->action_str[attrs->n_actions]) {
1423 			ret = -ENOMEM;
1424 			return ret;
1425 		}
1426 		attrs->n_actions++;
1427 		ret = 0;
1428 	}
1429 	return ret;
1430 }
1431 
1432 static int parse_assignment(struct trace_array *tr,
1433 			    char *str, struct hist_trigger_attrs *attrs)
1434 {
1435 	int len, ret = 0;
1436 
1437 	if ((len = str_has_prefix(str, "key=")) ||
1438 	    (len = str_has_prefix(str, "keys="))) {
1439 		attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1440 		if (!attrs->keys_str) {
1441 			ret = -ENOMEM;
1442 			goto out;
1443 		}
1444 	} else if ((len = str_has_prefix(str, "val=")) ||
1445 		   (len = str_has_prefix(str, "vals=")) ||
1446 		   (len = str_has_prefix(str, "values="))) {
1447 		attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1448 		if (!attrs->vals_str) {
1449 			ret = -ENOMEM;
1450 			goto out;
1451 		}
1452 	} else if ((len = str_has_prefix(str, "sort="))) {
1453 		attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1454 		if (!attrs->sort_key_str) {
1455 			ret = -ENOMEM;
1456 			goto out;
1457 		}
1458 	} else if (str_has_prefix(str, "name=")) {
1459 		attrs->name = kstrdup(str, GFP_KERNEL);
1460 		if (!attrs->name) {
1461 			ret = -ENOMEM;
1462 			goto out;
1463 		}
1464 	} else if ((len = str_has_prefix(str, "clock="))) {
1465 		str += len;
1466 
1467 		str = strstrip(str);
1468 		attrs->clock = kstrdup(str, GFP_KERNEL);
1469 		if (!attrs->clock) {
1470 			ret = -ENOMEM;
1471 			goto out;
1472 		}
1473 	} else if ((len = str_has_prefix(str, "size="))) {
1474 		int map_bits = parse_map_size(str + len);
1475 
1476 		if (map_bits < 0) {
1477 			ret = map_bits;
1478 			goto out;
1479 		}
1480 		attrs->map_bits = map_bits;
1481 	} else {
1482 		char *assignment;
1483 
1484 		if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1485 			hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1486 			ret = -EINVAL;
1487 			goto out;
1488 		}
1489 
1490 		assignment = kstrdup(str, GFP_KERNEL);
1491 		if (!assignment) {
1492 			ret = -ENOMEM;
1493 			goto out;
1494 		}
1495 
1496 		attrs->assignment_str[attrs->n_assignments++] = assignment;
1497 	}
1498  out:
1499 	return ret;
1500 }
1501 
1502 static struct hist_trigger_attrs *
1503 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1504 {
1505 	struct hist_trigger_attrs *attrs;
1506 	int ret = 0;
1507 
1508 	attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1509 	if (!attrs)
1510 		return ERR_PTR(-ENOMEM);
1511 
1512 	while (trigger_str) {
1513 		char *str = strsep(&trigger_str, ":");
1514 		char *rhs;
1515 
1516 		rhs = strchr(str, '=');
1517 		if (rhs) {
1518 			if (!strlen(++rhs)) {
1519 				ret = -EINVAL;
1520 				hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1521 				goto free;
1522 			}
1523 			ret = parse_assignment(tr, str, attrs);
1524 			if (ret)
1525 				goto free;
1526 		} else if (strcmp(str, "pause") == 0)
1527 			attrs->pause = true;
1528 		else if ((strcmp(str, "cont") == 0) ||
1529 			 (strcmp(str, "continue") == 0))
1530 			attrs->cont = true;
1531 		else if (strcmp(str, "clear") == 0)
1532 			attrs->clear = true;
1533 		else {
1534 			ret = parse_action(str, attrs);
1535 			if (ret)
1536 				goto free;
1537 		}
1538 	}
1539 
1540 	if (!attrs->keys_str) {
1541 		ret = -EINVAL;
1542 		goto free;
1543 	}
1544 
1545 	if (!attrs->clock) {
1546 		attrs->clock = kstrdup("global", GFP_KERNEL);
1547 		if (!attrs->clock) {
1548 			ret = -ENOMEM;
1549 			goto free;
1550 		}
1551 	}
1552 
1553 	return attrs;
1554  free:
1555 	destroy_hist_trigger_attrs(attrs);
1556 
1557 	return ERR_PTR(ret);
1558 }
1559 
1560 static inline void save_comm(char *comm, struct task_struct *task)
1561 {
1562 	if (!task->pid) {
1563 		strcpy(comm, "<idle>");
1564 		return;
1565 	}
1566 
1567 	if (WARN_ON_ONCE(task->pid < 0)) {
1568 		strcpy(comm, "<XXX>");
1569 		return;
1570 	}
1571 
1572 	strncpy(comm, task->comm, TASK_COMM_LEN);
1573 }
1574 
1575 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1576 {
1577 	unsigned int i;
1578 
1579 	for (i = 0; i < elt_data->n_field_var_str; i++)
1580 		kfree(elt_data->field_var_str[i]);
1581 
1582 	kfree(elt_data->field_var_str);
1583 
1584 	kfree(elt_data->comm);
1585 	kfree(elt_data);
1586 }
1587 
1588 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1589 {
1590 	struct hist_elt_data *elt_data = elt->private_data;
1591 
1592 	hist_elt_data_free(elt_data);
1593 }
1594 
1595 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1596 {
1597 	struct hist_trigger_data *hist_data = elt->map->private_data;
1598 	unsigned int size = TASK_COMM_LEN;
1599 	struct hist_elt_data *elt_data;
1600 	struct hist_field *hist_field;
1601 	unsigned int i, n_str;
1602 
1603 	elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1604 	if (!elt_data)
1605 		return -ENOMEM;
1606 
1607 	for_each_hist_field(i, hist_data) {
1608 		hist_field = hist_data->fields[i];
1609 
1610 		if (hist_field->flags & HIST_FIELD_FL_EXECNAME) {
1611 			elt_data->comm = kzalloc(size, GFP_KERNEL);
1612 			if (!elt_data->comm) {
1613 				kfree(elt_data);
1614 				return -ENOMEM;
1615 			}
1616 			break;
1617 		}
1618 	}
1619 
1620 	n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1621 		hist_data->n_var_str;
1622 	if (n_str > SYNTH_FIELDS_MAX) {
1623 		hist_elt_data_free(elt_data);
1624 		return -EINVAL;
1625 	}
1626 
1627 	BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1628 
1629 	size = STR_VAR_LEN_MAX;
1630 
1631 	elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL);
1632 	if (!elt_data->field_var_str) {
1633 		hist_elt_data_free(elt_data);
1634 		return -EINVAL;
1635 	}
1636 	elt_data->n_field_var_str = n_str;
1637 
1638 	for (i = 0; i < n_str; i++) {
1639 		elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1640 		if (!elt_data->field_var_str[i]) {
1641 			hist_elt_data_free(elt_data);
1642 			return -ENOMEM;
1643 		}
1644 	}
1645 
1646 	elt->private_data = elt_data;
1647 
1648 	return 0;
1649 }
1650 
1651 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1652 {
1653 	struct hist_elt_data *elt_data = elt->private_data;
1654 
1655 	if (elt_data->comm)
1656 		save_comm(elt_data->comm, current);
1657 }
1658 
1659 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1660 	.elt_alloc	= hist_trigger_elt_data_alloc,
1661 	.elt_free	= hist_trigger_elt_data_free,
1662 	.elt_init	= hist_trigger_elt_data_init,
1663 };
1664 
1665 static const char *get_hist_field_flags(struct hist_field *hist_field)
1666 {
1667 	const char *flags_str = NULL;
1668 
1669 	if (hist_field->flags & HIST_FIELD_FL_HEX)
1670 		flags_str = "hex";
1671 	else if (hist_field->flags & HIST_FIELD_FL_SYM)
1672 		flags_str = "sym";
1673 	else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1674 		flags_str = "sym-offset";
1675 	else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1676 		flags_str = "execname";
1677 	else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1678 		flags_str = "syscall";
1679 	else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1680 		flags_str = "log2";
1681 	else if (hist_field->flags & HIST_FIELD_FL_BUCKET)
1682 		flags_str = "buckets";
1683 	else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1684 		flags_str = "usecs";
1685 
1686 	return flags_str;
1687 }
1688 
1689 static void expr_field_str(struct hist_field *field, char *expr)
1690 {
1691 	if (field->flags & HIST_FIELD_FL_VAR_REF)
1692 		strcat(expr, "$");
1693 	else if (field->flags & HIST_FIELD_FL_CONST) {
1694 		char str[HIST_CONST_DIGITS_MAX];
1695 
1696 		snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant);
1697 		strcat(expr, str);
1698 	}
1699 
1700 	strcat(expr, hist_field_name(field, 0));
1701 
1702 	if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1703 		const char *flags_str = get_hist_field_flags(field);
1704 
1705 		if (flags_str) {
1706 			strcat(expr, ".");
1707 			strcat(expr, flags_str);
1708 		}
1709 	}
1710 }
1711 
1712 static char *expr_str(struct hist_field *field, unsigned int level)
1713 {
1714 	char *expr;
1715 
1716 	if (level > 1)
1717 		return NULL;
1718 
1719 	expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1720 	if (!expr)
1721 		return NULL;
1722 
1723 	if (!field->operands[0]) {
1724 		expr_field_str(field, expr);
1725 		return expr;
1726 	}
1727 
1728 	if (field->operator == FIELD_OP_UNARY_MINUS) {
1729 		char *subexpr;
1730 
1731 		strcat(expr, "-(");
1732 		subexpr = expr_str(field->operands[0], ++level);
1733 		if (!subexpr) {
1734 			kfree(expr);
1735 			return NULL;
1736 		}
1737 		strcat(expr, subexpr);
1738 		strcat(expr, ")");
1739 
1740 		kfree(subexpr);
1741 
1742 		return expr;
1743 	}
1744 
1745 	expr_field_str(field->operands[0], expr);
1746 
1747 	switch (field->operator) {
1748 	case FIELD_OP_MINUS:
1749 		strcat(expr, "-");
1750 		break;
1751 	case FIELD_OP_PLUS:
1752 		strcat(expr, "+");
1753 		break;
1754 	case FIELD_OP_DIV:
1755 		strcat(expr, "/");
1756 		break;
1757 	case FIELD_OP_MULT:
1758 		strcat(expr, "*");
1759 		break;
1760 	default:
1761 		kfree(expr);
1762 		return NULL;
1763 	}
1764 
1765 	expr_field_str(field->operands[1], expr);
1766 
1767 	return expr;
1768 }
1769 
1770 /*
1771  * If field_op != FIELD_OP_NONE, *sep points to the root operator
1772  * of the expression tree to be evaluated.
1773  */
1774 static int contains_operator(char *str, char **sep)
1775 {
1776 	enum field_op_id field_op = FIELD_OP_NONE;
1777 	char *minus_op, *plus_op, *div_op, *mult_op;
1778 
1779 
1780 	/*
1781 	 * Report the last occurrence of the operators first, so that the
1782 	 * expression is evaluated left to right. This is important since
1783 	 * subtraction and division are not associative.
1784 	 *
1785 	 *	e.g
1786 	 *		64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2
1787 	 *		14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2
1788 	 */
1789 
1790 	/*
1791 	 * First, find lower precedence addition and subtraction
1792 	 * since the expression will be evaluated recursively.
1793 	 */
1794 	minus_op = strrchr(str, '-');
1795 	if (minus_op) {
1796 		/*
1797 		 * Unary minus is not supported in sub-expressions. If
1798 		 * present, it is always the next root operator.
1799 		 */
1800 		if (minus_op == str) {
1801 			field_op = FIELD_OP_UNARY_MINUS;
1802 			goto out;
1803 		}
1804 
1805 		field_op = FIELD_OP_MINUS;
1806 	}
1807 
1808 	plus_op = strrchr(str, '+');
1809 	if (plus_op || minus_op) {
1810 		/*
1811 		 * For operators of the same precedence use to rightmost as the
1812 		 * root, so that the expression is evaluated left to right.
1813 		 */
1814 		if (plus_op > minus_op)
1815 			field_op = FIELD_OP_PLUS;
1816 		goto out;
1817 	}
1818 
1819 	/*
1820 	 * Multiplication and division have higher precedence than addition and
1821 	 * subtraction.
1822 	 */
1823 	div_op = strrchr(str, '/');
1824 	if (div_op)
1825 		field_op = FIELD_OP_DIV;
1826 
1827 	mult_op = strrchr(str, '*');
1828 	/*
1829 	 * For operators of the same precedence use to rightmost as the
1830 	 * root, so that the expression is evaluated left to right.
1831 	 */
1832 	if (mult_op > div_op)
1833 		field_op = FIELD_OP_MULT;
1834 
1835 out:
1836 	if (sep) {
1837 		switch (field_op) {
1838 		case FIELD_OP_UNARY_MINUS:
1839 		case FIELD_OP_MINUS:
1840 			*sep = minus_op;
1841 			break;
1842 		case FIELD_OP_PLUS:
1843 			*sep = plus_op;
1844 			break;
1845 		case FIELD_OP_DIV:
1846 			*sep = div_op;
1847 			break;
1848 		case FIELD_OP_MULT:
1849 			*sep = mult_op;
1850 			break;
1851 		case FIELD_OP_NONE:
1852 		default:
1853 			*sep = NULL;
1854 			break;
1855 		}
1856 	}
1857 
1858 	return field_op;
1859 }
1860 
1861 static void get_hist_field(struct hist_field *hist_field)
1862 {
1863 	hist_field->ref++;
1864 }
1865 
1866 static void __destroy_hist_field(struct hist_field *hist_field)
1867 {
1868 	if (--hist_field->ref > 1)
1869 		return;
1870 
1871 	kfree(hist_field->var.name);
1872 	kfree(hist_field->name);
1873 
1874 	/* Can likely be a const */
1875 	kfree_const(hist_field->type);
1876 
1877 	kfree(hist_field->system);
1878 	kfree(hist_field->event_name);
1879 
1880 	kfree(hist_field);
1881 }
1882 
1883 static void destroy_hist_field(struct hist_field *hist_field,
1884 			       unsigned int level)
1885 {
1886 	unsigned int i;
1887 
1888 	if (level > 3)
1889 		return;
1890 
1891 	if (!hist_field)
1892 		return;
1893 
1894 	if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1895 		return; /* var refs will be destroyed separately */
1896 
1897 	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1898 		destroy_hist_field(hist_field->operands[i], level + 1);
1899 
1900 	__destroy_hist_field(hist_field);
1901 }
1902 
1903 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1904 					    struct ftrace_event_field *field,
1905 					    unsigned long flags,
1906 					    char *var_name)
1907 {
1908 	struct hist_field *hist_field;
1909 
1910 	if (field && is_function_field(field))
1911 		return NULL;
1912 
1913 	hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1914 	if (!hist_field)
1915 		return NULL;
1916 
1917 	hist_field->ref = 1;
1918 
1919 	hist_field->hist_data = hist_data;
1920 
1921 	if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1922 		goto out; /* caller will populate */
1923 
1924 	if (flags & HIST_FIELD_FL_VAR_REF) {
1925 		hist_field->fn = hist_field_var_ref;
1926 		goto out;
1927 	}
1928 
1929 	if (flags & HIST_FIELD_FL_HITCOUNT) {
1930 		hist_field->fn = hist_field_counter;
1931 		hist_field->size = sizeof(u64);
1932 		hist_field->type = "u64";
1933 		goto out;
1934 	}
1935 
1936 	if (flags & HIST_FIELD_FL_CONST) {
1937 		hist_field->fn = hist_field_const;
1938 		hist_field->size = sizeof(u64);
1939 		hist_field->type = kstrdup("u64", GFP_KERNEL);
1940 		if (!hist_field->type)
1941 			goto free;
1942 		goto out;
1943 	}
1944 
1945 	if (flags & HIST_FIELD_FL_STACKTRACE) {
1946 		hist_field->fn = hist_field_none;
1947 		goto out;
1948 	}
1949 
1950 	if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
1951 		unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
1952 		hist_field->fn = flags & HIST_FIELD_FL_LOG2 ? hist_field_log2 :
1953 			hist_field_bucket;
1954 		hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1955 		hist_field->size = hist_field->operands[0]->size;
1956 		hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
1957 		if (!hist_field->type)
1958 			goto free;
1959 		goto out;
1960 	}
1961 
1962 	if (flags & HIST_FIELD_FL_TIMESTAMP) {
1963 		hist_field->fn = hist_field_timestamp;
1964 		hist_field->size = sizeof(u64);
1965 		hist_field->type = "u64";
1966 		goto out;
1967 	}
1968 
1969 	if (flags & HIST_FIELD_FL_CPU) {
1970 		hist_field->fn = hist_field_cpu;
1971 		hist_field->size = sizeof(int);
1972 		hist_field->type = "unsigned int";
1973 		goto out;
1974 	}
1975 
1976 	if (WARN_ON_ONCE(!field))
1977 		goto out;
1978 
1979 	/* Pointers to strings are just pointers and dangerous to dereference */
1980 	if (is_string_field(field) &&
1981 	    (field->filter_type != FILTER_PTR_STRING)) {
1982 		flags |= HIST_FIELD_FL_STRING;
1983 
1984 		hist_field->size = MAX_FILTER_STR_VAL;
1985 		hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
1986 		if (!hist_field->type)
1987 			goto free;
1988 
1989 		if (field->filter_type == FILTER_STATIC_STRING) {
1990 			hist_field->fn = hist_field_string;
1991 			hist_field->size = field->size;
1992 		} else if (field->filter_type == FILTER_DYN_STRING) {
1993 			hist_field->fn = hist_field_dynstring;
1994 		} else if (field->filter_type == FILTER_RDYN_STRING)
1995 			hist_field->fn = hist_field_reldynstring;
1996 		else
1997 			hist_field->fn = hist_field_pstring;
1998 	} else {
1999 		hist_field->size = field->size;
2000 		hist_field->is_signed = field->is_signed;
2001 		hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2002 		if (!hist_field->type)
2003 			goto free;
2004 
2005 		hist_field->fn = select_value_fn(field->size,
2006 						 field->is_signed);
2007 		if (!hist_field->fn) {
2008 			destroy_hist_field(hist_field, 0);
2009 			return NULL;
2010 		}
2011 	}
2012  out:
2013 	hist_field->field = field;
2014 	hist_field->flags = flags;
2015 
2016 	if (var_name) {
2017 		hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2018 		if (!hist_field->var.name)
2019 			goto free;
2020 	}
2021 
2022 	return hist_field;
2023  free:
2024 	destroy_hist_field(hist_field, 0);
2025 	return NULL;
2026 }
2027 
2028 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2029 {
2030 	unsigned int i;
2031 
2032 	for (i = 0; i < HIST_FIELDS_MAX; i++) {
2033 		if (hist_data->fields[i]) {
2034 			destroy_hist_field(hist_data->fields[i], 0);
2035 			hist_data->fields[i] = NULL;
2036 		}
2037 	}
2038 
2039 	for (i = 0; i < hist_data->n_var_refs; i++) {
2040 		WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2041 		__destroy_hist_field(hist_data->var_refs[i]);
2042 		hist_data->var_refs[i] = NULL;
2043 	}
2044 }
2045 
2046 static int init_var_ref(struct hist_field *ref_field,
2047 			struct hist_field *var_field,
2048 			char *system, char *event_name)
2049 {
2050 	int err = 0;
2051 
2052 	ref_field->var.idx = var_field->var.idx;
2053 	ref_field->var.hist_data = var_field->hist_data;
2054 	ref_field->size = var_field->size;
2055 	ref_field->is_signed = var_field->is_signed;
2056 	ref_field->flags |= var_field->flags &
2057 		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2058 
2059 	if (system) {
2060 		ref_field->system = kstrdup(system, GFP_KERNEL);
2061 		if (!ref_field->system)
2062 			return -ENOMEM;
2063 	}
2064 
2065 	if (event_name) {
2066 		ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2067 		if (!ref_field->event_name) {
2068 			err = -ENOMEM;
2069 			goto free;
2070 		}
2071 	}
2072 
2073 	if (var_field->var.name) {
2074 		ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2075 		if (!ref_field->name) {
2076 			err = -ENOMEM;
2077 			goto free;
2078 		}
2079 	} else if (var_field->name) {
2080 		ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2081 		if (!ref_field->name) {
2082 			err = -ENOMEM;
2083 			goto free;
2084 		}
2085 	}
2086 
2087 	ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL);
2088 	if (!ref_field->type) {
2089 		err = -ENOMEM;
2090 		goto free;
2091 	}
2092  out:
2093 	return err;
2094  free:
2095 	kfree(ref_field->system);
2096 	kfree(ref_field->event_name);
2097 	kfree(ref_field->name);
2098 
2099 	goto out;
2100 }
2101 
2102 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
2103 			    struct hist_field *var_field)
2104 {
2105 	struct hist_field *ref_field;
2106 	int i;
2107 
2108 	for (i = 0; i < hist_data->n_var_refs; i++) {
2109 		ref_field = hist_data->var_refs[i];
2110 		if (ref_field->var.idx == var_field->var.idx &&
2111 		    ref_field->var.hist_data == var_field->hist_data)
2112 			return i;
2113 	}
2114 
2115 	return -ENOENT;
2116 }
2117 
2118 /**
2119  * create_var_ref - Create a variable reference and attach it to trigger
2120  * @hist_data: The trigger that will be referencing the variable
2121  * @var_field: The VAR field to create a reference to
2122  * @system: The optional system string
2123  * @event_name: The optional event_name string
2124  *
2125  * Given a variable hist_field, create a VAR_REF hist_field that
2126  * represents a reference to it.
2127  *
2128  * This function also adds the reference to the trigger that
2129  * now references the variable.
2130  *
2131  * Return: The VAR_REF field if successful, NULL if not
2132  */
2133 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2134 					 struct hist_field *var_field,
2135 					 char *system, char *event_name)
2136 {
2137 	unsigned long flags = HIST_FIELD_FL_VAR_REF;
2138 	struct hist_field *ref_field;
2139 	int i;
2140 
2141 	/* Check if the variable already exists */
2142 	for (i = 0; i < hist_data->n_var_refs; i++) {
2143 		ref_field = hist_data->var_refs[i];
2144 		if (ref_field->var.idx == var_field->var.idx &&
2145 		    ref_field->var.hist_data == var_field->hist_data) {
2146 			get_hist_field(ref_field);
2147 			return ref_field;
2148 		}
2149 	}
2150 
2151 	ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2152 	if (ref_field) {
2153 		if (init_var_ref(ref_field, var_field, system, event_name)) {
2154 			destroy_hist_field(ref_field, 0);
2155 			return NULL;
2156 		}
2157 
2158 		hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2159 		ref_field->var_ref_idx = hist_data->n_var_refs++;
2160 	}
2161 
2162 	return ref_field;
2163 }
2164 
2165 static bool is_var_ref(char *var_name)
2166 {
2167 	if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2168 		return false;
2169 
2170 	return true;
2171 }
2172 
2173 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2174 				 char *var_name)
2175 {
2176 	char *name, *field;
2177 	unsigned int i;
2178 
2179 	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2180 		name = hist_data->attrs->var_defs.name[i];
2181 
2182 		if (strcmp(var_name, name) == 0) {
2183 			field = hist_data->attrs->var_defs.expr[i];
2184 			if (contains_operator(field, NULL) || is_var_ref(field))
2185 				continue;
2186 			return field;
2187 		}
2188 	}
2189 
2190 	return NULL;
2191 }
2192 
2193 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2194 				 char *system, char *event_name,
2195 				 char *var_name)
2196 {
2197 	struct trace_event_call *call;
2198 
2199 	if (system && event_name) {
2200 		call = hist_data->event_file->event_call;
2201 
2202 		if (strcmp(system, call->class->system) != 0)
2203 			return NULL;
2204 
2205 		if (strcmp(event_name, trace_event_name(call)) != 0)
2206 			return NULL;
2207 	}
2208 
2209 	if (!!system != !!event_name)
2210 		return NULL;
2211 
2212 	if (!is_var_ref(var_name))
2213 		return NULL;
2214 
2215 	var_name++;
2216 
2217 	return field_name_from_var(hist_data, var_name);
2218 }
2219 
2220 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2221 					char *system, char *event_name,
2222 					char *var_name)
2223 {
2224 	struct hist_field *var_field = NULL, *ref_field = NULL;
2225 	struct trace_array *tr = hist_data->event_file->tr;
2226 
2227 	if (!is_var_ref(var_name))
2228 		return NULL;
2229 
2230 	var_name++;
2231 
2232 	var_field = find_event_var(hist_data, system, event_name, var_name);
2233 	if (var_field)
2234 		ref_field = create_var_ref(hist_data, var_field,
2235 					   system, event_name);
2236 
2237 	if (!ref_field)
2238 		hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2239 
2240 	return ref_field;
2241 }
2242 
2243 static struct ftrace_event_field *
2244 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2245 	    char *field_str, unsigned long *flags, unsigned long *buckets)
2246 {
2247 	struct ftrace_event_field *field = NULL;
2248 	char *field_name, *modifier, *str;
2249 	struct trace_array *tr = file->tr;
2250 
2251 	modifier = str = kstrdup(field_str, GFP_KERNEL);
2252 	if (!modifier)
2253 		return ERR_PTR(-ENOMEM);
2254 
2255 	field_name = strsep(&modifier, ".");
2256 	if (modifier) {
2257 		if (strcmp(modifier, "hex") == 0)
2258 			*flags |= HIST_FIELD_FL_HEX;
2259 		else if (strcmp(modifier, "sym") == 0)
2260 			*flags |= HIST_FIELD_FL_SYM;
2261 		/*
2262 		 * 'sym-offset' occurrences in the trigger string are modified
2263 		 * to 'symXoffset' to simplify arithmetic expression parsing.
2264 		 */
2265 		else if (strcmp(modifier, "symXoffset") == 0)
2266 			*flags |= HIST_FIELD_FL_SYM_OFFSET;
2267 		else if ((strcmp(modifier, "execname") == 0) &&
2268 			 (strcmp(field_name, "common_pid") == 0))
2269 			*flags |= HIST_FIELD_FL_EXECNAME;
2270 		else if (strcmp(modifier, "syscall") == 0)
2271 			*flags |= HIST_FIELD_FL_SYSCALL;
2272 		else if (strcmp(modifier, "log2") == 0)
2273 			*flags |= HIST_FIELD_FL_LOG2;
2274 		else if (strcmp(modifier, "usecs") == 0)
2275 			*flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2276 		else if (strncmp(modifier, "bucket", 6) == 0) {
2277 			int ret;
2278 
2279 			modifier += 6;
2280 
2281 			if (*modifier == 's')
2282 				modifier++;
2283 			if (*modifier != '=')
2284 				goto error;
2285 			modifier++;
2286 			ret = kstrtoul(modifier, 0, buckets);
2287 			if (ret || !(*buckets))
2288 				goto error;
2289 			*flags |= HIST_FIELD_FL_BUCKET;
2290 		} else {
2291  error:
2292 			hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2293 			field = ERR_PTR(-EINVAL);
2294 			goto out;
2295 		}
2296 	}
2297 
2298 	if (strcmp(field_name, "common_timestamp") == 0) {
2299 		*flags |= HIST_FIELD_FL_TIMESTAMP;
2300 		hist_data->enable_timestamps = true;
2301 		if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2302 			hist_data->attrs->ts_in_usecs = true;
2303 	} else if (strcmp(field_name, "common_cpu") == 0)
2304 		*flags |= HIST_FIELD_FL_CPU;
2305 	else {
2306 		field = trace_find_event_field(file->event_call, field_name);
2307 		if (!field || !field->size) {
2308 			/*
2309 			 * For backward compatibility, if field_name
2310 			 * was "cpu", then we treat this the same as
2311 			 * common_cpu. This also works for "CPU".
2312 			 */
2313 			if (field && field->filter_type == FILTER_CPU) {
2314 				*flags |= HIST_FIELD_FL_CPU;
2315 			} else {
2316 				hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2317 					 errpos(field_name));
2318 				field = ERR_PTR(-EINVAL);
2319 				goto out;
2320 			}
2321 		}
2322 	}
2323  out:
2324 	kfree(str);
2325 
2326 	return field;
2327 }
2328 
2329 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2330 				       struct hist_field *var_ref,
2331 				       char *var_name)
2332 {
2333 	struct hist_field *alias = NULL;
2334 	unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2335 
2336 	alias = create_hist_field(hist_data, NULL, flags, var_name);
2337 	if (!alias)
2338 		return NULL;
2339 
2340 	alias->fn = var_ref->fn;
2341 	alias->operands[0] = var_ref;
2342 
2343 	if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2344 		destroy_hist_field(alias, 0);
2345 		return NULL;
2346 	}
2347 
2348 	alias->var_ref_idx = var_ref->var_ref_idx;
2349 
2350 	return alias;
2351 }
2352 
2353 static struct hist_field *parse_const(struct hist_trigger_data *hist_data,
2354 				      char *str, char *var_name,
2355 				      unsigned long *flags)
2356 {
2357 	struct trace_array *tr = hist_data->event_file->tr;
2358 	struct hist_field *field = NULL;
2359 	u64 constant;
2360 
2361 	if (kstrtoull(str, 0, &constant)) {
2362 		hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str));
2363 		return NULL;
2364 	}
2365 
2366 	*flags |= HIST_FIELD_FL_CONST;
2367 	field = create_hist_field(hist_data, NULL, *flags, var_name);
2368 	if (!field)
2369 		return NULL;
2370 
2371 	field->constant = constant;
2372 
2373 	return field;
2374 }
2375 
2376 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2377 				     struct trace_event_file *file, char *str,
2378 				     unsigned long *flags, char *var_name)
2379 {
2380 	char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2381 	struct ftrace_event_field *field = NULL;
2382 	struct hist_field *hist_field = NULL;
2383 	unsigned long buckets = 0;
2384 	int ret = 0;
2385 
2386 	if (isdigit(str[0])) {
2387 		hist_field = parse_const(hist_data, str, var_name, flags);
2388 		if (!hist_field) {
2389 			ret = -EINVAL;
2390 			goto out;
2391 		}
2392 		return hist_field;
2393 	}
2394 
2395 	s = strchr(str, '.');
2396 	if (s) {
2397 		s = strchr(++s, '.');
2398 		if (s) {
2399 			ref_system = strsep(&str, ".");
2400 			if (!str) {
2401 				ret = -EINVAL;
2402 				goto out;
2403 			}
2404 			ref_event = strsep(&str, ".");
2405 			if (!str) {
2406 				ret = -EINVAL;
2407 				goto out;
2408 			}
2409 			ref_var = str;
2410 		}
2411 	}
2412 
2413 	s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2414 	if (!s) {
2415 		hist_field = parse_var_ref(hist_data, ref_system,
2416 					   ref_event, ref_var);
2417 		if (hist_field) {
2418 			if (var_name) {
2419 				hist_field = create_alias(hist_data, hist_field, var_name);
2420 				if (!hist_field) {
2421 					ret = -ENOMEM;
2422 					goto out;
2423 				}
2424 			}
2425 			return hist_field;
2426 		}
2427 	} else
2428 		str = s;
2429 
2430 	field = parse_field(hist_data, file, str, flags, &buckets);
2431 	if (IS_ERR(field)) {
2432 		ret = PTR_ERR(field);
2433 		goto out;
2434 	}
2435 
2436 	hist_field = create_hist_field(hist_data, field, *flags, var_name);
2437 	if (!hist_field) {
2438 		ret = -ENOMEM;
2439 		goto out;
2440 	}
2441 	hist_field->buckets = buckets;
2442 
2443 	return hist_field;
2444  out:
2445 	return ERR_PTR(ret);
2446 }
2447 
2448 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2449 				     struct trace_event_file *file,
2450 				     char *str, unsigned long flags,
2451 				     char *var_name, unsigned int *n_subexprs);
2452 
2453 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2454 				      struct trace_event_file *file,
2455 				      char *str, unsigned long flags,
2456 				      char *var_name, unsigned int *n_subexprs)
2457 {
2458 	struct hist_field *operand1, *expr = NULL;
2459 	unsigned long operand_flags;
2460 	int ret = 0;
2461 	char *s;
2462 
2463 	/* Unary minus operator, increment n_subexprs */
2464 	++*n_subexprs;
2465 
2466 	/* we support only -(xxx) i.e. explicit parens required */
2467 
2468 	if (*n_subexprs > 3) {
2469 		hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2470 		ret = -EINVAL;
2471 		goto free;
2472 	}
2473 
2474 	str++; /* skip leading '-' */
2475 
2476 	s = strchr(str, '(');
2477 	if (s)
2478 		str++;
2479 	else {
2480 		ret = -EINVAL;
2481 		goto free;
2482 	}
2483 
2484 	s = strrchr(str, ')');
2485 	if (s) {
2486 		 /* unary minus not supported in sub-expressions */
2487 		if (*(s+1) != '\0') {
2488 			hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR,
2489 				 errpos(str));
2490 			ret = -EINVAL;
2491 			goto free;
2492 		}
2493 		*s = '\0';
2494 	}
2495 	else {
2496 		ret = -EINVAL; /* no closing ')' */
2497 		goto free;
2498 	}
2499 
2500 	flags |= HIST_FIELD_FL_EXPR;
2501 	expr = create_hist_field(hist_data, NULL, flags, var_name);
2502 	if (!expr) {
2503 		ret = -ENOMEM;
2504 		goto free;
2505 	}
2506 
2507 	operand_flags = 0;
2508 	operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2509 	if (IS_ERR(operand1)) {
2510 		ret = PTR_ERR(operand1);
2511 		goto free;
2512 	}
2513 	if (operand1->flags & HIST_FIELD_FL_STRING) {
2514 		/* String type can not be the operand of unary operator. */
2515 		hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2516 		destroy_hist_field(operand1, 0);
2517 		ret = -EINVAL;
2518 		goto free;
2519 	}
2520 
2521 	expr->flags |= operand1->flags &
2522 		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2523 	expr->fn = hist_field_unary_minus;
2524 	expr->operands[0] = operand1;
2525 	expr->size = operand1->size;
2526 	expr->is_signed = operand1->is_signed;
2527 	expr->operator = FIELD_OP_UNARY_MINUS;
2528 	expr->name = expr_str(expr, 0);
2529 	expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2530 	if (!expr->type) {
2531 		ret = -ENOMEM;
2532 		goto free;
2533 	}
2534 
2535 	return expr;
2536  free:
2537 	destroy_hist_field(expr, 0);
2538 	return ERR_PTR(ret);
2539 }
2540 
2541 /*
2542  * If the operands are var refs, return pointers the
2543  * variable(s) referenced in var1 and var2, else NULL.
2544  */
2545 static int check_expr_operands(struct trace_array *tr,
2546 			       struct hist_field *operand1,
2547 			       struct hist_field *operand2,
2548 			       struct hist_field **var1,
2549 			       struct hist_field **var2)
2550 {
2551 	unsigned long operand1_flags = operand1->flags;
2552 	unsigned long operand2_flags = operand2->flags;
2553 
2554 	if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2555 	    (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2556 		struct hist_field *var;
2557 
2558 		var = find_var_field(operand1->var.hist_data, operand1->name);
2559 		if (!var)
2560 			return -EINVAL;
2561 		operand1_flags = var->flags;
2562 		*var1 = var;
2563 	}
2564 
2565 	if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2566 	    (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2567 		struct hist_field *var;
2568 
2569 		var = find_var_field(operand2->var.hist_data, operand2->name);
2570 		if (!var)
2571 			return -EINVAL;
2572 		operand2_flags = var->flags;
2573 		*var2 = var;
2574 	}
2575 
2576 	if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2577 	    (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2578 		hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2579 		return -EINVAL;
2580 	}
2581 
2582 	return 0;
2583 }
2584 
2585 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2586 				     struct trace_event_file *file,
2587 				     char *str, unsigned long flags,
2588 				     char *var_name, unsigned int *n_subexprs)
2589 {
2590 	struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2591 	struct hist_field *var1 = NULL, *var2 = NULL;
2592 	unsigned long operand_flags, operand2_flags;
2593 	int field_op, ret = -EINVAL;
2594 	char *sep, *operand1_str;
2595 	hist_field_fn_t op_fn;
2596 	bool combine_consts;
2597 
2598 	if (*n_subexprs > 3) {
2599 		hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2600 		return ERR_PTR(-EINVAL);
2601 	}
2602 
2603 	field_op = contains_operator(str, &sep);
2604 
2605 	if (field_op == FIELD_OP_NONE)
2606 		return parse_atom(hist_data, file, str, &flags, var_name);
2607 
2608 	if (field_op == FIELD_OP_UNARY_MINUS)
2609 		return parse_unary(hist_data, file, str, flags, var_name, n_subexprs);
2610 
2611 	/* Binary operator found, increment n_subexprs */
2612 	++*n_subexprs;
2613 
2614 	/* Split the expression string at the root operator */
2615 	if (!sep)
2616 		return ERR_PTR(-EINVAL);
2617 
2618 	*sep = '\0';
2619 	operand1_str = str;
2620 	str = sep+1;
2621 
2622 	/* Binary operator requires both operands */
2623 	if (*operand1_str == '\0' || *str == '\0')
2624 		return ERR_PTR(-EINVAL);
2625 
2626 	operand_flags = 0;
2627 
2628 	/* LHS of string is an expression e.g. a+b in a+b+c */
2629 	operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs);
2630 	if (IS_ERR(operand1))
2631 		return ERR_CAST(operand1);
2632 
2633 	if (operand1->flags & HIST_FIELD_FL_STRING) {
2634 		hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2635 		ret = -EINVAL;
2636 		goto free_op1;
2637 	}
2638 
2639 	/* RHS of string is another expression e.g. c in a+b+c */
2640 	operand_flags = 0;
2641 	operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2642 	if (IS_ERR(operand2)) {
2643 		ret = PTR_ERR(operand2);
2644 		goto free_op1;
2645 	}
2646 	if (operand2->flags & HIST_FIELD_FL_STRING) {
2647 		hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2648 		ret = -EINVAL;
2649 		goto free_operands;
2650 	}
2651 
2652 	switch (field_op) {
2653 	case FIELD_OP_MINUS:
2654 		op_fn = hist_field_minus;
2655 		break;
2656 	case FIELD_OP_PLUS:
2657 		op_fn = hist_field_plus;
2658 		break;
2659 	case FIELD_OP_DIV:
2660 		op_fn = hist_field_div;
2661 		break;
2662 	case FIELD_OP_MULT:
2663 		op_fn = hist_field_mult;
2664 		break;
2665 	default:
2666 		ret = -EINVAL;
2667 		goto free_operands;
2668 	}
2669 
2670 	ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2);
2671 	if (ret)
2672 		goto free_operands;
2673 
2674 	operand_flags = var1 ? var1->flags : operand1->flags;
2675 	operand2_flags = var2 ? var2->flags : operand2->flags;
2676 
2677 	/*
2678 	 * If both operands are constant, the expression can be
2679 	 * collapsed to a single constant.
2680 	 */
2681 	combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST;
2682 
2683 	flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR;
2684 
2685 	flags |= operand1->flags &
2686 		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2687 
2688 	expr = create_hist_field(hist_data, NULL, flags, var_name);
2689 	if (!expr) {
2690 		ret = -ENOMEM;
2691 		goto free_operands;
2692 	}
2693 
2694 	operand1->read_once = true;
2695 	operand2->read_once = true;
2696 
2697 	/* The operands are now owned and free'd by 'expr' */
2698 	expr->operands[0] = operand1;
2699 	expr->operands[1] = operand2;
2700 
2701 	if (field_op == FIELD_OP_DIV &&
2702 			operand2_flags & HIST_FIELD_FL_CONST) {
2703 		u64 divisor = var2 ? var2->constant : operand2->constant;
2704 
2705 		if (!divisor) {
2706 			hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str));
2707 			ret = -EDOM;
2708 			goto free_expr;
2709 		}
2710 
2711 		/*
2712 		 * Copy the divisor here so we don't have to look it up
2713 		 * later if this is a var ref
2714 		 */
2715 		operand2->constant = divisor;
2716 		op_fn = hist_field_get_div_fn(operand2);
2717 	}
2718 
2719 	if (combine_consts) {
2720 		if (var1)
2721 			expr->operands[0] = var1;
2722 		if (var2)
2723 			expr->operands[1] = var2;
2724 
2725 		expr->constant = op_fn(expr, NULL, NULL, NULL, NULL);
2726 
2727 		expr->operands[0] = NULL;
2728 		expr->operands[1] = NULL;
2729 
2730 		/*
2731 		 * var refs won't be destroyed immediately
2732 		 * See: destroy_hist_field()
2733 		 */
2734 		destroy_hist_field(operand2, 0);
2735 		destroy_hist_field(operand1, 0);
2736 
2737 		expr->name = expr_str(expr, 0);
2738 	} else {
2739 		expr->fn = op_fn;
2740 
2741 		/* The operand sizes should be the same, so just pick one */
2742 		expr->size = operand1->size;
2743 		expr->is_signed = operand1->is_signed;
2744 
2745 		expr->operator = field_op;
2746 		expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2747 		if (!expr->type) {
2748 			ret = -ENOMEM;
2749 			goto free_expr;
2750 		}
2751 
2752 		expr->name = expr_str(expr, 0);
2753 	}
2754 
2755 	return expr;
2756 
2757 free_operands:
2758 	destroy_hist_field(operand2, 0);
2759 free_op1:
2760 	destroy_hist_field(operand1, 0);
2761 	return ERR_PTR(ret);
2762 
2763 free_expr:
2764 	destroy_hist_field(expr, 0);
2765 	return ERR_PTR(ret);
2766 }
2767 
2768 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2769 				 struct trace_event_file *file)
2770 {
2771 	struct event_trigger_data *test;
2772 
2773 	lockdep_assert_held(&event_mutex);
2774 
2775 	list_for_each_entry(test, &file->triggers, list) {
2776 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2777 			if (test->private_data == hist_data)
2778 				return test->filter_str;
2779 		}
2780 	}
2781 
2782 	return NULL;
2783 }
2784 
2785 static struct event_command trigger_hist_cmd;
2786 static int event_hist_trigger_parse(struct event_command *cmd_ops,
2787 				    struct trace_event_file *file,
2788 				    char *glob, char *cmd, char *param);
2789 
2790 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2791 			    struct hist_trigger_data *hist_data,
2792 			    unsigned int n_keys)
2793 {
2794 	struct hist_field *target_hist_field, *hist_field;
2795 	unsigned int n, i, j;
2796 
2797 	if (hist_data->n_fields - hist_data->n_vals != n_keys)
2798 		return false;
2799 
2800 	i = hist_data->n_vals;
2801 	j = target_hist_data->n_vals;
2802 
2803 	for (n = 0; n < n_keys; n++) {
2804 		hist_field = hist_data->fields[i + n];
2805 		target_hist_field = target_hist_data->fields[j + n];
2806 
2807 		if (strcmp(hist_field->type, target_hist_field->type) != 0)
2808 			return false;
2809 		if (hist_field->size != target_hist_field->size)
2810 			return false;
2811 		if (hist_field->is_signed != target_hist_field->is_signed)
2812 			return false;
2813 	}
2814 
2815 	return true;
2816 }
2817 
2818 static struct hist_trigger_data *
2819 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2820 		     struct trace_event_file *file)
2821 {
2822 	struct hist_trigger_data *hist_data;
2823 	struct event_trigger_data *test;
2824 	unsigned int n_keys;
2825 
2826 	lockdep_assert_held(&event_mutex);
2827 
2828 	n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2829 
2830 	list_for_each_entry(test, &file->triggers, list) {
2831 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2832 			hist_data = test->private_data;
2833 
2834 			if (compatible_keys(target_hist_data, hist_data, n_keys))
2835 				return hist_data;
2836 		}
2837 	}
2838 
2839 	return NULL;
2840 }
2841 
2842 static struct trace_event_file *event_file(struct trace_array *tr,
2843 					   char *system, char *event_name)
2844 {
2845 	struct trace_event_file *file;
2846 
2847 	file = __find_event_file(tr, system, event_name);
2848 	if (!file)
2849 		return ERR_PTR(-EINVAL);
2850 
2851 	return file;
2852 }
2853 
2854 static struct hist_field *
2855 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2856 			 char *system, char *event_name, char *field_name)
2857 {
2858 	struct hist_field *event_var;
2859 	char *synthetic_name;
2860 
2861 	synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2862 	if (!synthetic_name)
2863 		return ERR_PTR(-ENOMEM);
2864 
2865 	strcpy(synthetic_name, "synthetic_");
2866 	strcat(synthetic_name, field_name);
2867 
2868 	event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2869 
2870 	kfree(synthetic_name);
2871 
2872 	return event_var;
2873 }
2874 
2875 /**
2876  * create_field_var_hist - Automatically create a histogram and var for a field
2877  * @target_hist_data: The target hist trigger
2878  * @subsys_name: Optional subsystem name
2879  * @event_name: Optional event name
2880  * @field_name: The name of the field (and the resulting variable)
2881  *
2882  * Hist trigger actions fetch data from variables, not directly from
2883  * events.  However, for convenience, users are allowed to directly
2884  * specify an event field in an action, which will be automatically
2885  * converted into a variable on their behalf.
2886  *
2887  * If a user specifies a field on an event that isn't the event the
2888  * histogram currently being defined (the target event histogram), the
2889  * only way that can be accomplished is if a new hist trigger is
2890  * created and the field variable defined on that.
2891  *
2892  * This function creates a new histogram compatible with the target
2893  * event (meaning a histogram with the same key as the target
2894  * histogram), and creates a variable for the specified field, but
2895  * with 'synthetic_' prepended to the variable name in order to avoid
2896  * collision with normal field variables.
2897  *
2898  * Return: The variable created for the field.
2899  */
2900 static struct hist_field *
2901 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2902 		      char *subsys_name, char *event_name, char *field_name)
2903 {
2904 	struct trace_array *tr = target_hist_data->event_file->tr;
2905 	struct hist_trigger_data *hist_data;
2906 	unsigned int i, n, first = true;
2907 	struct field_var_hist *var_hist;
2908 	struct trace_event_file *file;
2909 	struct hist_field *key_field;
2910 	struct hist_field *event_var;
2911 	char *saved_filter;
2912 	char *cmd;
2913 	int ret;
2914 
2915 	if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2916 		hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2917 		return ERR_PTR(-EINVAL);
2918 	}
2919 
2920 	file = event_file(tr, subsys_name, event_name);
2921 
2922 	if (IS_ERR(file)) {
2923 		hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2924 		ret = PTR_ERR(file);
2925 		return ERR_PTR(ret);
2926 	}
2927 
2928 	/*
2929 	 * Look for a histogram compatible with target.  We'll use the
2930 	 * found histogram specification to create a new matching
2931 	 * histogram with our variable on it.  target_hist_data is not
2932 	 * yet a registered histogram so we can't use that.
2933 	 */
2934 	hist_data = find_compatible_hist(target_hist_data, file);
2935 	if (!hist_data) {
2936 		hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2937 		return ERR_PTR(-EINVAL);
2938 	}
2939 
2940 	/* See if a synthetic field variable has already been created */
2941 	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2942 					     event_name, field_name);
2943 	if (!IS_ERR_OR_NULL(event_var))
2944 		return event_var;
2945 
2946 	var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2947 	if (!var_hist)
2948 		return ERR_PTR(-ENOMEM);
2949 
2950 	cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2951 	if (!cmd) {
2952 		kfree(var_hist);
2953 		return ERR_PTR(-ENOMEM);
2954 	}
2955 
2956 	/* Use the same keys as the compatible histogram */
2957 	strcat(cmd, "keys=");
2958 
2959 	for_each_hist_key_field(i, hist_data) {
2960 		key_field = hist_data->fields[i];
2961 		if (!first)
2962 			strcat(cmd, ",");
2963 		strcat(cmd, key_field->field->name);
2964 		first = false;
2965 	}
2966 
2967 	/* Create the synthetic field variable specification */
2968 	strcat(cmd, ":synthetic_");
2969 	strcat(cmd, field_name);
2970 	strcat(cmd, "=");
2971 	strcat(cmd, field_name);
2972 
2973 	/* Use the same filter as the compatible histogram */
2974 	saved_filter = find_trigger_filter(hist_data, file);
2975 	if (saved_filter) {
2976 		strcat(cmd, " if ");
2977 		strcat(cmd, saved_filter);
2978 	}
2979 
2980 	var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2981 	if (!var_hist->cmd) {
2982 		kfree(cmd);
2983 		kfree(var_hist);
2984 		return ERR_PTR(-ENOMEM);
2985 	}
2986 
2987 	/* Save the compatible histogram information */
2988 	var_hist->hist_data = hist_data;
2989 
2990 	/* Create the new histogram with our variable */
2991 	ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
2992 				       "", "hist", cmd);
2993 	if (ret) {
2994 		kfree(cmd);
2995 		kfree(var_hist->cmd);
2996 		kfree(var_hist);
2997 		hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2998 		return ERR_PTR(ret);
2999 	}
3000 
3001 	kfree(cmd);
3002 
3003 	/* If we can't find the variable, something went wrong */
3004 	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3005 					     event_name, field_name);
3006 	if (IS_ERR_OR_NULL(event_var)) {
3007 		kfree(var_hist->cmd);
3008 		kfree(var_hist);
3009 		hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
3010 		return ERR_PTR(-EINVAL);
3011 	}
3012 
3013 	n = target_hist_data->n_field_var_hists;
3014 	target_hist_data->field_var_hists[n] = var_hist;
3015 	target_hist_data->n_field_var_hists++;
3016 
3017 	return event_var;
3018 }
3019 
3020 static struct hist_field *
3021 find_target_event_var(struct hist_trigger_data *hist_data,
3022 		      char *subsys_name, char *event_name, char *var_name)
3023 {
3024 	struct trace_event_file *file = hist_data->event_file;
3025 	struct hist_field *hist_field = NULL;
3026 
3027 	if (subsys_name) {
3028 		struct trace_event_call *call;
3029 
3030 		if (!event_name)
3031 			return NULL;
3032 
3033 		call = file->event_call;
3034 
3035 		if (strcmp(subsys_name, call->class->system) != 0)
3036 			return NULL;
3037 
3038 		if (strcmp(event_name, trace_event_name(call)) != 0)
3039 			return NULL;
3040 	}
3041 
3042 	hist_field = find_var_field(hist_data, var_name);
3043 
3044 	return hist_field;
3045 }
3046 
3047 static inline void __update_field_vars(struct tracing_map_elt *elt,
3048 				       struct trace_buffer *buffer,
3049 				       struct ring_buffer_event *rbe,
3050 				       void *rec,
3051 				       struct field_var **field_vars,
3052 				       unsigned int n_field_vars,
3053 				       unsigned int field_var_str_start)
3054 {
3055 	struct hist_elt_data *elt_data = elt->private_data;
3056 	unsigned int i, j, var_idx;
3057 	u64 var_val;
3058 
3059 	for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3060 		struct field_var *field_var = field_vars[i];
3061 		struct hist_field *var = field_var->var;
3062 		struct hist_field *val = field_var->val;
3063 
3064 		var_val = val->fn(val, elt, buffer, rbe, rec);
3065 		var_idx = var->var.idx;
3066 
3067 		if (val->flags & HIST_FIELD_FL_STRING) {
3068 			char *str = elt_data->field_var_str[j++];
3069 			char *val_str = (char *)(uintptr_t)var_val;
3070 			unsigned int size;
3071 
3072 			size = min(val->size, STR_VAR_LEN_MAX);
3073 			strscpy(str, val_str, size);
3074 			var_val = (u64)(uintptr_t)str;
3075 		}
3076 		tracing_map_set_var(elt, var_idx, var_val);
3077 	}
3078 }
3079 
3080 static void update_field_vars(struct hist_trigger_data *hist_data,
3081 			      struct tracing_map_elt *elt,
3082 			      struct trace_buffer *buffer,
3083 			      struct ring_buffer_event *rbe,
3084 			      void *rec)
3085 {
3086 	__update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
3087 			    hist_data->n_field_vars, 0);
3088 }
3089 
3090 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3091 				 struct tracing_map_elt *elt,
3092 				 struct trace_buffer *buffer,  void *rec,
3093 				 struct ring_buffer_event *rbe, void *key,
3094 				 struct action_data *data, u64 *var_ref_vals)
3095 {
3096 	__update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
3097 			    hist_data->n_save_vars, hist_data->n_field_var_str);
3098 }
3099 
3100 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3101 				     struct trace_event_file *file,
3102 				     char *name, int size, const char *type)
3103 {
3104 	struct hist_field *var;
3105 	int idx;
3106 
3107 	if (find_var(hist_data, file, name) && !hist_data->remove) {
3108 		var = ERR_PTR(-EINVAL);
3109 		goto out;
3110 	}
3111 
3112 	var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3113 	if (!var) {
3114 		var = ERR_PTR(-ENOMEM);
3115 		goto out;
3116 	}
3117 
3118 	idx = tracing_map_add_var(hist_data->map);
3119 	if (idx < 0) {
3120 		kfree(var);
3121 		var = ERR_PTR(-EINVAL);
3122 		goto out;
3123 	}
3124 
3125 	var->ref = 1;
3126 	var->flags = HIST_FIELD_FL_VAR;
3127 	var->var.idx = idx;
3128 	var->var.hist_data = var->hist_data = hist_data;
3129 	var->size = size;
3130 	var->var.name = kstrdup(name, GFP_KERNEL);
3131 	var->type = kstrdup_const(type, GFP_KERNEL);
3132 	if (!var->var.name || !var->type) {
3133 		kfree_const(var->type);
3134 		kfree(var->var.name);
3135 		kfree(var);
3136 		var = ERR_PTR(-ENOMEM);
3137 	}
3138  out:
3139 	return var;
3140 }
3141 
3142 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3143 					  struct trace_event_file *file,
3144 					  char *field_name)
3145 {
3146 	struct hist_field *val = NULL, *var = NULL;
3147 	unsigned long flags = HIST_FIELD_FL_VAR;
3148 	struct trace_array *tr = file->tr;
3149 	struct field_var *field_var;
3150 	int ret = 0;
3151 
3152 	if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3153 		hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3154 		ret = -EINVAL;
3155 		goto err;
3156 	}
3157 
3158 	val = parse_atom(hist_data, file, field_name, &flags, NULL);
3159 	if (IS_ERR(val)) {
3160 		hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3161 		ret = PTR_ERR(val);
3162 		goto err;
3163 	}
3164 
3165 	var = create_var(hist_data, file, field_name, val->size, val->type);
3166 	if (IS_ERR(var)) {
3167 		hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3168 		kfree(val);
3169 		ret = PTR_ERR(var);
3170 		goto err;
3171 	}
3172 
3173 	field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3174 	if (!field_var) {
3175 		kfree(val);
3176 		kfree(var);
3177 		ret =  -ENOMEM;
3178 		goto err;
3179 	}
3180 
3181 	field_var->var = var;
3182 	field_var->val = val;
3183  out:
3184 	return field_var;
3185  err:
3186 	field_var = ERR_PTR(ret);
3187 	goto out;
3188 }
3189 
3190 /**
3191  * create_target_field_var - Automatically create a variable for a field
3192  * @target_hist_data: The target hist trigger
3193  * @subsys_name: Optional subsystem name
3194  * @event_name: Optional event name
3195  * @var_name: The name of the field (and the resulting variable)
3196  *
3197  * Hist trigger actions fetch data from variables, not directly from
3198  * events.  However, for convenience, users are allowed to directly
3199  * specify an event field in an action, which will be automatically
3200  * converted into a variable on their behalf.
3201 
3202  * This function creates a field variable with the name var_name on
3203  * the hist trigger currently being defined on the target event.  If
3204  * subsys_name and event_name are specified, this function simply
3205  * verifies that they do in fact match the target event subsystem and
3206  * event name.
3207  *
3208  * Return: The variable created for the field.
3209  */
3210 static struct field_var *
3211 create_target_field_var(struct hist_trigger_data *target_hist_data,
3212 			char *subsys_name, char *event_name, char *var_name)
3213 {
3214 	struct trace_event_file *file = target_hist_data->event_file;
3215 
3216 	if (subsys_name) {
3217 		struct trace_event_call *call;
3218 
3219 		if (!event_name)
3220 			return NULL;
3221 
3222 		call = file->event_call;
3223 
3224 		if (strcmp(subsys_name, call->class->system) != 0)
3225 			return NULL;
3226 
3227 		if (strcmp(event_name, trace_event_name(call)) != 0)
3228 			return NULL;
3229 	}
3230 
3231 	return create_field_var(target_hist_data, file, var_name);
3232 }
3233 
3234 static bool check_track_val_max(u64 track_val, u64 var_val)
3235 {
3236 	if (var_val <= track_val)
3237 		return false;
3238 
3239 	return true;
3240 }
3241 
3242 static bool check_track_val_changed(u64 track_val, u64 var_val)
3243 {
3244 	if (var_val == track_val)
3245 		return false;
3246 
3247 	return true;
3248 }
3249 
3250 static u64 get_track_val(struct hist_trigger_data *hist_data,
3251 			 struct tracing_map_elt *elt,
3252 			 struct action_data *data)
3253 {
3254 	unsigned int track_var_idx = data->track_data.track_var->var.idx;
3255 	u64 track_val;
3256 
3257 	track_val = tracing_map_read_var(elt, track_var_idx);
3258 
3259 	return track_val;
3260 }
3261 
3262 static void save_track_val(struct hist_trigger_data *hist_data,
3263 			   struct tracing_map_elt *elt,
3264 			   struct action_data *data, u64 var_val)
3265 {
3266 	unsigned int track_var_idx = data->track_data.track_var->var.idx;
3267 
3268 	tracing_map_set_var(elt, track_var_idx, var_val);
3269 }
3270 
3271 static void save_track_data(struct hist_trigger_data *hist_data,
3272 			    struct tracing_map_elt *elt,
3273 			    struct trace_buffer *buffer, void *rec,
3274 			    struct ring_buffer_event *rbe, void *key,
3275 			    struct action_data *data, u64 *var_ref_vals)
3276 {
3277 	if (data->track_data.save_data)
3278 		data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
3279 					   key, data, var_ref_vals);
3280 }
3281 
3282 static bool check_track_val(struct tracing_map_elt *elt,
3283 			    struct action_data *data,
3284 			    u64 var_val)
3285 {
3286 	struct hist_trigger_data *hist_data;
3287 	u64 track_val;
3288 
3289 	hist_data = data->track_data.track_var->hist_data;
3290 	track_val = get_track_val(hist_data, elt, data);
3291 
3292 	return data->track_data.check_val(track_val, var_val);
3293 }
3294 
3295 #ifdef CONFIG_TRACER_SNAPSHOT
3296 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3297 {
3298 	/* called with tr->max_lock held */
3299 	struct track_data *track_data = tr->cond_snapshot->cond_data;
3300 	struct hist_elt_data *elt_data, *track_elt_data;
3301 	struct snapshot_context *context = cond_data;
3302 	struct action_data *action;
3303 	u64 track_val;
3304 
3305 	if (!track_data)
3306 		return false;
3307 
3308 	action = track_data->action_data;
3309 
3310 	track_val = get_track_val(track_data->hist_data, context->elt,
3311 				  track_data->action_data);
3312 
3313 	if (!action->track_data.check_val(track_data->track_val, track_val))
3314 		return false;
3315 
3316 	track_data->track_val = track_val;
3317 	memcpy(track_data->key, context->key, track_data->key_len);
3318 
3319 	elt_data = context->elt->private_data;
3320 	track_elt_data = track_data->elt.private_data;
3321 	if (elt_data->comm)
3322 		strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3323 
3324 	track_data->updated = true;
3325 
3326 	return true;
3327 }
3328 
3329 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3330 				     struct tracing_map_elt *elt,
3331 				     struct trace_buffer *buffer, void *rec,
3332 				     struct ring_buffer_event *rbe, void *key,
3333 				     struct action_data *data,
3334 				     u64 *var_ref_vals)
3335 {
3336 	struct trace_event_file *file = hist_data->event_file;
3337 	struct snapshot_context context;
3338 
3339 	context.elt = elt;
3340 	context.key = key;
3341 
3342 	tracing_snapshot_cond(file->tr, &context);
3343 }
3344 
3345 static void hist_trigger_print_key(struct seq_file *m,
3346 				   struct hist_trigger_data *hist_data,
3347 				   void *key,
3348 				   struct tracing_map_elt *elt);
3349 
3350 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3351 {
3352 	unsigned int i;
3353 
3354 	if (!hist_data->n_actions)
3355 		return NULL;
3356 
3357 	for (i = 0; i < hist_data->n_actions; i++) {
3358 		struct action_data *data = hist_data->actions[i];
3359 
3360 		if (data->action == ACTION_SNAPSHOT)
3361 			return data;
3362 	}
3363 
3364 	return NULL;
3365 }
3366 
3367 static void track_data_snapshot_print(struct seq_file *m,
3368 				      struct hist_trigger_data *hist_data)
3369 {
3370 	struct trace_event_file *file = hist_data->event_file;
3371 	struct track_data *track_data;
3372 	struct action_data *action;
3373 
3374 	track_data = tracing_cond_snapshot_data(file->tr);
3375 	if (!track_data)
3376 		return;
3377 
3378 	if (!track_data->updated)
3379 		return;
3380 
3381 	action = snapshot_action(hist_data);
3382 	if (!action)
3383 		return;
3384 
3385 	seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3386 	seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3387 		   action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3388 		   action->track_data.var_str, track_data->track_val);
3389 
3390 	seq_puts(m, "\ttriggered by event with key: ");
3391 	hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3392 	seq_putc(m, '\n');
3393 }
3394 #else
3395 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3396 {
3397 	return false;
3398 }
3399 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3400 				     struct tracing_map_elt *elt,
3401 				     struct trace_buffer *buffer, void *rec,
3402 				     struct ring_buffer_event *rbe, void *key,
3403 				     struct action_data *data,
3404 				     u64 *var_ref_vals) {}
3405 static void track_data_snapshot_print(struct seq_file *m,
3406 				      struct hist_trigger_data *hist_data) {}
3407 #endif /* CONFIG_TRACER_SNAPSHOT */
3408 
3409 static void track_data_print(struct seq_file *m,
3410 			     struct hist_trigger_data *hist_data,
3411 			     struct tracing_map_elt *elt,
3412 			     struct action_data *data)
3413 {
3414 	u64 track_val = get_track_val(hist_data, elt, data);
3415 	unsigned int i, save_var_idx;
3416 
3417 	if (data->handler == HANDLER_ONMAX)
3418 		seq_printf(m, "\n\tmax: %10llu", track_val);
3419 	else if (data->handler == HANDLER_ONCHANGE)
3420 		seq_printf(m, "\n\tchanged: %10llu", track_val);
3421 
3422 	if (data->action == ACTION_SNAPSHOT)
3423 		return;
3424 
3425 	for (i = 0; i < hist_data->n_save_vars; i++) {
3426 		struct hist_field *save_val = hist_data->save_vars[i]->val;
3427 		struct hist_field *save_var = hist_data->save_vars[i]->var;
3428 		u64 val;
3429 
3430 		save_var_idx = save_var->var.idx;
3431 
3432 		val = tracing_map_read_var(elt, save_var_idx);
3433 
3434 		if (save_val->flags & HIST_FIELD_FL_STRING) {
3435 			seq_printf(m, "  %s: %-32s", save_var->var.name,
3436 				   (char *)(uintptr_t)(val));
3437 		} else
3438 			seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3439 	}
3440 }
3441 
3442 static void ontrack_action(struct hist_trigger_data *hist_data,
3443 			   struct tracing_map_elt *elt,
3444 			   struct trace_buffer *buffer, void *rec,
3445 			   struct ring_buffer_event *rbe, void *key,
3446 			   struct action_data *data, u64 *var_ref_vals)
3447 {
3448 	u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3449 
3450 	if (check_track_val(elt, data, var_val)) {
3451 		save_track_val(hist_data, elt, data, var_val);
3452 		save_track_data(hist_data, elt, buffer, rec, rbe,
3453 				key, data, var_ref_vals);
3454 	}
3455 }
3456 
3457 static void action_data_destroy(struct action_data *data)
3458 {
3459 	unsigned int i;
3460 
3461 	lockdep_assert_held(&event_mutex);
3462 
3463 	kfree(data->action_name);
3464 
3465 	for (i = 0; i < data->n_params; i++)
3466 		kfree(data->params[i]);
3467 
3468 	if (data->synth_event)
3469 		data->synth_event->ref--;
3470 
3471 	kfree(data->synth_event_name);
3472 
3473 	kfree(data);
3474 }
3475 
3476 static void track_data_destroy(struct hist_trigger_data *hist_data,
3477 			       struct action_data *data)
3478 {
3479 	struct trace_event_file *file = hist_data->event_file;
3480 
3481 	destroy_hist_field(data->track_data.track_var, 0);
3482 
3483 	if (data->action == ACTION_SNAPSHOT) {
3484 		struct track_data *track_data;
3485 
3486 		track_data = tracing_cond_snapshot_data(file->tr);
3487 		if (track_data && track_data->hist_data == hist_data) {
3488 			tracing_snapshot_cond_disable(file->tr);
3489 			track_data_free(track_data);
3490 		}
3491 	}
3492 
3493 	kfree(data->track_data.var_str);
3494 
3495 	action_data_destroy(data);
3496 }
3497 
3498 static int action_create(struct hist_trigger_data *hist_data,
3499 			 struct action_data *data);
3500 
3501 static int track_data_create(struct hist_trigger_data *hist_data,
3502 			     struct action_data *data)
3503 {
3504 	struct hist_field *var_field, *ref_field, *track_var = NULL;
3505 	struct trace_event_file *file = hist_data->event_file;
3506 	struct trace_array *tr = file->tr;
3507 	char *track_data_var_str;
3508 	int ret = 0;
3509 
3510 	track_data_var_str = data->track_data.var_str;
3511 	if (track_data_var_str[0] != '$') {
3512 		hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3513 		return -EINVAL;
3514 	}
3515 	track_data_var_str++;
3516 
3517 	var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3518 	if (!var_field) {
3519 		hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3520 		return -EINVAL;
3521 	}
3522 
3523 	ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3524 	if (!ref_field)
3525 		return -ENOMEM;
3526 
3527 	data->track_data.var_ref = ref_field;
3528 
3529 	if (data->handler == HANDLER_ONMAX)
3530 		track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3531 	if (IS_ERR(track_var)) {
3532 		hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3533 		ret = PTR_ERR(track_var);
3534 		goto out;
3535 	}
3536 
3537 	if (data->handler == HANDLER_ONCHANGE)
3538 		track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3539 	if (IS_ERR(track_var)) {
3540 		hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3541 		ret = PTR_ERR(track_var);
3542 		goto out;
3543 	}
3544 	data->track_data.track_var = track_var;
3545 
3546 	ret = action_create(hist_data, data);
3547  out:
3548 	return ret;
3549 }
3550 
3551 static int parse_action_params(struct trace_array *tr, char *params,
3552 			       struct action_data *data)
3553 {
3554 	char *param, *saved_param;
3555 	bool first_param = true;
3556 	int ret = 0;
3557 
3558 	while (params) {
3559 		if (data->n_params >= SYNTH_FIELDS_MAX) {
3560 			hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3561 			goto out;
3562 		}
3563 
3564 		param = strsep(&params, ",");
3565 		if (!param) {
3566 			hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3567 			ret = -EINVAL;
3568 			goto out;
3569 		}
3570 
3571 		param = strstrip(param);
3572 		if (strlen(param) < 2) {
3573 			hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3574 			ret = -EINVAL;
3575 			goto out;
3576 		}
3577 
3578 		saved_param = kstrdup(param, GFP_KERNEL);
3579 		if (!saved_param) {
3580 			ret = -ENOMEM;
3581 			goto out;
3582 		}
3583 
3584 		if (first_param && data->use_trace_keyword) {
3585 			data->synth_event_name = saved_param;
3586 			first_param = false;
3587 			continue;
3588 		}
3589 		first_param = false;
3590 
3591 		data->params[data->n_params++] = saved_param;
3592 	}
3593  out:
3594 	return ret;
3595 }
3596 
3597 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3598 			enum handler_id handler)
3599 {
3600 	char *action_name;
3601 	int ret = 0;
3602 
3603 	strsep(&str, ".");
3604 	if (!str) {
3605 		hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3606 		ret = -EINVAL;
3607 		goto out;
3608 	}
3609 
3610 	action_name = strsep(&str, "(");
3611 	if (!action_name || !str) {
3612 		hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3613 		ret = -EINVAL;
3614 		goto out;
3615 	}
3616 
3617 	if (str_has_prefix(action_name, "save")) {
3618 		char *params = strsep(&str, ")");
3619 
3620 		if (!params) {
3621 			hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3622 			ret = -EINVAL;
3623 			goto out;
3624 		}
3625 
3626 		ret = parse_action_params(tr, params, data);
3627 		if (ret)
3628 			goto out;
3629 
3630 		if (handler == HANDLER_ONMAX)
3631 			data->track_data.check_val = check_track_val_max;
3632 		else if (handler == HANDLER_ONCHANGE)
3633 			data->track_data.check_val = check_track_val_changed;
3634 		else {
3635 			hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3636 			ret = -EINVAL;
3637 			goto out;
3638 		}
3639 
3640 		data->track_data.save_data = save_track_data_vars;
3641 		data->fn = ontrack_action;
3642 		data->action = ACTION_SAVE;
3643 	} else if (str_has_prefix(action_name, "snapshot")) {
3644 		char *params = strsep(&str, ")");
3645 
3646 		if (!str) {
3647 			hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3648 			ret = -EINVAL;
3649 			goto out;
3650 		}
3651 
3652 		if (handler == HANDLER_ONMAX)
3653 			data->track_data.check_val = check_track_val_max;
3654 		else if (handler == HANDLER_ONCHANGE)
3655 			data->track_data.check_val = check_track_val_changed;
3656 		else {
3657 			hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3658 			ret = -EINVAL;
3659 			goto out;
3660 		}
3661 
3662 		data->track_data.save_data = save_track_data_snapshot;
3663 		data->fn = ontrack_action;
3664 		data->action = ACTION_SNAPSHOT;
3665 	} else {
3666 		char *params = strsep(&str, ")");
3667 
3668 		if (str_has_prefix(action_name, "trace"))
3669 			data->use_trace_keyword = true;
3670 
3671 		if (params) {
3672 			ret = parse_action_params(tr, params, data);
3673 			if (ret)
3674 				goto out;
3675 		}
3676 
3677 		if (handler == HANDLER_ONMAX)
3678 			data->track_data.check_val = check_track_val_max;
3679 		else if (handler == HANDLER_ONCHANGE)
3680 			data->track_data.check_val = check_track_val_changed;
3681 
3682 		if (handler != HANDLER_ONMATCH) {
3683 			data->track_data.save_data = action_trace;
3684 			data->fn = ontrack_action;
3685 		} else
3686 			data->fn = action_trace;
3687 
3688 		data->action = ACTION_TRACE;
3689 	}
3690 
3691 	data->action_name = kstrdup(action_name, GFP_KERNEL);
3692 	if (!data->action_name) {
3693 		ret = -ENOMEM;
3694 		goto out;
3695 	}
3696 
3697 	data->handler = handler;
3698  out:
3699 	return ret;
3700 }
3701 
3702 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3703 					    char *str, enum handler_id handler)
3704 {
3705 	struct action_data *data;
3706 	int ret = -EINVAL;
3707 	char *var_str;
3708 
3709 	data = kzalloc(sizeof(*data), GFP_KERNEL);
3710 	if (!data)
3711 		return ERR_PTR(-ENOMEM);
3712 
3713 	var_str = strsep(&str, ")");
3714 	if (!var_str || !str) {
3715 		ret = -EINVAL;
3716 		goto free;
3717 	}
3718 
3719 	data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3720 	if (!data->track_data.var_str) {
3721 		ret = -ENOMEM;
3722 		goto free;
3723 	}
3724 
3725 	ret = action_parse(hist_data->event_file->tr, str, data, handler);
3726 	if (ret)
3727 		goto free;
3728  out:
3729 	return data;
3730  free:
3731 	track_data_destroy(hist_data, data);
3732 	data = ERR_PTR(ret);
3733 	goto out;
3734 }
3735 
3736 static void onmatch_destroy(struct action_data *data)
3737 {
3738 	kfree(data->match_data.event);
3739 	kfree(data->match_data.event_system);
3740 
3741 	action_data_destroy(data);
3742 }
3743 
3744 static void destroy_field_var(struct field_var *field_var)
3745 {
3746 	if (!field_var)
3747 		return;
3748 
3749 	destroy_hist_field(field_var->var, 0);
3750 	destroy_hist_field(field_var->val, 0);
3751 
3752 	kfree(field_var);
3753 }
3754 
3755 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3756 {
3757 	unsigned int i;
3758 
3759 	for (i = 0; i < hist_data->n_field_vars; i++)
3760 		destroy_field_var(hist_data->field_vars[i]);
3761 
3762 	for (i = 0; i < hist_data->n_save_vars; i++)
3763 		destroy_field_var(hist_data->save_vars[i]);
3764 }
3765 
3766 static void save_field_var(struct hist_trigger_data *hist_data,
3767 			   struct field_var *field_var)
3768 {
3769 	hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3770 
3771 	if (field_var->val->flags & HIST_FIELD_FL_STRING)
3772 		hist_data->n_field_var_str++;
3773 }
3774 
3775 
3776 static int check_synth_field(struct synth_event *event,
3777 			     struct hist_field *hist_field,
3778 			     unsigned int field_pos)
3779 {
3780 	struct synth_field *field;
3781 
3782 	if (field_pos >= event->n_fields)
3783 		return -EINVAL;
3784 
3785 	field = event->fields[field_pos];
3786 
3787 	/*
3788 	 * A dynamic string synth field can accept static or
3789 	 * dynamic. A static string synth field can only accept a
3790 	 * same-sized static string, which is checked for later.
3791 	 */
3792 	if (strstr(hist_field->type, "char[") && field->is_string
3793 	    && field->is_dynamic)
3794 		return 0;
3795 
3796 	if (strcmp(field->type, hist_field->type) != 0) {
3797 		if (field->size != hist_field->size ||
3798 		    (!field->is_string && field->is_signed != hist_field->is_signed))
3799 			return -EINVAL;
3800 	}
3801 
3802 	return 0;
3803 }
3804 
3805 static struct hist_field *
3806 trace_action_find_var(struct hist_trigger_data *hist_data,
3807 		      struct action_data *data,
3808 		      char *system, char *event, char *var)
3809 {
3810 	struct trace_array *tr = hist_data->event_file->tr;
3811 	struct hist_field *hist_field;
3812 
3813 	var++; /* skip '$' */
3814 
3815 	hist_field = find_target_event_var(hist_data, system, event, var);
3816 	if (!hist_field) {
3817 		if (!system && data->handler == HANDLER_ONMATCH) {
3818 			system = data->match_data.event_system;
3819 			event = data->match_data.event;
3820 		}
3821 
3822 		hist_field = find_event_var(hist_data, system, event, var);
3823 	}
3824 
3825 	if (!hist_field)
3826 		hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3827 
3828 	return hist_field;
3829 }
3830 
3831 static struct hist_field *
3832 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3833 			      struct action_data *data, char *system,
3834 			      char *event, char *var)
3835 {
3836 	struct hist_field *hist_field = NULL;
3837 	struct field_var *field_var;
3838 
3839 	/*
3840 	 * First try to create a field var on the target event (the
3841 	 * currently being defined).  This will create a variable for
3842 	 * unqualified fields on the target event, or if qualified,
3843 	 * target fields that have qualified names matching the target.
3844 	 */
3845 	field_var = create_target_field_var(hist_data, system, event, var);
3846 
3847 	if (field_var && !IS_ERR(field_var)) {
3848 		save_field_var(hist_data, field_var);
3849 		hist_field = field_var->var;
3850 	} else {
3851 		field_var = NULL;
3852 		/*
3853 		 * If no explicit system.event is specified, default to
3854 		 * looking for fields on the onmatch(system.event.xxx)
3855 		 * event.
3856 		 */
3857 		if (!system && data->handler == HANDLER_ONMATCH) {
3858 			system = data->match_data.event_system;
3859 			event = data->match_data.event;
3860 		}
3861 
3862 		if (!event)
3863 			goto free;
3864 		/*
3865 		 * At this point, we're looking at a field on another
3866 		 * event.  Because we can't modify a hist trigger on
3867 		 * another event to add a variable for a field, we need
3868 		 * to create a new trigger on that event and create the
3869 		 * variable at the same time.
3870 		 */
3871 		hist_field = create_field_var_hist(hist_data, system, event, var);
3872 		if (IS_ERR(hist_field))
3873 			goto free;
3874 	}
3875  out:
3876 	return hist_field;
3877  free:
3878 	destroy_field_var(field_var);
3879 	hist_field = NULL;
3880 	goto out;
3881 }
3882 
3883 static int trace_action_create(struct hist_trigger_data *hist_data,
3884 			       struct action_data *data)
3885 {
3886 	struct trace_array *tr = hist_data->event_file->tr;
3887 	char *event_name, *param, *system = NULL;
3888 	struct hist_field *hist_field, *var_ref;
3889 	unsigned int i;
3890 	unsigned int field_pos = 0;
3891 	struct synth_event *event;
3892 	char *synth_event_name;
3893 	int var_ref_idx, ret = 0;
3894 
3895 	lockdep_assert_held(&event_mutex);
3896 
3897 	if (data->use_trace_keyword)
3898 		synth_event_name = data->synth_event_name;
3899 	else
3900 		synth_event_name = data->action_name;
3901 
3902 	event = find_synth_event(synth_event_name);
3903 	if (!event) {
3904 		hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3905 		return -EINVAL;
3906 	}
3907 
3908 	event->ref++;
3909 
3910 	for (i = 0; i < data->n_params; i++) {
3911 		char *p;
3912 
3913 		p = param = kstrdup(data->params[i], GFP_KERNEL);
3914 		if (!param) {
3915 			ret = -ENOMEM;
3916 			goto err;
3917 		}
3918 
3919 		system = strsep(&param, ".");
3920 		if (!param) {
3921 			param = (char *)system;
3922 			system = event_name = NULL;
3923 		} else {
3924 			event_name = strsep(&param, ".");
3925 			if (!param) {
3926 				kfree(p);
3927 				ret = -EINVAL;
3928 				goto err;
3929 			}
3930 		}
3931 
3932 		if (param[0] == '$')
3933 			hist_field = trace_action_find_var(hist_data, data,
3934 							   system, event_name,
3935 							   param);
3936 		else
3937 			hist_field = trace_action_create_field_var(hist_data,
3938 								   data,
3939 								   system,
3940 								   event_name,
3941 								   param);
3942 
3943 		if (!hist_field) {
3944 			kfree(p);
3945 			ret = -EINVAL;
3946 			goto err;
3947 		}
3948 
3949 		if (check_synth_field(event, hist_field, field_pos) == 0) {
3950 			var_ref = create_var_ref(hist_data, hist_field,
3951 						 system, event_name);
3952 			if (!var_ref) {
3953 				kfree(p);
3954 				ret = -ENOMEM;
3955 				goto err;
3956 			}
3957 
3958 			var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3959 			if (WARN_ON(var_ref_idx < 0)) {
3960 				kfree(p);
3961 				ret = var_ref_idx;
3962 				goto err;
3963 			}
3964 
3965 			data->var_ref_idx[i] = var_ref_idx;
3966 
3967 			field_pos++;
3968 			kfree(p);
3969 			continue;
3970 		}
3971 
3972 		hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3973 		kfree(p);
3974 		ret = -EINVAL;
3975 		goto err;
3976 	}
3977 
3978 	if (field_pos != event->n_fields) {
3979 		hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3980 		ret = -EINVAL;
3981 		goto err;
3982 	}
3983 
3984 	data->synth_event = event;
3985  out:
3986 	return ret;
3987  err:
3988 	event->ref--;
3989 
3990 	goto out;
3991 }
3992 
3993 static int action_create(struct hist_trigger_data *hist_data,
3994 			 struct action_data *data)
3995 {
3996 	struct trace_event_file *file = hist_data->event_file;
3997 	struct trace_array *tr = file->tr;
3998 	struct track_data *track_data;
3999 	struct field_var *field_var;
4000 	unsigned int i;
4001 	char *param;
4002 	int ret = 0;
4003 
4004 	if (data->action == ACTION_TRACE)
4005 		return trace_action_create(hist_data, data);
4006 
4007 	if (data->action == ACTION_SNAPSHOT) {
4008 		track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4009 		if (IS_ERR(track_data)) {
4010 			ret = PTR_ERR(track_data);
4011 			goto out;
4012 		}
4013 
4014 		ret = tracing_snapshot_cond_enable(file->tr, track_data,
4015 						   cond_snapshot_update);
4016 		if (ret)
4017 			track_data_free(track_data);
4018 
4019 		goto out;
4020 	}
4021 
4022 	if (data->action == ACTION_SAVE) {
4023 		if (hist_data->n_save_vars) {
4024 			ret = -EEXIST;
4025 			hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4026 			goto out;
4027 		}
4028 
4029 		for (i = 0; i < data->n_params; i++) {
4030 			param = kstrdup(data->params[i], GFP_KERNEL);
4031 			if (!param) {
4032 				ret = -ENOMEM;
4033 				goto out;
4034 			}
4035 
4036 			field_var = create_target_field_var(hist_data, NULL, NULL, param);
4037 			if (IS_ERR(field_var)) {
4038 				hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4039 					 errpos(param));
4040 				ret = PTR_ERR(field_var);
4041 				kfree(param);
4042 				goto out;
4043 			}
4044 
4045 			hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4046 			if (field_var->val->flags & HIST_FIELD_FL_STRING)
4047 				hist_data->n_save_var_str++;
4048 			kfree(param);
4049 		}
4050 	}
4051  out:
4052 	return ret;
4053 }
4054 
4055 static int onmatch_create(struct hist_trigger_data *hist_data,
4056 			  struct action_data *data)
4057 {
4058 	return action_create(hist_data, data);
4059 }
4060 
4061 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4062 {
4063 	char *match_event, *match_event_system;
4064 	struct action_data *data;
4065 	int ret = -EINVAL;
4066 
4067 	data = kzalloc(sizeof(*data), GFP_KERNEL);
4068 	if (!data)
4069 		return ERR_PTR(-ENOMEM);
4070 
4071 	match_event = strsep(&str, ")");
4072 	if (!match_event || !str) {
4073 		hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4074 		goto free;
4075 	}
4076 
4077 	match_event_system = strsep(&match_event, ".");
4078 	if (!match_event) {
4079 		hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4080 		goto free;
4081 	}
4082 
4083 	if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4084 		hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4085 		goto free;
4086 	}
4087 
4088 	data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4089 	if (!data->match_data.event) {
4090 		ret = -ENOMEM;
4091 		goto free;
4092 	}
4093 
4094 	data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4095 	if (!data->match_data.event_system) {
4096 		ret = -ENOMEM;
4097 		goto free;
4098 	}
4099 
4100 	ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4101 	if (ret)
4102 		goto free;
4103  out:
4104 	return data;
4105  free:
4106 	onmatch_destroy(data);
4107 	data = ERR_PTR(ret);
4108 	goto out;
4109 }
4110 
4111 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4112 {
4113 	hist_data->fields[HITCOUNT_IDX] =
4114 		create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4115 	if (!hist_data->fields[HITCOUNT_IDX])
4116 		return -ENOMEM;
4117 
4118 	hist_data->n_vals++;
4119 	hist_data->n_fields++;
4120 
4121 	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4122 		return -EINVAL;
4123 
4124 	return 0;
4125 }
4126 
4127 static int __create_val_field(struct hist_trigger_data *hist_data,
4128 			      unsigned int val_idx,
4129 			      struct trace_event_file *file,
4130 			      char *var_name, char *field_str,
4131 			      unsigned long flags)
4132 {
4133 	struct hist_field *hist_field;
4134 	int ret = 0, n_subexprs = 0;
4135 
4136 	hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs);
4137 	if (IS_ERR(hist_field)) {
4138 		ret = PTR_ERR(hist_field);
4139 		goto out;
4140 	}
4141 
4142 	hist_data->fields[val_idx] = hist_field;
4143 
4144 	++hist_data->n_vals;
4145 	++hist_data->n_fields;
4146 
4147 	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4148 		ret = -EINVAL;
4149  out:
4150 	return ret;
4151 }
4152 
4153 static int create_val_field(struct hist_trigger_data *hist_data,
4154 			    unsigned int val_idx,
4155 			    struct trace_event_file *file,
4156 			    char *field_str)
4157 {
4158 	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4159 		return -EINVAL;
4160 
4161 	return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4162 }
4163 
4164 static const char *no_comm = "(no comm)";
4165 
4166 static u64 hist_field_execname(struct hist_field *hist_field,
4167 			       struct tracing_map_elt *elt,
4168 			       struct trace_buffer *buffer,
4169 			       struct ring_buffer_event *rbe,
4170 			       void *event)
4171 {
4172 	struct hist_elt_data *elt_data;
4173 
4174 	if (WARN_ON_ONCE(!elt))
4175 		return (u64)(unsigned long)no_comm;
4176 
4177 	elt_data = elt->private_data;
4178 
4179 	if (WARN_ON_ONCE(!elt_data->comm))
4180 		return (u64)(unsigned long)no_comm;
4181 
4182 	return (u64)(unsigned long)(elt_data->comm);
4183 }
4184 
4185 /* Convert a var that points to common_pid.execname to a string */
4186 static void update_var_execname(struct hist_field *hist_field)
4187 {
4188 	hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR |
4189 		HIST_FIELD_FL_EXECNAME;
4190 	hist_field->size = MAX_FILTER_STR_VAL;
4191 	hist_field->is_signed = 0;
4192 
4193 	kfree_const(hist_field->type);
4194 	hist_field->type = "char[]";
4195 
4196 	hist_field->fn = hist_field_execname;
4197 }
4198 
4199 static int create_var_field(struct hist_trigger_data *hist_data,
4200 			    unsigned int val_idx,
4201 			    struct trace_event_file *file,
4202 			    char *var_name, char *expr_str)
4203 {
4204 	struct trace_array *tr = hist_data->event_file->tr;
4205 	unsigned long flags = 0;
4206 	int ret;
4207 
4208 	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4209 		return -EINVAL;
4210 
4211 	if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4212 		hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4213 		return -EINVAL;
4214 	}
4215 
4216 	flags |= HIST_FIELD_FL_VAR;
4217 	hist_data->n_vars++;
4218 	if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4219 		return -EINVAL;
4220 
4221 	ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4222 
4223 	if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME)
4224 		update_var_execname(hist_data->fields[val_idx]);
4225 
4226 	if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_STRING)
4227 		hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
4228 
4229 	return ret;
4230 }
4231 
4232 static int create_val_fields(struct hist_trigger_data *hist_data,
4233 			     struct trace_event_file *file)
4234 {
4235 	char *fields_str, *field_str;
4236 	unsigned int i, j = 1;
4237 	int ret;
4238 
4239 	ret = create_hitcount_val(hist_data);
4240 	if (ret)
4241 		goto out;
4242 
4243 	fields_str = hist_data->attrs->vals_str;
4244 	if (!fields_str)
4245 		goto out;
4246 
4247 	for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4248 		     j < TRACING_MAP_VALS_MAX; i++) {
4249 		field_str = strsep(&fields_str, ",");
4250 		if (!field_str)
4251 			break;
4252 
4253 		if (strcmp(field_str, "hitcount") == 0)
4254 			continue;
4255 
4256 		ret = create_val_field(hist_data, j++, file, field_str);
4257 		if (ret)
4258 			goto out;
4259 	}
4260 
4261 	if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4262 		ret = -EINVAL;
4263  out:
4264 	return ret;
4265 }
4266 
4267 static int create_key_field(struct hist_trigger_data *hist_data,
4268 			    unsigned int key_idx,
4269 			    unsigned int key_offset,
4270 			    struct trace_event_file *file,
4271 			    char *field_str)
4272 {
4273 	struct trace_array *tr = hist_data->event_file->tr;
4274 	struct hist_field *hist_field = NULL;
4275 	unsigned long flags = 0;
4276 	unsigned int key_size;
4277 	int ret = 0, n_subexprs = 0;
4278 
4279 	if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4280 		return -EINVAL;
4281 
4282 	flags |= HIST_FIELD_FL_KEY;
4283 
4284 	if (strcmp(field_str, "stacktrace") == 0) {
4285 		flags |= HIST_FIELD_FL_STACKTRACE;
4286 		key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4287 		hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4288 	} else {
4289 		hist_field = parse_expr(hist_data, file, field_str, flags,
4290 					NULL, &n_subexprs);
4291 		if (IS_ERR(hist_field)) {
4292 			ret = PTR_ERR(hist_field);
4293 			goto out;
4294 		}
4295 
4296 		if (field_has_hist_vars(hist_field, 0))	{
4297 			hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4298 			destroy_hist_field(hist_field, 0);
4299 			ret = -EINVAL;
4300 			goto out;
4301 		}
4302 
4303 		key_size = hist_field->size;
4304 	}
4305 
4306 	hist_data->fields[key_idx] = hist_field;
4307 
4308 	key_size = ALIGN(key_size, sizeof(u64));
4309 	hist_data->fields[key_idx]->size = key_size;
4310 	hist_data->fields[key_idx]->offset = key_offset;
4311 
4312 	hist_data->key_size += key_size;
4313 
4314 	if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4315 		ret = -EINVAL;
4316 		goto out;
4317 	}
4318 
4319 	hist_data->n_keys++;
4320 	hist_data->n_fields++;
4321 
4322 	if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4323 		return -EINVAL;
4324 
4325 	ret = key_size;
4326  out:
4327 	return ret;
4328 }
4329 
4330 static int create_key_fields(struct hist_trigger_data *hist_data,
4331 			     struct trace_event_file *file)
4332 {
4333 	unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4334 	char *fields_str, *field_str;
4335 	int ret = -EINVAL;
4336 
4337 	fields_str = hist_data->attrs->keys_str;
4338 	if (!fields_str)
4339 		goto out;
4340 
4341 	for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4342 		field_str = strsep(&fields_str, ",");
4343 		if (!field_str)
4344 			break;
4345 		ret = create_key_field(hist_data, i, key_offset,
4346 				       file, field_str);
4347 		if (ret < 0)
4348 			goto out;
4349 		key_offset += ret;
4350 	}
4351 	if (fields_str) {
4352 		ret = -EINVAL;
4353 		goto out;
4354 	}
4355 	ret = 0;
4356  out:
4357 	return ret;
4358 }
4359 
4360 static int create_var_fields(struct hist_trigger_data *hist_data,
4361 			     struct trace_event_file *file)
4362 {
4363 	unsigned int i, j = hist_data->n_vals;
4364 	int ret = 0;
4365 
4366 	unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4367 
4368 	for (i = 0; i < n_vars; i++) {
4369 		char *var_name = hist_data->attrs->var_defs.name[i];
4370 		char *expr = hist_data->attrs->var_defs.expr[i];
4371 
4372 		ret = create_var_field(hist_data, j++, file, var_name, expr);
4373 		if (ret)
4374 			goto out;
4375 	}
4376  out:
4377 	return ret;
4378 }
4379 
4380 static void free_var_defs(struct hist_trigger_data *hist_data)
4381 {
4382 	unsigned int i;
4383 
4384 	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4385 		kfree(hist_data->attrs->var_defs.name[i]);
4386 		kfree(hist_data->attrs->var_defs.expr[i]);
4387 	}
4388 
4389 	hist_data->attrs->var_defs.n_vars = 0;
4390 }
4391 
4392 static int parse_var_defs(struct hist_trigger_data *hist_data)
4393 {
4394 	struct trace_array *tr = hist_data->event_file->tr;
4395 	char *s, *str, *var_name, *field_str;
4396 	unsigned int i, j, n_vars = 0;
4397 	int ret = 0;
4398 
4399 	for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4400 		str = hist_data->attrs->assignment_str[i];
4401 		for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4402 			field_str = strsep(&str, ",");
4403 			if (!field_str)
4404 				break;
4405 
4406 			var_name = strsep(&field_str, "=");
4407 			if (!var_name || !field_str) {
4408 				hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4409 					 errpos(var_name));
4410 				ret = -EINVAL;
4411 				goto free;
4412 			}
4413 
4414 			if (n_vars == TRACING_MAP_VARS_MAX) {
4415 				hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4416 				ret = -EINVAL;
4417 				goto free;
4418 			}
4419 
4420 			s = kstrdup(var_name, GFP_KERNEL);
4421 			if (!s) {
4422 				ret = -ENOMEM;
4423 				goto free;
4424 			}
4425 			hist_data->attrs->var_defs.name[n_vars] = s;
4426 
4427 			s = kstrdup(field_str, GFP_KERNEL);
4428 			if (!s) {
4429 				ret = -ENOMEM;
4430 				goto free;
4431 			}
4432 			hist_data->attrs->var_defs.expr[n_vars++] = s;
4433 
4434 			hist_data->attrs->var_defs.n_vars = n_vars;
4435 		}
4436 	}
4437 
4438 	return ret;
4439  free:
4440 	free_var_defs(hist_data);
4441 
4442 	return ret;
4443 }
4444 
4445 static int create_hist_fields(struct hist_trigger_data *hist_data,
4446 			      struct trace_event_file *file)
4447 {
4448 	int ret;
4449 
4450 	ret = parse_var_defs(hist_data);
4451 	if (ret)
4452 		goto out;
4453 
4454 	ret = create_val_fields(hist_data, file);
4455 	if (ret)
4456 		goto out;
4457 
4458 	ret = create_var_fields(hist_data, file);
4459 	if (ret)
4460 		goto out;
4461 
4462 	ret = create_key_fields(hist_data, file);
4463 	if (ret)
4464 		goto out;
4465  out:
4466 	free_var_defs(hist_data);
4467 
4468 	return ret;
4469 }
4470 
4471 static int is_descending(struct trace_array *tr, const char *str)
4472 {
4473 	if (!str)
4474 		return 0;
4475 
4476 	if (strcmp(str, "descending") == 0)
4477 		return 1;
4478 
4479 	if (strcmp(str, "ascending") == 0)
4480 		return 0;
4481 
4482 	hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4483 
4484 	return -EINVAL;
4485 }
4486 
4487 static int create_sort_keys(struct hist_trigger_data *hist_data)
4488 {
4489 	struct trace_array *tr = hist_data->event_file->tr;
4490 	char *fields_str = hist_data->attrs->sort_key_str;
4491 	struct tracing_map_sort_key *sort_key;
4492 	int descending, ret = 0;
4493 	unsigned int i, j, k;
4494 
4495 	hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4496 
4497 	if (!fields_str)
4498 		goto out;
4499 
4500 	for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4501 		struct hist_field *hist_field;
4502 		char *field_str, *field_name;
4503 		const char *test_name;
4504 
4505 		sort_key = &hist_data->sort_keys[i];
4506 
4507 		field_str = strsep(&fields_str, ",");
4508 		if (!field_str)
4509 			break;
4510 
4511 		if (!*field_str) {
4512 			ret = -EINVAL;
4513 			hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4514 			break;
4515 		}
4516 
4517 		if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4518 			hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4519 			ret = -EINVAL;
4520 			break;
4521 		}
4522 
4523 		field_name = strsep(&field_str, ".");
4524 		if (!field_name || !*field_name) {
4525 			ret = -EINVAL;
4526 			hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4527 			break;
4528 		}
4529 
4530 		if (strcmp(field_name, "hitcount") == 0) {
4531 			descending = is_descending(tr, field_str);
4532 			if (descending < 0) {
4533 				ret = descending;
4534 				break;
4535 			}
4536 			sort_key->descending = descending;
4537 			continue;
4538 		}
4539 
4540 		for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4541 			unsigned int idx;
4542 
4543 			hist_field = hist_data->fields[j];
4544 			if (hist_field->flags & HIST_FIELD_FL_VAR)
4545 				continue;
4546 
4547 			idx = k++;
4548 
4549 			test_name = hist_field_name(hist_field, 0);
4550 
4551 			if (strcmp(field_name, test_name) == 0) {
4552 				sort_key->field_idx = idx;
4553 				descending = is_descending(tr, field_str);
4554 				if (descending < 0) {
4555 					ret = descending;
4556 					goto out;
4557 				}
4558 				sort_key->descending = descending;
4559 				break;
4560 			}
4561 		}
4562 		if (j == hist_data->n_fields) {
4563 			ret = -EINVAL;
4564 			hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4565 			break;
4566 		}
4567 	}
4568 
4569 	hist_data->n_sort_keys = i;
4570  out:
4571 	return ret;
4572 }
4573 
4574 static void destroy_actions(struct hist_trigger_data *hist_data)
4575 {
4576 	unsigned int i;
4577 
4578 	for (i = 0; i < hist_data->n_actions; i++) {
4579 		struct action_data *data = hist_data->actions[i];
4580 
4581 		if (data->handler == HANDLER_ONMATCH)
4582 			onmatch_destroy(data);
4583 		else if (data->handler == HANDLER_ONMAX ||
4584 			 data->handler == HANDLER_ONCHANGE)
4585 			track_data_destroy(hist_data, data);
4586 		else
4587 			kfree(data);
4588 	}
4589 }
4590 
4591 static int parse_actions(struct hist_trigger_data *hist_data)
4592 {
4593 	struct trace_array *tr = hist_data->event_file->tr;
4594 	struct action_data *data;
4595 	unsigned int i;
4596 	int ret = 0;
4597 	char *str;
4598 	int len;
4599 
4600 	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4601 		str = hist_data->attrs->action_str[i];
4602 
4603 		if ((len = str_has_prefix(str, "onmatch("))) {
4604 			char *action_str = str + len;
4605 
4606 			data = onmatch_parse(tr, action_str);
4607 			if (IS_ERR(data)) {
4608 				ret = PTR_ERR(data);
4609 				break;
4610 			}
4611 		} else if ((len = str_has_prefix(str, "onmax("))) {
4612 			char *action_str = str + len;
4613 
4614 			data = track_data_parse(hist_data, action_str,
4615 						HANDLER_ONMAX);
4616 			if (IS_ERR(data)) {
4617 				ret = PTR_ERR(data);
4618 				break;
4619 			}
4620 		} else if ((len = str_has_prefix(str, "onchange("))) {
4621 			char *action_str = str + len;
4622 
4623 			data = track_data_parse(hist_data, action_str,
4624 						HANDLER_ONCHANGE);
4625 			if (IS_ERR(data)) {
4626 				ret = PTR_ERR(data);
4627 				break;
4628 			}
4629 		} else {
4630 			ret = -EINVAL;
4631 			break;
4632 		}
4633 
4634 		hist_data->actions[hist_data->n_actions++] = data;
4635 	}
4636 
4637 	return ret;
4638 }
4639 
4640 static int create_actions(struct hist_trigger_data *hist_data)
4641 {
4642 	struct action_data *data;
4643 	unsigned int i;
4644 	int ret = 0;
4645 
4646 	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4647 		data = hist_data->actions[i];
4648 
4649 		if (data->handler == HANDLER_ONMATCH) {
4650 			ret = onmatch_create(hist_data, data);
4651 			if (ret)
4652 				break;
4653 		} else if (data->handler == HANDLER_ONMAX ||
4654 			   data->handler == HANDLER_ONCHANGE) {
4655 			ret = track_data_create(hist_data, data);
4656 			if (ret)
4657 				break;
4658 		} else {
4659 			ret = -EINVAL;
4660 			break;
4661 		}
4662 	}
4663 
4664 	return ret;
4665 }
4666 
4667 static void print_actions(struct seq_file *m,
4668 			  struct hist_trigger_data *hist_data,
4669 			  struct tracing_map_elt *elt)
4670 {
4671 	unsigned int i;
4672 
4673 	for (i = 0; i < hist_data->n_actions; i++) {
4674 		struct action_data *data = hist_data->actions[i];
4675 
4676 		if (data->action == ACTION_SNAPSHOT)
4677 			continue;
4678 
4679 		if (data->handler == HANDLER_ONMAX ||
4680 		    data->handler == HANDLER_ONCHANGE)
4681 			track_data_print(m, hist_data, elt, data);
4682 	}
4683 }
4684 
4685 static void print_action_spec(struct seq_file *m,
4686 			      struct hist_trigger_data *hist_data,
4687 			      struct action_data *data)
4688 {
4689 	unsigned int i;
4690 
4691 	if (data->action == ACTION_SAVE) {
4692 		for (i = 0; i < hist_data->n_save_vars; i++) {
4693 			seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4694 			if (i < hist_data->n_save_vars - 1)
4695 				seq_puts(m, ",");
4696 		}
4697 	} else if (data->action == ACTION_TRACE) {
4698 		if (data->use_trace_keyword)
4699 			seq_printf(m, "%s", data->synth_event_name);
4700 		for (i = 0; i < data->n_params; i++) {
4701 			if (i || data->use_trace_keyword)
4702 				seq_puts(m, ",");
4703 			seq_printf(m, "%s", data->params[i]);
4704 		}
4705 	}
4706 }
4707 
4708 static void print_track_data_spec(struct seq_file *m,
4709 				  struct hist_trigger_data *hist_data,
4710 				  struct action_data *data)
4711 {
4712 	if (data->handler == HANDLER_ONMAX)
4713 		seq_puts(m, ":onmax(");
4714 	else if (data->handler == HANDLER_ONCHANGE)
4715 		seq_puts(m, ":onchange(");
4716 	seq_printf(m, "%s", data->track_data.var_str);
4717 	seq_printf(m, ").%s(", data->action_name);
4718 
4719 	print_action_spec(m, hist_data, data);
4720 
4721 	seq_puts(m, ")");
4722 }
4723 
4724 static void print_onmatch_spec(struct seq_file *m,
4725 			       struct hist_trigger_data *hist_data,
4726 			       struct action_data *data)
4727 {
4728 	seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4729 		   data->match_data.event);
4730 
4731 	seq_printf(m, "%s(", data->action_name);
4732 
4733 	print_action_spec(m, hist_data, data);
4734 
4735 	seq_puts(m, ")");
4736 }
4737 
4738 static bool actions_match(struct hist_trigger_data *hist_data,
4739 			  struct hist_trigger_data *hist_data_test)
4740 {
4741 	unsigned int i, j;
4742 
4743 	if (hist_data->n_actions != hist_data_test->n_actions)
4744 		return false;
4745 
4746 	for (i = 0; i < hist_data->n_actions; i++) {
4747 		struct action_data *data = hist_data->actions[i];
4748 		struct action_data *data_test = hist_data_test->actions[i];
4749 		char *action_name, *action_name_test;
4750 
4751 		if (data->handler != data_test->handler)
4752 			return false;
4753 		if (data->action != data_test->action)
4754 			return false;
4755 
4756 		if (data->n_params != data_test->n_params)
4757 			return false;
4758 
4759 		for (j = 0; j < data->n_params; j++) {
4760 			if (strcmp(data->params[j], data_test->params[j]) != 0)
4761 				return false;
4762 		}
4763 
4764 		if (data->use_trace_keyword)
4765 			action_name = data->synth_event_name;
4766 		else
4767 			action_name = data->action_name;
4768 
4769 		if (data_test->use_trace_keyword)
4770 			action_name_test = data_test->synth_event_name;
4771 		else
4772 			action_name_test = data_test->action_name;
4773 
4774 		if (strcmp(action_name, action_name_test) != 0)
4775 			return false;
4776 
4777 		if (data->handler == HANDLER_ONMATCH) {
4778 			if (strcmp(data->match_data.event_system,
4779 				   data_test->match_data.event_system) != 0)
4780 				return false;
4781 			if (strcmp(data->match_data.event,
4782 				   data_test->match_data.event) != 0)
4783 				return false;
4784 		} else if (data->handler == HANDLER_ONMAX ||
4785 			   data->handler == HANDLER_ONCHANGE) {
4786 			if (strcmp(data->track_data.var_str,
4787 				   data_test->track_data.var_str) != 0)
4788 				return false;
4789 		}
4790 	}
4791 
4792 	return true;
4793 }
4794 
4795 
4796 static void print_actions_spec(struct seq_file *m,
4797 			       struct hist_trigger_data *hist_data)
4798 {
4799 	unsigned int i;
4800 
4801 	for (i = 0; i < hist_data->n_actions; i++) {
4802 		struct action_data *data = hist_data->actions[i];
4803 
4804 		if (data->handler == HANDLER_ONMATCH)
4805 			print_onmatch_spec(m, hist_data, data);
4806 		else if (data->handler == HANDLER_ONMAX ||
4807 			 data->handler == HANDLER_ONCHANGE)
4808 			print_track_data_spec(m, hist_data, data);
4809 	}
4810 }
4811 
4812 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4813 {
4814 	unsigned int i;
4815 
4816 	for (i = 0; i < hist_data->n_field_var_hists; i++) {
4817 		kfree(hist_data->field_var_hists[i]->cmd);
4818 		kfree(hist_data->field_var_hists[i]);
4819 	}
4820 }
4821 
4822 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4823 {
4824 	if (!hist_data)
4825 		return;
4826 
4827 	destroy_hist_trigger_attrs(hist_data->attrs);
4828 	destroy_hist_fields(hist_data);
4829 	tracing_map_destroy(hist_data->map);
4830 
4831 	destroy_actions(hist_data);
4832 	destroy_field_vars(hist_data);
4833 	destroy_field_var_hists(hist_data);
4834 
4835 	kfree(hist_data);
4836 }
4837 
4838 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4839 {
4840 	struct tracing_map *map = hist_data->map;
4841 	struct ftrace_event_field *field;
4842 	struct hist_field *hist_field;
4843 	int i, idx = 0;
4844 
4845 	for_each_hist_field(i, hist_data) {
4846 		hist_field = hist_data->fields[i];
4847 		if (hist_field->flags & HIST_FIELD_FL_KEY) {
4848 			tracing_map_cmp_fn_t cmp_fn;
4849 
4850 			field = hist_field->field;
4851 
4852 			if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4853 				cmp_fn = tracing_map_cmp_none;
4854 			else if (!field || hist_field->flags & HIST_FIELD_FL_CPU)
4855 				cmp_fn = tracing_map_cmp_num(hist_field->size,
4856 							     hist_field->is_signed);
4857 			else if (is_string_field(field))
4858 				cmp_fn = tracing_map_cmp_string;
4859 			else
4860 				cmp_fn = tracing_map_cmp_num(field->size,
4861 							     field->is_signed);
4862 			idx = tracing_map_add_key_field(map,
4863 							hist_field->offset,
4864 							cmp_fn);
4865 		} else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4866 			idx = tracing_map_add_sum_field(map);
4867 
4868 		if (idx < 0)
4869 			return idx;
4870 
4871 		if (hist_field->flags & HIST_FIELD_FL_VAR) {
4872 			idx = tracing_map_add_var(map);
4873 			if (idx < 0)
4874 				return idx;
4875 			hist_field->var.idx = idx;
4876 			hist_field->var.hist_data = hist_data;
4877 		}
4878 	}
4879 
4880 	return 0;
4881 }
4882 
4883 static struct hist_trigger_data *
4884 create_hist_data(unsigned int map_bits,
4885 		 struct hist_trigger_attrs *attrs,
4886 		 struct trace_event_file *file,
4887 		 bool remove)
4888 {
4889 	const struct tracing_map_ops *map_ops = NULL;
4890 	struct hist_trigger_data *hist_data;
4891 	int ret = 0;
4892 
4893 	hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4894 	if (!hist_data)
4895 		return ERR_PTR(-ENOMEM);
4896 
4897 	hist_data->attrs = attrs;
4898 	hist_data->remove = remove;
4899 	hist_data->event_file = file;
4900 
4901 	ret = parse_actions(hist_data);
4902 	if (ret)
4903 		goto free;
4904 
4905 	ret = create_hist_fields(hist_data, file);
4906 	if (ret)
4907 		goto free;
4908 
4909 	ret = create_sort_keys(hist_data);
4910 	if (ret)
4911 		goto free;
4912 
4913 	map_ops = &hist_trigger_elt_data_ops;
4914 
4915 	hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4916 					    map_ops, hist_data);
4917 	if (IS_ERR(hist_data->map)) {
4918 		ret = PTR_ERR(hist_data->map);
4919 		hist_data->map = NULL;
4920 		goto free;
4921 	}
4922 
4923 	ret = create_tracing_map_fields(hist_data);
4924 	if (ret)
4925 		goto free;
4926  out:
4927 	return hist_data;
4928  free:
4929 	hist_data->attrs = NULL;
4930 
4931 	destroy_hist_data(hist_data);
4932 
4933 	hist_data = ERR_PTR(ret);
4934 
4935 	goto out;
4936 }
4937 
4938 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4939 				    struct tracing_map_elt *elt,
4940 				    struct trace_buffer *buffer, void *rec,
4941 				    struct ring_buffer_event *rbe,
4942 				    u64 *var_ref_vals)
4943 {
4944 	struct hist_elt_data *elt_data;
4945 	struct hist_field *hist_field;
4946 	unsigned int i, var_idx;
4947 	u64 hist_val;
4948 
4949 	elt_data = elt->private_data;
4950 	elt_data->var_ref_vals = var_ref_vals;
4951 
4952 	for_each_hist_val_field(i, hist_data) {
4953 		hist_field = hist_data->fields[i];
4954 		hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4955 		if (hist_field->flags & HIST_FIELD_FL_VAR) {
4956 			var_idx = hist_field->var.idx;
4957 
4958 			if (hist_field->flags & HIST_FIELD_FL_STRING) {
4959 				unsigned int str_start, var_str_idx, idx;
4960 				char *str, *val_str;
4961 				unsigned int size;
4962 
4963 				str_start = hist_data->n_field_var_str +
4964 					hist_data->n_save_var_str;
4965 				var_str_idx = hist_field->var_str_idx;
4966 				idx = str_start + var_str_idx;
4967 
4968 				str = elt_data->field_var_str[idx];
4969 				val_str = (char *)(uintptr_t)hist_val;
4970 
4971 				size = min(hist_field->size, STR_VAR_LEN_MAX);
4972 				strscpy(str, val_str, size);
4973 
4974 				hist_val = (u64)(uintptr_t)str;
4975 			}
4976 			tracing_map_set_var(elt, var_idx, hist_val);
4977 			continue;
4978 		}
4979 		tracing_map_update_sum(elt, i, hist_val);
4980 	}
4981 
4982 	for_each_hist_key_field(i, hist_data) {
4983 		hist_field = hist_data->fields[i];
4984 		if (hist_field->flags & HIST_FIELD_FL_VAR) {
4985 			hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4986 			var_idx = hist_field->var.idx;
4987 			tracing_map_set_var(elt, var_idx, hist_val);
4988 		}
4989 	}
4990 
4991 	update_field_vars(hist_data, elt, buffer, rbe, rec);
4992 }
4993 
4994 static inline void add_to_key(char *compound_key, void *key,
4995 			      struct hist_field *key_field, void *rec)
4996 {
4997 	size_t size = key_field->size;
4998 
4999 	if (key_field->flags & HIST_FIELD_FL_STRING) {
5000 		struct ftrace_event_field *field;
5001 
5002 		field = key_field->field;
5003 		if (field->filter_type == FILTER_DYN_STRING ||
5004 		    field->filter_type == FILTER_RDYN_STRING)
5005 			size = *(u32 *)(rec + field->offset) >> 16;
5006 		else if (field->filter_type == FILTER_STATIC_STRING)
5007 			size = field->size;
5008 
5009 		/* ensure NULL-termination */
5010 		if (size > key_field->size - 1)
5011 			size = key_field->size - 1;
5012 
5013 		strncpy(compound_key + key_field->offset, (char *)key, size);
5014 	} else
5015 		memcpy(compound_key + key_field->offset, key, size);
5016 }
5017 
5018 static void
5019 hist_trigger_actions(struct hist_trigger_data *hist_data,
5020 		     struct tracing_map_elt *elt,
5021 		     struct trace_buffer *buffer, void *rec,
5022 		     struct ring_buffer_event *rbe, void *key,
5023 		     u64 *var_ref_vals)
5024 {
5025 	struct action_data *data;
5026 	unsigned int i;
5027 
5028 	for (i = 0; i < hist_data->n_actions; i++) {
5029 		data = hist_data->actions[i];
5030 		data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
5031 	}
5032 }
5033 
5034 static void event_hist_trigger(struct event_trigger_data *data,
5035 			       struct trace_buffer *buffer, void *rec,
5036 			       struct ring_buffer_event *rbe)
5037 {
5038 	struct hist_trigger_data *hist_data = data->private_data;
5039 	bool use_compound_key = (hist_data->n_keys > 1);
5040 	unsigned long entries[HIST_STACKTRACE_DEPTH];
5041 	u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5042 	char compound_key[HIST_KEY_SIZE_MAX];
5043 	struct tracing_map_elt *elt = NULL;
5044 	struct hist_field *key_field;
5045 	u64 field_contents;
5046 	void *key = NULL;
5047 	unsigned int i;
5048 
5049 	memset(compound_key, 0, hist_data->key_size);
5050 
5051 	for_each_hist_key_field(i, hist_data) {
5052 		key_field = hist_data->fields[i];
5053 
5054 		if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5055 			memset(entries, 0, HIST_STACKTRACE_SIZE);
5056 			stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5057 					 HIST_STACKTRACE_SKIP);
5058 			key = entries;
5059 		} else {
5060 			field_contents = key_field->fn(key_field, elt, buffer, rbe, rec);
5061 			if (key_field->flags & HIST_FIELD_FL_STRING) {
5062 				key = (void *)(unsigned long)field_contents;
5063 				use_compound_key = true;
5064 			} else
5065 				key = (void *)&field_contents;
5066 		}
5067 
5068 		if (use_compound_key)
5069 			add_to_key(compound_key, key, key_field, rec);
5070 	}
5071 
5072 	if (use_compound_key)
5073 		key = compound_key;
5074 
5075 	if (hist_data->n_var_refs &&
5076 	    !resolve_var_refs(hist_data, key, var_ref_vals, false))
5077 		return;
5078 
5079 	elt = tracing_map_insert(hist_data->map, key);
5080 	if (!elt)
5081 		return;
5082 
5083 	hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
5084 
5085 	if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5086 		hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
5087 }
5088 
5089 static void hist_trigger_stacktrace_print(struct seq_file *m,
5090 					  unsigned long *stacktrace_entries,
5091 					  unsigned int max_entries)
5092 {
5093 	unsigned int spaces = 8;
5094 	unsigned int i;
5095 
5096 	for (i = 0; i < max_entries; i++) {
5097 		if (!stacktrace_entries[i])
5098 			return;
5099 
5100 		seq_printf(m, "%*c", 1 + spaces, ' ');
5101 		seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]);
5102 	}
5103 }
5104 
5105 static void hist_trigger_print_key(struct seq_file *m,
5106 				   struct hist_trigger_data *hist_data,
5107 				   void *key,
5108 				   struct tracing_map_elt *elt)
5109 {
5110 	struct hist_field *key_field;
5111 	bool multiline = false;
5112 	const char *field_name;
5113 	unsigned int i;
5114 	u64 uval;
5115 
5116 	seq_puts(m, "{ ");
5117 
5118 	for_each_hist_key_field(i, hist_data) {
5119 		key_field = hist_data->fields[i];
5120 
5121 		if (i > hist_data->n_vals)
5122 			seq_puts(m, ", ");
5123 
5124 		field_name = hist_field_name(key_field, 0);
5125 
5126 		if (key_field->flags & HIST_FIELD_FL_HEX) {
5127 			uval = *(u64 *)(key + key_field->offset);
5128 			seq_printf(m, "%s: %llx", field_name, uval);
5129 		} else if (key_field->flags & HIST_FIELD_FL_SYM) {
5130 			uval = *(u64 *)(key + key_field->offset);
5131 			seq_printf(m, "%s: [%llx] %-45ps", field_name,
5132 				   uval, (void *)(uintptr_t)uval);
5133 		} else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5134 			uval = *(u64 *)(key + key_field->offset);
5135 			seq_printf(m, "%s: [%llx] %-55pS", field_name,
5136 				   uval, (void *)(uintptr_t)uval);
5137 		} else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5138 			struct hist_elt_data *elt_data = elt->private_data;
5139 			char *comm;
5140 
5141 			if (WARN_ON_ONCE(!elt_data))
5142 				return;
5143 
5144 			comm = elt_data->comm;
5145 
5146 			uval = *(u64 *)(key + key_field->offset);
5147 			seq_printf(m, "%s: %-16s[%10llu]", field_name,
5148 				   comm, uval);
5149 		} else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5150 			const char *syscall_name;
5151 
5152 			uval = *(u64 *)(key + key_field->offset);
5153 			syscall_name = get_syscall_name(uval);
5154 			if (!syscall_name)
5155 				syscall_name = "unknown_syscall";
5156 
5157 			seq_printf(m, "%s: %-30s[%3llu]", field_name,
5158 				   syscall_name, uval);
5159 		} else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5160 			seq_puts(m, "stacktrace:\n");
5161 			hist_trigger_stacktrace_print(m,
5162 						      key + key_field->offset,
5163 						      HIST_STACKTRACE_DEPTH);
5164 			multiline = true;
5165 		} else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5166 			seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5167 				   *(u64 *)(key + key_field->offset));
5168 		} else if (key_field->flags & HIST_FIELD_FL_BUCKET) {
5169 			unsigned long buckets = key_field->buckets;
5170 			uval = *(u64 *)(key + key_field->offset);
5171 			seq_printf(m, "%s: ~ %llu-%llu", field_name,
5172 				   uval, uval + buckets -1);
5173 		} else if (key_field->flags & HIST_FIELD_FL_STRING) {
5174 			seq_printf(m, "%s: %-50s", field_name,
5175 				   (char *)(key + key_field->offset));
5176 		} else {
5177 			uval = *(u64 *)(key + key_field->offset);
5178 			seq_printf(m, "%s: %10llu", field_name, uval);
5179 		}
5180 	}
5181 
5182 	if (!multiline)
5183 		seq_puts(m, " ");
5184 
5185 	seq_puts(m, "}");
5186 }
5187 
5188 static void hist_trigger_entry_print(struct seq_file *m,
5189 				     struct hist_trigger_data *hist_data,
5190 				     void *key,
5191 				     struct tracing_map_elt *elt)
5192 {
5193 	const char *field_name;
5194 	unsigned int i;
5195 
5196 	hist_trigger_print_key(m, hist_data, key, elt);
5197 
5198 	seq_printf(m, " hitcount: %10llu",
5199 		   tracing_map_read_sum(elt, HITCOUNT_IDX));
5200 
5201 	for (i = 1; i < hist_data->n_vals; i++) {
5202 		field_name = hist_field_name(hist_data->fields[i], 0);
5203 
5204 		if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
5205 		    hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
5206 			continue;
5207 
5208 		if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
5209 			seq_printf(m, "  %s: %10llx", field_name,
5210 				   tracing_map_read_sum(elt, i));
5211 		} else {
5212 			seq_printf(m, "  %s: %10llu", field_name,
5213 				   tracing_map_read_sum(elt, i));
5214 		}
5215 	}
5216 
5217 	print_actions(m, hist_data, elt);
5218 
5219 	seq_puts(m, "\n");
5220 }
5221 
5222 static int print_entries(struct seq_file *m,
5223 			 struct hist_trigger_data *hist_data)
5224 {
5225 	struct tracing_map_sort_entry **sort_entries = NULL;
5226 	struct tracing_map *map = hist_data->map;
5227 	int i, n_entries;
5228 
5229 	n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5230 					     hist_data->n_sort_keys,
5231 					     &sort_entries);
5232 	if (n_entries < 0)
5233 		return n_entries;
5234 
5235 	for (i = 0; i < n_entries; i++)
5236 		hist_trigger_entry_print(m, hist_data,
5237 					 sort_entries[i]->key,
5238 					 sort_entries[i]->elt);
5239 
5240 	tracing_map_destroy_sort_entries(sort_entries, n_entries);
5241 
5242 	return n_entries;
5243 }
5244 
5245 static void hist_trigger_show(struct seq_file *m,
5246 			      struct event_trigger_data *data, int n)
5247 {
5248 	struct hist_trigger_data *hist_data;
5249 	int n_entries;
5250 
5251 	if (n > 0)
5252 		seq_puts(m, "\n\n");
5253 
5254 	seq_puts(m, "# event histogram\n#\n# trigger info: ");
5255 	data->ops->print(m, data->ops, data);
5256 	seq_puts(m, "#\n\n");
5257 
5258 	hist_data = data->private_data;
5259 	n_entries = print_entries(m, hist_data);
5260 	if (n_entries < 0)
5261 		n_entries = 0;
5262 
5263 	track_data_snapshot_print(m, hist_data);
5264 
5265 	seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5266 		   (u64)atomic64_read(&hist_data->map->hits),
5267 		   n_entries, (u64)atomic64_read(&hist_data->map->drops));
5268 }
5269 
5270 static int hist_show(struct seq_file *m, void *v)
5271 {
5272 	struct event_trigger_data *data;
5273 	struct trace_event_file *event_file;
5274 	int n = 0, ret = 0;
5275 
5276 	mutex_lock(&event_mutex);
5277 
5278 	event_file = event_file_data(m->private);
5279 	if (unlikely(!event_file)) {
5280 		ret = -ENODEV;
5281 		goto out_unlock;
5282 	}
5283 
5284 	list_for_each_entry(data, &event_file->triggers, list) {
5285 		if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5286 			hist_trigger_show(m, data, n++);
5287 	}
5288 
5289  out_unlock:
5290 	mutex_unlock(&event_mutex);
5291 
5292 	return ret;
5293 }
5294 
5295 static int event_hist_open(struct inode *inode, struct file *file)
5296 {
5297 	int ret;
5298 
5299 	ret = security_locked_down(LOCKDOWN_TRACEFS);
5300 	if (ret)
5301 		return ret;
5302 
5303 	return single_open(file, hist_show, file);
5304 }
5305 
5306 const struct file_operations event_hist_fops = {
5307 	.open = event_hist_open,
5308 	.read = seq_read,
5309 	.llseek = seq_lseek,
5310 	.release = single_release,
5311 };
5312 
5313 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
5314 static void hist_field_debug_show_flags(struct seq_file *m,
5315 					unsigned long flags)
5316 {
5317 	seq_puts(m, "      flags:\n");
5318 
5319 	if (flags & HIST_FIELD_FL_KEY)
5320 		seq_puts(m, "        HIST_FIELD_FL_KEY\n");
5321 	else if (flags & HIST_FIELD_FL_HITCOUNT)
5322 		seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
5323 	else if (flags & HIST_FIELD_FL_VAR)
5324 		seq_puts(m, "        HIST_FIELD_FL_VAR\n");
5325 	else if (flags & HIST_FIELD_FL_VAR_REF)
5326 		seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
5327 	else
5328 		seq_puts(m, "        VAL: normal u64 value\n");
5329 
5330 	if (flags & HIST_FIELD_FL_ALIAS)
5331 		seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
5332 	else if (flags & HIST_FIELD_FL_CONST)
5333 		seq_puts(m, "        HIST_FIELD_FL_CONST\n");
5334 }
5335 
5336 static int hist_field_debug_show(struct seq_file *m,
5337 				 struct hist_field *field, unsigned long flags)
5338 {
5339 	if ((field->flags & flags) != flags) {
5340 		seq_printf(m, "ERROR: bad flags - %lx\n", flags);
5341 		return -EINVAL;
5342 	}
5343 
5344 	hist_field_debug_show_flags(m, field->flags);
5345 	if (field->field)
5346 		seq_printf(m, "      ftrace_event_field name: %s\n",
5347 			   field->field->name);
5348 
5349 	if (field->flags & HIST_FIELD_FL_VAR) {
5350 		seq_printf(m, "      var.name: %s\n", field->var.name);
5351 		seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5352 			   field->var.idx);
5353 	}
5354 
5355 	if (field->flags & HIST_FIELD_FL_CONST)
5356 		seq_printf(m, "      constant: %llu\n", field->constant);
5357 
5358 	if (field->flags & HIST_FIELD_FL_ALIAS)
5359 		seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5360 			   field->var_ref_idx);
5361 
5362 	if (field->flags & HIST_FIELD_FL_VAR_REF) {
5363 		seq_printf(m, "      name: %s\n", field->name);
5364 		seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5365 			   field->var.idx);
5366 		seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
5367 		seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5368 			   field->var_ref_idx);
5369 		if (field->system)
5370 			seq_printf(m, "      system: %s\n", field->system);
5371 		if (field->event_name)
5372 			seq_printf(m, "      event_name: %s\n", field->event_name);
5373 	}
5374 
5375 	seq_printf(m, "      type: %s\n", field->type);
5376 	seq_printf(m, "      size: %u\n", field->size);
5377 	seq_printf(m, "      is_signed: %u\n", field->is_signed);
5378 
5379 	return 0;
5380 }
5381 
5382 static int field_var_debug_show(struct seq_file *m,
5383 				struct field_var *field_var, unsigned int i,
5384 				bool save_vars)
5385 {
5386 	const char *vars_name = save_vars ? "save_vars" : "field_vars";
5387 	struct hist_field *field;
5388 	int ret = 0;
5389 
5390 	seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
5391 
5392 	field = field_var->var;
5393 
5394 	seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
5395 
5396 	hist_field_debug_show_flags(m, field->flags);
5397 	seq_printf(m, "      var.name: %s\n", field->var.name);
5398 	seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5399 		   field->var.idx);
5400 
5401 	field = field_var->val;
5402 
5403 	seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
5404 	if (field->field)
5405 		seq_printf(m, "      ftrace_event_field name: %s\n",
5406 			   field->field->name);
5407 	else {
5408 		ret = -EINVAL;
5409 		goto out;
5410 	}
5411 
5412 	seq_printf(m, "      type: %s\n", field->type);
5413 	seq_printf(m, "      size: %u\n", field->size);
5414 	seq_printf(m, "      is_signed: %u\n", field->is_signed);
5415 out:
5416 	return ret;
5417 }
5418 
5419 static int hist_action_debug_show(struct seq_file *m,
5420 				  struct action_data *data, int i)
5421 {
5422 	int ret = 0;
5423 
5424 	if (data->handler == HANDLER_ONMAX ||
5425 	    data->handler == HANDLER_ONCHANGE) {
5426 		seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
5427 		ret = hist_field_debug_show(m, data->track_data.var_ref,
5428 					    HIST_FIELD_FL_VAR_REF);
5429 		if (ret)
5430 			goto out;
5431 
5432 		seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
5433 		ret = hist_field_debug_show(m, data->track_data.track_var,
5434 					    HIST_FIELD_FL_VAR);
5435 		if (ret)
5436 			goto out;
5437 	}
5438 
5439 	if (data->handler == HANDLER_ONMATCH) {
5440 		seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
5441 			   i, data->match_data.event_system);
5442 		seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
5443 			   i, data->match_data.event);
5444 	}
5445 out:
5446 	return ret;
5447 }
5448 
5449 static int hist_actions_debug_show(struct seq_file *m,
5450 				   struct hist_trigger_data *hist_data)
5451 {
5452 	int i, ret = 0;
5453 
5454 	if (hist_data->n_actions)
5455 		seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
5456 
5457 	for (i = 0; i < hist_data->n_actions; i++) {
5458 		struct action_data *action = hist_data->actions[i];
5459 
5460 		ret = hist_action_debug_show(m, action, i);
5461 		if (ret)
5462 			goto out;
5463 	}
5464 
5465 	if (hist_data->n_save_vars)
5466 		seq_puts(m, "\n  save action variables (save() params):\n");
5467 
5468 	for (i = 0; i < hist_data->n_save_vars; i++) {
5469 		ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
5470 		if (ret)
5471 			goto out;
5472 	}
5473 out:
5474 	return ret;
5475 }
5476 
5477 static void hist_trigger_debug_show(struct seq_file *m,
5478 				    struct event_trigger_data *data, int n)
5479 {
5480 	struct hist_trigger_data *hist_data;
5481 	int i, ret;
5482 
5483 	if (n > 0)
5484 		seq_puts(m, "\n\n");
5485 
5486 	seq_puts(m, "# event histogram\n#\n# trigger info: ");
5487 	data->ops->print(m, data->ops, data);
5488 	seq_puts(m, "#\n\n");
5489 
5490 	hist_data = data->private_data;
5491 
5492 	seq_printf(m, "hist_data: %p\n\n", hist_data);
5493 	seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
5494 	seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
5495 	seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
5496 
5497 	seq_puts(m, "\n  val fields:\n\n");
5498 
5499 	seq_puts(m, "    hist_data->fields[0]:\n");
5500 	ret = hist_field_debug_show(m, hist_data->fields[0],
5501 				    HIST_FIELD_FL_HITCOUNT);
5502 	if (ret)
5503 		return;
5504 
5505 	for (i = 1; i < hist_data->n_vals; i++) {
5506 		seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5507 		ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5508 		if (ret)
5509 			return;
5510 	}
5511 
5512 	seq_puts(m, "\n  key fields:\n");
5513 
5514 	for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5515 		seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5516 		ret = hist_field_debug_show(m, hist_data->fields[i],
5517 					    HIST_FIELD_FL_KEY);
5518 		if (ret)
5519 			return;
5520 	}
5521 
5522 	if (hist_data->n_var_refs)
5523 		seq_puts(m, "\n  variable reference fields:\n");
5524 
5525 	for (i = 0; i < hist_data->n_var_refs; i++) {
5526 		seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5527 		ret = hist_field_debug_show(m, hist_data->var_refs[i],
5528 					    HIST_FIELD_FL_VAR_REF);
5529 		if (ret)
5530 			return;
5531 	}
5532 
5533 	if (hist_data->n_field_vars)
5534 		seq_puts(m, "\n  field variables:\n");
5535 
5536 	for (i = 0; i < hist_data->n_field_vars; i++) {
5537 		ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5538 		if (ret)
5539 			return;
5540 	}
5541 
5542 	ret = hist_actions_debug_show(m, hist_data);
5543 	if (ret)
5544 		return;
5545 }
5546 
5547 static int hist_debug_show(struct seq_file *m, void *v)
5548 {
5549 	struct event_trigger_data *data;
5550 	struct trace_event_file *event_file;
5551 	int n = 0, ret = 0;
5552 
5553 	mutex_lock(&event_mutex);
5554 
5555 	event_file = event_file_data(m->private);
5556 	if (unlikely(!event_file)) {
5557 		ret = -ENODEV;
5558 		goto out_unlock;
5559 	}
5560 
5561 	list_for_each_entry(data, &event_file->triggers, list) {
5562 		if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5563 			hist_trigger_debug_show(m, data, n++);
5564 	}
5565 
5566  out_unlock:
5567 	mutex_unlock(&event_mutex);
5568 
5569 	return ret;
5570 }
5571 
5572 static int event_hist_debug_open(struct inode *inode, struct file *file)
5573 {
5574 	int ret;
5575 
5576 	ret = security_locked_down(LOCKDOWN_TRACEFS);
5577 	if (ret)
5578 		return ret;
5579 
5580 	return single_open(file, hist_debug_show, file);
5581 }
5582 
5583 const struct file_operations event_hist_debug_fops = {
5584 	.open = event_hist_debug_open,
5585 	.read = seq_read,
5586 	.llseek = seq_lseek,
5587 	.release = single_release,
5588 };
5589 #endif
5590 
5591 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5592 {
5593 	const char *field_name = hist_field_name(hist_field, 0);
5594 
5595 	if (hist_field->var.name)
5596 		seq_printf(m, "%s=", hist_field->var.name);
5597 
5598 	if (hist_field->flags & HIST_FIELD_FL_CPU)
5599 		seq_puts(m, "common_cpu");
5600 	else if (hist_field->flags & HIST_FIELD_FL_CONST)
5601 		seq_printf(m, "%llu", hist_field->constant);
5602 	else if (field_name) {
5603 		if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5604 		    hist_field->flags & HIST_FIELD_FL_ALIAS)
5605 			seq_putc(m, '$');
5606 		seq_printf(m, "%s", field_name);
5607 	} else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5608 		seq_puts(m, "common_timestamp");
5609 
5610 	if (hist_field->flags) {
5611 		if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5612 		    !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5613 			const char *flags = get_hist_field_flags(hist_field);
5614 
5615 			if (flags)
5616 				seq_printf(m, ".%s", flags);
5617 		}
5618 	}
5619 	if (hist_field->buckets)
5620 		seq_printf(m, "=%ld", hist_field->buckets);
5621 }
5622 
5623 static int event_hist_trigger_print(struct seq_file *m,
5624 				    struct event_trigger_ops *ops,
5625 				    struct event_trigger_data *data)
5626 {
5627 	struct hist_trigger_data *hist_data = data->private_data;
5628 	struct hist_field *field;
5629 	bool have_var = false;
5630 	unsigned int i;
5631 
5632 	seq_puts(m, HIST_PREFIX);
5633 
5634 	if (data->name)
5635 		seq_printf(m, "%s:", data->name);
5636 
5637 	seq_puts(m, "keys=");
5638 
5639 	for_each_hist_key_field(i, hist_data) {
5640 		field = hist_data->fields[i];
5641 
5642 		if (i > hist_data->n_vals)
5643 			seq_puts(m, ",");
5644 
5645 		if (field->flags & HIST_FIELD_FL_STACKTRACE)
5646 			seq_puts(m, "stacktrace");
5647 		else
5648 			hist_field_print(m, field);
5649 	}
5650 
5651 	seq_puts(m, ":vals=");
5652 
5653 	for_each_hist_val_field(i, hist_data) {
5654 		field = hist_data->fields[i];
5655 		if (field->flags & HIST_FIELD_FL_VAR) {
5656 			have_var = true;
5657 			continue;
5658 		}
5659 
5660 		if (i == HITCOUNT_IDX)
5661 			seq_puts(m, "hitcount");
5662 		else {
5663 			seq_puts(m, ",");
5664 			hist_field_print(m, field);
5665 		}
5666 	}
5667 
5668 	if (have_var) {
5669 		unsigned int n = 0;
5670 
5671 		seq_puts(m, ":");
5672 
5673 		for_each_hist_val_field(i, hist_data) {
5674 			field = hist_data->fields[i];
5675 
5676 			if (field->flags & HIST_FIELD_FL_VAR) {
5677 				if (n++)
5678 					seq_puts(m, ",");
5679 				hist_field_print(m, field);
5680 			}
5681 		}
5682 	}
5683 
5684 	seq_puts(m, ":sort=");
5685 
5686 	for (i = 0; i < hist_data->n_sort_keys; i++) {
5687 		struct tracing_map_sort_key *sort_key;
5688 		unsigned int idx, first_key_idx;
5689 
5690 		/* skip VAR vals */
5691 		first_key_idx = hist_data->n_vals - hist_data->n_vars;
5692 
5693 		sort_key = &hist_data->sort_keys[i];
5694 		idx = sort_key->field_idx;
5695 
5696 		if (WARN_ON(idx >= HIST_FIELDS_MAX))
5697 			return -EINVAL;
5698 
5699 		if (i > 0)
5700 			seq_puts(m, ",");
5701 
5702 		if (idx == HITCOUNT_IDX)
5703 			seq_puts(m, "hitcount");
5704 		else {
5705 			if (idx >= first_key_idx)
5706 				idx += hist_data->n_vars;
5707 			hist_field_print(m, hist_data->fields[idx]);
5708 		}
5709 
5710 		if (sort_key->descending)
5711 			seq_puts(m, ".descending");
5712 	}
5713 	seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5714 	if (hist_data->enable_timestamps)
5715 		seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5716 
5717 	print_actions_spec(m, hist_data);
5718 
5719 	if (data->filter_str)
5720 		seq_printf(m, " if %s", data->filter_str);
5721 
5722 	if (data->paused)
5723 		seq_puts(m, " [paused]");
5724 	else
5725 		seq_puts(m, " [active]");
5726 
5727 	seq_putc(m, '\n');
5728 
5729 	return 0;
5730 }
5731 
5732 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5733 				   struct event_trigger_data *data)
5734 {
5735 	struct hist_trigger_data *hist_data = data->private_data;
5736 
5737 	if (!data->ref && hist_data->attrs->name)
5738 		save_named_trigger(hist_data->attrs->name, data);
5739 
5740 	data->ref++;
5741 
5742 	return 0;
5743 }
5744 
5745 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5746 {
5747 	struct trace_event_file *file;
5748 	unsigned int i;
5749 	char *cmd;
5750 	int ret;
5751 
5752 	for (i = 0; i < hist_data->n_field_var_hists; i++) {
5753 		file = hist_data->field_var_hists[i]->hist_data->event_file;
5754 		cmd = hist_data->field_var_hists[i]->cmd;
5755 		ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
5756 					       "!hist", "hist", cmd);
5757 		WARN_ON_ONCE(ret < 0);
5758 	}
5759 }
5760 
5761 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5762 				    struct event_trigger_data *data)
5763 {
5764 	struct hist_trigger_data *hist_data = data->private_data;
5765 
5766 	if (WARN_ON_ONCE(data->ref <= 0))
5767 		return;
5768 
5769 	data->ref--;
5770 	if (!data->ref) {
5771 		if (data->name)
5772 			del_named_trigger(data);
5773 
5774 		trigger_data_free(data);
5775 
5776 		remove_hist_vars(hist_data);
5777 
5778 		unregister_field_var_hists(hist_data);
5779 
5780 		destroy_hist_data(hist_data);
5781 	}
5782 }
5783 
5784 static struct event_trigger_ops event_hist_trigger_ops = {
5785 	.trigger		= event_hist_trigger,
5786 	.print			= event_hist_trigger_print,
5787 	.init			= event_hist_trigger_init,
5788 	.free			= event_hist_trigger_free,
5789 };
5790 
5791 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5792 					 struct event_trigger_data *data)
5793 {
5794 	data->ref++;
5795 
5796 	save_named_trigger(data->named_data->name, data);
5797 
5798 	event_hist_trigger_init(ops, data->named_data);
5799 
5800 	return 0;
5801 }
5802 
5803 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5804 					  struct event_trigger_data *data)
5805 {
5806 	if (WARN_ON_ONCE(data->ref <= 0))
5807 		return;
5808 
5809 	event_hist_trigger_free(ops, data->named_data);
5810 
5811 	data->ref--;
5812 	if (!data->ref) {
5813 		del_named_trigger(data);
5814 		trigger_data_free(data);
5815 	}
5816 }
5817 
5818 static struct event_trigger_ops event_hist_trigger_named_ops = {
5819 	.trigger		= event_hist_trigger,
5820 	.print			= event_hist_trigger_print,
5821 	.init			= event_hist_trigger_named_init,
5822 	.free			= event_hist_trigger_named_free,
5823 };
5824 
5825 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5826 							    char *param)
5827 {
5828 	return &event_hist_trigger_ops;
5829 }
5830 
5831 static void hist_clear(struct event_trigger_data *data)
5832 {
5833 	struct hist_trigger_data *hist_data = data->private_data;
5834 
5835 	if (data->name)
5836 		pause_named_trigger(data);
5837 
5838 	tracepoint_synchronize_unregister();
5839 
5840 	tracing_map_clear(hist_data->map);
5841 
5842 	if (data->name)
5843 		unpause_named_trigger(data);
5844 }
5845 
5846 static bool compatible_field(struct ftrace_event_field *field,
5847 			     struct ftrace_event_field *test_field)
5848 {
5849 	if (field == test_field)
5850 		return true;
5851 	if (field == NULL || test_field == NULL)
5852 		return false;
5853 	if (strcmp(field->name, test_field->name) != 0)
5854 		return false;
5855 	if (strcmp(field->type, test_field->type) != 0)
5856 		return false;
5857 	if (field->size != test_field->size)
5858 		return false;
5859 	if (field->is_signed != test_field->is_signed)
5860 		return false;
5861 
5862 	return true;
5863 }
5864 
5865 static bool hist_trigger_match(struct event_trigger_data *data,
5866 			       struct event_trigger_data *data_test,
5867 			       struct event_trigger_data *named_data,
5868 			       bool ignore_filter)
5869 {
5870 	struct tracing_map_sort_key *sort_key, *sort_key_test;
5871 	struct hist_trigger_data *hist_data, *hist_data_test;
5872 	struct hist_field *key_field, *key_field_test;
5873 	unsigned int i;
5874 
5875 	if (named_data && (named_data != data_test) &&
5876 	    (named_data != data_test->named_data))
5877 		return false;
5878 
5879 	if (!named_data && is_named_trigger(data_test))
5880 		return false;
5881 
5882 	hist_data = data->private_data;
5883 	hist_data_test = data_test->private_data;
5884 
5885 	if (hist_data->n_vals != hist_data_test->n_vals ||
5886 	    hist_data->n_fields != hist_data_test->n_fields ||
5887 	    hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5888 		return false;
5889 
5890 	if (!ignore_filter) {
5891 		if ((data->filter_str && !data_test->filter_str) ||
5892 		   (!data->filter_str && data_test->filter_str))
5893 			return false;
5894 	}
5895 
5896 	for_each_hist_field(i, hist_data) {
5897 		key_field = hist_data->fields[i];
5898 		key_field_test = hist_data_test->fields[i];
5899 
5900 		if (key_field->flags != key_field_test->flags)
5901 			return false;
5902 		if (!compatible_field(key_field->field, key_field_test->field))
5903 			return false;
5904 		if (key_field->offset != key_field_test->offset)
5905 			return false;
5906 		if (key_field->size != key_field_test->size)
5907 			return false;
5908 		if (key_field->is_signed != key_field_test->is_signed)
5909 			return false;
5910 		if (!!key_field->var.name != !!key_field_test->var.name)
5911 			return false;
5912 		if (key_field->var.name &&
5913 		    strcmp(key_field->var.name, key_field_test->var.name) != 0)
5914 			return false;
5915 	}
5916 
5917 	for (i = 0; i < hist_data->n_sort_keys; i++) {
5918 		sort_key = &hist_data->sort_keys[i];
5919 		sort_key_test = &hist_data_test->sort_keys[i];
5920 
5921 		if (sort_key->field_idx != sort_key_test->field_idx ||
5922 		    sort_key->descending != sort_key_test->descending)
5923 			return false;
5924 	}
5925 
5926 	if (!ignore_filter && data->filter_str &&
5927 	    (strcmp(data->filter_str, data_test->filter_str) != 0))
5928 		return false;
5929 
5930 	if (!actions_match(hist_data, hist_data_test))
5931 		return false;
5932 
5933 	return true;
5934 }
5935 
5936 static int hist_register_trigger(char *glob,
5937 				 struct event_trigger_data *data,
5938 				 struct trace_event_file *file)
5939 {
5940 	struct hist_trigger_data *hist_data = data->private_data;
5941 	struct event_trigger_data *test, *named_data = NULL;
5942 	struct trace_array *tr = file->tr;
5943 	int ret = 0;
5944 
5945 	if (hist_data->attrs->name) {
5946 		named_data = find_named_trigger(hist_data->attrs->name);
5947 		if (named_data) {
5948 			if (!hist_trigger_match(data, named_data, named_data,
5949 						true)) {
5950 				hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5951 				ret = -EINVAL;
5952 				goto out;
5953 			}
5954 		}
5955 	}
5956 
5957 	if (hist_data->attrs->name && !named_data)
5958 		goto new;
5959 
5960 	lockdep_assert_held(&event_mutex);
5961 
5962 	list_for_each_entry(test, &file->triggers, list) {
5963 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5964 			if (!hist_trigger_match(data, test, named_data, false))
5965 				continue;
5966 			if (hist_data->attrs->pause)
5967 				test->paused = true;
5968 			else if (hist_data->attrs->cont)
5969 				test->paused = false;
5970 			else if (hist_data->attrs->clear)
5971 				hist_clear(test);
5972 			else {
5973 				hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5974 				ret = -EEXIST;
5975 			}
5976 			goto out;
5977 		}
5978 	}
5979  new:
5980 	if (hist_data->attrs->cont || hist_data->attrs->clear) {
5981 		hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5982 		ret = -ENOENT;
5983 		goto out;
5984 	}
5985 
5986 	if (hist_data->attrs->pause)
5987 		data->paused = true;
5988 
5989 	if (named_data) {
5990 		data->private_data = named_data->private_data;
5991 		set_named_trigger_data(data, named_data);
5992 		data->ops = &event_hist_trigger_named_ops;
5993 	}
5994 
5995 	if (data->ops->init) {
5996 		ret = data->ops->init(data->ops, data);
5997 		if (ret < 0)
5998 			goto out;
5999 	}
6000 
6001 	if (hist_data->enable_timestamps) {
6002 		char *clock = hist_data->attrs->clock;
6003 
6004 		ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
6005 		if (ret) {
6006 			hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
6007 			goto out;
6008 		}
6009 
6010 		tracing_set_filter_buffering(file->tr, true);
6011 	}
6012 
6013 	if (named_data)
6014 		destroy_hist_data(hist_data);
6015 
6016 	ret++;
6017  out:
6018 	return ret;
6019 }
6020 
6021 static int hist_trigger_enable(struct event_trigger_data *data,
6022 			       struct trace_event_file *file)
6023 {
6024 	int ret = 0;
6025 
6026 	list_add_tail_rcu(&data->list, &file->triggers);
6027 
6028 	update_cond_flag(file);
6029 
6030 	if (trace_event_trigger_enable_disable(file, 1) < 0) {
6031 		list_del_rcu(&data->list);
6032 		update_cond_flag(file);
6033 		ret--;
6034 	}
6035 
6036 	return ret;
6037 }
6038 
6039 static bool have_hist_trigger_match(struct event_trigger_data *data,
6040 				    struct trace_event_file *file)
6041 {
6042 	struct hist_trigger_data *hist_data = data->private_data;
6043 	struct event_trigger_data *test, *named_data = NULL;
6044 	bool match = false;
6045 
6046 	lockdep_assert_held(&event_mutex);
6047 
6048 	if (hist_data->attrs->name)
6049 		named_data = find_named_trigger(hist_data->attrs->name);
6050 
6051 	list_for_each_entry(test, &file->triggers, list) {
6052 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6053 			if (hist_trigger_match(data, test, named_data, false)) {
6054 				match = true;
6055 				break;
6056 			}
6057 		}
6058 	}
6059 
6060 	return match;
6061 }
6062 
6063 static bool hist_trigger_check_refs(struct event_trigger_data *data,
6064 				    struct trace_event_file *file)
6065 {
6066 	struct hist_trigger_data *hist_data = data->private_data;
6067 	struct event_trigger_data *test, *named_data = NULL;
6068 
6069 	lockdep_assert_held(&event_mutex);
6070 
6071 	if (hist_data->attrs->name)
6072 		named_data = find_named_trigger(hist_data->attrs->name);
6073 
6074 	list_for_each_entry(test, &file->triggers, list) {
6075 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6076 			if (!hist_trigger_match(data, test, named_data, false))
6077 				continue;
6078 			hist_data = test->private_data;
6079 			if (check_var_refs(hist_data))
6080 				return true;
6081 			break;
6082 		}
6083 	}
6084 
6085 	return false;
6086 }
6087 
6088 static void hist_unregister_trigger(char *glob,
6089 				    struct event_trigger_data *data,
6090 				    struct trace_event_file *file)
6091 {
6092 	struct hist_trigger_data *hist_data = data->private_data;
6093 	struct event_trigger_data *test, *named_data = NULL;
6094 	bool unregistered = false;
6095 
6096 	lockdep_assert_held(&event_mutex);
6097 
6098 	if (hist_data->attrs->name)
6099 		named_data = find_named_trigger(hist_data->attrs->name);
6100 
6101 	list_for_each_entry(test, &file->triggers, list) {
6102 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6103 			if (!hist_trigger_match(data, test, named_data, false))
6104 				continue;
6105 			unregistered = true;
6106 			list_del_rcu(&test->list);
6107 			trace_event_trigger_enable_disable(file, 0);
6108 			update_cond_flag(file);
6109 			break;
6110 		}
6111 	}
6112 
6113 	if (unregistered && test->ops->free)
6114 		test->ops->free(test->ops, test);
6115 
6116 	if (hist_data->enable_timestamps) {
6117 		if (!hist_data->remove || unregistered)
6118 			tracing_set_filter_buffering(file->tr, false);
6119 	}
6120 }
6121 
6122 static bool hist_file_check_refs(struct trace_event_file *file)
6123 {
6124 	struct hist_trigger_data *hist_data;
6125 	struct event_trigger_data *test;
6126 
6127 	lockdep_assert_held(&event_mutex);
6128 
6129 	list_for_each_entry(test, &file->triggers, list) {
6130 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6131 			hist_data = test->private_data;
6132 			if (check_var_refs(hist_data))
6133 				return true;
6134 		}
6135 	}
6136 
6137 	return false;
6138 }
6139 
6140 static void hist_unreg_all(struct trace_event_file *file)
6141 {
6142 	struct event_trigger_data *test, *n;
6143 	struct hist_trigger_data *hist_data;
6144 	struct synth_event *se;
6145 	const char *se_name;
6146 
6147 	lockdep_assert_held(&event_mutex);
6148 
6149 	if (hist_file_check_refs(file))
6150 		return;
6151 
6152 	list_for_each_entry_safe(test, n, &file->triggers, list) {
6153 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6154 			hist_data = test->private_data;
6155 			list_del_rcu(&test->list);
6156 			trace_event_trigger_enable_disable(file, 0);
6157 
6158 			se_name = trace_event_name(file->event_call);
6159 			se = find_synth_event(se_name);
6160 			if (se)
6161 				se->ref--;
6162 
6163 			update_cond_flag(file);
6164 			if (hist_data->enable_timestamps)
6165 				tracing_set_filter_buffering(file->tr, false);
6166 			if (test->ops->free)
6167 				test->ops->free(test->ops, test);
6168 		}
6169 	}
6170 }
6171 
6172 static int event_hist_trigger_parse(struct event_command *cmd_ops,
6173 				    struct trace_event_file *file,
6174 				    char *glob, char *cmd, char *param)
6175 {
6176 	unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6177 	struct event_trigger_data *trigger_data;
6178 	struct hist_trigger_attrs *attrs;
6179 	struct event_trigger_ops *trigger_ops;
6180 	struct hist_trigger_data *hist_data;
6181 	struct synth_event *se;
6182 	const char *se_name;
6183 	bool remove = false;
6184 	char *trigger, *p, *start;
6185 	int ret = 0;
6186 
6187 	lockdep_assert_held(&event_mutex);
6188 
6189 	WARN_ON(!glob);
6190 
6191 	if (strlen(glob)) {
6192 		hist_err_clear();
6193 		last_cmd_set(file, param);
6194 	}
6195 
6196 	if (!param)
6197 		return -EINVAL;
6198 
6199 	if (glob[0] == '!')
6200 		remove = true;
6201 
6202 	/*
6203 	 * separate the trigger from the filter (k:v [if filter])
6204 	 * allowing for whitespace in the trigger
6205 	 */
6206 	p = trigger = param;
6207 	do {
6208 		p = strstr(p, "if");
6209 		if (!p)
6210 			break;
6211 		if (p == param)
6212 			return -EINVAL;
6213 		if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6214 			p++;
6215 			continue;
6216 		}
6217 		if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
6218 			return -EINVAL;
6219 		if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6220 			p++;
6221 			continue;
6222 		}
6223 		break;
6224 	} while (1);
6225 
6226 	if (!p)
6227 		param = NULL;
6228 	else {
6229 		*(p - 1) = '\0';
6230 		param = strstrip(p);
6231 		trigger = strstrip(trigger);
6232 	}
6233 
6234 	/*
6235 	 * To simplify arithmetic expression parsing, replace occurrences of
6236 	 * '.sym-offset' modifier with '.symXoffset'
6237 	 */
6238 	start = strstr(trigger, ".sym-offset");
6239 	while (start) {
6240 		*(start + 4) = 'X';
6241 		start = strstr(start + 11, ".sym-offset");
6242 	}
6243 
6244 	attrs = parse_hist_trigger_attrs(file->tr, trigger);
6245 	if (IS_ERR(attrs))
6246 		return PTR_ERR(attrs);
6247 
6248 	if (attrs->map_bits)
6249 		hist_trigger_bits = attrs->map_bits;
6250 
6251 	hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6252 	if (IS_ERR(hist_data)) {
6253 		destroy_hist_trigger_attrs(attrs);
6254 		return PTR_ERR(hist_data);
6255 	}
6256 
6257 	trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
6258 
6259 	trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
6260 	if (!trigger_data) {
6261 		ret = -ENOMEM;
6262 		goto out_free;
6263 	}
6264 
6265 	trigger_data->count = -1;
6266 	trigger_data->ops = trigger_ops;
6267 	trigger_data->cmd_ops = cmd_ops;
6268 
6269 	INIT_LIST_HEAD(&trigger_data->list);
6270 	RCU_INIT_POINTER(trigger_data->filter, NULL);
6271 
6272 	trigger_data->private_data = hist_data;
6273 
6274 	/* if param is non-empty, it's supposed to be a filter */
6275 	if (param && cmd_ops->set_filter) {
6276 		ret = cmd_ops->set_filter(param, trigger_data, file);
6277 		if (ret < 0)
6278 			goto out_free;
6279 	}
6280 
6281 	if (remove) {
6282 		if (!have_hist_trigger_match(trigger_data, file))
6283 			goto out_free;
6284 
6285 		if (hist_trigger_check_refs(trigger_data, file)) {
6286 			ret = -EBUSY;
6287 			goto out_free;
6288 		}
6289 
6290 		cmd_ops->unreg(glob+1, trigger_data, file);
6291 		se_name = trace_event_name(file->event_call);
6292 		se = find_synth_event(se_name);
6293 		if (se)
6294 			se->ref--;
6295 		ret = 0;
6296 		goto out_free;
6297 	}
6298 
6299 	ret = cmd_ops->reg(glob, trigger_data, file);
6300 	/*
6301 	 * The above returns on success the # of triggers registered,
6302 	 * but if it didn't register any it returns zero.  Consider no
6303 	 * triggers registered a failure too.
6304 	 */
6305 	if (!ret) {
6306 		if (!(attrs->pause || attrs->cont || attrs->clear))
6307 			ret = -ENOENT;
6308 		goto out_free;
6309 	} else if (ret < 0)
6310 		goto out_free;
6311 
6312 	if (get_named_trigger_data(trigger_data))
6313 		goto enable;
6314 
6315 	if (has_hist_vars(hist_data))
6316 		save_hist_vars(hist_data);
6317 
6318 	ret = create_actions(hist_data);
6319 	if (ret)
6320 		goto out_unreg;
6321 
6322 	ret = tracing_map_init(hist_data->map);
6323 	if (ret)
6324 		goto out_unreg;
6325 enable:
6326 	ret = hist_trigger_enable(trigger_data, file);
6327 	if (ret)
6328 		goto out_unreg;
6329 
6330 	se_name = trace_event_name(file->event_call);
6331 	se = find_synth_event(se_name);
6332 	if (se)
6333 		se->ref++;
6334 	/* Just return zero, not the number of registered triggers */
6335 	ret = 0;
6336  out:
6337 	if (ret == 0)
6338 		hist_err_clear();
6339 
6340 	return ret;
6341  out_unreg:
6342 	cmd_ops->unreg(glob+1, trigger_data, file);
6343  out_free:
6344 	if (cmd_ops->set_filter)
6345 		cmd_ops->set_filter(NULL, trigger_data, NULL);
6346 
6347 	remove_hist_vars(hist_data);
6348 
6349 	kfree(trigger_data);
6350 
6351 	destroy_hist_data(hist_data);
6352 	goto out;
6353 }
6354 
6355 static struct event_command trigger_hist_cmd = {
6356 	.name			= "hist",
6357 	.trigger_type		= ETT_EVENT_HIST,
6358 	.flags			= EVENT_CMD_FL_NEEDS_REC,
6359 	.parse			= event_hist_trigger_parse,
6360 	.reg			= hist_register_trigger,
6361 	.unreg			= hist_unregister_trigger,
6362 	.unreg_all		= hist_unreg_all,
6363 	.get_trigger_ops	= event_hist_get_trigger_ops,
6364 	.set_filter		= set_trigger_filter,
6365 };
6366 
6367 __init int register_trigger_hist_cmd(void)
6368 {
6369 	int ret;
6370 
6371 	ret = register_event_command(&trigger_hist_cmd);
6372 	WARN_ON(ret < 0);
6373 
6374 	return ret;
6375 }
6376 
6377 static void
6378 hist_enable_trigger(struct event_trigger_data *data,
6379 		    struct trace_buffer *buffer,  void *rec,
6380 		    struct ring_buffer_event *event)
6381 {
6382 	struct enable_trigger_data *enable_data = data->private_data;
6383 	struct event_trigger_data *test;
6384 
6385 	list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6386 				lockdep_is_held(&event_mutex)) {
6387 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6388 			if (enable_data->enable)
6389 				test->paused = false;
6390 			else
6391 				test->paused = true;
6392 		}
6393 	}
6394 }
6395 
6396 static void
6397 hist_enable_count_trigger(struct event_trigger_data *data,
6398 			  struct trace_buffer *buffer,  void *rec,
6399 			  struct ring_buffer_event *event)
6400 {
6401 	if (!data->count)
6402 		return;
6403 
6404 	if (data->count != -1)
6405 		(data->count)--;
6406 
6407 	hist_enable_trigger(data, buffer, rec, event);
6408 }
6409 
6410 static struct event_trigger_ops hist_enable_trigger_ops = {
6411 	.trigger		= hist_enable_trigger,
6412 	.print			= event_enable_trigger_print,
6413 	.init			= event_trigger_init,
6414 	.free			= event_enable_trigger_free,
6415 };
6416 
6417 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6418 	.trigger		= hist_enable_count_trigger,
6419 	.print			= event_enable_trigger_print,
6420 	.init			= event_trigger_init,
6421 	.free			= event_enable_trigger_free,
6422 };
6423 
6424 static struct event_trigger_ops hist_disable_trigger_ops = {
6425 	.trigger		= hist_enable_trigger,
6426 	.print			= event_enable_trigger_print,
6427 	.init			= event_trigger_init,
6428 	.free			= event_enable_trigger_free,
6429 };
6430 
6431 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6432 	.trigger		= hist_enable_count_trigger,
6433 	.print			= event_enable_trigger_print,
6434 	.init			= event_trigger_init,
6435 	.free			= event_enable_trigger_free,
6436 };
6437 
6438 static struct event_trigger_ops *
6439 hist_enable_get_trigger_ops(char *cmd, char *param)
6440 {
6441 	struct event_trigger_ops *ops;
6442 	bool enable;
6443 
6444 	enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6445 
6446 	if (enable)
6447 		ops = param ? &hist_enable_count_trigger_ops :
6448 			&hist_enable_trigger_ops;
6449 	else
6450 		ops = param ? &hist_disable_count_trigger_ops :
6451 			&hist_disable_trigger_ops;
6452 
6453 	return ops;
6454 }
6455 
6456 static void hist_enable_unreg_all(struct trace_event_file *file)
6457 {
6458 	struct event_trigger_data *test, *n;
6459 
6460 	list_for_each_entry_safe(test, n, &file->triggers, list) {
6461 		if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6462 			list_del_rcu(&test->list);
6463 			update_cond_flag(file);
6464 			trace_event_trigger_enable_disable(file, 0);
6465 			if (test->ops->free)
6466 				test->ops->free(test->ops, test);
6467 		}
6468 	}
6469 }
6470 
6471 static struct event_command trigger_hist_enable_cmd = {
6472 	.name			= ENABLE_HIST_STR,
6473 	.trigger_type		= ETT_HIST_ENABLE,
6474 	.parse			= event_enable_trigger_parse,
6475 	.reg			= event_enable_register_trigger,
6476 	.unreg			= event_enable_unregister_trigger,
6477 	.unreg_all		= hist_enable_unreg_all,
6478 	.get_trigger_ops	= hist_enable_get_trigger_ops,
6479 	.set_filter		= set_trigger_filter,
6480 };
6481 
6482 static struct event_command trigger_hist_disable_cmd = {
6483 	.name			= DISABLE_HIST_STR,
6484 	.trigger_type		= ETT_HIST_ENABLE,
6485 	.parse			= event_enable_trigger_parse,
6486 	.reg			= event_enable_register_trigger,
6487 	.unreg			= event_enable_unregister_trigger,
6488 	.unreg_all		= hist_enable_unreg_all,
6489 	.get_trigger_ops	= hist_enable_get_trigger_ops,
6490 	.set_filter		= set_trigger_filter,
6491 };
6492 
6493 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6494 {
6495 	unregister_event_command(&trigger_hist_enable_cmd);
6496 	unregister_event_command(&trigger_hist_disable_cmd);
6497 }
6498 
6499 __init int register_trigger_hist_enable_disable_cmds(void)
6500 {
6501 	int ret;
6502 
6503 	ret = register_event_command(&trigger_hist_enable_cmd);
6504 	if (WARN_ON(ret < 0))
6505 		return ret;
6506 	ret = register_event_command(&trigger_hist_disable_cmd);
6507 	if (WARN_ON(ret < 0))
6508 		unregister_trigger_hist_enable_disable_cmds();
6509 
6510 	return ret;
6511 }
6512