1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * trace_events_hist - trace event hist triggers
4  *
5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
6  */
7 
8 #include <linux/module.h>
9 #include <linux/kallsyms.h>
10 #include <linux/security.h>
11 #include <linux/mutex.h>
12 #include <linux/slab.h>
13 #include <linux/stacktrace.h>
14 #include <linux/rculist.h>
15 #include <linux/tracefs.h>
16 
17 /* for gfp flag names */
18 #include <linux/trace_events.h>
19 #include <trace/events/mmflags.h>
20 
21 #include "tracing_map.h"
22 #include "trace_synth.h"
23 
24 #define ERRORS								\
25 	C(NONE,			"No error"),				\
26 	C(DUPLICATE_VAR,	"Variable already defined"),		\
27 	C(VAR_NOT_UNIQUE,	"Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
28 	C(TOO_MANY_VARS,	"Too many variables defined"),		\
29 	C(MALFORMED_ASSIGNMENT,	"Malformed assignment"),		\
30 	C(NAMED_MISMATCH,	"Named hist trigger doesn't match existing named trigger (includes variables)"), \
31 	C(TRIGGER_EEXIST,	"Hist trigger already exists"),		\
32 	C(TRIGGER_ENOENT_CLEAR,	"Can't clear or continue a nonexistent hist trigger"), \
33 	C(SET_CLOCK_FAIL,	"Couldn't set trace_clock"),		\
34 	C(BAD_FIELD_MODIFIER,	"Invalid field modifier"),		\
35 	C(TOO_MANY_SUBEXPR,	"Too many subexpressions (3 max)"),	\
36 	C(TIMESTAMP_MISMATCH,	"Timestamp units in expression don't match"), \
37 	C(TOO_MANY_FIELD_VARS,	"Too many field variables defined"),	\
38 	C(EVENT_FILE_NOT_FOUND,	"Event file not found"),		\
39 	C(HIST_NOT_FOUND,	"Matching event histogram not found"),	\
40 	C(HIST_CREATE_FAIL,	"Couldn't create histogram for field"),	\
41 	C(SYNTH_VAR_NOT_FOUND,	"Couldn't find synthetic variable"),	\
42 	C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),	\
43 	C(SYNTH_TYPE_MISMATCH,	"Param type doesn't match synthetic event field type"), \
44 	C(SYNTH_COUNT_MISMATCH,	"Param count doesn't match synthetic event field count"), \
45 	C(FIELD_VAR_PARSE_FAIL,	"Couldn't parse field variable"),	\
46 	C(VAR_CREATE_FIND_FAIL,	"Couldn't create or find variable"),	\
47 	C(ONX_NOT_VAR,		"For onmax(x) or onchange(x), x must be a variable"), \
48 	C(ONX_VAR_NOT_FOUND,	"Couldn't find onmax or onchange variable"), \
49 	C(ONX_VAR_CREATE_FAIL,	"Couldn't create onmax or onchange variable"), \
50 	C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),	\
51 	C(TOO_MANY_PARAMS,	"Too many action params"),		\
52 	C(PARAM_NOT_FOUND,	"Couldn't find param"),			\
53 	C(INVALID_PARAM,	"Invalid action param"),		\
54 	C(ACTION_NOT_FOUND,	"No action found"),			\
55 	C(NO_SAVE_PARAMS,	"No params found for save()"),		\
56 	C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
57 	C(ACTION_MISMATCH,	"Handler doesn't support action"),	\
58 	C(NO_CLOSING_PAREN,	"No closing paren found"),		\
59 	C(SUBSYS_NOT_FOUND,	"Missing subsystem"),			\
60 	C(INVALID_SUBSYS_EVENT,	"Invalid subsystem or event name"),	\
61 	C(INVALID_REF_KEY,	"Using variable references in keys not supported"), \
62 	C(VAR_NOT_FOUND,	"Couldn't find variable"),		\
63 	C(FIELD_NOT_FOUND,	"Couldn't find field"),			\
64 	C(EMPTY_ASSIGNMENT,	"Empty assignment"),			\
65 	C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),		\
66 	C(EMPTY_SORT_FIELD,	"Empty sort field"),			\
67 	C(TOO_MANY_SORT_FIELDS,	"Too many sort fields (Max = 2)"),	\
68 	C(INVALID_SORT_FIELD,	"Sort field must be a key or a val"),	\
69 	C(INVALID_STR_OPERAND,	"String type can not be an operand in expression"), \
70 	C(EXPECT_NUMBER,	"Expecting numeric literal"),		\
71 	C(UNARY_MINUS_SUBEXPR,	"Unary minus not supported in sub-expressions"), \
72 	C(DIVISION_BY_ZERO,	"Division by zero"),			\
73 	C(NEED_NOHC_VAL,	"Non-hitcount value is required for 'nohitcount'"),
74 
75 #undef C
76 #define C(a, b)		HIST_ERR_##a
77 
78 enum { ERRORS };
79 
80 #undef C
81 #define C(a, b)		b
82 
83 static const char *err_text[] = { ERRORS };
84 
85 struct hist_field;
86 
87 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
88 				struct tracing_map_elt *elt,
89 				struct trace_buffer *buffer,
90 				struct ring_buffer_event *rbe,
91 				void *event);
92 
93 #define HIST_FIELD_OPERANDS_MAX	2
94 #define HIST_FIELDS_MAX		(TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
95 #define HIST_ACTIONS_MAX	8
96 #define HIST_CONST_DIGITS_MAX	21
97 #define HIST_DIV_SHIFT		20  /* For optimizing division by constants */
98 
99 enum field_op_id {
100 	FIELD_OP_NONE,
101 	FIELD_OP_PLUS,
102 	FIELD_OP_MINUS,
103 	FIELD_OP_UNARY_MINUS,
104 	FIELD_OP_DIV,
105 	FIELD_OP_MULT,
106 };
107 
108 enum hist_field_fn {
109 	HIST_FIELD_FN_NOP,
110 	HIST_FIELD_FN_VAR_REF,
111 	HIST_FIELD_FN_COUNTER,
112 	HIST_FIELD_FN_CONST,
113 	HIST_FIELD_FN_LOG2,
114 	HIST_FIELD_FN_BUCKET,
115 	HIST_FIELD_FN_TIMESTAMP,
116 	HIST_FIELD_FN_CPU,
117 	HIST_FIELD_FN_STRING,
118 	HIST_FIELD_FN_DYNSTRING,
119 	HIST_FIELD_FN_RELDYNSTRING,
120 	HIST_FIELD_FN_PSTRING,
121 	HIST_FIELD_FN_S64,
122 	HIST_FIELD_FN_U64,
123 	HIST_FIELD_FN_S32,
124 	HIST_FIELD_FN_U32,
125 	HIST_FIELD_FN_S16,
126 	HIST_FIELD_FN_U16,
127 	HIST_FIELD_FN_S8,
128 	HIST_FIELD_FN_U8,
129 	HIST_FIELD_FN_UMINUS,
130 	HIST_FIELD_FN_MINUS,
131 	HIST_FIELD_FN_PLUS,
132 	HIST_FIELD_FN_DIV,
133 	HIST_FIELD_FN_MULT,
134 	HIST_FIELD_FN_DIV_POWER2,
135 	HIST_FIELD_FN_DIV_NOT_POWER2,
136 	HIST_FIELD_FN_DIV_MULT_SHIFT,
137 	HIST_FIELD_FN_EXECNAME,
138 	HIST_FIELD_FN_STACK,
139 };
140 
141 /*
142  * A hist_var (histogram variable) contains variable information for
143  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
144  * flag set.  A hist_var has a variable name e.g. ts0, and is
145  * associated with a given histogram trigger, as specified by
146  * hist_data.  The hist_var idx is the unique index assigned to the
147  * variable by the hist trigger's tracing_map.  The idx is what is
148  * used to set a variable's value and, by a variable reference, to
149  * retrieve it.
150  */
151 struct hist_var {
152 	char				*name;
153 	struct hist_trigger_data	*hist_data;
154 	unsigned int			idx;
155 };
156 
157 struct hist_field {
158 	struct ftrace_event_field	*field;
159 	unsigned long			flags;
160 	unsigned long			buckets;
161 	const char			*type;
162 	struct hist_field		*operands[HIST_FIELD_OPERANDS_MAX];
163 	struct hist_trigger_data	*hist_data;
164 	enum hist_field_fn		fn_num;
165 	unsigned int			ref;
166 	unsigned int			size;
167 	unsigned int			offset;
168 	unsigned int                    is_signed;
169 
170 	/*
171 	 * Variable fields contain variable-specific info in var.
172 	 */
173 	struct hist_var			var;
174 	enum field_op_id		operator;
175 	char				*system;
176 	char				*event_name;
177 
178 	/*
179 	 * The name field is used for EXPR and VAR_REF fields.  VAR
180 	 * fields contain the variable name in var.name.
181 	 */
182 	char				*name;
183 
184 	/*
185 	 * When a histogram trigger is hit, if it has any references
186 	 * to variables, the values of those variables are collected
187 	 * into a var_ref_vals array by resolve_var_refs().  The
188 	 * current value of each variable is read from the tracing_map
189 	 * using the hist field's hist_var.idx and entered into the
190 	 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
191 	 */
192 	unsigned int			var_ref_idx;
193 	bool                            read_once;
194 
195 	unsigned int			var_str_idx;
196 
197 	/* Numeric literals are represented as u64 */
198 	u64				constant;
199 	/* Used to optimize division by constants */
200 	u64				div_multiplier;
201 };
202 
203 static u64 hist_fn_call(struct hist_field *hist_field,
204 			struct tracing_map_elt *elt,
205 			struct trace_buffer *buffer,
206 			struct ring_buffer_event *rbe,
207 			void *event);
208 
209 static u64 hist_field_const(struct hist_field *field,
210 			   struct tracing_map_elt *elt,
211 			   struct trace_buffer *buffer,
212 			   struct ring_buffer_event *rbe,
213 			   void *event)
214 {
215 	return field->constant;
216 }
217 
218 static u64 hist_field_counter(struct hist_field *field,
219 			      struct tracing_map_elt *elt,
220 			      struct trace_buffer *buffer,
221 			      struct ring_buffer_event *rbe,
222 			      void *event)
223 {
224 	return 1;
225 }
226 
227 static u64 hist_field_string(struct hist_field *hist_field,
228 			     struct tracing_map_elt *elt,
229 			     struct trace_buffer *buffer,
230 			     struct ring_buffer_event *rbe,
231 			     void *event)
232 {
233 	char *addr = (char *)(event + hist_field->field->offset);
234 
235 	return (u64)(unsigned long)addr;
236 }
237 
238 static u64 hist_field_dynstring(struct hist_field *hist_field,
239 				struct tracing_map_elt *elt,
240 				struct trace_buffer *buffer,
241 				struct ring_buffer_event *rbe,
242 				void *event)
243 {
244 	u32 str_item = *(u32 *)(event + hist_field->field->offset);
245 	int str_loc = str_item & 0xffff;
246 	char *addr = (char *)(event + str_loc);
247 
248 	return (u64)(unsigned long)addr;
249 }
250 
251 static u64 hist_field_reldynstring(struct hist_field *hist_field,
252 				   struct tracing_map_elt *elt,
253 				   struct trace_buffer *buffer,
254 				   struct ring_buffer_event *rbe,
255 				   void *event)
256 {
257 	u32 *item = event + hist_field->field->offset;
258 	u32 str_item = *item;
259 	int str_loc = str_item & 0xffff;
260 	char *addr = (char *)&item[1] + str_loc;
261 
262 	return (u64)(unsigned long)addr;
263 }
264 
265 static u64 hist_field_pstring(struct hist_field *hist_field,
266 			      struct tracing_map_elt *elt,
267 			      struct trace_buffer *buffer,
268 			      struct ring_buffer_event *rbe,
269 			      void *event)
270 {
271 	char **addr = (char **)(event + hist_field->field->offset);
272 
273 	return (u64)(unsigned long)*addr;
274 }
275 
276 static u64 hist_field_log2(struct hist_field *hist_field,
277 			   struct tracing_map_elt *elt,
278 			   struct trace_buffer *buffer,
279 			   struct ring_buffer_event *rbe,
280 			   void *event)
281 {
282 	struct hist_field *operand = hist_field->operands[0];
283 
284 	u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
285 
286 	return (u64) ilog2(roundup_pow_of_two(val));
287 }
288 
289 static u64 hist_field_bucket(struct hist_field *hist_field,
290 			     struct tracing_map_elt *elt,
291 			     struct trace_buffer *buffer,
292 			     struct ring_buffer_event *rbe,
293 			     void *event)
294 {
295 	struct hist_field *operand = hist_field->operands[0];
296 	unsigned long buckets = hist_field->buckets;
297 
298 	u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
299 
300 	if (WARN_ON_ONCE(!buckets))
301 		return val;
302 
303 	if (val >= LONG_MAX)
304 		val = div64_ul(val, buckets);
305 	else
306 		val = (u64)((unsigned long)val / buckets);
307 	return val * buckets;
308 }
309 
310 static u64 hist_field_plus(struct hist_field *hist_field,
311 			   struct tracing_map_elt *elt,
312 			   struct trace_buffer *buffer,
313 			   struct ring_buffer_event *rbe,
314 			   void *event)
315 {
316 	struct hist_field *operand1 = hist_field->operands[0];
317 	struct hist_field *operand2 = hist_field->operands[1];
318 
319 	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
320 	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
321 
322 	return val1 + val2;
323 }
324 
325 static u64 hist_field_minus(struct hist_field *hist_field,
326 			    struct tracing_map_elt *elt,
327 			    struct trace_buffer *buffer,
328 			    struct ring_buffer_event *rbe,
329 			    void *event)
330 {
331 	struct hist_field *operand1 = hist_field->operands[0];
332 	struct hist_field *operand2 = hist_field->operands[1];
333 
334 	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
335 	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
336 
337 	return val1 - val2;
338 }
339 
340 static u64 hist_field_div(struct hist_field *hist_field,
341 			   struct tracing_map_elt *elt,
342 			   struct trace_buffer *buffer,
343 			   struct ring_buffer_event *rbe,
344 			   void *event)
345 {
346 	struct hist_field *operand1 = hist_field->operands[0];
347 	struct hist_field *operand2 = hist_field->operands[1];
348 
349 	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
350 	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
351 
352 	/* Return -1 for the undefined case */
353 	if (!val2)
354 		return -1;
355 
356 	/* Use shift if the divisor is a power of 2 */
357 	if (!(val2 & (val2 - 1)))
358 		return val1 >> __ffs64(val2);
359 
360 	return div64_u64(val1, val2);
361 }
362 
363 static u64 div_by_power_of_two(struct hist_field *hist_field,
364 				struct tracing_map_elt *elt,
365 				struct trace_buffer *buffer,
366 				struct ring_buffer_event *rbe,
367 				void *event)
368 {
369 	struct hist_field *operand1 = hist_field->operands[0];
370 	struct hist_field *operand2 = hist_field->operands[1];
371 
372 	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
373 
374 	return val1 >> __ffs64(operand2->constant);
375 }
376 
377 static u64 div_by_not_power_of_two(struct hist_field *hist_field,
378 				struct tracing_map_elt *elt,
379 				struct trace_buffer *buffer,
380 				struct ring_buffer_event *rbe,
381 				void *event)
382 {
383 	struct hist_field *operand1 = hist_field->operands[0];
384 	struct hist_field *operand2 = hist_field->operands[1];
385 
386 	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
387 
388 	return div64_u64(val1, operand2->constant);
389 }
390 
391 static u64 div_by_mult_and_shift(struct hist_field *hist_field,
392 				struct tracing_map_elt *elt,
393 				struct trace_buffer *buffer,
394 				struct ring_buffer_event *rbe,
395 				void *event)
396 {
397 	struct hist_field *operand1 = hist_field->operands[0];
398 	struct hist_field *operand2 = hist_field->operands[1];
399 
400 	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
401 
402 	/*
403 	 * If the divisor is a constant, do a multiplication and shift instead.
404 	 *
405 	 * Choose Z = some power of 2. If Y <= Z, then:
406 	 *     X / Y = (X * (Z / Y)) / Z
407 	 *
408 	 * (Z / Y) is a constant (mult) which is calculated at parse time, so:
409 	 *     X / Y = (X * mult) / Z
410 	 *
411 	 * The division by Z can be replaced by a shift since Z is a power of 2:
412 	 *     X / Y = (X * mult) >> HIST_DIV_SHIFT
413 	 *
414 	 * As long, as X < Z the results will not be off by more than 1.
415 	 */
416 	if (val1 < (1 << HIST_DIV_SHIFT)) {
417 		u64 mult = operand2->div_multiplier;
418 
419 		return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT;
420 	}
421 
422 	return div64_u64(val1, operand2->constant);
423 }
424 
425 static u64 hist_field_mult(struct hist_field *hist_field,
426 			   struct tracing_map_elt *elt,
427 			   struct trace_buffer *buffer,
428 			   struct ring_buffer_event *rbe,
429 			   void *event)
430 {
431 	struct hist_field *operand1 = hist_field->operands[0];
432 	struct hist_field *operand2 = hist_field->operands[1];
433 
434 	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
435 	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
436 
437 	return val1 * val2;
438 }
439 
440 static u64 hist_field_unary_minus(struct hist_field *hist_field,
441 				  struct tracing_map_elt *elt,
442 				  struct trace_buffer *buffer,
443 				  struct ring_buffer_event *rbe,
444 				  void *event)
445 {
446 	struct hist_field *operand = hist_field->operands[0];
447 
448 	s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event);
449 	u64 val = (u64)-sval;
450 
451 	return val;
452 }
453 
454 #define DEFINE_HIST_FIELD_FN(type)					\
455 	static u64 hist_field_##type(struct hist_field *hist_field,	\
456 				     struct tracing_map_elt *elt,	\
457 				     struct trace_buffer *buffer,	\
458 				     struct ring_buffer_event *rbe,	\
459 				     void *event)			\
460 {									\
461 	type *addr = (type *)(event + hist_field->field->offset);	\
462 									\
463 	return (u64)(unsigned long)*addr;				\
464 }
465 
466 DEFINE_HIST_FIELD_FN(s64);
467 DEFINE_HIST_FIELD_FN(u64);
468 DEFINE_HIST_FIELD_FN(s32);
469 DEFINE_HIST_FIELD_FN(u32);
470 DEFINE_HIST_FIELD_FN(s16);
471 DEFINE_HIST_FIELD_FN(u16);
472 DEFINE_HIST_FIELD_FN(s8);
473 DEFINE_HIST_FIELD_FN(u8);
474 
475 #define for_each_hist_field(i, hist_data)	\
476 	for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
477 
478 #define for_each_hist_val_field(i, hist_data)	\
479 	for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
480 
481 #define for_each_hist_key_field(i, hist_data)	\
482 	for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
483 
484 #define HITCOUNT_IDX		0
485 #define HIST_KEY_SIZE_MAX	(MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
486 
487 enum hist_field_flags {
488 	HIST_FIELD_FL_HITCOUNT		= 1 << 0,
489 	HIST_FIELD_FL_KEY		= 1 << 1,
490 	HIST_FIELD_FL_STRING		= 1 << 2,
491 	HIST_FIELD_FL_HEX		= 1 << 3,
492 	HIST_FIELD_FL_SYM		= 1 << 4,
493 	HIST_FIELD_FL_SYM_OFFSET	= 1 << 5,
494 	HIST_FIELD_FL_EXECNAME		= 1 << 6,
495 	HIST_FIELD_FL_SYSCALL		= 1 << 7,
496 	HIST_FIELD_FL_STACKTRACE	= 1 << 8,
497 	HIST_FIELD_FL_LOG2		= 1 << 9,
498 	HIST_FIELD_FL_TIMESTAMP		= 1 << 10,
499 	HIST_FIELD_FL_TIMESTAMP_USECS	= 1 << 11,
500 	HIST_FIELD_FL_VAR		= 1 << 12,
501 	HIST_FIELD_FL_EXPR		= 1 << 13,
502 	HIST_FIELD_FL_VAR_REF		= 1 << 14,
503 	HIST_FIELD_FL_CPU		= 1 << 15,
504 	HIST_FIELD_FL_ALIAS		= 1 << 16,
505 	HIST_FIELD_FL_BUCKET		= 1 << 17,
506 	HIST_FIELD_FL_CONST		= 1 << 18,
507 	HIST_FIELD_FL_PERCENT		= 1 << 19,
508 	HIST_FIELD_FL_GRAPH		= 1 << 20,
509 };
510 
511 struct var_defs {
512 	unsigned int	n_vars;
513 	char		*name[TRACING_MAP_VARS_MAX];
514 	char		*expr[TRACING_MAP_VARS_MAX];
515 };
516 
517 struct hist_trigger_attrs {
518 	char		*keys_str;
519 	char		*vals_str;
520 	char		*sort_key_str;
521 	char		*name;
522 	char		*clock;
523 	bool		pause;
524 	bool		cont;
525 	bool		clear;
526 	bool		ts_in_usecs;
527 	bool		no_hitcount;
528 	unsigned int	map_bits;
529 
530 	char		*assignment_str[TRACING_MAP_VARS_MAX];
531 	unsigned int	n_assignments;
532 
533 	char		*action_str[HIST_ACTIONS_MAX];
534 	unsigned int	n_actions;
535 
536 	struct var_defs	var_defs;
537 };
538 
539 struct field_var {
540 	struct hist_field	*var;
541 	struct hist_field	*val;
542 };
543 
544 struct field_var_hist {
545 	struct hist_trigger_data	*hist_data;
546 	char				*cmd;
547 };
548 
549 struct hist_trigger_data {
550 	struct hist_field               *fields[HIST_FIELDS_MAX];
551 	unsigned int			n_vals;
552 	unsigned int			n_keys;
553 	unsigned int			n_fields;
554 	unsigned int			n_vars;
555 	unsigned int			n_var_str;
556 	unsigned int			key_size;
557 	struct tracing_map_sort_key	sort_keys[TRACING_MAP_SORT_KEYS_MAX];
558 	unsigned int			n_sort_keys;
559 	struct trace_event_file		*event_file;
560 	struct hist_trigger_attrs	*attrs;
561 	struct tracing_map		*map;
562 	bool				enable_timestamps;
563 	bool				remove;
564 	struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
565 	unsigned int			n_var_refs;
566 
567 	struct action_data		*actions[HIST_ACTIONS_MAX];
568 	unsigned int			n_actions;
569 
570 	struct field_var		*field_vars[SYNTH_FIELDS_MAX];
571 	unsigned int			n_field_vars;
572 	unsigned int			n_field_var_str;
573 	struct field_var_hist		*field_var_hists[SYNTH_FIELDS_MAX];
574 	unsigned int			n_field_var_hists;
575 
576 	struct field_var		*save_vars[SYNTH_FIELDS_MAX];
577 	unsigned int			n_save_vars;
578 	unsigned int			n_save_var_str;
579 };
580 
581 struct action_data;
582 
583 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
584 			     struct tracing_map_elt *elt,
585 			     struct trace_buffer *buffer, void *rec,
586 			     struct ring_buffer_event *rbe, void *key,
587 			     struct action_data *data, u64 *var_ref_vals);
588 
589 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
590 
591 enum handler_id {
592 	HANDLER_ONMATCH = 1,
593 	HANDLER_ONMAX,
594 	HANDLER_ONCHANGE,
595 };
596 
597 enum action_id {
598 	ACTION_SAVE = 1,
599 	ACTION_TRACE,
600 	ACTION_SNAPSHOT,
601 };
602 
603 struct action_data {
604 	enum handler_id		handler;
605 	enum action_id		action;
606 	char			*action_name;
607 	action_fn_t		fn;
608 
609 	unsigned int		n_params;
610 	char			*params[SYNTH_FIELDS_MAX];
611 
612 	/*
613 	 * When a histogram trigger is hit, the values of any
614 	 * references to variables, including variables being passed
615 	 * as parameters to synthetic events, are collected into a
616 	 * var_ref_vals array.  This var_ref_idx array is an array of
617 	 * indices into the var_ref_vals array, one for each synthetic
618 	 * event param, and is passed to the synthetic event
619 	 * invocation.
620 	 */
621 	unsigned int		var_ref_idx[SYNTH_FIELDS_MAX];
622 	struct synth_event	*synth_event;
623 	bool			use_trace_keyword;
624 	char			*synth_event_name;
625 
626 	union {
627 		struct {
628 			char			*event;
629 			char			*event_system;
630 		} match_data;
631 
632 		struct {
633 			/*
634 			 * var_str contains the $-unstripped variable
635 			 * name referenced by var_ref, and used when
636 			 * printing the action.  Because var_ref
637 			 * creation is deferred to create_actions(),
638 			 * we need a per-action way to save it until
639 			 * then, thus var_str.
640 			 */
641 			char			*var_str;
642 
643 			/*
644 			 * var_ref refers to the variable being
645 			 * tracked e.g onmax($var).
646 			 */
647 			struct hist_field	*var_ref;
648 
649 			/*
650 			 * track_var contains the 'invisible' tracking
651 			 * variable created to keep the current
652 			 * e.g. max value.
653 			 */
654 			struct hist_field	*track_var;
655 
656 			check_track_val_fn_t	check_val;
657 			action_fn_t		save_data;
658 		} track_data;
659 	};
660 };
661 
662 struct track_data {
663 	u64				track_val;
664 	bool				updated;
665 
666 	unsigned int			key_len;
667 	void				*key;
668 	struct tracing_map_elt		elt;
669 
670 	struct action_data		*action_data;
671 	struct hist_trigger_data	*hist_data;
672 };
673 
674 struct hist_elt_data {
675 	char *comm;
676 	u64 *var_ref_vals;
677 	char **field_var_str;
678 	int n_field_var_str;
679 };
680 
681 struct snapshot_context {
682 	struct tracing_map_elt	*elt;
683 	void			*key;
684 };
685 
686 /*
687  * Returns the specific division function to use if the divisor
688  * is constant. This avoids extra branches when the trigger is hit.
689  */
690 static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor)
691 {
692 	u64 div = divisor->constant;
693 
694 	if (!(div & (div - 1)))
695 		return HIST_FIELD_FN_DIV_POWER2;
696 
697 	/* If the divisor is too large, do a regular division */
698 	if (div > (1 << HIST_DIV_SHIFT))
699 		return HIST_FIELD_FN_DIV_NOT_POWER2;
700 
701 	divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
702 	return HIST_FIELD_FN_DIV_MULT_SHIFT;
703 }
704 
705 static void track_data_free(struct track_data *track_data)
706 {
707 	struct hist_elt_data *elt_data;
708 
709 	if (!track_data)
710 		return;
711 
712 	kfree(track_data->key);
713 
714 	elt_data = track_data->elt.private_data;
715 	if (elt_data) {
716 		kfree(elt_data->comm);
717 		kfree(elt_data);
718 	}
719 
720 	kfree(track_data);
721 }
722 
723 static struct track_data *track_data_alloc(unsigned int key_len,
724 					   struct action_data *action_data,
725 					   struct hist_trigger_data *hist_data)
726 {
727 	struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
728 	struct hist_elt_data *elt_data;
729 
730 	if (!data)
731 		return ERR_PTR(-ENOMEM);
732 
733 	data->key = kzalloc(key_len, GFP_KERNEL);
734 	if (!data->key) {
735 		track_data_free(data);
736 		return ERR_PTR(-ENOMEM);
737 	}
738 
739 	data->key_len = key_len;
740 	data->action_data = action_data;
741 	data->hist_data = hist_data;
742 
743 	elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
744 	if (!elt_data) {
745 		track_data_free(data);
746 		return ERR_PTR(-ENOMEM);
747 	}
748 
749 	data->elt.private_data = elt_data;
750 
751 	elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
752 	if (!elt_data->comm) {
753 		track_data_free(data);
754 		return ERR_PTR(-ENOMEM);
755 	}
756 
757 	return data;
758 }
759 
760 #define HIST_PREFIX "hist:"
761 
762 static char *last_cmd;
763 static char last_cmd_loc[MAX_FILTER_STR_VAL];
764 
765 static int errpos(char *str)
766 {
767 	if (!str || !last_cmd)
768 		return 0;
769 
770 	return err_pos(last_cmd, str);
771 }
772 
773 static void last_cmd_set(struct trace_event_file *file, char *str)
774 {
775 	const char *system = NULL, *name = NULL;
776 	struct trace_event_call *call;
777 	int len;
778 
779 	if (!str)
780 		return;
781 
782 	/* sizeof() contains the nul byte */
783 	len = sizeof(HIST_PREFIX) + strlen(str);
784 	kfree(last_cmd);
785 	last_cmd = kzalloc(len, GFP_KERNEL);
786 	if (!last_cmd)
787 		return;
788 
789 	strcpy(last_cmd, HIST_PREFIX);
790 	/* Again, sizeof() contains the nul byte */
791 	len -= sizeof(HIST_PREFIX);
792 	strncat(last_cmd, str, len);
793 
794 	if (file) {
795 		call = file->event_call;
796 		system = call->class->system;
797 		if (system) {
798 			name = trace_event_name(call);
799 			if (!name)
800 				system = NULL;
801 		}
802 	}
803 
804 	if (system)
805 		snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, HIST_PREFIX "%s:%s", system, name);
806 }
807 
808 static void hist_err(struct trace_array *tr, u8 err_type, u16 err_pos)
809 {
810 	if (!last_cmd)
811 		return;
812 
813 	tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
814 			err_type, err_pos);
815 }
816 
817 static void hist_err_clear(void)
818 {
819 	if (last_cmd)
820 		last_cmd[0] = '\0';
821 	last_cmd_loc[0] = '\0';
822 }
823 
824 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
825 				    unsigned int *var_ref_idx);
826 
827 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
828 			       unsigned int *var_ref_idx)
829 {
830 	struct tracepoint *tp = event->tp;
831 
832 	if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
833 		struct tracepoint_func *probe_func_ptr;
834 		synth_probe_func_t probe_func;
835 		void *__data;
836 
837 		if (!(cpu_online(raw_smp_processor_id())))
838 			return;
839 
840 		probe_func_ptr = rcu_dereference_sched((tp)->funcs);
841 		if (probe_func_ptr) {
842 			do {
843 				probe_func = probe_func_ptr->func;
844 				__data = probe_func_ptr->data;
845 				probe_func(__data, var_ref_vals, var_ref_idx);
846 			} while ((++probe_func_ptr)->func);
847 		}
848 	}
849 }
850 
851 static void action_trace(struct hist_trigger_data *hist_data,
852 			 struct tracing_map_elt *elt,
853 			 struct trace_buffer *buffer, void *rec,
854 			 struct ring_buffer_event *rbe, void *key,
855 			 struct action_data *data, u64 *var_ref_vals)
856 {
857 	struct synth_event *event = data->synth_event;
858 
859 	trace_synth(event, var_ref_vals, data->var_ref_idx);
860 }
861 
862 struct hist_var_data {
863 	struct list_head list;
864 	struct hist_trigger_data *hist_data;
865 };
866 
867 static u64 hist_field_timestamp(struct hist_field *hist_field,
868 				struct tracing_map_elt *elt,
869 				struct trace_buffer *buffer,
870 				struct ring_buffer_event *rbe,
871 				void *event)
872 {
873 	struct hist_trigger_data *hist_data = hist_field->hist_data;
874 	struct trace_array *tr = hist_data->event_file->tr;
875 
876 	u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
877 
878 	if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
879 		ts = ns2usecs(ts);
880 
881 	return ts;
882 }
883 
884 static u64 hist_field_cpu(struct hist_field *hist_field,
885 			  struct tracing_map_elt *elt,
886 			  struct trace_buffer *buffer,
887 			  struct ring_buffer_event *rbe,
888 			  void *event)
889 {
890 	int cpu = smp_processor_id();
891 
892 	return cpu;
893 }
894 
895 /**
896  * check_field_for_var_ref - Check if a VAR_REF field references a variable
897  * @hist_field: The VAR_REF field to check
898  * @var_data: The hist trigger that owns the variable
899  * @var_idx: The trigger variable identifier
900  *
901  * Check the given VAR_REF field to see whether or not it references
902  * the given variable associated with the given trigger.
903  *
904  * Return: The VAR_REF field if it does reference the variable, NULL if not
905  */
906 static struct hist_field *
907 check_field_for_var_ref(struct hist_field *hist_field,
908 			struct hist_trigger_data *var_data,
909 			unsigned int var_idx)
910 {
911 	WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
912 
913 	if (hist_field && hist_field->var.idx == var_idx &&
914 	    hist_field->var.hist_data == var_data)
915 		return hist_field;
916 
917 	return NULL;
918 }
919 
920 /**
921  * find_var_ref - Check if a trigger has a reference to a trigger variable
922  * @hist_data: The hist trigger that might have a reference to the variable
923  * @var_data: The hist trigger that owns the variable
924  * @var_idx: The trigger variable identifier
925  *
926  * Check the list of var_refs[] on the first hist trigger to see
927  * whether any of them are references to the variable on the second
928  * trigger.
929  *
930  * Return: The VAR_REF field referencing the variable if so, NULL if not
931  */
932 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
933 				       struct hist_trigger_data *var_data,
934 				       unsigned int var_idx)
935 {
936 	struct hist_field *hist_field;
937 	unsigned int i;
938 
939 	for (i = 0; i < hist_data->n_var_refs; i++) {
940 		hist_field = hist_data->var_refs[i];
941 		if (check_field_for_var_ref(hist_field, var_data, var_idx))
942 			return hist_field;
943 	}
944 
945 	return NULL;
946 }
947 
948 /**
949  * find_any_var_ref - Check if there is a reference to a given trigger variable
950  * @hist_data: The hist trigger
951  * @var_idx: The trigger variable identifier
952  *
953  * Check to see whether the given variable is currently referenced by
954  * any other trigger.
955  *
956  * The trigger the variable is defined on is explicitly excluded - the
957  * assumption being that a self-reference doesn't prevent a trigger
958  * from being removed.
959  *
960  * Return: The VAR_REF field referencing the variable if so, NULL if not
961  */
962 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
963 					   unsigned int var_idx)
964 {
965 	struct trace_array *tr = hist_data->event_file->tr;
966 	struct hist_field *found = NULL;
967 	struct hist_var_data *var_data;
968 
969 	list_for_each_entry(var_data, &tr->hist_vars, list) {
970 		if (var_data->hist_data == hist_data)
971 			continue;
972 		found = find_var_ref(var_data->hist_data, hist_data, var_idx);
973 		if (found)
974 			break;
975 	}
976 
977 	return found;
978 }
979 
980 /**
981  * check_var_refs - Check if there is a reference to any of trigger's variables
982  * @hist_data: The hist trigger
983  *
984  * A trigger can define one or more variables.  If any one of them is
985  * currently referenced by any other trigger, this function will
986  * determine that.
987  *
988  * Typically used to determine whether or not a trigger can be removed
989  * - if there are any references to a trigger's variables, it cannot.
990  *
991  * Return: True if there is a reference to any of trigger's variables
992  */
993 static bool check_var_refs(struct hist_trigger_data *hist_data)
994 {
995 	struct hist_field *field;
996 	bool found = false;
997 	int i;
998 
999 	for_each_hist_field(i, hist_data) {
1000 		field = hist_data->fields[i];
1001 		if (field && field->flags & HIST_FIELD_FL_VAR) {
1002 			if (find_any_var_ref(hist_data, field->var.idx)) {
1003 				found = true;
1004 				break;
1005 			}
1006 		}
1007 	}
1008 
1009 	return found;
1010 }
1011 
1012 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1013 {
1014 	struct trace_array *tr = hist_data->event_file->tr;
1015 	struct hist_var_data *var_data, *found = NULL;
1016 
1017 	list_for_each_entry(var_data, &tr->hist_vars, list) {
1018 		if (var_data->hist_data == hist_data) {
1019 			found = var_data;
1020 			break;
1021 		}
1022 	}
1023 
1024 	return found;
1025 }
1026 
1027 static bool field_has_hist_vars(struct hist_field *hist_field,
1028 				unsigned int level)
1029 {
1030 	int i;
1031 
1032 	if (level > 3)
1033 		return false;
1034 
1035 	if (!hist_field)
1036 		return false;
1037 
1038 	if (hist_field->flags & HIST_FIELD_FL_VAR ||
1039 	    hist_field->flags & HIST_FIELD_FL_VAR_REF)
1040 		return true;
1041 
1042 	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1043 		struct hist_field *operand;
1044 
1045 		operand = hist_field->operands[i];
1046 		if (field_has_hist_vars(operand, level + 1))
1047 			return true;
1048 	}
1049 
1050 	return false;
1051 }
1052 
1053 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1054 {
1055 	struct hist_field *hist_field;
1056 	int i;
1057 
1058 	for_each_hist_field(i, hist_data) {
1059 		hist_field = hist_data->fields[i];
1060 		if (field_has_hist_vars(hist_field, 0))
1061 			return true;
1062 	}
1063 
1064 	return false;
1065 }
1066 
1067 static int save_hist_vars(struct hist_trigger_data *hist_data)
1068 {
1069 	struct trace_array *tr = hist_data->event_file->tr;
1070 	struct hist_var_data *var_data;
1071 
1072 	var_data = find_hist_vars(hist_data);
1073 	if (var_data)
1074 		return 0;
1075 
1076 	if (tracing_check_open_get_tr(tr))
1077 		return -ENODEV;
1078 
1079 	var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1080 	if (!var_data) {
1081 		trace_array_put(tr);
1082 		return -ENOMEM;
1083 	}
1084 
1085 	var_data->hist_data = hist_data;
1086 	list_add(&var_data->list, &tr->hist_vars);
1087 
1088 	return 0;
1089 }
1090 
1091 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1092 {
1093 	struct trace_array *tr = hist_data->event_file->tr;
1094 	struct hist_var_data *var_data;
1095 
1096 	var_data = find_hist_vars(hist_data);
1097 	if (!var_data)
1098 		return;
1099 
1100 	if (WARN_ON(check_var_refs(hist_data)))
1101 		return;
1102 
1103 	list_del(&var_data->list);
1104 
1105 	kfree(var_data);
1106 
1107 	trace_array_put(tr);
1108 }
1109 
1110 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1111 					 const char *var_name)
1112 {
1113 	struct hist_field *hist_field, *found = NULL;
1114 	int i;
1115 
1116 	for_each_hist_field(i, hist_data) {
1117 		hist_field = hist_data->fields[i];
1118 		if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1119 		    strcmp(hist_field->var.name, var_name) == 0) {
1120 			found = hist_field;
1121 			break;
1122 		}
1123 	}
1124 
1125 	return found;
1126 }
1127 
1128 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1129 				   struct trace_event_file *file,
1130 				   const char *var_name)
1131 {
1132 	struct hist_trigger_data *test_data;
1133 	struct event_trigger_data *test;
1134 	struct hist_field *hist_field;
1135 
1136 	lockdep_assert_held(&event_mutex);
1137 
1138 	hist_field = find_var_field(hist_data, var_name);
1139 	if (hist_field)
1140 		return hist_field;
1141 
1142 	list_for_each_entry(test, &file->triggers, list) {
1143 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1144 			test_data = test->private_data;
1145 			hist_field = find_var_field(test_data, var_name);
1146 			if (hist_field)
1147 				return hist_field;
1148 		}
1149 	}
1150 
1151 	return NULL;
1152 }
1153 
1154 static struct trace_event_file *find_var_file(struct trace_array *tr,
1155 					      char *system,
1156 					      char *event_name,
1157 					      char *var_name)
1158 {
1159 	struct hist_trigger_data *var_hist_data;
1160 	struct hist_var_data *var_data;
1161 	struct trace_event_file *file, *found = NULL;
1162 
1163 	if (system)
1164 		return find_event_file(tr, system, event_name);
1165 
1166 	list_for_each_entry(var_data, &tr->hist_vars, list) {
1167 		var_hist_data = var_data->hist_data;
1168 		file = var_hist_data->event_file;
1169 		if (file == found)
1170 			continue;
1171 
1172 		if (find_var_field(var_hist_data, var_name)) {
1173 			if (found) {
1174 				hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1175 				return NULL;
1176 			}
1177 
1178 			found = file;
1179 		}
1180 	}
1181 
1182 	return found;
1183 }
1184 
1185 static struct hist_field *find_file_var(struct trace_event_file *file,
1186 					const char *var_name)
1187 {
1188 	struct hist_trigger_data *test_data;
1189 	struct event_trigger_data *test;
1190 	struct hist_field *hist_field;
1191 
1192 	lockdep_assert_held(&event_mutex);
1193 
1194 	list_for_each_entry(test, &file->triggers, list) {
1195 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1196 			test_data = test->private_data;
1197 			hist_field = find_var_field(test_data, var_name);
1198 			if (hist_field)
1199 				return hist_field;
1200 		}
1201 	}
1202 
1203 	return NULL;
1204 }
1205 
1206 static struct hist_field *
1207 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1208 {
1209 	struct trace_array *tr = hist_data->event_file->tr;
1210 	struct hist_field *hist_field, *found = NULL;
1211 	struct trace_event_file *file;
1212 	unsigned int i;
1213 
1214 	for (i = 0; i < hist_data->n_actions; i++) {
1215 		struct action_data *data = hist_data->actions[i];
1216 
1217 		if (data->handler == HANDLER_ONMATCH) {
1218 			char *system = data->match_data.event_system;
1219 			char *event_name = data->match_data.event;
1220 
1221 			file = find_var_file(tr, system, event_name, var_name);
1222 			if (!file)
1223 				continue;
1224 			hist_field = find_file_var(file, var_name);
1225 			if (hist_field) {
1226 				if (found) {
1227 					hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1228 						 errpos(var_name));
1229 					return ERR_PTR(-EINVAL);
1230 				}
1231 
1232 				found = hist_field;
1233 			}
1234 		}
1235 	}
1236 	return found;
1237 }
1238 
1239 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1240 					 char *system,
1241 					 char *event_name,
1242 					 char *var_name)
1243 {
1244 	struct trace_array *tr = hist_data->event_file->tr;
1245 	struct hist_field *hist_field = NULL;
1246 	struct trace_event_file *file;
1247 
1248 	if (!system || !event_name) {
1249 		hist_field = find_match_var(hist_data, var_name);
1250 		if (IS_ERR(hist_field))
1251 			return NULL;
1252 		if (hist_field)
1253 			return hist_field;
1254 	}
1255 
1256 	file = find_var_file(tr, system, event_name, var_name);
1257 	if (!file)
1258 		return NULL;
1259 
1260 	hist_field = find_file_var(file, var_name);
1261 
1262 	return hist_field;
1263 }
1264 
1265 static u64 hist_field_var_ref(struct hist_field *hist_field,
1266 			      struct tracing_map_elt *elt,
1267 			      struct trace_buffer *buffer,
1268 			      struct ring_buffer_event *rbe,
1269 			      void *event)
1270 {
1271 	struct hist_elt_data *elt_data;
1272 	u64 var_val = 0;
1273 
1274 	if (WARN_ON_ONCE(!elt))
1275 		return var_val;
1276 
1277 	elt_data = elt->private_data;
1278 	var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1279 
1280 	return var_val;
1281 }
1282 
1283 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1284 			     u64 *var_ref_vals, bool self)
1285 {
1286 	struct hist_trigger_data *var_data;
1287 	struct tracing_map_elt *var_elt;
1288 	struct hist_field *hist_field;
1289 	unsigned int i, var_idx;
1290 	bool resolved = true;
1291 	u64 var_val = 0;
1292 
1293 	for (i = 0; i < hist_data->n_var_refs; i++) {
1294 		hist_field = hist_data->var_refs[i];
1295 		var_idx = hist_field->var.idx;
1296 		var_data = hist_field->var.hist_data;
1297 
1298 		if (var_data == NULL) {
1299 			resolved = false;
1300 			break;
1301 		}
1302 
1303 		if ((self && var_data != hist_data) ||
1304 		    (!self && var_data == hist_data))
1305 			continue;
1306 
1307 		var_elt = tracing_map_lookup(var_data->map, key);
1308 		if (!var_elt) {
1309 			resolved = false;
1310 			break;
1311 		}
1312 
1313 		if (!tracing_map_var_set(var_elt, var_idx)) {
1314 			resolved = false;
1315 			break;
1316 		}
1317 
1318 		if (self || !hist_field->read_once)
1319 			var_val = tracing_map_read_var(var_elt, var_idx);
1320 		else
1321 			var_val = tracing_map_read_var_once(var_elt, var_idx);
1322 
1323 		var_ref_vals[i] = var_val;
1324 	}
1325 
1326 	return resolved;
1327 }
1328 
1329 static const char *hist_field_name(struct hist_field *field,
1330 				   unsigned int level)
1331 {
1332 	const char *field_name = "";
1333 
1334 	if (WARN_ON_ONCE(!field))
1335 		return field_name;
1336 
1337 	if (level > 1)
1338 		return field_name;
1339 
1340 	if (field->field)
1341 		field_name = field->field->name;
1342 	else if (field->flags & HIST_FIELD_FL_LOG2 ||
1343 		 field->flags & HIST_FIELD_FL_ALIAS ||
1344 		 field->flags & HIST_FIELD_FL_BUCKET)
1345 		field_name = hist_field_name(field->operands[0], ++level);
1346 	else if (field->flags & HIST_FIELD_FL_CPU)
1347 		field_name = "common_cpu";
1348 	else if (field->flags & HIST_FIELD_FL_EXPR ||
1349 		 field->flags & HIST_FIELD_FL_VAR_REF) {
1350 		if (field->system) {
1351 			static char full_name[MAX_FILTER_STR_VAL];
1352 
1353 			strcat(full_name, field->system);
1354 			strcat(full_name, ".");
1355 			strcat(full_name, field->event_name);
1356 			strcat(full_name, ".");
1357 			strcat(full_name, field->name);
1358 			field_name = full_name;
1359 		} else
1360 			field_name = field->name;
1361 	} else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1362 		field_name = "common_timestamp";
1363 	else if (field->flags & HIST_FIELD_FL_STACKTRACE) {
1364 		if (field->field)
1365 			field_name = field->field->name;
1366 		else
1367 			field_name = "common_stacktrace";
1368 	} else if (field->flags & HIST_FIELD_FL_HITCOUNT)
1369 		field_name = "hitcount";
1370 
1371 	if (field_name == NULL)
1372 		field_name = "";
1373 
1374 	return field_name;
1375 }
1376 
1377 static enum hist_field_fn select_value_fn(int field_size, int field_is_signed)
1378 {
1379 	switch (field_size) {
1380 	case 8:
1381 		if (field_is_signed)
1382 			return HIST_FIELD_FN_S64;
1383 		else
1384 			return HIST_FIELD_FN_U64;
1385 	case 4:
1386 		if (field_is_signed)
1387 			return HIST_FIELD_FN_S32;
1388 		else
1389 			return HIST_FIELD_FN_U32;
1390 	case 2:
1391 		if (field_is_signed)
1392 			return HIST_FIELD_FN_S16;
1393 		else
1394 			return HIST_FIELD_FN_U16;
1395 	case 1:
1396 		if (field_is_signed)
1397 			return HIST_FIELD_FN_S8;
1398 		else
1399 			return HIST_FIELD_FN_U8;
1400 	}
1401 
1402 	return HIST_FIELD_FN_NOP;
1403 }
1404 
1405 static int parse_map_size(char *str)
1406 {
1407 	unsigned long size, map_bits;
1408 	int ret;
1409 
1410 	ret = kstrtoul(str, 0, &size);
1411 	if (ret)
1412 		goto out;
1413 
1414 	map_bits = ilog2(roundup_pow_of_two(size));
1415 	if (map_bits < TRACING_MAP_BITS_MIN ||
1416 	    map_bits > TRACING_MAP_BITS_MAX)
1417 		ret = -EINVAL;
1418 	else
1419 		ret = map_bits;
1420  out:
1421 	return ret;
1422 }
1423 
1424 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1425 {
1426 	unsigned int i;
1427 
1428 	if (!attrs)
1429 		return;
1430 
1431 	for (i = 0; i < attrs->n_assignments; i++)
1432 		kfree(attrs->assignment_str[i]);
1433 
1434 	for (i = 0; i < attrs->n_actions; i++)
1435 		kfree(attrs->action_str[i]);
1436 
1437 	kfree(attrs->name);
1438 	kfree(attrs->sort_key_str);
1439 	kfree(attrs->keys_str);
1440 	kfree(attrs->vals_str);
1441 	kfree(attrs->clock);
1442 	kfree(attrs);
1443 }
1444 
1445 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1446 {
1447 	int ret = -EINVAL;
1448 
1449 	if (attrs->n_actions >= HIST_ACTIONS_MAX)
1450 		return ret;
1451 
1452 	if ((str_has_prefix(str, "onmatch(")) ||
1453 	    (str_has_prefix(str, "onmax(")) ||
1454 	    (str_has_prefix(str, "onchange("))) {
1455 		attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1456 		if (!attrs->action_str[attrs->n_actions]) {
1457 			ret = -ENOMEM;
1458 			return ret;
1459 		}
1460 		attrs->n_actions++;
1461 		ret = 0;
1462 	}
1463 	return ret;
1464 }
1465 
1466 static int parse_assignment(struct trace_array *tr,
1467 			    char *str, struct hist_trigger_attrs *attrs)
1468 {
1469 	int len, ret = 0;
1470 
1471 	if ((len = str_has_prefix(str, "key=")) ||
1472 	    (len = str_has_prefix(str, "keys="))) {
1473 		attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1474 		if (!attrs->keys_str) {
1475 			ret = -ENOMEM;
1476 			goto out;
1477 		}
1478 	} else if ((len = str_has_prefix(str, "val=")) ||
1479 		   (len = str_has_prefix(str, "vals=")) ||
1480 		   (len = str_has_prefix(str, "values="))) {
1481 		attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1482 		if (!attrs->vals_str) {
1483 			ret = -ENOMEM;
1484 			goto out;
1485 		}
1486 	} else if ((len = str_has_prefix(str, "sort="))) {
1487 		attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1488 		if (!attrs->sort_key_str) {
1489 			ret = -ENOMEM;
1490 			goto out;
1491 		}
1492 	} else if (str_has_prefix(str, "name=")) {
1493 		attrs->name = kstrdup(str, GFP_KERNEL);
1494 		if (!attrs->name) {
1495 			ret = -ENOMEM;
1496 			goto out;
1497 		}
1498 	} else if ((len = str_has_prefix(str, "clock="))) {
1499 		str += len;
1500 
1501 		str = strstrip(str);
1502 		attrs->clock = kstrdup(str, GFP_KERNEL);
1503 		if (!attrs->clock) {
1504 			ret = -ENOMEM;
1505 			goto out;
1506 		}
1507 	} else if ((len = str_has_prefix(str, "size="))) {
1508 		int map_bits = parse_map_size(str + len);
1509 
1510 		if (map_bits < 0) {
1511 			ret = map_bits;
1512 			goto out;
1513 		}
1514 		attrs->map_bits = map_bits;
1515 	} else {
1516 		char *assignment;
1517 
1518 		if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1519 			hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1520 			ret = -EINVAL;
1521 			goto out;
1522 		}
1523 
1524 		assignment = kstrdup(str, GFP_KERNEL);
1525 		if (!assignment) {
1526 			ret = -ENOMEM;
1527 			goto out;
1528 		}
1529 
1530 		attrs->assignment_str[attrs->n_assignments++] = assignment;
1531 	}
1532  out:
1533 	return ret;
1534 }
1535 
1536 static struct hist_trigger_attrs *
1537 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1538 {
1539 	struct hist_trigger_attrs *attrs;
1540 	int ret = 0;
1541 
1542 	attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1543 	if (!attrs)
1544 		return ERR_PTR(-ENOMEM);
1545 
1546 	while (trigger_str) {
1547 		char *str = strsep(&trigger_str, ":");
1548 		char *rhs;
1549 
1550 		rhs = strchr(str, '=');
1551 		if (rhs) {
1552 			if (!strlen(++rhs)) {
1553 				ret = -EINVAL;
1554 				hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1555 				goto free;
1556 			}
1557 			ret = parse_assignment(tr, str, attrs);
1558 			if (ret)
1559 				goto free;
1560 		} else if (strcmp(str, "nohitcount") == 0 ||
1561 			   strcmp(str, "NOHC") == 0)
1562 			attrs->no_hitcount = true;
1563 		else if (strcmp(str, "pause") == 0)
1564 			attrs->pause = true;
1565 		else if ((strcmp(str, "cont") == 0) ||
1566 			 (strcmp(str, "continue") == 0))
1567 			attrs->cont = true;
1568 		else if (strcmp(str, "clear") == 0)
1569 			attrs->clear = true;
1570 		else {
1571 			ret = parse_action(str, attrs);
1572 			if (ret)
1573 				goto free;
1574 		}
1575 	}
1576 
1577 	if (!attrs->keys_str) {
1578 		ret = -EINVAL;
1579 		goto free;
1580 	}
1581 
1582 	if (!attrs->clock) {
1583 		attrs->clock = kstrdup("global", GFP_KERNEL);
1584 		if (!attrs->clock) {
1585 			ret = -ENOMEM;
1586 			goto free;
1587 		}
1588 	}
1589 
1590 	return attrs;
1591  free:
1592 	destroy_hist_trigger_attrs(attrs);
1593 
1594 	return ERR_PTR(ret);
1595 }
1596 
1597 static inline void save_comm(char *comm, struct task_struct *task)
1598 {
1599 	if (!task->pid) {
1600 		strcpy(comm, "<idle>");
1601 		return;
1602 	}
1603 
1604 	if (WARN_ON_ONCE(task->pid < 0)) {
1605 		strcpy(comm, "<XXX>");
1606 		return;
1607 	}
1608 
1609 	strncpy(comm, task->comm, TASK_COMM_LEN);
1610 }
1611 
1612 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1613 {
1614 	unsigned int i;
1615 
1616 	for (i = 0; i < elt_data->n_field_var_str; i++)
1617 		kfree(elt_data->field_var_str[i]);
1618 
1619 	kfree(elt_data->field_var_str);
1620 
1621 	kfree(elt_data->comm);
1622 	kfree(elt_data);
1623 }
1624 
1625 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1626 {
1627 	struct hist_elt_data *elt_data = elt->private_data;
1628 
1629 	hist_elt_data_free(elt_data);
1630 }
1631 
1632 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1633 {
1634 	struct hist_trigger_data *hist_data = elt->map->private_data;
1635 	unsigned int size = TASK_COMM_LEN;
1636 	struct hist_elt_data *elt_data;
1637 	struct hist_field *hist_field;
1638 	unsigned int i, n_str;
1639 
1640 	elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1641 	if (!elt_data)
1642 		return -ENOMEM;
1643 
1644 	for_each_hist_field(i, hist_data) {
1645 		hist_field = hist_data->fields[i];
1646 
1647 		if (hist_field->flags & HIST_FIELD_FL_EXECNAME) {
1648 			elt_data->comm = kzalloc(size, GFP_KERNEL);
1649 			if (!elt_data->comm) {
1650 				kfree(elt_data);
1651 				return -ENOMEM;
1652 			}
1653 			break;
1654 		}
1655 	}
1656 
1657 	n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1658 		hist_data->n_var_str;
1659 	if (n_str > SYNTH_FIELDS_MAX) {
1660 		hist_elt_data_free(elt_data);
1661 		return -EINVAL;
1662 	}
1663 
1664 	BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1665 
1666 	size = STR_VAR_LEN_MAX;
1667 
1668 	elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL);
1669 	if (!elt_data->field_var_str) {
1670 		hist_elt_data_free(elt_data);
1671 		return -EINVAL;
1672 	}
1673 	elt_data->n_field_var_str = n_str;
1674 
1675 	for (i = 0; i < n_str; i++) {
1676 		elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1677 		if (!elt_data->field_var_str[i]) {
1678 			hist_elt_data_free(elt_data);
1679 			return -ENOMEM;
1680 		}
1681 	}
1682 
1683 	elt->private_data = elt_data;
1684 
1685 	return 0;
1686 }
1687 
1688 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1689 {
1690 	struct hist_elt_data *elt_data = elt->private_data;
1691 
1692 	if (elt_data->comm)
1693 		save_comm(elt_data->comm, current);
1694 }
1695 
1696 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1697 	.elt_alloc	= hist_trigger_elt_data_alloc,
1698 	.elt_free	= hist_trigger_elt_data_free,
1699 	.elt_init	= hist_trigger_elt_data_init,
1700 };
1701 
1702 static const char *get_hist_field_flags(struct hist_field *hist_field)
1703 {
1704 	const char *flags_str = NULL;
1705 
1706 	if (hist_field->flags & HIST_FIELD_FL_HEX)
1707 		flags_str = "hex";
1708 	else if (hist_field->flags & HIST_FIELD_FL_SYM)
1709 		flags_str = "sym";
1710 	else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1711 		flags_str = "sym-offset";
1712 	else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1713 		flags_str = "execname";
1714 	else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1715 		flags_str = "syscall";
1716 	else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1717 		flags_str = "log2";
1718 	else if (hist_field->flags & HIST_FIELD_FL_BUCKET)
1719 		flags_str = "buckets";
1720 	else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1721 		flags_str = "usecs";
1722 	else if (hist_field->flags & HIST_FIELD_FL_PERCENT)
1723 		flags_str = "percent";
1724 	else if (hist_field->flags & HIST_FIELD_FL_GRAPH)
1725 		flags_str = "graph";
1726 	else if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
1727 		flags_str = "stacktrace";
1728 
1729 	return flags_str;
1730 }
1731 
1732 static void expr_field_str(struct hist_field *field, char *expr)
1733 {
1734 	if (field->flags & HIST_FIELD_FL_VAR_REF)
1735 		strcat(expr, "$");
1736 	else if (field->flags & HIST_FIELD_FL_CONST) {
1737 		char str[HIST_CONST_DIGITS_MAX];
1738 
1739 		snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant);
1740 		strcat(expr, str);
1741 	}
1742 
1743 	strcat(expr, hist_field_name(field, 0));
1744 
1745 	if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1746 		const char *flags_str = get_hist_field_flags(field);
1747 
1748 		if (flags_str) {
1749 			strcat(expr, ".");
1750 			strcat(expr, flags_str);
1751 		}
1752 	}
1753 }
1754 
1755 static char *expr_str(struct hist_field *field, unsigned int level)
1756 {
1757 	char *expr;
1758 
1759 	if (level > 1)
1760 		return NULL;
1761 
1762 	expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1763 	if (!expr)
1764 		return NULL;
1765 
1766 	if (!field->operands[0]) {
1767 		expr_field_str(field, expr);
1768 		return expr;
1769 	}
1770 
1771 	if (field->operator == FIELD_OP_UNARY_MINUS) {
1772 		char *subexpr;
1773 
1774 		strcat(expr, "-(");
1775 		subexpr = expr_str(field->operands[0], ++level);
1776 		if (!subexpr) {
1777 			kfree(expr);
1778 			return NULL;
1779 		}
1780 		strcat(expr, subexpr);
1781 		strcat(expr, ")");
1782 
1783 		kfree(subexpr);
1784 
1785 		return expr;
1786 	}
1787 
1788 	expr_field_str(field->operands[0], expr);
1789 
1790 	switch (field->operator) {
1791 	case FIELD_OP_MINUS:
1792 		strcat(expr, "-");
1793 		break;
1794 	case FIELD_OP_PLUS:
1795 		strcat(expr, "+");
1796 		break;
1797 	case FIELD_OP_DIV:
1798 		strcat(expr, "/");
1799 		break;
1800 	case FIELD_OP_MULT:
1801 		strcat(expr, "*");
1802 		break;
1803 	default:
1804 		kfree(expr);
1805 		return NULL;
1806 	}
1807 
1808 	expr_field_str(field->operands[1], expr);
1809 
1810 	return expr;
1811 }
1812 
1813 /*
1814  * If field_op != FIELD_OP_NONE, *sep points to the root operator
1815  * of the expression tree to be evaluated.
1816  */
1817 static int contains_operator(char *str, char **sep)
1818 {
1819 	enum field_op_id field_op = FIELD_OP_NONE;
1820 	char *minus_op, *plus_op, *div_op, *mult_op;
1821 
1822 
1823 	/*
1824 	 * Report the last occurrence of the operators first, so that the
1825 	 * expression is evaluated left to right. This is important since
1826 	 * subtraction and division are not associative.
1827 	 *
1828 	 *	e.g
1829 	 *		64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2
1830 	 *		14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2
1831 	 */
1832 
1833 	/*
1834 	 * First, find lower precedence addition and subtraction
1835 	 * since the expression will be evaluated recursively.
1836 	 */
1837 	minus_op = strrchr(str, '-');
1838 	if (minus_op) {
1839 		/*
1840 		 * Unary minus is not supported in sub-expressions. If
1841 		 * present, it is always the next root operator.
1842 		 */
1843 		if (minus_op == str) {
1844 			field_op = FIELD_OP_UNARY_MINUS;
1845 			goto out;
1846 		}
1847 
1848 		field_op = FIELD_OP_MINUS;
1849 	}
1850 
1851 	plus_op = strrchr(str, '+');
1852 	if (plus_op || minus_op) {
1853 		/*
1854 		 * For operators of the same precedence use to rightmost as the
1855 		 * root, so that the expression is evaluated left to right.
1856 		 */
1857 		if (plus_op > minus_op)
1858 			field_op = FIELD_OP_PLUS;
1859 		goto out;
1860 	}
1861 
1862 	/*
1863 	 * Multiplication and division have higher precedence than addition and
1864 	 * subtraction.
1865 	 */
1866 	div_op = strrchr(str, '/');
1867 	if (div_op)
1868 		field_op = FIELD_OP_DIV;
1869 
1870 	mult_op = strrchr(str, '*');
1871 	/*
1872 	 * For operators of the same precedence use to rightmost as the
1873 	 * root, so that the expression is evaluated left to right.
1874 	 */
1875 	if (mult_op > div_op)
1876 		field_op = FIELD_OP_MULT;
1877 
1878 out:
1879 	if (sep) {
1880 		switch (field_op) {
1881 		case FIELD_OP_UNARY_MINUS:
1882 		case FIELD_OP_MINUS:
1883 			*sep = minus_op;
1884 			break;
1885 		case FIELD_OP_PLUS:
1886 			*sep = plus_op;
1887 			break;
1888 		case FIELD_OP_DIV:
1889 			*sep = div_op;
1890 			break;
1891 		case FIELD_OP_MULT:
1892 			*sep = mult_op;
1893 			break;
1894 		case FIELD_OP_NONE:
1895 		default:
1896 			*sep = NULL;
1897 			break;
1898 		}
1899 	}
1900 
1901 	return field_op;
1902 }
1903 
1904 static void get_hist_field(struct hist_field *hist_field)
1905 {
1906 	hist_field->ref++;
1907 }
1908 
1909 static void __destroy_hist_field(struct hist_field *hist_field)
1910 {
1911 	if (--hist_field->ref > 1)
1912 		return;
1913 
1914 	kfree(hist_field->var.name);
1915 	kfree(hist_field->name);
1916 
1917 	/* Can likely be a const */
1918 	kfree_const(hist_field->type);
1919 
1920 	kfree(hist_field->system);
1921 	kfree(hist_field->event_name);
1922 
1923 	kfree(hist_field);
1924 }
1925 
1926 static void destroy_hist_field(struct hist_field *hist_field,
1927 			       unsigned int level)
1928 {
1929 	unsigned int i;
1930 
1931 	if (level > 3)
1932 		return;
1933 
1934 	if (!hist_field)
1935 		return;
1936 
1937 	if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1938 		return; /* var refs will be destroyed separately */
1939 
1940 	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1941 		destroy_hist_field(hist_field->operands[i], level + 1);
1942 
1943 	__destroy_hist_field(hist_field);
1944 }
1945 
1946 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1947 					    struct ftrace_event_field *field,
1948 					    unsigned long flags,
1949 					    char *var_name)
1950 {
1951 	struct hist_field *hist_field;
1952 
1953 	if (field && is_function_field(field))
1954 		return NULL;
1955 
1956 	hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1957 	if (!hist_field)
1958 		return NULL;
1959 
1960 	hist_field->ref = 1;
1961 
1962 	hist_field->hist_data = hist_data;
1963 
1964 	if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1965 		goto out; /* caller will populate */
1966 
1967 	if (flags & HIST_FIELD_FL_VAR_REF) {
1968 		hist_field->fn_num = HIST_FIELD_FN_VAR_REF;
1969 		goto out;
1970 	}
1971 
1972 	if (flags & HIST_FIELD_FL_HITCOUNT) {
1973 		hist_field->fn_num = HIST_FIELD_FN_COUNTER;
1974 		hist_field->size = sizeof(u64);
1975 		hist_field->type = "u64";
1976 		goto out;
1977 	}
1978 
1979 	if (flags & HIST_FIELD_FL_CONST) {
1980 		hist_field->fn_num = HIST_FIELD_FN_CONST;
1981 		hist_field->size = sizeof(u64);
1982 		hist_field->type = kstrdup("u64", GFP_KERNEL);
1983 		if (!hist_field->type)
1984 			goto free;
1985 		goto out;
1986 	}
1987 
1988 	if (flags & HIST_FIELD_FL_STACKTRACE) {
1989 		if (field)
1990 			hist_field->fn_num = HIST_FIELD_FN_STACK;
1991 		else
1992 			hist_field->fn_num = HIST_FIELD_FN_NOP;
1993 		hist_field->size = HIST_STACKTRACE_SIZE;
1994 		hist_field->type = kstrdup_const("unsigned long[]", GFP_KERNEL);
1995 		if (!hist_field->type)
1996 			goto free;
1997 		goto out;
1998 	}
1999 
2000 	if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
2001 		unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
2002 		hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 :
2003 			HIST_FIELD_FN_BUCKET;
2004 		hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2005 		if (!hist_field->operands[0])
2006 			goto free;
2007 		hist_field->size = hist_field->operands[0]->size;
2008 		hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
2009 		if (!hist_field->type)
2010 			goto free;
2011 		goto out;
2012 	}
2013 
2014 	if (flags & HIST_FIELD_FL_TIMESTAMP) {
2015 		hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP;
2016 		hist_field->size = sizeof(u64);
2017 		hist_field->type = "u64";
2018 		goto out;
2019 	}
2020 
2021 	if (flags & HIST_FIELD_FL_CPU) {
2022 		hist_field->fn_num = HIST_FIELD_FN_CPU;
2023 		hist_field->size = sizeof(int);
2024 		hist_field->type = "unsigned int";
2025 		goto out;
2026 	}
2027 
2028 	if (WARN_ON_ONCE(!field))
2029 		goto out;
2030 
2031 	/* Pointers to strings are just pointers and dangerous to dereference */
2032 	if (is_string_field(field) &&
2033 	    (field->filter_type != FILTER_PTR_STRING)) {
2034 		flags |= HIST_FIELD_FL_STRING;
2035 
2036 		hist_field->size = MAX_FILTER_STR_VAL;
2037 		hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2038 		if (!hist_field->type)
2039 			goto free;
2040 
2041 		if (field->filter_type == FILTER_STATIC_STRING) {
2042 			hist_field->fn_num = HIST_FIELD_FN_STRING;
2043 			hist_field->size = field->size;
2044 		} else if (field->filter_type == FILTER_DYN_STRING) {
2045 			hist_field->fn_num = HIST_FIELD_FN_DYNSTRING;
2046 		} else if (field->filter_type == FILTER_RDYN_STRING)
2047 			hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING;
2048 		else
2049 			hist_field->fn_num = HIST_FIELD_FN_PSTRING;
2050 	} else {
2051 		hist_field->size = field->size;
2052 		hist_field->is_signed = field->is_signed;
2053 		hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2054 		if (!hist_field->type)
2055 			goto free;
2056 
2057 		hist_field->fn_num = select_value_fn(field->size,
2058 						     field->is_signed);
2059 		if (hist_field->fn_num == HIST_FIELD_FN_NOP) {
2060 			destroy_hist_field(hist_field, 0);
2061 			return NULL;
2062 		}
2063 	}
2064  out:
2065 	hist_field->field = field;
2066 	hist_field->flags = flags;
2067 
2068 	if (var_name) {
2069 		hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2070 		if (!hist_field->var.name)
2071 			goto free;
2072 	}
2073 
2074 	return hist_field;
2075  free:
2076 	destroy_hist_field(hist_field, 0);
2077 	return NULL;
2078 }
2079 
2080 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2081 {
2082 	unsigned int i;
2083 
2084 	for (i = 0; i < HIST_FIELDS_MAX; i++) {
2085 		if (hist_data->fields[i]) {
2086 			destroy_hist_field(hist_data->fields[i], 0);
2087 			hist_data->fields[i] = NULL;
2088 		}
2089 	}
2090 
2091 	for (i = 0; i < hist_data->n_var_refs; i++) {
2092 		WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2093 		__destroy_hist_field(hist_data->var_refs[i]);
2094 		hist_data->var_refs[i] = NULL;
2095 	}
2096 }
2097 
2098 static int init_var_ref(struct hist_field *ref_field,
2099 			struct hist_field *var_field,
2100 			char *system, char *event_name)
2101 {
2102 	int err = 0;
2103 
2104 	ref_field->var.idx = var_field->var.idx;
2105 	ref_field->var.hist_data = var_field->hist_data;
2106 	ref_field->size = var_field->size;
2107 	ref_field->is_signed = var_field->is_signed;
2108 	ref_field->flags |= var_field->flags &
2109 		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2110 
2111 	if (system) {
2112 		ref_field->system = kstrdup(system, GFP_KERNEL);
2113 		if (!ref_field->system)
2114 			return -ENOMEM;
2115 	}
2116 
2117 	if (event_name) {
2118 		ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2119 		if (!ref_field->event_name) {
2120 			err = -ENOMEM;
2121 			goto free;
2122 		}
2123 	}
2124 
2125 	if (var_field->var.name) {
2126 		ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2127 		if (!ref_field->name) {
2128 			err = -ENOMEM;
2129 			goto free;
2130 		}
2131 	} else if (var_field->name) {
2132 		ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2133 		if (!ref_field->name) {
2134 			err = -ENOMEM;
2135 			goto free;
2136 		}
2137 	}
2138 
2139 	ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL);
2140 	if (!ref_field->type) {
2141 		err = -ENOMEM;
2142 		goto free;
2143 	}
2144  out:
2145 	return err;
2146  free:
2147 	kfree(ref_field->system);
2148 	ref_field->system = NULL;
2149 	kfree(ref_field->event_name);
2150 	ref_field->event_name = NULL;
2151 	kfree(ref_field->name);
2152 	ref_field->name = NULL;
2153 
2154 	goto out;
2155 }
2156 
2157 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
2158 			    struct hist_field *var_field)
2159 {
2160 	struct hist_field *ref_field;
2161 	int i;
2162 
2163 	for (i = 0; i < hist_data->n_var_refs; i++) {
2164 		ref_field = hist_data->var_refs[i];
2165 		if (ref_field->var.idx == var_field->var.idx &&
2166 		    ref_field->var.hist_data == var_field->hist_data)
2167 			return i;
2168 	}
2169 
2170 	return -ENOENT;
2171 }
2172 
2173 /**
2174  * create_var_ref - Create a variable reference and attach it to trigger
2175  * @hist_data: The trigger that will be referencing the variable
2176  * @var_field: The VAR field to create a reference to
2177  * @system: The optional system string
2178  * @event_name: The optional event_name string
2179  *
2180  * Given a variable hist_field, create a VAR_REF hist_field that
2181  * represents a reference to it.
2182  *
2183  * This function also adds the reference to the trigger that
2184  * now references the variable.
2185  *
2186  * Return: The VAR_REF field if successful, NULL if not
2187  */
2188 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2189 					 struct hist_field *var_field,
2190 					 char *system, char *event_name)
2191 {
2192 	unsigned long flags = HIST_FIELD_FL_VAR_REF;
2193 	struct hist_field *ref_field;
2194 	int i;
2195 
2196 	/* Check if the variable already exists */
2197 	for (i = 0; i < hist_data->n_var_refs; i++) {
2198 		ref_field = hist_data->var_refs[i];
2199 		if (ref_field->var.idx == var_field->var.idx &&
2200 		    ref_field->var.hist_data == var_field->hist_data) {
2201 			get_hist_field(ref_field);
2202 			return ref_field;
2203 		}
2204 	}
2205 	/* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */
2206 	if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX)
2207 		return NULL;
2208 	ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2209 	if (ref_field) {
2210 		if (init_var_ref(ref_field, var_field, system, event_name)) {
2211 			destroy_hist_field(ref_field, 0);
2212 			return NULL;
2213 		}
2214 
2215 		hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2216 		ref_field->var_ref_idx = hist_data->n_var_refs++;
2217 	}
2218 
2219 	return ref_field;
2220 }
2221 
2222 static bool is_var_ref(char *var_name)
2223 {
2224 	if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2225 		return false;
2226 
2227 	return true;
2228 }
2229 
2230 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2231 				 char *var_name)
2232 {
2233 	char *name, *field;
2234 	unsigned int i;
2235 
2236 	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2237 		name = hist_data->attrs->var_defs.name[i];
2238 
2239 		if (strcmp(var_name, name) == 0) {
2240 			field = hist_data->attrs->var_defs.expr[i];
2241 			if (contains_operator(field, NULL) || is_var_ref(field))
2242 				continue;
2243 			return field;
2244 		}
2245 	}
2246 
2247 	return NULL;
2248 }
2249 
2250 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2251 				 char *system, char *event_name,
2252 				 char *var_name)
2253 {
2254 	struct trace_event_call *call;
2255 
2256 	if (system && event_name) {
2257 		call = hist_data->event_file->event_call;
2258 
2259 		if (strcmp(system, call->class->system) != 0)
2260 			return NULL;
2261 
2262 		if (strcmp(event_name, trace_event_name(call)) != 0)
2263 			return NULL;
2264 	}
2265 
2266 	if (!!system != !!event_name)
2267 		return NULL;
2268 
2269 	if (!is_var_ref(var_name))
2270 		return NULL;
2271 
2272 	var_name++;
2273 
2274 	return field_name_from_var(hist_data, var_name);
2275 }
2276 
2277 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2278 					char *system, char *event_name,
2279 					char *var_name)
2280 {
2281 	struct hist_field *var_field = NULL, *ref_field = NULL;
2282 	struct trace_array *tr = hist_data->event_file->tr;
2283 
2284 	if (!is_var_ref(var_name))
2285 		return NULL;
2286 
2287 	var_name++;
2288 
2289 	var_field = find_event_var(hist_data, system, event_name, var_name);
2290 	if (var_field)
2291 		ref_field = create_var_ref(hist_data, var_field,
2292 					   system, event_name);
2293 
2294 	if (!ref_field)
2295 		hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2296 
2297 	return ref_field;
2298 }
2299 
2300 static struct ftrace_event_field *
2301 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2302 	    char *field_str, unsigned long *flags, unsigned long *buckets)
2303 {
2304 	struct ftrace_event_field *field = NULL;
2305 	char *field_name, *modifier, *str;
2306 	struct trace_array *tr = file->tr;
2307 
2308 	modifier = str = kstrdup(field_str, GFP_KERNEL);
2309 	if (!modifier)
2310 		return ERR_PTR(-ENOMEM);
2311 
2312 	field_name = strsep(&modifier, ".");
2313 	if (modifier) {
2314 		if (strcmp(modifier, "hex") == 0)
2315 			*flags |= HIST_FIELD_FL_HEX;
2316 		else if (strcmp(modifier, "sym") == 0)
2317 			*flags |= HIST_FIELD_FL_SYM;
2318 		/*
2319 		 * 'sym-offset' occurrences in the trigger string are modified
2320 		 * to 'symXoffset' to simplify arithmetic expression parsing.
2321 		 */
2322 		else if (strcmp(modifier, "symXoffset") == 0)
2323 			*flags |= HIST_FIELD_FL_SYM_OFFSET;
2324 		else if ((strcmp(modifier, "execname") == 0) &&
2325 			 (strcmp(field_name, "common_pid") == 0))
2326 			*flags |= HIST_FIELD_FL_EXECNAME;
2327 		else if (strcmp(modifier, "syscall") == 0)
2328 			*flags |= HIST_FIELD_FL_SYSCALL;
2329 		else if (strcmp(modifier, "stacktrace") == 0)
2330 			*flags |= HIST_FIELD_FL_STACKTRACE;
2331 		else if (strcmp(modifier, "log2") == 0)
2332 			*flags |= HIST_FIELD_FL_LOG2;
2333 		else if (strcmp(modifier, "usecs") == 0)
2334 			*flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2335 		else if (strncmp(modifier, "bucket", 6) == 0) {
2336 			int ret;
2337 
2338 			modifier += 6;
2339 
2340 			if (*modifier == 's')
2341 				modifier++;
2342 			if (*modifier != '=')
2343 				goto error;
2344 			modifier++;
2345 			ret = kstrtoul(modifier, 0, buckets);
2346 			if (ret || !(*buckets))
2347 				goto error;
2348 			*flags |= HIST_FIELD_FL_BUCKET;
2349 		} else if (strncmp(modifier, "percent", 7) == 0) {
2350 			if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2351 				goto error;
2352 			*flags |= HIST_FIELD_FL_PERCENT;
2353 		} else if (strncmp(modifier, "graph", 5) == 0) {
2354 			if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2355 				goto error;
2356 			*flags |= HIST_FIELD_FL_GRAPH;
2357 		} else {
2358  error:
2359 			hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2360 			field = ERR_PTR(-EINVAL);
2361 			goto out;
2362 		}
2363 	}
2364 
2365 	if (strcmp(field_name, "common_timestamp") == 0) {
2366 		*flags |= HIST_FIELD_FL_TIMESTAMP;
2367 		hist_data->enable_timestamps = true;
2368 		if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2369 			hist_data->attrs->ts_in_usecs = true;
2370 	} else if (strcmp(field_name, "common_stacktrace") == 0) {
2371 		*flags |= HIST_FIELD_FL_STACKTRACE;
2372 	} else if (strcmp(field_name, "common_cpu") == 0)
2373 		*flags |= HIST_FIELD_FL_CPU;
2374 	else if (strcmp(field_name, "hitcount") == 0)
2375 		*flags |= HIST_FIELD_FL_HITCOUNT;
2376 	else {
2377 		field = trace_find_event_field(file->event_call, field_name);
2378 		if (!field || !field->size) {
2379 			/*
2380 			 * For backward compatibility, if field_name
2381 			 * was "cpu" or "stacktrace", then we treat this
2382 			 * the same as common_cpu and common_stacktrace
2383 			 * respectively. This also works for "CPU", and
2384 			 * "STACKTRACE".
2385 			 */
2386 			if (field && field->filter_type == FILTER_CPU) {
2387 				*flags |= HIST_FIELD_FL_CPU;
2388 			} else if (field && field->filter_type == FILTER_STACKTRACE) {
2389 				*flags |= HIST_FIELD_FL_STACKTRACE;
2390 			} else {
2391 				hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2392 					 errpos(field_name));
2393 				field = ERR_PTR(-EINVAL);
2394 				goto out;
2395 			}
2396 		}
2397 	}
2398  out:
2399 	kfree(str);
2400 
2401 	return field;
2402 }
2403 
2404 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2405 				       struct hist_field *var_ref,
2406 				       char *var_name)
2407 {
2408 	struct hist_field *alias = NULL;
2409 	unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2410 
2411 	alias = create_hist_field(hist_data, NULL, flags, var_name);
2412 	if (!alias)
2413 		return NULL;
2414 
2415 	alias->fn_num = var_ref->fn_num;
2416 	alias->operands[0] = var_ref;
2417 
2418 	if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2419 		destroy_hist_field(alias, 0);
2420 		return NULL;
2421 	}
2422 
2423 	alias->var_ref_idx = var_ref->var_ref_idx;
2424 
2425 	return alias;
2426 }
2427 
2428 static struct hist_field *parse_const(struct hist_trigger_data *hist_data,
2429 				      char *str, char *var_name,
2430 				      unsigned long *flags)
2431 {
2432 	struct trace_array *tr = hist_data->event_file->tr;
2433 	struct hist_field *field = NULL;
2434 	u64 constant;
2435 
2436 	if (kstrtoull(str, 0, &constant)) {
2437 		hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str));
2438 		return NULL;
2439 	}
2440 
2441 	*flags |= HIST_FIELD_FL_CONST;
2442 	field = create_hist_field(hist_data, NULL, *flags, var_name);
2443 	if (!field)
2444 		return NULL;
2445 
2446 	field->constant = constant;
2447 
2448 	return field;
2449 }
2450 
2451 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2452 				     struct trace_event_file *file, char *str,
2453 				     unsigned long *flags, char *var_name)
2454 {
2455 	char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2456 	struct ftrace_event_field *field = NULL;
2457 	struct hist_field *hist_field = NULL;
2458 	unsigned long buckets = 0;
2459 	int ret = 0;
2460 
2461 	if (isdigit(str[0])) {
2462 		hist_field = parse_const(hist_data, str, var_name, flags);
2463 		if (!hist_field) {
2464 			ret = -EINVAL;
2465 			goto out;
2466 		}
2467 		return hist_field;
2468 	}
2469 
2470 	s = strchr(str, '.');
2471 	if (s) {
2472 		s = strchr(++s, '.');
2473 		if (s) {
2474 			ref_system = strsep(&str, ".");
2475 			if (!str) {
2476 				ret = -EINVAL;
2477 				goto out;
2478 			}
2479 			ref_event = strsep(&str, ".");
2480 			if (!str) {
2481 				ret = -EINVAL;
2482 				goto out;
2483 			}
2484 			ref_var = str;
2485 		}
2486 	}
2487 
2488 	s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2489 	if (!s) {
2490 		hist_field = parse_var_ref(hist_data, ref_system,
2491 					   ref_event, ref_var);
2492 		if (hist_field) {
2493 			if (var_name) {
2494 				hist_field = create_alias(hist_data, hist_field, var_name);
2495 				if (!hist_field) {
2496 					ret = -ENOMEM;
2497 					goto out;
2498 				}
2499 			}
2500 			return hist_field;
2501 		}
2502 	} else
2503 		str = s;
2504 
2505 	field = parse_field(hist_data, file, str, flags, &buckets);
2506 	if (IS_ERR(field)) {
2507 		ret = PTR_ERR(field);
2508 		goto out;
2509 	}
2510 
2511 	hist_field = create_hist_field(hist_data, field, *flags, var_name);
2512 	if (!hist_field) {
2513 		ret = -ENOMEM;
2514 		goto out;
2515 	}
2516 	hist_field->buckets = buckets;
2517 
2518 	return hist_field;
2519  out:
2520 	return ERR_PTR(ret);
2521 }
2522 
2523 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2524 				     struct trace_event_file *file,
2525 				     char *str, unsigned long flags,
2526 				     char *var_name, unsigned int *n_subexprs);
2527 
2528 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2529 				      struct trace_event_file *file,
2530 				      char *str, unsigned long flags,
2531 				      char *var_name, unsigned int *n_subexprs)
2532 {
2533 	struct hist_field *operand1, *expr = NULL;
2534 	unsigned long operand_flags;
2535 	int ret = 0;
2536 	char *s;
2537 
2538 	/* Unary minus operator, increment n_subexprs */
2539 	++*n_subexprs;
2540 
2541 	/* we support only -(xxx) i.e. explicit parens required */
2542 
2543 	if (*n_subexprs > 3) {
2544 		hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2545 		ret = -EINVAL;
2546 		goto free;
2547 	}
2548 
2549 	str++; /* skip leading '-' */
2550 
2551 	s = strchr(str, '(');
2552 	if (s)
2553 		str++;
2554 	else {
2555 		ret = -EINVAL;
2556 		goto free;
2557 	}
2558 
2559 	s = strrchr(str, ')');
2560 	if (s) {
2561 		 /* unary minus not supported in sub-expressions */
2562 		if (*(s+1) != '\0') {
2563 			hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR,
2564 				 errpos(str));
2565 			ret = -EINVAL;
2566 			goto free;
2567 		}
2568 		*s = '\0';
2569 	}
2570 	else {
2571 		ret = -EINVAL; /* no closing ')' */
2572 		goto free;
2573 	}
2574 
2575 	flags |= HIST_FIELD_FL_EXPR;
2576 	expr = create_hist_field(hist_data, NULL, flags, var_name);
2577 	if (!expr) {
2578 		ret = -ENOMEM;
2579 		goto free;
2580 	}
2581 
2582 	operand_flags = 0;
2583 	operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2584 	if (IS_ERR(operand1)) {
2585 		ret = PTR_ERR(operand1);
2586 		goto free;
2587 	}
2588 	if (operand1->flags & HIST_FIELD_FL_STRING) {
2589 		/* String type can not be the operand of unary operator. */
2590 		hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2591 		destroy_hist_field(operand1, 0);
2592 		ret = -EINVAL;
2593 		goto free;
2594 	}
2595 
2596 	expr->flags |= operand1->flags &
2597 		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2598 	expr->fn_num = HIST_FIELD_FN_UMINUS;
2599 	expr->operands[0] = operand1;
2600 	expr->size = operand1->size;
2601 	expr->is_signed = operand1->is_signed;
2602 	expr->operator = FIELD_OP_UNARY_MINUS;
2603 	expr->name = expr_str(expr, 0);
2604 	expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2605 	if (!expr->type) {
2606 		ret = -ENOMEM;
2607 		goto free;
2608 	}
2609 
2610 	return expr;
2611  free:
2612 	destroy_hist_field(expr, 0);
2613 	return ERR_PTR(ret);
2614 }
2615 
2616 /*
2617  * If the operands are var refs, return pointers the
2618  * variable(s) referenced in var1 and var2, else NULL.
2619  */
2620 static int check_expr_operands(struct trace_array *tr,
2621 			       struct hist_field *operand1,
2622 			       struct hist_field *operand2,
2623 			       struct hist_field **var1,
2624 			       struct hist_field **var2)
2625 {
2626 	unsigned long operand1_flags = operand1->flags;
2627 	unsigned long operand2_flags = operand2->flags;
2628 
2629 	if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2630 	    (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2631 		struct hist_field *var;
2632 
2633 		var = find_var_field(operand1->var.hist_data, operand1->name);
2634 		if (!var)
2635 			return -EINVAL;
2636 		operand1_flags = var->flags;
2637 		*var1 = var;
2638 	}
2639 
2640 	if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2641 	    (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2642 		struct hist_field *var;
2643 
2644 		var = find_var_field(operand2->var.hist_data, operand2->name);
2645 		if (!var)
2646 			return -EINVAL;
2647 		operand2_flags = var->flags;
2648 		*var2 = var;
2649 	}
2650 
2651 	if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2652 	    (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2653 		hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2654 		return -EINVAL;
2655 	}
2656 
2657 	return 0;
2658 }
2659 
2660 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2661 				     struct trace_event_file *file,
2662 				     char *str, unsigned long flags,
2663 				     char *var_name, unsigned int *n_subexprs)
2664 {
2665 	struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2666 	struct hist_field *var1 = NULL, *var2 = NULL;
2667 	unsigned long operand_flags, operand2_flags;
2668 	int field_op, ret = -EINVAL;
2669 	char *sep, *operand1_str;
2670 	enum hist_field_fn op_fn;
2671 	bool combine_consts;
2672 
2673 	if (*n_subexprs > 3) {
2674 		hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2675 		return ERR_PTR(-EINVAL);
2676 	}
2677 
2678 	field_op = contains_operator(str, &sep);
2679 
2680 	if (field_op == FIELD_OP_NONE)
2681 		return parse_atom(hist_data, file, str, &flags, var_name);
2682 
2683 	if (field_op == FIELD_OP_UNARY_MINUS)
2684 		return parse_unary(hist_data, file, str, flags, var_name, n_subexprs);
2685 
2686 	/* Binary operator found, increment n_subexprs */
2687 	++*n_subexprs;
2688 
2689 	/* Split the expression string at the root operator */
2690 	if (!sep)
2691 		return ERR_PTR(-EINVAL);
2692 
2693 	*sep = '\0';
2694 	operand1_str = str;
2695 	str = sep+1;
2696 
2697 	/* Binary operator requires both operands */
2698 	if (*operand1_str == '\0' || *str == '\0')
2699 		return ERR_PTR(-EINVAL);
2700 
2701 	operand_flags = 0;
2702 
2703 	/* LHS of string is an expression e.g. a+b in a+b+c */
2704 	operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs);
2705 	if (IS_ERR(operand1))
2706 		return ERR_CAST(operand1);
2707 
2708 	if (operand1->flags & HIST_FIELD_FL_STRING) {
2709 		hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2710 		ret = -EINVAL;
2711 		goto free_op1;
2712 	}
2713 
2714 	/* RHS of string is another expression e.g. c in a+b+c */
2715 	operand_flags = 0;
2716 	operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2717 	if (IS_ERR(operand2)) {
2718 		ret = PTR_ERR(operand2);
2719 		goto free_op1;
2720 	}
2721 	if (operand2->flags & HIST_FIELD_FL_STRING) {
2722 		hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2723 		ret = -EINVAL;
2724 		goto free_operands;
2725 	}
2726 
2727 	switch (field_op) {
2728 	case FIELD_OP_MINUS:
2729 		op_fn = HIST_FIELD_FN_MINUS;
2730 		break;
2731 	case FIELD_OP_PLUS:
2732 		op_fn = HIST_FIELD_FN_PLUS;
2733 		break;
2734 	case FIELD_OP_DIV:
2735 		op_fn = HIST_FIELD_FN_DIV;
2736 		break;
2737 	case FIELD_OP_MULT:
2738 		op_fn = HIST_FIELD_FN_MULT;
2739 		break;
2740 	default:
2741 		ret = -EINVAL;
2742 		goto free_operands;
2743 	}
2744 
2745 	ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2);
2746 	if (ret)
2747 		goto free_operands;
2748 
2749 	operand_flags = var1 ? var1->flags : operand1->flags;
2750 	operand2_flags = var2 ? var2->flags : operand2->flags;
2751 
2752 	/*
2753 	 * If both operands are constant, the expression can be
2754 	 * collapsed to a single constant.
2755 	 */
2756 	combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST;
2757 
2758 	flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR;
2759 
2760 	flags |= operand1->flags &
2761 		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2762 
2763 	expr = create_hist_field(hist_data, NULL, flags, var_name);
2764 	if (!expr) {
2765 		ret = -ENOMEM;
2766 		goto free_operands;
2767 	}
2768 
2769 	operand1->read_once = true;
2770 	operand2->read_once = true;
2771 
2772 	/* The operands are now owned and free'd by 'expr' */
2773 	expr->operands[0] = operand1;
2774 	expr->operands[1] = operand2;
2775 
2776 	if (field_op == FIELD_OP_DIV &&
2777 			operand2_flags & HIST_FIELD_FL_CONST) {
2778 		u64 divisor = var2 ? var2->constant : operand2->constant;
2779 
2780 		if (!divisor) {
2781 			hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str));
2782 			ret = -EDOM;
2783 			goto free_expr;
2784 		}
2785 
2786 		/*
2787 		 * Copy the divisor here so we don't have to look it up
2788 		 * later if this is a var ref
2789 		 */
2790 		operand2->constant = divisor;
2791 		op_fn = hist_field_get_div_fn(operand2);
2792 	}
2793 
2794 	expr->fn_num = op_fn;
2795 
2796 	if (combine_consts) {
2797 		if (var1)
2798 			expr->operands[0] = var1;
2799 		if (var2)
2800 			expr->operands[1] = var2;
2801 
2802 		expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL);
2803 		expr->fn_num = HIST_FIELD_FN_CONST;
2804 
2805 		expr->operands[0] = NULL;
2806 		expr->operands[1] = NULL;
2807 
2808 		/*
2809 		 * var refs won't be destroyed immediately
2810 		 * See: destroy_hist_field()
2811 		 */
2812 		destroy_hist_field(operand2, 0);
2813 		destroy_hist_field(operand1, 0);
2814 
2815 		expr->name = expr_str(expr, 0);
2816 	} else {
2817 		/* The operand sizes should be the same, so just pick one */
2818 		expr->size = operand1->size;
2819 		expr->is_signed = operand1->is_signed;
2820 
2821 		expr->operator = field_op;
2822 		expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2823 		if (!expr->type) {
2824 			ret = -ENOMEM;
2825 			goto free_expr;
2826 		}
2827 
2828 		expr->name = expr_str(expr, 0);
2829 	}
2830 
2831 	return expr;
2832 
2833 free_operands:
2834 	destroy_hist_field(operand2, 0);
2835 free_op1:
2836 	destroy_hist_field(operand1, 0);
2837 	return ERR_PTR(ret);
2838 
2839 free_expr:
2840 	destroy_hist_field(expr, 0);
2841 	return ERR_PTR(ret);
2842 }
2843 
2844 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2845 				 struct trace_event_file *file)
2846 {
2847 	struct event_trigger_data *test;
2848 
2849 	lockdep_assert_held(&event_mutex);
2850 
2851 	list_for_each_entry(test, &file->triggers, list) {
2852 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2853 			if (test->private_data == hist_data)
2854 				return test->filter_str;
2855 		}
2856 	}
2857 
2858 	return NULL;
2859 }
2860 
2861 static struct event_command trigger_hist_cmd;
2862 static int event_hist_trigger_parse(struct event_command *cmd_ops,
2863 				    struct trace_event_file *file,
2864 				    char *glob, char *cmd,
2865 				    char *param_and_filter);
2866 
2867 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2868 			    struct hist_trigger_data *hist_data,
2869 			    unsigned int n_keys)
2870 {
2871 	struct hist_field *target_hist_field, *hist_field;
2872 	unsigned int n, i, j;
2873 
2874 	if (hist_data->n_fields - hist_data->n_vals != n_keys)
2875 		return false;
2876 
2877 	i = hist_data->n_vals;
2878 	j = target_hist_data->n_vals;
2879 
2880 	for (n = 0; n < n_keys; n++) {
2881 		hist_field = hist_data->fields[i + n];
2882 		target_hist_field = target_hist_data->fields[j + n];
2883 
2884 		if (strcmp(hist_field->type, target_hist_field->type) != 0)
2885 			return false;
2886 		if (hist_field->size != target_hist_field->size)
2887 			return false;
2888 		if (hist_field->is_signed != target_hist_field->is_signed)
2889 			return false;
2890 	}
2891 
2892 	return true;
2893 }
2894 
2895 static struct hist_trigger_data *
2896 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2897 		     struct trace_event_file *file)
2898 {
2899 	struct hist_trigger_data *hist_data;
2900 	struct event_trigger_data *test;
2901 	unsigned int n_keys;
2902 
2903 	lockdep_assert_held(&event_mutex);
2904 
2905 	n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2906 
2907 	list_for_each_entry(test, &file->triggers, list) {
2908 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2909 			hist_data = test->private_data;
2910 
2911 			if (compatible_keys(target_hist_data, hist_data, n_keys))
2912 				return hist_data;
2913 		}
2914 	}
2915 
2916 	return NULL;
2917 }
2918 
2919 static struct trace_event_file *event_file(struct trace_array *tr,
2920 					   char *system, char *event_name)
2921 {
2922 	struct trace_event_file *file;
2923 
2924 	file = __find_event_file(tr, system, event_name);
2925 	if (!file)
2926 		return ERR_PTR(-EINVAL);
2927 
2928 	return file;
2929 }
2930 
2931 static struct hist_field *
2932 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2933 			 char *system, char *event_name, char *field_name)
2934 {
2935 	struct hist_field *event_var;
2936 	char *synthetic_name;
2937 
2938 	synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2939 	if (!synthetic_name)
2940 		return ERR_PTR(-ENOMEM);
2941 
2942 	strcpy(synthetic_name, "synthetic_");
2943 	strcat(synthetic_name, field_name);
2944 
2945 	event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2946 
2947 	kfree(synthetic_name);
2948 
2949 	return event_var;
2950 }
2951 
2952 /**
2953  * create_field_var_hist - Automatically create a histogram and var for a field
2954  * @target_hist_data: The target hist trigger
2955  * @subsys_name: Optional subsystem name
2956  * @event_name: Optional event name
2957  * @field_name: The name of the field (and the resulting variable)
2958  *
2959  * Hist trigger actions fetch data from variables, not directly from
2960  * events.  However, for convenience, users are allowed to directly
2961  * specify an event field in an action, which will be automatically
2962  * converted into a variable on their behalf.
2963  *
2964  * If a user specifies a field on an event that isn't the event the
2965  * histogram currently being defined (the target event histogram), the
2966  * only way that can be accomplished is if a new hist trigger is
2967  * created and the field variable defined on that.
2968  *
2969  * This function creates a new histogram compatible with the target
2970  * event (meaning a histogram with the same key as the target
2971  * histogram), and creates a variable for the specified field, but
2972  * with 'synthetic_' prepended to the variable name in order to avoid
2973  * collision with normal field variables.
2974  *
2975  * Return: The variable created for the field.
2976  */
2977 static struct hist_field *
2978 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2979 		      char *subsys_name, char *event_name, char *field_name)
2980 {
2981 	struct trace_array *tr = target_hist_data->event_file->tr;
2982 	struct hist_trigger_data *hist_data;
2983 	unsigned int i, n, first = true;
2984 	struct field_var_hist *var_hist;
2985 	struct trace_event_file *file;
2986 	struct hist_field *key_field;
2987 	struct hist_field *event_var;
2988 	char *saved_filter;
2989 	char *cmd;
2990 	int ret;
2991 
2992 	if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2993 		hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2994 		return ERR_PTR(-EINVAL);
2995 	}
2996 
2997 	file = event_file(tr, subsys_name, event_name);
2998 
2999 	if (IS_ERR(file)) {
3000 		hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
3001 		ret = PTR_ERR(file);
3002 		return ERR_PTR(ret);
3003 	}
3004 
3005 	/*
3006 	 * Look for a histogram compatible with target.  We'll use the
3007 	 * found histogram specification to create a new matching
3008 	 * histogram with our variable on it.  target_hist_data is not
3009 	 * yet a registered histogram so we can't use that.
3010 	 */
3011 	hist_data = find_compatible_hist(target_hist_data, file);
3012 	if (!hist_data) {
3013 		hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
3014 		return ERR_PTR(-EINVAL);
3015 	}
3016 
3017 	/* See if a synthetic field variable has already been created */
3018 	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3019 					     event_name, field_name);
3020 	if (!IS_ERR_OR_NULL(event_var))
3021 		return event_var;
3022 
3023 	var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3024 	if (!var_hist)
3025 		return ERR_PTR(-ENOMEM);
3026 
3027 	cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3028 	if (!cmd) {
3029 		kfree(var_hist);
3030 		return ERR_PTR(-ENOMEM);
3031 	}
3032 
3033 	/* Use the same keys as the compatible histogram */
3034 	strcat(cmd, "keys=");
3035 
3036 	for_each_hist_key_field(i, hist_data) {
3037 		key_field = hist_data->fields[i];
3038 		if (!first)
3039 			strcat(cmd, ",");
3040 		strcat(cmd, key_field->field->name);
3041 		first = false;
3042 	}
3043 
3044 	/* Create the synthetic field variable specification */
3045 	strcat(cmd, ":synthetic_");
3046 	strcat(cmd, field_name);
3047 	strcat(cmd, "=");
3048 	strcat(cmd, field_name);
3049 
3050 	/* Use the same filter as the compatible histogram */
3051 	saved_filter = find_trigger_filter(hist_data, file);
3052 	if (saved_filter) {
3053 		strcat(cmd, " if ");
3054 		strcat(cmd, saved_filter);
3055 	}
3056 
3057 	var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3058 	if (!var_hist->cmd) {
3059 		kfree(cmd);
3060 		kfree(var_hist);
3061 		return ERR_PTR(-ENOMEM);
3062 	}
3063 
3064 	/* Save the compatible histogram information */
3065 	var_hist->hist_data = hist_data;
3066 
3067 	/* Create the new histogram with our variable */
3068 	ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
3069 				       "", "hist", cmd);
3070 	if (ret) {
3071 		kfree(cmd);
3072 		kfree(var_hist->cmd);
3073 		kfree(var_hist);
3074 		hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
3075 		return ERR_PTR(ret);
3076 	}
3077 
3078 	kfree(cmd);
3079 
3080 	/* If we can't find the variable, something went wrong */
3081 	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3082 					     event_name, field_name);
3083 	if (IS_ERR_OR_NULL(event_var)) {
3084 		kfree(var_hist->cmd);
3085 		kfree(var_hist);
3086 		hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
3087 		return ERR_PTR(-EINVAL);
3088 	}
3089 
3090 	n = target_hist_data->n_field_var_hists;
3091 	target_hist_data->field_var_hists[n] = var_hist;
3092 	target_hist_data->n_field_var_hists++;
3093 
3094 	return event_var;
3095 }
3096 
3097 static struct hist_field *
3098 find_target_event_var(struct hist_trigger_data *hist_data,
3099 		      char *subsys_name, char *event_name, char *var_name)
3100 {
3101 	struct trace_event_file *file = hist_data->event_file;
3102 	struct hist_field *hist_field = NULL;
3103 
3104 	if (subsys_name) {
3105 		struct trace_event_call *call;
3106 
3107 		if (!event_name)
3108 			return NULL;
3109 
3110 		call = file->event_call;
3111 
3112 		if (strcmp(subsys_name, call->class->system) != 0)
3113 			return NULL;
3114 
3115 		if (strcmp(event_name, trace_event_name(call)) != 0)
3116 			return NULL;
3117 	}
3118 
3119 	hist_field = find_var_field(hist_data, var_name);
3120 
3121 	return hist_field;
3122 }
3123 
3124 static inline void __update_field_vars(struct tracing_map_elt *elt,
3125 				       struct trace_buffer *buffer,
3126 				       struct ring_buffer_event *rbe,
3127 				       void *rec,
3128 				       struct field_var **field_vars,
3129 				       unsigned int n_field_vars,
3130 				       unsigned int field_var_str_start)
3131 {
3132 	struct hist_elt_data *elt_data = elt->private_data;
3133 	unsigned int i, j, var_idx;
3134 	u64 var_val;
3135 
3136 	/* Make sure stacktrace can fit in the string variable length */
3137 	BUILD_BUG_ON((HIST_STACKTRACE_DEPTH + 1) * sizeof(long) >= STR_VAR_LEN_MAX);
3138 
3139 	for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3140 		struct field_var *field_var = field_vars[i];
3141 		struct hist_field *var = field_var->var;
3142 		struct hist_field *val = field_var->val;
3143 
3144 		var_val = hist_fn_call(val, elt, buffer, rbe, rec);
3145 		var_idx = var->var.idx;
3146 
3147 		if (val->flags & (HIST_FIELD_FL_STRING |
3148 				  HIST_FIELD_FL_STACKTRACE)) {
3149 			char *str = elt_data->field_var_str[j++];
3150 			char *val_str = (char *)(uintptr_t)var_val;
3151 			unsigned int size;
3152 
3153 			if (val->flags & HIST_FIELD_FL_STRING) {
3154 				size = min(val->size, STR_VAR_LEN_MAX);
3155 				strscpy(str, val_str, size);
3156 			} else {
3157 				char *stack_start = str + sizeof(unsigned long);
3158 				int e;
3159 
3160 				e = stack_trace_save((void *)stack_start,
3161 						     HIST_STACKTRACE_DEPTH,
3162 						     HIST_STACKTRACE_SKIP);
3163 				if (e < HIST_STACKTRACE_DEPTH - 1)
3164 					((unsigned long *)stack_start)[e] = 0;
3165 				*((unsigned long *)str) = e;
3166 			}
3167 			var_val = (u64)(uintptr_t)str;
3168 		}
3169 		tracing_map_set_var(elt, var_idx, var_val);
3170 	}
3171 }
3172 
3173 static void update_field_vars(struct hist_trigger_data *hist_data,
3174 			      struct tracing_map_elt *elt,
3175 			      struct trace_buffer *buffer,
3176 			      struct ring_buffer_event *rbe,
3177 			      void *rec)
3178 {
3179 	__update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
3180 			    hist_data->n_field_vars, 0);
3181 }
3182 
3183 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3184 				 struct tracing_map_elt *elt,
3185 				 struct trace_buffer *buffer,  void *rec,
3186 				 struct ring_buffer_event *rbe, void *key,
3187 				 struct action_data *data, u64 *var_ref_vals)
3188 {
3189 	__update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
3190 			    hist_data->n_save_vars, hist_data->n_field_var_str);
3191 }
3192 
3193 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3194 				     struct trace_event_file *file,
3195 				     char *name, int size, const char *type)
3196 {
3197 	struct hist_field *var;
3198 	int idx;
3199 
3200 	if (find_var(hist_data, file, name) && !hist_data->remove) {
3201 		var = ERR_PTR(-EINVAL);
3202 		goto out;
3203 	}
3204 
3205 	var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3206 	if (!var) {
3207 		var = ERR_PTR(-ENOMEM);
3208 		goto out;
3209 	}
3210 
3211 	idx = tracing_map_add_var(hist_data->map);
3212 	if (idx < 0) {
3213 		kfree(var);
3214 		var = ERR_PTR(-EINVAL);
3215 		goto out;
3216 	}
3217 
3218 	var->ref = 1;
3219 	var->flags = HIST_FIELD_FL_VAR;
3220 	var->var.idx = idx;
3221 	var->var.hist_data = var->hist_data = hist_data;
3222 	var->size = size;
3223 	var->var.name = kstrdup(name, GFP_KERNEL);
3224 	var->type = kstrdup_const(type, GFP_KERNEL);
3225 	if (!var->var.name || !var->type) {
3226 		kfree_const(var->type);
3227 		kfree(var->var.name);
3228 		kfree(var);
3229 		var = ERR_PTR(-ENOMEM);
3230 	}
3231  out:
3232 	return var;
3233 }
3234 
3235 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3236 					  struct trace_event_file *file,
3237 					  char *field_name)
3238 {
3239 	struct hist_field *val = NULL, *var = NULL;
3240 	unsigned long flags = HIST_FIELD_FL_VAR;
3241 	struct trace_array *tr = file->tr;
3242 	struct field_var *field_var;
3243 	int ret = 0;
3244 
3245 	if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3246 		hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3247 		ret = -EINVAL;
3248 		goto err;
3249 	}
3250 
3251 	val = parse_atom(hist_data, file, field_name, &flags, NULL);
3252 	if (IS_ERR(val)) {
3253 		hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3254 		ret = PTR_ERR(val);
3255 		goto err;
3256 	}
3257 
3258 	var = create_var(hist_data, file, field_name, val->size, val->type);
3259 	if (IS_ERR(var)) {
3260 		hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3261 		kfree(val);
3262 		ret = PTR_ERR(var);
3263 		goto err;
3264 	}
3265 
3266 	field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3267 	if (!field_var) {
3268 		kfree(val);
3269 		kfree(var);
3270 		ret =  -ENOMEM;
3271 		goto err;
3272 	}
3273 
3274 	field_var->var = var;
3275 	field_var->val = val;
3276  out:
3277 	return field_var;
3278  err:
3279 	field_var = ERR_PTR(ret);
3280 	goto out;
3281 }
3282 
3283 /**
3284  * create_target_field_var - Automatically create a variable for a field
3285  * @target_hist_data: The target hist trigger
3286  * @subsys_name: Optional subsystem name
3287  * @event_name: Optional event name
3288  * @var_name: The name of the field (and the resulting variable)
3289  *
3290  * Hist trigger actions fetch data from variables, not directly from
3291  * events.  However, for convenience, users are allowed to directly
3292  * specify an event field in an action, which will be automatically
3293  * converted into a variable on their behalf.
3294  *
3295  * This function creates a field variable with the name var_name on
3296  * the hist trigger currently being defined on the target event.  If
3297  * subsys_name and event_name are specified, this function simply
3298  * verifies that they do in fact match the target event subsystem and
3299  * event name.
3300  *
3301  * Return: The variable created for the field.
3302  */
3303 static struct field_var *
3304 create_target_field_var(struct hist_trigger_data *target_hist_data,
3305 			char *subsys_name, char *event_name, char *var_name)
3306 {
3307 	struct trace_event_file *file = target_hist_data->event_file;
3308 
3309 	if (subsys_name) {
3310 		struct trace_event_call *call;
3311 
3312 		if (!event_name)
3313 			return NULL;
3314 
3315 		call = file->event_call;
3316 
3317 		if (strcmp(subsys_name, call->class->system) != 0)
3318 			return NULL;
3319 
3320 		if (strcmp(event_name, trace_event_name(call)) != 0)
3321 			return NULL;
3322 	}
3323 
3324 	return create_field_var(target_hist_data, file, var_name);
3325 }
3326 
3327 static bool check_track_val_max(u64 track_val, u64 var_val)
3328 {
3329 	if (var_val <= track_val)
3330 		return false;
3331 
3332 	return true;
3333 }
3334 
3335 static bool check_track_val_changed(u64 track_val, u64 var_val)
3336 {
3337 	if (var_val == track_val)
3338 		return false;
3339 
3340 	return true;
3341 }
3342 
3343 static u64 get_track_val(struct hist_trigger_data *hist_data,
3344 			 struct tracing_map_elt *elt,
3345 			 struct action_data *data)
3346 {
3347 	unsigned int track_var_idx = data->track_data.track_var->var.idx;
3348 	u64 track_val;
3349 
3350 	track_val = tracing_map_read_var(elt, track_var_idx);
3351 
3352 	return track_val;
3353 }
3354 
3355 static void save_track_val(struct hist_trigger_data *hist_data,
3356 			   struct tracing_map_elt *elt,
3357 			   struct action_data *data, u64 var_val)
3358 {
3359 	unsigned int track_var_idx = data->track_data.track_var->var.idx;
3360 
3361 	tracing_map_set_var(elt, track_var_idx, var_val);
3362 }
3363 
3364 static void save_track_data(struct hist_trigger_data *hist_data,
3365 			    struct tracing_map_elt *elt,
3366 			    struct trace_buffer *buffer, void *rec,
3367 			    struct ring_buffer_event *rbe, void *key,
3368 			    struct action_data *data, u64 *var_ref_vals)
3369 {
3370 	if (data->track_data.save_data)
3371 		data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
3372 					   key, data, var_ref_vals);
3373 }
3374 
3375 static bool check_track_val(struct tracing_map_elt *elt,
3376 			    struct action_data *data,
3377 			    u64 var_val)
3378 {
3379 	struct hist_trigger_data *hist_data;
3380 	u64 track_val;
3381 
3382 	hist_data = data->track_data.track_var->hist_data;
3383 	track_val = get_track_val(hist_data, elt, data);
3384 
3385 	return data->track_data.check_val(track_val, var_val);
3386 }
3387 
3388 #ifdef CONFIG_TRACER_SNAPSHOT
3389 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3390 {
3391 	/* called with tr->max_lock held */
3392 	struct track_data *track_data = tr->cond_snapshot->cond_data;
3393 	struct hist_elt_data *elt_data, *track_elt_data;
3394 	struct snapshot_context *context = cond_data;
3395 	struct action_data *action;
3396 	u64 track_val;
3397 
3398 	if (!track_data)
3399 		return false;
3400 
3401 	action = track_data->action_data;
3402 
3403 	track_val = get_track_val(track_data->hist_data, context->elt,
3404 				  track_data->action_data);
3405 
3406 	if (!action->track_data.check_val(track_data->track_val, track_val))
3407 		return false;
3408 
3409 	track_data->track_val = track_val;
3410 	memcpy(track_data->key, context->key, track_data->key_len);
3411 
3412 	elt_data = context->elt->private_data;
3413 	track_elt_data = track_data->elt.private_data;
3414 	if (elt_data->comm)
3415 		strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3416 
3417 	track_data->updated = true;
3418 
3419 	return true;
3420 }
3421 
3422 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3423 				     struct tracing_map_elt *elt,
3424 				     struct trace_buffer *buffer, void *rec,
3425 				     struct ring_buffer_event *rbe, void *key,
3426 				     struct action_data *data,
3427 				     u64 *var_ref_vals)
3428 {
3429 	struct trace_event_file *file = hist_data->event_file;
3430 	struct snapshot_context context;
3431 
3432 	context.elt = elt;
3433 	context.key = key;
3434 
3435 	tracing_snapshot_cond(file->tr, &context);
3436 }
3437 
3438 static void hist_trigger_print_key(struct seq_file *m,
3439 				   struct hist_trigger_data *hist_data,
3440 				   void *key,
3441 				   struct tracing_map_elt *elt);
3442 
3443 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3444 {
3445 	unsigned int i;
3446 
3447 	if (!hist_data->n_actions)
3448 		return NULL;
3449 
3450 	for (i = 0; i < hist_data->n_actions; i++) {
3451 		struct action_data *data = hist_data->actions[i];
3452 
3453 		if (data->action == ACTION_SNAPSHOT)
3454 			return data;
3455 	}
3456 
3457 	return NULL;
3458 }
3459 
3460 static void track_data_snapshot_print(struct seq_file *m,
3461 				      struct hist_trigger_data *hist_data)
3462 {
3463 	struct trace_event_file *file = hist_data->event_file;
3464 	struct track_data *track_data;
3465 	struct action_data *action;
3466 
3467 	track_data = tracing_cond_snapshot_data(file->tr);
3468 	if (!track_data)
3469 		return;
3470 
3471 	if (!track_data->updated)
3472 		return;
3473 
3474 	action = snapshot_action(hist_data);
3475 	if (!action)
3476 		return;
3477 
3478 	seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3479 	seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3480 		   action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3481 		   action->track_data.var_str, track_data->track_val);
3482 
3483 	seq_puts(m, "\ttriggered by event with key: ");
3484 	hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3485 	seq_putc(m, '\n');
3486 }
3487 #else
3488 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3489 {
3490 	return false;
3491 }
3492 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3493 				     struct tracing_map_elt *elt,
3494 				     struct trace_buffer *buffer, void *rec,
3495 				     struct ring_buffer_event *rbe, void *key,
3496 				     struct action_data *data,
3497 				     u64 *var_ref_vals) {}
3498 static void track_data_snapshot_print(struct seq_file *m,
3499 				      struct hist_trigger_data *hist_data) {}
3500 #endif /* CONFIG_TRACER_SNAPSHOT */
3501 
3502 static void track_data_print(struct seq_file *m,
3503 			     struct hist_trigger_data *hist_data,
3504 			     struct tracing_map_elt *elt,
3505 			     struct action_data *data)
3506 {
3507 	u64 track_val = get_track_val(hist_data, elt, data);
3508 	unsigned int i, save_var_idx;
3509 
3510 	if (data->handler == HANDLER_ONMAX)
3511 		seq_printf(m, "\n\tmax: %10llu", track_val);
3512 	else if (data->handler == HANDLER_ONCHANGE)
3513 		seq_printf(m, "\n\tchanged: %10llu", track_val);
3514 
3515 	if (data->action == ACTION_SNAPSHOT)
3516 		return;
3517 
3518 	for (i = 0; i < hist_data->n_save_vars; i++) {
3519 		struct hist_field *save_val = hist_data->save_vars[i]->val;
3520 		struct hist_field *save_var = hist_data->save_vars[i]->var;
3521 		u64 val;
3522 
3523 		save_var_idx = save_var->var.idx;
3524 
3525 		val = tracing_map_read_var(elt, save_var_idx);
3526 
3527 		if (save_val->flags & HIST_FIELD_FL_STRING) {
3528 			seq_printf(m, "  %s: %-32s", save_var->var.name,
3529 				   (char *)(uintptr_t)(val));
3530 		} else
3531 			seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3532 	}
3533 }
3534 
3535 static void ontrack_action(struct hist_trigger_data *hist_data,
3536 			   struct tracing_map_elt *elt,
3537 			   struct trace_buffer *buffer, void *rec,
3538 			   struct ring_buffer_event *rbe, void *key,
3539 			   struct action_data *data, u64 *var_ref_vals)
3540 {
3541 	u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3542 
3543 	if (check_track_val(elt, data, var_val)) {
3544 		save_track_val(hist_data, elt, data, var_val);
3545 		save_track_data(hist_data, elt, buffer, rec, rbe,
3546 				key, data, var_ref_vals);
3547 	}
3548 }
3549 
3550 static void action_data_destroy(struct action_data *data)
3551 {
3552 	unsigned int i;
3553 
3554 	lockdep_assert_held(&event_mutex);
3555 
3556 	kfree(data->action_name);
3557 
3558 	for (i = 0; i < data->n_params; i++)
3559 		kfree(data->params[i]);
3560 
3561 	if (data->synth_event)
3562 		data->synth_event->ref--;
3563 
3564 	kfree(data->synth_event_name);
3565 
3566 	kfree(data);
3567 }
3568 
3569 static void track_data_destroy(struct hist_trigger_data *hist_data,
3570 			       struct action_data *data)
3571 {
3572 	struct trace_event_file *file = hist_data->event_file;
3573 
3574 	destroy_hist_field(data->track_data.track_var, 0);
3575 
3576 	if (data->action == ACTION_SNAPSHOT) {
3577 		struct track_data *track_data;
3578 
3579 		track_data = tracing_cond_snapshot_data(file->tr);
3580 		if (track_data && track_data->hist_data == hist_data) {
3581 			tracing_snapshot_cond_disable(file->tr);
3582 			track_data_free(track_data);
3583 		}
3584 	}
3585 
3586 	kfree(data->track_data.var_str);
3587 
3588 	action_data_destroy(data);
3589 }
3590 
3591 static int action_create(struct hist_trigger_data *hist_data,
3592 			 struct action_data *data);
3593 
3594 static int track_data_create(struct hist_trigger_data *hist_data,
3595 			     struct action_data *data)
3596 {
3597 	struct hist_field *var_field, *ref_field, *track_var = NULL;
3598 	struct trace_event_file *file = hist_data->event_file;
3599 	struct trace_array *tr = file->tr;
3600 	char *track_data_var_str;
3601 	int ret = 0;
3602 
3603 	track_data_var_str = data->track_data.var_str;
3604 	if (track_data_var_str[0] != '$') {
3605 		hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3606 		return -EINVAL;
3607 	}
3608 	track_data_var_str++;
3609 
3610 	var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3611 	if (!var_field) {
3612 		hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3613 		return -EINVAL;
3614 	}
3615 
3616 	ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3617 	if (!ref_field)
3618 		return -ENOMEM;
3619 
3620 	data->track_data.var_ref = ref_field;
3621 
3622 	if (data->handler == HANDLER_ONMAX)
3623 		track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3624 	if (IS_ERR(track_var)) {
3625 		hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3626 		ret = PTR_ERR(track_var);
3627 		goto out;
3628 	}
3629 
3630 	if (data->handler == HANDLER_ONCHANGE)
3631 		track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3632 	if (IS_ERR(track_var)) {
3633 		hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3634 		ret = PTR_ERR(track_var);
3635 		goto out;
3636 	}
3637 	data->track_data.track_var = track_var;
3638 
3639 	ret = action_create(hist_data, data);
3640  out:
3641 	return ret;
3642 }
3643 
3644 static int parse_action_params(struct trace_array *tr, char *params,
3645 			       struct action_data *data)
3646 {
3647 	char *param, *saved_param;
3648 	bool first_param = true;
3649 	int ret = 0;
3650 
3651 	while (params) {
3652 		if (data->n_params >= SYNTH_FIELDS_MAX) {
3653 			hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3654 			ret = -EINVAL;
3655 			goto out;
3656 		}
3657 
3658 		param = strsep(&params, ",");
3659 		if (!param) {
3660 			hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3661 			ret = -EINVAL;
3662 			goto out;
3663 		}
3664 
3665 		param = strstrip(param);
3666 		if (strlen(param) < 2) {
3667 			hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3668 			ret = -EINVAL;
3669 			goto out;
3670 		}
3671 
3672 		saved_param = kstrdup(param, GFP_KERNEL);
3673 		if (!saved_param) {
3674 			ret = -ENOMEM;
3675 			goto out;
3676 		}
3677 
3678 		if (first_param && data->use_trace_keyword) {
3679 			data->synth_event_name = saved_param;
3680 			first_param = false;
3681 			continue;
3682 		}
3683 		first_param = false;
3684 
3685 		data->params[data->n_params++] = saved_param;
3686 	}
3687  out:
3688 	return ret;
3689 }
3690 
3691 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3692 			enum handler_id handler)
3693 {
3694 	char *action_name;
3695 	int ret = 0;
3696 
3697 	strsep(&str, ".");
3698 	if (!str) {
3699 		hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3700 		ret = -EINVAL;
3701 		goto out;
3702 	}
3703 
3704 	action_name = strsep(&str, "(");
3705 	if (!action_name || !str) {
3706 		hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3707 		ret = -EINVAL;
3708 		goto out;
3709 	}
3710 
3711 	if (str_has_prefix(action_name, "save")) {
3712 		char *params = strsep(&str, ")");
3713 
3714 		if (!params) {
3715 			hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3716 			ret = -EINVAL;
3717 			goto out;
3718 		}
3719 
3720 		ret = parse_action_params(tr, params, data);
3721 		if (ret)
3722 			goto out;
3723 
3724 		if (handler == HANDLER_ONMAX)
3725 			data->track_data.check_val = check_track_val_max;
3726 		else if (handler == HANDLER_ONCHANGE)
3727 			data->track_data.check_val = check_track_val_changed;
3728 		else {
3729 			hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3730 			ret = -EINVAL;
3731 			goto out;
3732 		}
3733 
3734 		data->track_data.save_data = save_track_data_vars;
3735 		data->fn = ontrack_action;
3736 		data->action = ACTION_SAVE;
3737 	} else if (str_has_prefix(action_name, "snapshot")) {
3738 		char *params = strsep(&str, ")");
3739 
3740 		if (!str) {
3741 			hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3742 			ret = -EINVAL;
3743 			goto out;
3744 		}
3745 
3746 		if (handler == HANDLER_ONMAX)
3747 			data->track_data.check_val = check_track_val_max;
3748 		else if (handler == HANDLER_ONCHANGE)
3749 			data->track_data.check_val = check_track_val_changed;
3750 		else {
3751 			hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3752 			ret = -EINVAL;
3753 			goto out;
3754 		}
3755 
3756 		data->track_data.save_data = save_track_data_snapshot;
3757 		data->fn = ontrack_action;
3758 		data->action = ACTION_SNAPSHOT;
3759 	} else {
3760 		char *params = strsep(&str, ")");
3761 
3762 		if (str_has_prefix(action_name, "trace"))
3763 			data->use_trace_keyword = true;
3764 
3765 		if (params) {
3766 			ret = parse_action_params(tr, params, data);
3767 			if (ret)
3768 				goto out;
3769 		}
3770 
3771 		if (handler == HANDLER_ONMAX)
3772 			data->track_data.check_val = check_track_val_max;
3773 		else if (handler == HANDLER_ONCHANGE)
3774 			data->track_data.check_val = check_track_val_changed;
3775 
3776 		if (handler != HANDLER_ONMATCH) {
3777 			data->track_data.save_data = action_trace;
3778 			data->fn = ontrack_action;
3779 		} else
3780 			data->fn = action_trace;
3781 
3782 		data->action = ACTION_TRACE;
3783 	}
3784 
3785 	data->action_name = kstrdup(action_name, GFP_KERNEL);
3786 	if (!data->action_name) {
3787 		ret = -ENOMEM;
3788 		goto out;
3789 	}
3790 
3791 	data->handler = handler;
3792  out:
3793 	return ret;
3794 }
3795 
3796 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3797 					    char *str, enum handler_id handler)
3798 {
3799 	struct action_data *data;
3800 	int ret = -EINVAL;
3801 	char *var_str;
3802 
3803 	data = kzalloc(sizeof(*data), GFP_KERNEL);
3804 	if (!data)
3805 		return ERR_PTR(-ENOMEM);
3806 
3807 	var_str = strsep(&str, ")");
3808 	if (!var_str || !str) {
3809 		ret = -EINVAL;
3810 		goto free;
3811 	}
3812 
3813 	data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3814 	if (!data->track_data.var_str) {
3815 		ret = -ENOMEM;
3816 		goto free;
3817 	}
3818 
3819 	ret = action_parse(hist_data->event_file->tr, str, data, handler);
3820 	if (ret)
3821 		goto free;
3822  out:
3823 	return data;
3824  free:
3825 	track_data_destroy(hist_data, data);
3826 	data = ERR_PTR(ret);
3827 	goto out;
3828 }
3829 
3830 static void onmatch_destroy(struct action_data *data)
3831 {
3832 	kfree(data->match_data.event);
3833 	kfree(data->match_data.event_system);
3834 
3835 	action_data_destroy(data);
3836 }
3837 
3838 static void destroy_field_var(struct field_var *field_var)
3839 {
3840 	if (!field_var)
3841 		return;
3842 
3843 	destroy_hist_field(field_var->var, 0);
3844 	destroy_hist_field(field_var->val, 0);
3845 
3846 	kfree(field_var);
3847 }
3848 
3849 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3850 {
3851 	unsigned int i;
3852 
3853 	for (i = 0; i < hist_data->n_field_vars; i++)
3854 		destroy_field_var(hist_data->field_vars[i]);
3855 
3856 	for (i = 0; i < hist_data->n_save_vars; i++)
3857 		destroy_field_var(hist_data->save_vars[i]);
3858 }
3859 
3860 static void save_field_var(struct hist_trigger_data *hist_data,
3861 			   struct field_var *field_var)
3862 {
3863 	hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3864 
3865 	/* Stack traces are saved in the string storage too */
3866 	if (field_var->val->flags & (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
3867 		hist_data->n_field_var_str++;
3868 }
3869 
3870 
3871 static int check_synth_field(struct synth_event *event,
3872 			     struct hist_field *hist_field,
3873 			     unsigned int field_pos)
3874 {
3875 	struct synth_field *field;
3876 
3877 	if (field_pos >= event->n_fields)
3878 		return -EINVAL;
3879 
3880 	field = event->fields[field_pos];
3881 
3882 	/*
3883 	 * A dynamic string synth field can accept static or
3884 	 * dynamic. A static string synth field can only accept a
3885 	 * same-sized static string, which is checked for later.
3886 	 */
3887 	if (strstr(hist_field->type, "char[") && field->is_string
3888 	    && field->is_dynamic)
3889 		return 0;
3890 
3891 	if (strstr(hist_field->type, "long[") && field->is_stack)
3892 		return 0;
3893 
3894 	if (strcmp(field->type, hist_field->type) != 0) {
3895 		if (field->size != hist_field->size ||
3896 		    (!field->is_string && field->is_signed != hist_field->is_signed))
3897 			return -EINVAL;
3898 	}
3899 
3900 	return 0;
3901 }
3902 
3903 static struct hist_field *
3904 trace_action_find_var(struct hist_trigger_data *hist_data,
3905 		      struct action_data *data,
3906 		      char *system, char *event, char *var)
3907 {
3908 	struct trace_array *tr = hist_data->event_file->tr;
3909 	struct hist_field *hist_field;
3910 
3911 	var++; /* skip '$' */
3912 
3913 	hist_field = find_target_event_var(hist_data, system, event, var);
3914 	if (!hist_field) {
3915 		if (!system && data->handler == HANDLER_ONMATCH) {
3916 			system = data->match_data.event_system;
3917 			event = data->match_data.event;
3918 		}
3919 
3920 		hist_field = find_event_var(hist_data, system, event, var);
3921 	}
3922 
3923 	if (!hist_field)
3924 		hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3925 
3926 	return hist_field;
3927 }
3928 
3929 static struct hist_field *
3930 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3931 			      struct action_data *data, char *system,
3932 			      char *event, char *var)
3933 {
3934 	struct hist_field *hist_field = NULL;
3935 	struct field_var *field_var;
3936 
3937 	/*
3938 	 * First try to create a field var on the target event (the
3939 	 * currently being defined).  This will create a variable for
3940 	 * unqualified fields on the target event, or if qualified,
3941 	 * target fields that have qualified names matching the target.
3942 	 */
3943 	field_var = create_target_field_var(hist_data, system, event, var);
3944 
3945 	if (field_var && !IS_ERR(field_var)) {
3946 		save_field_var(hist_data, field_var);
3947 		hist_field = field_var->var;
3948 	} else {
3949 		field_var = NULL;
3950 		/*
3951 		 * If no explicit system.event is specified, default to
3952 		 * looking for fields on the onmatch(system.event.xxx)
3953 		 * event.
3954 		 */
3955 		if (!system && data->handler == HANDLER_ONMATCH) {
3956 			system = data->match_data.event_system;
3957 			event = data->match_data.event;
3958 		}
3959 
3960 		if (!event)
3961 			goto free;
3962 		/*
3963 		 * At this point, we're looking at a field on another
3964 		 * event.  Because we can't modify a hist trigger on
3965 		 * another event to add a variable for a field, we need
3966 		 * to create a new trigger on that event and create the
3967 		 * variable at the same time.
3968 		 */
3969 		hist_field = create_field_var_hist(hist_data, system, event, var);
3970 		if (IS_ERR(hist_field))
3971 			goto free;
3972 	}
3973  out:
3974 	return hist_field;
3975  free:
3976 	destroy_field_var(field_var);
3977 	hist_field = NULL;
3978 	goto out;
3979 }
3980 
3981 static int trace_action_create(struct hist_trigger_data *hist_data,
3982 			       struct action_data *data)
3983 {
3984 	struct trace_array *tr = hist_data->event_file->tr;
3985 	char *event_name, *param, *system = NULL;
3986 	struct hist_field *hist_field, *var_ref;
3987 	unsigned int i;
3988 	unsigned int field_pos = 0;
3989 	struct synth_event *event;
3990 	char *synth_event_name;
3991 	int var_ref_idx, ret = 0;
3992 
3993 	lockdep_assert_held(&event_mutex);
3994 
3995 	/* Sanity check to avoid out-of-bound write on 'data->var_ref_idx' */
3996 	if (data->n_params > SYNTH_FIELDS_MAX)
3997 		return -EINVAL;
3998 
3999 	if (data->use_trace_keyword)
4000 		synth_event_name = data->synth_event_name;
4001 	else
4002 		synth_event_name = data->action_name;
4003 
4004 	event = find_synth_event(synth_event_name);
4005 	if (!event) {
4006 		hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
4007 		return -EINVAL;
4008 	}
4009 
4010 	event->ref++;
4011 
4012 	for (i = 0; i < data->n_params; i++) {
4013 		char *p;
4014 
4015 		p = param = kstrdup(data->params[i], GFP_KERNEL);
4016 		if (!param) {
4017 			ret = -ENOMEM;
4018 			goto err;
4019 		}
4020 
4021 		system = strsep(&param, ".");
4022 		if (!param) {
4023 			param = (char *)system;
4024 			system = event_name = NULL;
4025 		} else {
4026 			event_name = strsep(&param, ".");
4027 			if (!param) {
4028 				kfree(p);
4029 				ret = -EINVAL;
4030 				goto err;
4031 			}
4032 		}
4033 
4034 		if (param[0] == '$')
4035 			hist_field = trace_action_find_var(hist_data, data,
4036 							   system, event_name,
4037 							   param);
4038 		else
4039 			hist_field = trace_action_create_field_var(hist_data,
4040 								   data,
4041 								   system,
4042 								   event_name,
4043 								   param);
4044 
4045 		if (!hist_field) {
4046 			kfree(p);
4047 			ret = -EINVAL;
4048 			goto err;
4049 		}
4050 
4051 		if (check_synth_field(event, hist_field, field_pos) == 0) {
4052 			var_ref = create_var_ref(hist_data, hist_field,
4053 						 system, event_name);
4054 			if (!var_ref) {
4055 				kfree(p);
4056 				ret = -ENOMEM;
4057 				goto err;
4058 			}
4059 
4060 			var_ref_idx = find_var_ref_idx(hist_data, var_ref);
4061 			if (WARN_ON(var_ref_idx < 0)) {
4062 				kfree(p);
4063 				ret = var_ref_idx;
4064 				goto err;
4065 			}
4066 
4067 			data->var_ref_idx[i] = var_ref_idx;
4068 
4069 			field_pos++;
4070 			kfree(p);
4071 			continue;
4072 		}
4073 
4074 		hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
4075 		kfree(p);
4076 		ret = -EINVAL;
4077 		goto err;
4078 	}
4079 
4080 	if (field_pos != event->n_fields) {
4081 		hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
4082 		ret = -EINVAL;
4083 		goto err;
4084 	}
4085 
4086 	data->synth_event = event;
4087  out:
4088 	return ret;
4089  err:
4090 	event->ref--;
4091 
4092 	goto out;
4093 }
4094 
4095 static int action_create(struct hist_trigger_data *hist_data,
4096 			 struct action_data *data)
4097 {
4098 	struct trace_event_file *file = hist_data->event_file;
4099 	struct trace_array *tr = file->tr;
4100 	struct track_data *track_data;
4101 	struct field_var *field_var;
4102 	unsigned int i;
4103 	char *param;
4104 	int ret = 0;
4105 
4106 	if (data->action == ACTION_TRACE)
4107 		return trace_action_create(hist_data, data);
4108 
4109 	if (data->action == ACTION_SNAPSHOT) {
4110 		track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4111 		if (IS_ERR(track_data)) {
4112 			ret = PTR_ERR(track_data);
4113 			goto out;
4114 		}
4115 
4116 		ret = tracing_snapshot_cond_enable(file->tr, track_data,
4117 						   cond_snapshot_update);
4118 		if (ret)
4119 			track_data_free(track_data);
4120 
4121 		goto out;
4122 	}
4123 
4124 	if (data->action == ACTION_SAVE) {
4125 		if (hist_data->n_save_vars) {
4126 			ret = -EEXIST;
4127 			hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4128 			goto out;
4129 		}
4130 
4131 		for (i = 0; i < data->n_params; i++) {
4132 			param = kstrdup(data->params[i], GFP_KERNEL);
4133 			if (!param) {
4134 				ret = -ENOMEM;
4135 				goto out;
4136 			}
4137 
4138 			field_var = create_target_field_var(hist_data, NULL, NULL, param);
4139 			if (IS_ERR(field_var)) {
4140 				hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4141 					 errpos(param));
4142 				ret = PTR_ERR(field_var);
4143 				kfree(param);
4144 				goto out;
4145 			}
4146 
4147 			hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4148 			if (field_var->val->flags &
4149 			    (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
4150 				hist_data->n_save_var_str++;
4151 			kfree(param);
4152 		}
4153 	}
4154  out:
4155 	return ret;
4156 }
4157 
4158 static int onmatch_create(struct hist_trigger_data *hist_data,
4159 			  struct action_data *data)
4160 {
4161 	return action_create(hist_data, data);
4162 }
4163 
4164 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4165 {
4166 	char *match_event, *match_event_system;
4167 	struct action_data *data;
4168 	int ret = -EINVAL;
4169 
4170 	data = kzalloc(sizeof(*data), GFP_KERNEL);
4171 	if (!data)
4172 		return ERR_PTR(-ENOMEM);
4173 
4174 	match_event = strsep(&str, ")");
4175 	if (!match_event || !str) {
4176 		hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4177 		goto free;
4178 	}
4179 
4180 	match_event_system = strsep(&match_event, ".");
4181 	if (!match_event) {
4182 		hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4183 		goto free;
4184 	}
4185 
4186 	if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4187 		hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4188 		goto free;
4189 	}
4190 
4191 	data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4192 	if (!data->match_data.event) {
4193 		ret = -ENOMEM;
4194 		goto free;
4195 	}
4196 
4197 	data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4198 	if (!data->match_data.event_system) {
4199 		ret = -ENOMEM;
4200 		goto free;
4201 	}
4202 
4203 	ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4204 	if (ret)
4205 		goto free;
4206  out:
4207 	return data;
4208  free:
4209 	onmatch_destroy(data);
4210 	data = ERR_PTR(ret);
4211 	goto out;
4212 }
4213 
4214 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4215 {
4216 	hist_data->fields[HITCOUNT_IDX] =
4217 		create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4218 	if (!hist_data->fields[HITCOUNT_IDX])
4219 		return -ENOMEM;
4220 
4221 	hist_data->n_vals++;
4222 	hist_data->n_fields++;
4223 
4224 	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4225 		return -EINVAL;
4226 
4227 	return 0;
4228 }
4229 
4230 static int __create_val_field(struct hist_trigger_data *hist_data,
4231 			      unsigned int val_idx,
4232 			      struct trace_event_file *file,
4233 			      char *var_name, char *field_str,
4234 			      unsigned long flags)
4235 {
4236 	struct hist_field *hist_field;
4237 	int ret = 0, n_subexprs = 0;
4238 
4239 	hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs);
4240 	if (IS_ERR(hist_field)) {
4241 		ret = PTR_ERR(hist_field);
4242 		goto out;
4243 	}
4244 
4245 	/* values and variables should not have some modifiers */
4246 	if (hist_field->flags & HIST_FIELD_FL_VAR) {
4247 		/* Variable */
4248 		if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT |
4249 					 HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2))
4250 			goto err;
4251 	} else {
4252 		/* Value */
4253 		if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT |
4254 					 HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2 |
4255 					 HIST_FIELD_FL_SYM | HIST_FIELD_FL_SYM_OFFSET |
4256 					 HIST_FIELD_FL_SYSCALL | HIST_FIELD_FL_STACKTRACE))
4257 			goto err;
4258 	}
4259 
4260 	hist_data->fields[val_idx] = hist_field;
4261 
4262 	++hist_data->n_vals;
4263 	++hist_data->n_fields;
4264 
4265 	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4266 		ret = -EINVAL;
4267  out:
4268 	return ret;
4269  err:
4270 	hist_err(file->tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str));
4271 	return -EINVAL;
4272 }
4273 
4274 static int create_val_field(struct hist_trigger_data *hist_data,
4275 			    unsigned int val_idx,
4276 			    struct trace_event_file *file,
4277 			    char *field_str)
4278 {
4279 	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4280 		return -EINVAL;
4281 
4282 	return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4283 }
4284 
4285 static const char no_comm[] = "(no comm)";
4286 
4287 static u64 hist_field_execname(struct hist_field *hist_field,
4288 			       struct tracing_map_elt *elt,
4289 			       struct trace_buffer *buffer,
4290 			       struct ring_buffer_event *rbe,
4291 			       void *event)
4292 {
4293 	struct hist_elt_data *elt_data;
4294 
4295 	if (WARN_ON_ONCE(!elt))
4296 		return (u64)(unsigned long)no_comm;
4297 
4298 	elt_data = elt->private_data;
4299 
4300 	if (WARN_ON_ONCE(!elt_data->comm))
4301 		return (u64)(unsigned long)no_comm;
4302 
4303 	return (u64)(unsigned long)(elt_data->comm);
4304 }
4305 
4306 static u64 hist_field_stack(struct hist_field *hist_field,
4307 			    struct tracing_map_elt *elt,
4308 			    struct trace_buffer *buffer,
4309 			    struct ring_buffer_event *rbe,
4310 			    void *event)
4311 {
4312 	u32 str_item = *(u32 *)(event + hist_field->field->offset);
4313 	int str_loc = str_item & 0xffff;
4314 	char *addr = (char *)(event + str_loc);
4315 
4316 	return (u64)(unsigned long)addr;
4317 }
4318 
4319 static u64 hist_fn_call(struct hist_field *hist_field,
4320 			struct tracing_map_elt *elt,
4321 			struct trace_buffer *buffer,
4322 			struct ring_buffer_event *rbe,
4323 			void *event)
4324 {
4325 	switch (hist_field->fn_num) {
4326 	case HIST_FIELD_FN_VAR_REF:
4327 		return hist_field_var_ref(hist_field, elt, buffer, rbe, event);
4328 	case HIST_FIELD_FN_COUNTER:
4329 		return hist_field_counter(hist_field, elt, buffer, rbe, event);
4330 	case HIST_FIELD_FN_CONST:
4331 		return hist_field_const(hist_field, elt, buffer, rbe, event);
4332 	case HIST_FIELD_FN_LOG2:
4333 		return hist_field_log2(hist_field, elt, buffer, rbe, event);
4334 	case HIST_FIELD_FN_BUCKET:
4335 		return hist_field_bucket(hist_field, elt, buffer, rbe, event);
4336 	case HIST_FIELD_FN_TIMESTAMP:
4337 		return hist_field_timestamp(hist_field, elt, buffer, rbe, event);
4338 	case HIST_FIELD_FN_CPU:
4339 		return hist_field_cpu(hist_field, elt, buffer, rbe, event);
4340 	case HIST_FIELD_FN_STRING:
4341 		return hist_field_string(hist_field, elt, buffer, rbe, event);
4342 	case HIST_FIELD_FN_DYNSTRING:
4343 		return hist_field_dynstring(hist_field, elt, buffer, rbe, event);
4344 	case HIST_FIELD_FN_RELDYNSTRING:
4345 		return hist_field_reldynstring(hist_field, elt, buffer, rbe, event);
4346 	case HIST_FIELD_FN_PSTRING:
4347 		return hist_field_pstring(hist_field, elt, buffer, rbe, event);
4348 	case HIST_FIELD_FN_S64:
4349 		return hist_field_s64(hist_field, elt, buffer, rbe, event);
4350 	case HIST_FIELD_FN_U64:
4351 		return hist_field_u64(hist_field, elt, buffer, rbe, event);
4352 	case HIST_FIELD_FN_S32:
4353 		return hist_field_s32(hist_field, elt, buffer, rbe, event);
4354 	case HIST_FIELD_FN_U32:
4355 		return hist_field_u32(hist_field, elt, buffer, rbe, event);
4356 	case HIST_FIELD_FN_S16:
4357 		return hist_field_s16(hist_field, elt, buffer, rbe, event);
4358 	case HIST_FIELD_FN_U16:
4359 		return hist_field_u16(hist_field, elt, buffer, rbe, event);
4360 	case HIST_FIELD_FN_S8:
4361 		return hist_field_s8(hist_field, elt, buffer, rbe, event);
4362 	case HIST_FIELD_FN_U8:
4363 		return hist_field_u8(hist_field, elt, buffer, rbe, event);
4364 	case HIST_FIELD_FN_UMINUS:
4365 		return hist_field_unary_minus(hist_field, elt, buffer, rbe, event);
4366 	case HIST_FIELD_FN_MINUS:
4367 		return hist_field_minus(hist_field, elt, buffer, rbe, event);
4368 	case HIST_FIELD_FN_PLUS:
4369 		return hist_field_plus(hist_field, elt, buffer, rbe, event);
4370 	case HIST_FIELD_FN_DIV:
4371 		return hist_field_div(hist_field, elt, buffer, rbe, event);
4372 	case HIST_FIELD_FN_MULT:
4373 		return hist_field_mult(hist_field, elt, buffer, rbe, event);
4374 	case HIST_FIELD_FN_DIV_POWER2:
4375 		return div_by_power_of_two(hist_field, elt, buffer, rbe, event);
4376 	case HIST_FIELD_FN_DIV_NOT_POWER2:
4377 		return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event);
4378 	case HIST_FIELD_FN_DIV_MULT_SHIFT:
4379 		return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event);
4380 	case HIST_FIELD_FN_EXECNAME:
4381 		return hist_field_execname(hist_field, elt, buffer, rbe, event);
4382 	case HIST_FIELD_FN_STACK:
4383 		return hist_field_stack(hist_field, elt, buffer, rbe, event);
4384 	default:
4385 		return 0;
4386 	}
4387 }
4388 
4389 /* Convert a var that points to common_pid.execname to a string */
4390 static void update_var_execname(struct hist_field *hist_field)
4391 {
4392 	hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR |
4393 		HIST_FIELD_FL_EXECNAME;
4394 	hist_field->size = MAX_FILTER_STR_VAL;
4395 	hist_field->is_signed = 0;
4396 
4397 	kfree_const(hist_field->type);
4398 	hist_field->type = "char[]";
4399 
4400 	hist_field->fn_num = HIST_FIELD_FN_EXECNAME;
4401 }
4402 
4403 static int create_var_field(struct hist_trigger_data *hist_data,
4404 			    unsigned int val_idx,
4405 			    struct trace_event_file *file,
4406 			    char *var_name, char *expr_str)
4407 {
4408 	struct trace_array *tr = hist_data->event_file->tr;
4409 	unsigned long flags = 0;
4410 	int ret;
4411 
4412 	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4413 		return -EINVAL;
4414 
4415 	if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4416 		hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4417 		return -EINVAL;
4418 	}
4419 
4420 	flags |= HIST_FIELD_FL_VAR;
4421 	hist_data->n_vars++;
4422 	if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4423 		return -EINVAL;
4424 
4425 	ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4426 
4427 	if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME)
4428 		update_var_execname(hist_data->fields[val_idx]);
4429 
4430 	if (!ret && hist_data->fields[val_idx]->flags &
4431 	    (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
4432 		hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
4433 
4434 	return ret;
4435 }
4436 
4437 static int create_val_fields(struct hist_trigger_data *hist_data,
4438 			     struct trace_event_file *file)
4439 {
4440 	unsigned int i, j = 1, n_hitcount = 0;
4441 	char *fields_str, *field_str;
4442 	int ret;
4443 
4444 	ret = create_hitcount_val(hist_data);
4445 	if (ret)
4446 		goto out;
4447 
4448 	fields_str = hist_data->attrs->vals_str;
4449 	if (!fields_str)
4450 		goto out;
4451 
4452 	for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4453 		     j < TRACING_MAP_VALS_MAX; i++) {
4454 		field_str = strsep(&fields_str, ",");
4455 		if (!field_str)
4456 			break;
4457 
4458 		if (strcmp(field_str, "hitcount") == 0) {
4459 			if (!n_hitcount++)
4460 				continue;
4461 		}
4462 
4463 		ret = create_val_field(hist_data, j++, file, field_str);
4464 		if (ret)
4465 			goto out;
4466 	}
4467 
4468 	if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4469 		ret = -EINVAL;
4470  out:
4471 	/* There is only raw hitcount but nohitcount suppresses it. */
4472 	if (j == 1 && hist_data->attrs->no_hitcount) {
4473 		hist_err(hist_data->event_file->tr, HIST_ERR_NEED_NOHC_VAL, 0);
4474 		ret = -ENOENT;
4475 	}
4476 
4477 	return ret;
4478 }
4479 
4480 static int create_key_field(struct hist_trigger_data *hist_data,
4481 			    unsigned int key_idx,
4482 			    unsigned int key_offset,
4483 			    struct trace_event_file *file,
4484 			    char *field_str)
4485 {
4486 	struct trace_array *tr = hist_data->event_file->tr;
4487 	struct hist_field *hist_field = NULL;
4488 	unsigned long flags = 0;
4489 	unsigned int key_size;
4490 	int ret = 0, n_subexprs = 0;
4491 
4492 	if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4493 		return -EINVAL;
4494 
4495 	flags |= HIST_FIELD_FL_KEY;
4496 
4497 	if (strcmp(field_str, "stacktrace") == 0) {
4498 		flags |= HIST_FIELD_FL_STACKTRACE;
4499 		key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4500 		hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4501 	} else {
4502 		hist_field = parse_expr(hist_data, file, field_str, flags,
4503 					NULL, &n_subexprs);
4504 		if (IS_ERR(hist_field)) {
4505 			ret = PTR_ERR(hist_field);
4506 			goto out;
4507 		}
4508 
4509 		if (field_has_hist_vars(hist_field, 0))	{
4510 			hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4511 			destroy_hist_field(hist_field, 0);
4512 			ret = -EINVAL;
4513 			goto out;
4514 		}
4515 
4516 		key_size = hist_field->size;
4517 	}
4518 
4519 	hist_data->fields[key_idx] = hist_field;
4520 
4521 	key_size = ALIGN(key_size, sizeof(u64));
4522 	hist_data->fields[key_idx]->size = key_size;
4523 	hist_data->fields[key_idx]->offset = key_offset;
4524 
4525 	hist_data->key_size += key_size;
4526 
4527 	if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4528 		ret = -EINVAL;
4529 		goto out;
4530 	}
4531 
4532 	hist_data->n_keys++;
4533 	hist_data->n_fields++;
4534 
4535 	if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4536 		return -EINVAL;
4537 
4538 	ret = key_size;
4539  out:
4540 	return ret;
4541 }
4542 
4543 static int create_key_fields(struct hist_trigger_data *hist_data,
4544 			     struct trace_event_file *file)
4545 {
4546 	unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4547 	char *fields_str, *field_str;
4548 	int ret = -EINVAL;
4549 
4550 	fields_str = hist_data->attrs->keys_str;
4551 	if (!fields_str)
4552 		goto out;
4553 
4554 	for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4555 		field_str = strsep(&fields_str, ",");
4556 		if (!field_str)
4557 			break;
4558 		ret = create_key_field(hist_data, i, key_offset,
4559 				       file, field_str);
4560 		if (ret < 0)
4561 			goto out;
4562 		key_offset += ret;
4563 	}
4564 	if (fields_str) {
4565 		ret = -EINVAL;
4566 		goto out;
4567 	}
4568 	ret = 0;
4569  out:
4570 	return ret;
4571 }
4572 
4573 static int create_var_fields(struct hist_trigger_data *hist_data,
4574 			     struct trace_event_file *file)
4575 {
4576 	unsigned int i, j = hist_data->n_vals;
4577 	int ret = 0;
4578 
4579 	unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4580 
4581 	for (i = 0; i < n_vars; i++) {
4582 		char *var_name = hist_data->attrs->var_defs.name[i];
4583 		char *expr = hist_data->attrs->var_defs.expr[i];
4584 
4585 		ret = create_var_field(hist_data, j++, file, var_name, expr);
4586 		if (ret)
4587 			goto out;
4588 	}
4589  out:
4590 	return ret;
4591 }
4592 
4593 static void free_var_defs(struct hist_trigger_data *hist_data)
4594 {
4595 	unsigned int i;
4596 
4597 	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4598 		kfree(hist_data->attrs->var_defs.name[i]);
4599 		kfree(hist_data->attrs->var_defs.expr[i]);
4600 	}
4601 
4602 	hist_data->attrs->var_defs.n_vars = 0;
4603 }
4604 
4605 static int parse_var_defs(struct hist_trigger_data *hist_data)
4606 {
4607 	struct trace_array *tr = hist_data->event_file->tr;
4608 	char *s, *str, *var_name, *field_str;
4609 	unsigned int i, j, n_vars = 0;
4610 	int ret = 0;
4611 
4612 	for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4613 		str = hist_data->attrs->assignment_str[i];
4614 		for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4615 			field_str = strsep(&str, ",");
4616 			if (!field_str)
4617 				break;
4618 
4619 			var_name = strsep(&field_str, "=");
4620 			if (!var_name || !field_str) {
4621 				hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4622 					 errpos(var_name));
4623 				ret = -EINVAL;
4624 				goto free;
4625 			}
4626 
4627 			if (n_vars == TRACING_MAP_VARS_MAX) {
4628 				hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4629 				ret = -EINVAL;
4630 				goto free;
4631 			}
4632 
4633 			s = kstrdup(var_name, GFP_KERNEL);
4634 			if (!s) {
4635 				ret = -ENOMEM;
4636 				goto free;
4637 			}
4638 			hist_data->attrs->var_defs.name[n_vars] = s;
4639 
4640 			s = kstrdup(field_str, GFP_KERNEL);
4641 			if (!s) {
4642 				kfree(hist_data->attrs->var_defs.name[n_vars]);
4643 				hist_data->attrs->var_defs.name[n_vars] = NULL;
4644 				ret = -ENOMEM;
4645 				goto free;
4646 			}
4647 			hist_data->attrs->var_defs.expr[n_vars++] = s;
4648 
4649 			hist_data->attrs->var_defs.n_vars = n_vars;
4650 		}
4651 	}
4652 
4653 	return ret;
4654  free:
4655 	free_var_defs(hist_data);
4656 
4657 	return ret;
4658 }
4659 
4660 static int create_hist_fields(struct hist_trigger_data *hist_data,
4661 			      struct trace_event_file *file)
4662 {
4663 	int ret;
4664 
4665 	ret = parse_var_defs(hist_data);
4666 	if (ret)
4667 		return ret;
4668 
4669 	ret = create_val_fields(hist_data, file);
4670 	if (ret)
4671 		goto out;
4672 
4673 	ret = create_var_fields(hist_data, file);
4674 	if (ret)
4675 		goto out;
4676 
4677 	ret = create_key_fields(hist_data, file);
4678 
4679  out:
4680 	free_var_defs(hist_data);
4681 
4682 	return ret;
4683 }
4684 
4685 static int is_descending(struct trace_array *tr, const char *str)
4686 {
4687 	if (!str)
4688 		return 0;
4689 
4690 	if (strcmp(str, "descending") == 0)
4691 		return 1;
4692 
4693 	if (strcmp(str, "ascending") == 0)
4694 		return 0;
4695 
4696 	hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4697 
4698 	return -EINVAL;
4699 }
4700 
4701 static int create_sort_keys(struct hist_trigger_data *hist_data)
4702 {
4703 	struct trace_array *tr = hist_data->event_file->tr;
4704 	char *fields_str = hist_data->attrs->sort_key_str;
4705 	struct tracing_map_sort_key *sort_key;
4706 	int descending, ret = 0;
4707 	unsigned int i, j, k;
4708 
4709 	hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4710 
4711 	if (!fields_str)
4712 		goto out;
4713 
4714 	for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4715 		struct hist_field *hist_field;
4716 		char *field_str, *field_name;
4717 		const char *test_name;
4718 
4719 		sort_key = &hist_data->sort_keys[i];
4720 
4721 		field_str = strsep(&fields_str, ",");
4722 		if (!field_str)
4723 			break;
4724 
4725 		if (!*field_str) {
4726 			ret = -EINVAL;
4727 			hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4728 			break;
4729 		}
4730 
4731 		if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4732 			hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4733 			ret = -EINVAL;
4734 			break;
4735 		}
4736 
4737 		field_name = strsep(&field_str, ".");
4738 		if (!field_name || !*field_name) {
4739 			ret = -EINVAL;
4740 			hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4741 			break;
4742 		}
4743 
4744 		if (strcmp(field_name, "hitcount") == 0) {
4745 			descending = is_descending(tr, field_str);
4746 			if (descending < 0) {
4747 				ret = descending;
4748 				break;
4749 			}
4750 			sort_key->descending = descending;
4751 			continue;
4752 		}
4753 
4754 		for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4755 			unsigned int idx;
4756 
4757 			hist_field = hist_data->fields[j];
4758 			if (hist_field->flags & HIST_FIELD_FL_VAR)
4759 				continue;
4760 
4761 			idx = k++;
4762 
4763 			test_name = hist_field_name(hist_field, 0);
4764 
4765 			if (strcmp(field_name, test_name) == 0) {
4766 				sort_key->field_idx = idx;
4767 				descending = is_descending(tr, field_str);
4768 				if (descending < 0) {
4769 					ret = descending;
4770 					goto out;
4771 				}
4772 				sort_key->descending = descending;
4773 				break;
4774 			}
4775 		}
4776 		if (j == hist_data->n_fields) {
4777 			ret = -EINVAL;
4778 			hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4779 			break;
4780 		}
4781 	}
4782 
4783 	hist_data->n_sort_keys = i;
4784  out:
4785 	return ret;
4786 }
4787 
4788 static void destroy_actions(struct hist_trigger_data *hist_data)
4789 {
4790 	unsigned int i;
4791 
4792 	for (i = 0; i < hist_data->n_actions; i++) {
4793 		struct action_data *data = hist_data->actions[i];
4794 
4795 		if (data->handler == HANDLER_ONMATCH)
4796 			onmatch_destroy(data);
4797 		else if (data->handler == HANDLER_ONMAX ||
4798 			 data->handler == HANDLER_ONCHANGE)
4799 			track_data_destroy(hist_data, data);
4800 		else
4801 			kfree(data);
4802 	}
4803 }
4804 
4805 static int parse_actions(struct hist_trigger_data *hist_data)
4806 {
4807 	struct trace_array *tr = hist_data->event_file->tr;
4808 	struct action_data *data;
4809 	unsigned int i;
4810 	int ret = 0;
4811 	char *str;
4812 	int len;
4813 
4814 	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4815 		str = hist_data->attrs->action_str[i];
4816 
4817 		if ((len = str_has_prefix(str, "onmatch("))) {
4818 			char *action_str = str + len;
4819 
4820 			data = onmatch_parse(tr, action_str);
4821 			if (IS_ERR(data)) {
4822 				ret = PTR_ERR(data);
4823 				break;
4824 			}
4825 		} else if ((len = str_has_prefix(str, "onmax("))) {
4826 			char *action_str = str + len;
4827 
4828 			data = track_data_parse(hist_data, action_str,
4829 						HANDLER_ONMAX);
4830 			if (IS_ERR(data)) {
4831 				ret = PTR_ERR(data);
4832 				break;
4833 			}
4834 		} else if ((len = str_has_prefix(str, "onchange("))) {
4835 			char *action_str = str + len;
4836 
4837 			data = track_data_parse(hist_data, action_str,
4838 						HANDLER_ONCHANGE);
4839 			if (IS_ERR(data)) {
4840 				ret = PTR_ERR(data);
4841 				break;
4842 			}
4843 		} else {
4844 			ret = -EINVAL;
4845 			break;
4846 		}
4847 
4848 		hist_data->actions[hist_data->n_actions++] = data;
4849 	}
4850 
4851 	return ret;
4852 }
4853 
4854 static int create_actions(struct hist_trigger_data *hist_data)
4855 {
4856 	struct action_data *data;
4857 	unsigned int i;
4858 	int ret = 0;
4859 
4860 	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4861 		data = hist_data->actions[i];
4862 
4863 		if (data->handler == HANDLER_ONMATCH) {
4864 			ret = onmatch_create(hist_data, data);
4865 			if (ret)
4866 				break;
4867 		} else if (data->handler == HANDLER_ONMAX ||
4868 			   data->handler == HANDLER_ONCHANGE) {
4869 			ret = track_data_create(hist_data, data);
4870 			if (ret)
4871 				break;
4872 		} else {
4873 			ret = -EINVAL;
4874 			break;
4875 		}
4876 	}
4877 
4878 	return ret;
4879 }
4880 
4881 static void print_actions(struct seq_file *m,
4882 			  struct hist_trigger_data *hist_data,
4883 			  struct tracing_map_elt *elt)
4884 {
4885 	unsigned int i;
4886 
4887 	for (i = 0; i < hist_data->n_actions; i++) {
4888 		struct action_data *data = hist_data->actions[i];
4889 
4890 		if (data->action == ACTION_SNAPSHOT)
4891 			continue;
4892 
4893 		if (data->handler == HANDLER_ONMAX ||
4894 		    data->handler == HANDLER_ONCHANGE)
4895 			track_data_print(m, hist_data, elt, data);
4896 	}
4897 }
4898 
4899 static void print_action_spec(struct seq_file *m,
4900 			      struct hist_trigger_data *hist_data,
4901 			      struct action_data *data)
4902 {
4903 	unsigned int i;
4904 
4905 	if (data->action == ACTION_SAVE) {
4906 		for (i = 0; i < hist_data->n_save_vars; i++) {
4907 			seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4908 			if (i < hist_data->n_save_vars - 1)
4909 				seq_puts(m, ",");
4910 		}
4911 	} else if (data->action == ACTION_TRACE) {
4912 		if (data->use_trace_keyword)
4913 			seq_printf(m, "%s", data->synth_event_name);
4914 		for (i = 0; i < data->n_params; i++) {
4915 			if (i || data->use_trace_keyword)
4916 				seq_puts(m, ",");
4917 			seq_printf(m, "%s", data->params[i]);
4918 		}
4919 	}
4920 }
4921 
4922 static void print_track_data_spec(struct seq_file *m,
4923 				  struct hist_trigger_data *hist_data,
4924 				  struct action_data *data)
4925 {
4926 	if (data->handler == HANDLER_ONMAX)
4927 		seq_puts(m, ":onmax(");
4928 	else if (data->handler == HANDLER_ONCHANGE)
4929 		seq_puts(m, ":onchange(");
4930 	seq_printf(m, "%s", data->track_data.var_str);
4931 	seq_printf(m, ").%s(", data->action_name);
4932 
4933 	print_action_spec(m, hist_data, data);
4934 
4935 	seq_puts(m, ")");
4936 }
4937 
4938 static void print_onmatch_spec(struct seq_file *m,
4939 			       struct hist_trigger_data *hist_data,
4940 			       struct action_data *data)
4941 {
4942 	seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4943 		   data->match_data.event);
4944 
4945 	seq_printf(m, "%s(", data->action_name);
4946 
4947 	print_action_spec(m, hist_data, data);
4948 
4949 	seq_puts(m, ")");
4950 }
4951 
4952 static bool actions_match(struct hist_trigger_data *hist_data,
4953 			  struct hist_trigger_data *hist_data_test)
4954 {
4955 	unsigned int i, j;
4956 
4957 	if (hist_data->n_actions != hist_data_test->n_actions)
4958 		return false;
4959 
4960 	for (i = 0; i < hist_data->n_actions; i++) {
4961 		struct action_data *data = hist_data->actions[i];
4962 		struct action_data *data_test = hist_data_test->actions[i];
4963 		char *action_name, *action_name_test;
4964 
4965 		if (data->handler != data_test->handler)
4966 			return false;
4967 		if (data->action != data_test->action)
4968 			return false;
4969 
4970 		if (data->n_params != data_test->n_params)
4971 			return false;
4972 
4973 		for (j = 0; j < data->n_params; j++) {
4974 			if (strcmp(data->params[j], data_test->params[j]) != 0)
4975 				return false;
4976 		}
4977 
4978 		if (data->use_trace_keyword)
4979 			action_name = data->synth_event_name;
4980 		else
4981 			action_name = data->action_name;
4982 
4983 		if (data_test->use_trace_keyword)
4984 			action_name_test = data_test->synth_event_name;
4985 		else
4986 			action_name_test = data_test->action_name;
4987 
4988 		if (strcmp(action_name, action_name_test) != 0)
4989 			return false;
4990 
4991 		if (data->handler == HANDLER_ONMATCH) {
4992 			if (strcmp(data->match_data.event_system,
4993 				   data_test->match_data.event_system) != 0)
4994 				return false;
4995 			if (strcmp(data->match_data.event,
4996 				   data_test->match_data.event) != 0)
4997 				return false;
4998 		} else if (data->handler == HANDLER_ONMAX ||
4999 			   data->handler == HANDLER_ONCHANGE) {
5000 			if (strcmp(data->track_data.var_str,
5001 				   data_test->track_data.var_str) != 0)
5002 				return false;
5003 		}
5004 	}
5005 
5006 	return true;
5007 }
5008 
5009 
5010 static void print_actions_spec(struct seq_file *m,
5011 			       struct hist_trigger_data *hist_data)
5012 {
5013 	unsigned int i;
5014 
5015 	for (i = 0; i < hist_data->n_actions; i++) {
5016 		struct action_data *data = hist_data->actions[i];
5017 
5018 		if (data->handler == HANDLER_ONMATCH)
5019 			print_onmatch_spec(m, hist_data, data);
5020 		else if (data->handler == HANDLER_ONMAX ||
5021 			 data->handler == HANDLER_ONCHANGE)
5022 			print_track_data_spec(m, hist_data, data);
5023 	}
5024 }
5025 
5026 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
5027 {
5028 	unsigned int i;
5029 
5030 	for (i = 0; i < hist_data->n_field_var_hists; i++) {
5031 		kfree(hist_data->field_var_hists[i]->cmd);
5032 		kfree(hist_data->field_var_hists[i]);
5033 	}
5034 }
5035 
5036 static void destroy_hist_data(struct hist_trigger_data *hist_data)
5037 {
5038 	if (!hist_data)
5039 		return;
5040 
5041 	destroy_hist_trigger_attrs(hist_data->attrs);
5042 	destroy_hist_fields(hist_data);
5043 	tracing_map_destroy(hist_data->map);
5044 
5045 	destroy_actions(hist_data);
5046 	destroy_field_vars(hist_data);
5047 	destroy_field_var_hists(hist_data);
5048 
5049 	kfree(hist_data);
5050 }
5051 
5052 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
5053 {
5054 	struct tracing_map *map = hist_data->map;
5055 	struct ftrace_event_field *field;
5056 	struct hist_field *hist_field;
5057 	int i, idx = 0;
5058 
5059 	for_each_hist_field(i, hist_data) {
5060 		hist_field = hist_data->fields[i];
5061 		if (hist_field->flags & HIST_FIELD_FL_KEY) {
5062 			tracing_map_cmp_fn_t cmp_fn;
5063 
5064 			field = hist_field->field;
5065 
5066 			if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
5067 				cmp_fn = tracing_map_cmp_none;
5068 			else if (!field || hist_field->flags & HIST_FIELD_FL_CPU)
5069 				cmp_fn = tracing_map_cmp_num(hist_field->size,
5070 							     hist_field->is_signed);
5071 			else if (is_string_field(field))
5072 				cmp_fn = tracing_map_cmp_string;
5073 			else
5074 				cmp_fn = tracing_map_cmp_num(field->size,
5075 							     field->is_signed);
5076 			idx = tracing_map_add_key_field(map,
5077 							hist_field->offset,
5078 							cmp_fn);
5079 		} else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5080 			idx = tracing_map_add_sum_field(map);
5081 
5082 		if (idx < 0)
5083 			return idx;
5084 
5085 		if (hist_field->flags & HIST_FIELD_FL_VAR) {
5086 			idx = tracing_map_add_var(map);
5087 			if (idx < 0)
5088 				return idx;
5089 			hist_field->var.idx = idx;
5090 			hist_field->var.hist_data = hist_data;
5091 		}
5092 	}
5093 
5094 	return 0;
5095 }
5096 
5097 static struct hist_trigger_data *
5098 create_hist_data(unsigned int map_bits,
5099 		 struct hist_trigger_attrs *attrs,
5100 		 struct trace_event_file *file,
5101 		 bool remove)
5102 {
5103 	const struct tracing_map_ops *map_ops = NULL;
5104 	struct hist_trigger_data *hist_data;
5105 	int ret = 0;
5106 
5107 	hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5108 	if (!hist_data)
5109 		return ERR_PTR(-ENOMEM);
5110 
5111 	hist_data->attrs = attrs;
5112 	hist_data->remove = remove;
5113 	hist_data->event_file = file;
5114 
5115 	ret = parse_actions(hist_data);
5116 	if (ret)
5117 		goto free;
5118 
5119 	ret = create_hist_fields(hist_data, file);
5120 	if (ret)
5121 		goto free;
5122 
5123 	ret = create_sort_keys(hist_data);
5124 	if (ret)
5125 		goto free;
5126 
5127 	map_ops = &hist_trigger_elt_data_ops;
5128 
5129 	hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5130 					    map_ops, hist_data);
5131 	if (IS_ERR(hist_data->map)) {
5132 		ret = PTR_ERR(hist_data->map);
5133 		hist_data->map = NULL;
5134 		goto free;
5135 	}
5136 
5137 	ret = create_tracing_map_fields(hist_data);
5138 	if (ret)
5139 		goto free;
5140  out:
5141 	return hist_data;
5142  free:
5143 	hist_data->attrs = NULL;
5144 
5145 	destroy_hist_data(hist_data);
5146 
5147 	hist_data = ERR_PTR(ret);
5148 
5149 	goto out;
5150 }
5151 
5152 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5153 				    struct tracing_map_elt *elt,
5154 				    struct trace_buffer *buffer, void *rec,
5155 				    struct ring_buffer_event *rbe,
5156 				    u64 *var_ref_vals)
5157 {
5158 	struct hist_elt_data *elt_data;
5159 	struct hist_field *hist_field;
5160 	unsigned int i, var_idx;
5161 	u64 hist_val;
5162 
5163 	elt_data = elt->private_data;
5164 	elt_data->var_ref_vals = var_ref_vals;
5165 
5166 	for_each_hist_val_field(i, hist_data) {
5167 		hist_field = hist_data->fields[i];
5168 		hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5169 		if (hist_field->flags & HIST_FIELD_FL_VAR) {
5170 			var_idx = hist_field->var.idx;
5171 
5172 			if (hist_field->flags &
5173 			    (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) {
5174 				unsigned int str_start, var_str_idx, idx;
5175 				char *str, *val_str;
5176 				unsigned int size;
5177 
5178 				str_start = hist_data->n_field_var_str +
5179 					hist_data->n_save_var_str;
5180 				var_str_idx = hist_field->var_str_idx;
5181 				idx = str_start + var_str_idx;
5182 
5183 				str = elt_data->field_var_str[idx];
5184 				val_str = (char *)(uintptr_t)hist_val;
5185 
5186 				if (hist_field->flags & HIST_FIELD_FL_STRING) {
5187 					size = min(hist_field->size, STR_VAR_LEN_MAX);
5188 					strscpy(str, val_str, size);
5189 				} else {
5190 					char *stack_start = str + sizeof(unsigned long);
5191 					int e;
5192 
5193 					e = stack_trace_save((void *)stack_start,
5194 							     HIST_STACKTRACE_DEPTH,
5195 							     HIST_STACKTRACE_SKIP);
5196 					if (e < HIST_STACKTRACE_DEPTH - 1)
5197 						((unsigned long *)stack_start)[e] = 0;
5198 					*((unsigned long *)str) = e;
5199 				}
5200 				hist_val = (u64)(uintptr_t)str;
5201 			}
5202 			tracing_map_set_var(elt, var_idx, hist_val);
5203 			continue;
5204 		}
5205 		tracing_map_update_sum(elt, i, hist_val);
5206 	}
5207 
5208 	for_each_hist_key_field(i, hist_data) {
5209 		hist_field = hist_data->fields[i];
5210 		if (hist_field->flags & HIST_FIELD_FL_VAR) {
5211 			hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5212 			var_idx = hist_field->var.idx;
5213 			tracing_map_set_var(elt, var_idx, hist_val);
5214 		}
5215 	}
5216 
5217 	update_field_vars(hist_data, elt, buffer, rbe, rec);
5218 }
5219 
5220 static inline void add_to_key(char *compound_key, void *key,
5221 			      struct hist_field *key_field, void *rec)
5222 {
5223 	size_t size = key_field->size;
5224 
5225 	if (key_field->flags & HIST_FIELD_FL_STRING) {
5226 		struct ftrace_event_field *field;
5227 
5228 		field = key_field->field;
5229 		if (field->filter_type == FILTER_DYN_STRING ||
5230 		    field->filter_type == FILTER_RDYN_STRING)
5231 			size = *(u32 *)(rec + field->offset) >> 16;
5232 		else if (field->filter_type == FILTER_STATIC_STRING)
5233 			size = field->size;
5234 
5235 		/* ensure NULL-termination */
5236 		if (size > key_field->size - 1)
5237 			size = key_field->size - 1;
5238 
5239 		strncpy(compound_key + key_field->offset, (char *)key, size);
5240 	} else
5241 		memcpy(compound_key + key_field->offset, key, size);
5242 }
5243 
5244 static void
5245 hist_trigger_actions(struct hist_trigger_data *hist_data,
5246 		     struct tracing_map_elt *elt,
5247 		     struct trace_buffer *buffer, void *rec,
5248 		     struct ring_buffer_event *rbe, void *key,
5249 		     u64 *var_ref_vals)
5250 {
5251 	struct action_data *data;
5252 	unsigned int i;
5253 
5254 	for (i = 0; i < hist_data->n_actions; i++) {
5255 		data = hist_data->actions[i];
5256 		data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
5257 	}
5258 }
5259 
5260 static void event_hist_trigger(struct event_trigger_data *data,
5261 			       struct trace_buffer *buffer, void *rec,
5262 			       struct ring_buffer_event *rbe)
5263 {
5264 	struct hist_trigger_data *hist_data = data->private_data;
5265 	bool use_compound_key = (hist_data->n_keys > 1);
5266 	unsigned long entries[HIST_STACKTRACE_DEPTH];
5267 	u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5268 	char compound_key[HIST_KEY_SIZE_MAX];
5269 	struct tracing_map_elt *elt = NULL;
5270 	struct hist_field *key_field;
5271 	u64 field_contents;
5272 	void *key = NULL;
5273 	unsigned int i;
5274 
5275 	if (unlikely(!rbe))
5276 		return;
5277 
5278 	memset(compound_key, 0, hist_data->key_size);
5279 
5280 	for_each_hist_key_field(i, hist_data) {
5281 		key_field = hist_data->fields[i];
5282 
5283 		if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5284 			memset(entries, 0, HIST_STACKTRACE_SIZE);
5285 			if (key_field->field) {
5286 				unsigned long *stack, n_entries;
5287 
5288 				field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
5289 				stack = (unsigned long *)(long)field_contents;
5290 				n_entries = *stack;
5291 				memcpy(entries, ++stack, n_entries * sizeof(unsigned long));
5292 			} else {
5293 				stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5294 						 HIST_STACKTRACE_SKIP);
5295 			}
5296 			key = entries;
5297 		} else {
5298 			field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
5299 			if (key_field->flags & HIST_FIELD_FL_STRING) {
5300 				key = (void *)(unsigned long)field_contents;
5301 				use_compound_key = true;
5302 			} else
5303 				key = (void *)&field_contents;
5304 		}
5305 
5306 		if (use_compound_key)
5307 			add_to_key(compound_key, key, key_field, rec);
5308 	}
5309 
5310 	if (use_compound_key)
5311 		key = compound_key;
5312 
5313 	if (hist_data->n_var_refs &&
5314 	    !resolve_var_refs(hist_data, key, var_ref_vals, false))
5315 		return;
5316 
5317 	elt = tracing_map_insert(hist_data->map, key);
5318 	if (!elt)
5319 		return;
5320 
5321 	hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
5322 
5323 	if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5324 		hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
5325 }
5326 
5327 static void hist_trigger_stacktrace_print(struct seq_file *m,
5328 					  unsigned long *stacktrace_entries,
5329 					  unsigned int max_entries)
5330 {
5331 	unsigned int spaces = 8;
5332 	unsigned int i;
5333 
5334 	for (i = 0; i < max_entries; i++) {
5335 		if (!stacktrace_entries[i])
5336 			return;
5337 
5338 		seq_printf(m, "%*c", 1 + spaces, ' ');
5339 		seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]);
5340 	}
5341 }
5342 
5343 static void hist_trigger_print_key(struct seq_file *m,
5344 				   struct hist_trigger_data *hist_data,
5345 				   void *key,
5346 				   struct tracing_map_elt *elt)
5347 {
5348 	struct hist_field *key_field;
5349 	bool multiline = false;
5350 	const char *field_name;
5351 	unsigned int i;
5352 	u64 uval;
5353 
5354 	seq_puts(m, "{ ");
5355 
5356 	for_each_hist_key_field(i, hist_data) {
5357 		key_field = hist_data->fields[i];
5358 
5359 		if (i > hist_data->n_vals)
5360 			seq_puts(m, ", ");
5361 
5362 		field_name = hist_field_name(key_field, 0);
5363 
5364 		if (key_field->flags & HIST_FIELD_FL_HEX) {
5365 			uval = *(u64 *)(key + key_field->offset);
5366 			seq_printf(m, "%s: %llx", field_name, uval);
5367 		} else if (key_field->flags & HIST_FIELD_FL_SYM) {
5368 			uval = *(u64 *)(key + key_field->offset);
5369 			seq_printf(m, "%s: [%llx] %-45ps", field_name,
5370 				   uval, (void *)(uintptr_t)uval);
5371 		} else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5372 			uval = *(u64 *)(key + key_field->offset);
5373 			seq_printf(m, "%s: [%llx] %-55pS", field_name,
5374 				   uval, (void *)(uintptr_t)uval);
5375 		} else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5376 			struct hist_elt_data *elt_data = elt->private_data;
5377 			char *comm;
5378 
5379 			if (WARN_ON_ONCE(!elt_data))
5380 				return;
5381 
5382 			comm = elt_data->comm;
5383 
5384 			uval = *(u64 *)(key + key_field->offset);
5385 			seq_printf(m, "%s: %-16s[%10llu]", field_name,
5386 				   comm, uval);
5387 		} else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5388 			const char *syscall_name;
5389 
5390 			uval = *(u64 *)(key + key_field->offset);
5391 			syscall_name = get_syscall_name(uval);
5392 			if (!syscall_name)
5393 				syscall_name = "unknown_syscall";
5394 
5395 			seq_printf(m, "%s: %-30s[%3llu]", field_name,
5396 				   syscall_name, uval);
5397 		} else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5398 			if (key_field->field)
5399 				seq_printf(m, "%s.stacktrace", key_field->field->name);
5400 			else
5401 				seq_puts(m, "common_stacktrace:\n");
5402 			hist_trigger_stacktrace_print(m,
5403 						      key + key_field->offset,
5404 						      HIST_STACKTRACE_DEPTH);
5405 			multiline = true;
5406 		} else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5407 			seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5408 				   *(u64 *)(key + key_field->offset));
5409 		} else if (key_field->flags & HIST_FIELD_FL_BUCKET) {
5410 			unsigned long buckets = key_field->buckets;
5411 			uval = *(u64 *)(key + key_field->offset);
5412 			seq_printf(m, "%s: ~ %llu-%llu", field_name,
5413 				   uval, uval + buckets -1);
5414 		} else if (key_field->flags & HIST_FIELD_FL_STRING) {
5415 			seq_printf(m, "%s: %-50s", field_name,
5416 				   (char *)(key + key_field->offset));
5417 		} else {
5418 			uval = *(u64 *)(key + key_field->offset);
5419 			seq_printf(m, "%s: %10llu", field_name, uval);
5420 		}
5421 	}
5422 
5423 	if (!multiline)
5424 		seq_puts(m, " ");
5425 
5426 	seq_puts(m, "}");
5427 }
5428 
5429 /* Get the 100 times of the percentage of @val in @total */
5430 static inline unsigned int __get_percentage(u64 val, u64 total)
5431 {
5432 	if (!total)
5433 		goto div0;
5434 
5435 	if (val < (U64_MAX / 10000))
5436 		return (unsigned int)div64_ul(val * 10000, total);
5437 
5438 	total = div64_u64(total, 10000);
5439 	if (!total)
5440 		goto div0;
5441 
5442 	return (unsigned int)div64_ul(val, total);
5443 div0:
5444 	return val ? UINT_MAX : 0;
5445 }
5446 
5447 #define BAR_CHAR '#'
5448 
5449 static inline const char *__fill_bar_str(char *buf, int size, u64 val, u64 max)
5450 {
5451 	unsigned int len = __get_percentage(val, max);
5452 	int i;
5453 
5454 	if (len == UINT_MAX) {
5455 		snprintf(buf, size, "[ERROR]");
5456 		return buf;
5457 	}
5458 
5459 	len = len * size / 10000;
5460 	for (i = 0; i < len && i < size; i++)
5461 		buf[i] = BAR_CHAR;
5462 	while (i < size)
5463 		buf[i++] = ' ';
5464 	buf[size] = '\0';
5465 
5466 	return buf;
5467 }
5468 
5469 struct hist_val_stat {
5470 	u64 max;
5471 	u64 total;
5472 };
5473 
5474 static void hist_trigger_print_val(struct seq_file *m, unsigned int idx,
5475 				   const char *field_name, unsigned long flags,
5476 				   struct hist_val_stat *stats,
5477 				   struct tracing_map_elt *elt)
5478 {
5479 	u64 val = tracing_map_read_sum(elt, idx);
5480 	unsigned int pc;
5481 	char bar[21];
5482 
5483 	if (flags & HIST_FIELD_FL_PERCENT) {
5484 		pc = __get_percentage(val, stats[idx].total);
5485 		if (pc == UINT_MAX)
5486 			seq_printf(m, " %s (%%):[ERROR]", field_name);
5487 		else
5488 			seq_printf(m, " %s (%%): %3u.%02u", field_name,
5489 					pc / 100, pc % 100);
5490 	} else if (flags & HIST_FIELD_FL_GRAPH) {
5491 		seq_printf(m, " %s: %20s", field_name,
5492 			   __fill_bar_str(bar, 20, val, stats[idx].max));
5493 	} else if (flags & HIST_FIELD_FL_HEX) {
5494 		seq_printf(m, " %s: %10llx", field_name, val);
5495 	} else {
5496 		seq_printf(m, " %s: %10llu", field_name, val);
5497 	}
5498 }
5499 
5500 static void hist_trigger_entry_print(struct seq_file *m,
5501 				     struct hist_trigger_data *hist_data,
5502 				     struct hist_val_stat *stats,
5503 				     void *key,
5504 				     struct tracing_map_elt *elt)
5505 {
5506 	const char *field_name;
5507 	unsigned int i = HITCOUNT_IDX;
5508 	unsigned long flags;
5509 
5510 	hist_trigger_print_key(m, hist_data, key, elt);
5511 
5512 	/* At first, show the raw hitcount if !nohitcount */
5513 	if (!hist_data->attrs->no_hitcount)
5514 		hist_trigger_print_val(m, i, "hitcount", 0, stats, elt);
5515 
5516 	for (i = 1; i < hist_data->n_vals; i++) {
5517 		field_name = hist_field_name(hist_data->fields[i], 0);
5518 		flags = hist_data->fields[i]->flags;
5519 		if (flags & HIST_FIELD_FL_VAR || flags & HIST_FIELD_FL_EXPR)
5520 			continue;
5521 
5522 		seq_puts(m, " ");
5523 		hist_trigger_print_val(m, i, field_name, flags, stats, elt);
5524 	}
5525 
5526 	print_actions(m, hist_data, elt);
5527 
5528 	seq_puts(m, "\n");
5529 }
5530 
5531 static int print_entries(struct seq_file *m,
5532 			 struct hist_trigger_data *hist_data)
5533 {
5534 	struct tracing_map_sort_entry **sort_entries = NULL;
5535 	struct tracing_map *map = hist_data->map;
5536 	int i, j, n_entries;
5537 	struct hist_val_stat *stats = NULL;
5538 	u64 val;
5539 
5540 	n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5541 					     hist_data->n_sort_keys,
5542 					     &sort_entries);
5543 	if (n_entries < 0)
5544 		return n_entries;
5545 
5546 	/* Calculate the max and the total for each field if needed. */
5547 	for (j = 0; j < hist_data->n_vals; j++) {
5548 		if (!(hist_data->fields[j]->flags &
5549 			(HIST_FIELD_FL_PERCENT | HIST_FIELD_FL_GRAPH)))
5550 			continue;
5551 		if (!stats) {
5552 			stats = kcalloc(hist_data->n_vals, sizeof(*stats),
5553 				       GFP_KERNEL);
5554 			if (!stats) {
5555 				n_entries = -ENOMEM;
5556 				goto out;
5557 			}
5558 		}
5559 		for (i = 0; i < n_entries; i++) {
5560 			val = tracing_map_read_sum(sort_entries[i]->elt, j);
5561 			stats[j].total += val;
5562 			if (stats[j].max < val)
5563 				stats[j].max = val;
5564 		}
5565 	}
5566 
5567 	for (i = 0; i < n_entries; i++)
5568 		hist_trigger_entry_print(m, hist_data, stats,
5569 					 sort_entries[i]->key,
5570 					 sort_entries[i]->elt);
5571 
5572 	kfree(stats);
5573 out:
5574 	tracing_map_destroy_sort_entries(sort_entries, n_entries);
5575 
5576 	return n_entries;
5577 }
5578 
5579 static void hist_trigger_show(struct seq_file *m,
5580 			      struct event_trigger_data *data, int n)
5581 {
5582 	struct hist_trigger_data *hist_data;
5583 	int n_entries;
5584 
5585 	if (n > 0)
5586 		seq_puts(m, "\n\n");
5587 
5588 	seq_puts(m, "# event histogram\n#\n# trigger info: ");
5589 	data->ops->print(m, data);
5590 	seq_puts(m, "#\n\n");
5591 
5592 	hist_data = data->private_data;
5593 	n_entries = print_entries(m, hist_data);
5594 	if (n_entries < 0)
5595 		n_entries = 0;
5596 
5597 	track_data_snapshot_print(m, hist_data);
5598 
5599 	seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5600 		   (u64)atomic64_read(&hist_data->map->hits),
5601 		   n_entries, (u64)atomic64_read(&hist_data->map->drops));
5602 }
5603 
5604 static int hist_show(struct seq_file *m, void *v)
5605 {
5606 	struct event_trigger_data *data;
5607 	struct trace_event_file *event_file;
5608 	int n = 0, ret = 0;
5609 
5610 	mutex_lock(&event_mutex);
5611 
5612 	event_file = event_file_data(m->private);
5613 	if (unlikely(!event_file)) {
5614 		ret = -ENODEV;
5615 		goto out_unlock;
5616 	}
5617 
5618 	list_for_each_entry(data, &event_file->triggers, list) {
5619 		if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5620 			hist_trigger_show(m, data, n++);
5621 	}
5622 
5623  out_unlock:
5624 	mutex_unlock(&event_mutex);
5625 
5626 	return ret;
5627 }
5628 
5629 static int event_hist_open(struct inode *inode, struct file *file)
5630 {
5631 	int ret;
5632 
5633 	ret = tracing_open_file_tr(inode, file);
5634 	if (ret)
5635 		return ret;
5636 
5637 	/* Clear private_data to avoid warning in single_open() */
5638 	file->private_data = NULL;
5639 	return single_open(file, hist_show, file);
5640 }
5641 
5642 const struct file_operations event_hist_fops = {
5643 	.open = event_hist_open,
5644 	.read = seq_read,
5645 	.llseek = seq_lseek,
5646 	.release = tracing_single_release_file_tr,
5647 };
5648 
5649 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
5650 static void hist_field_debug_show_flags(struct seq_file *m,
5651 					unsigned long flags)
5652 {
5653 	seq_puts(m, "      flags:\n");
5654 
5655 	if (flags & HIST_FIELD_FL_KEY)
5656 		seq_puts(m, "        HIST_FIELD_FL_KEY\n");
5657 	else if (flags & HIST_FIELD_FL_HITCOUNT)
5658 		seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
5659 	else if (flags & HIST_FIELD_FL_VAR)
5660 		seq_puts(m, "        HIST_FIELD_FL_VAR\n");
5661 	else if (flags & HIST_FIELD_FL_VAR_REF)
5662 		seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
5663 	else
5664 		seq_puts(m, "        VAL: normal u64 value\n");
5665 
5666 	if (flags & HIST_FIELD_FL_ALIAS)
5667 		seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
5668 	else if (flags & HIST_FIELD_FL_CONST)
5669 		seq_puts(m, "        HIST_FIELD_FL_CONST\n");
5670 }
5671 
5672 static int hist_field_debug_show(struct seq_file *m,
5673 				 struct hist_field *field, unsigned long flags)
5674 {
5675 	if ((field->flags & flags) != flags) {
5676 		seq_printf(m, "ERROR: bad flags - %lx\n", flags);
5677 		return -EINVAL;
5678 	}
5679 
5680 	hist_field_debug_show_flags(m, field->flags);
5681 	if (field->field)
5682 		seq_printf(m, "      ftrace_event_field name: %s\n",
5683 			   field->field->name);
5684 
5685 	if (field->flags & HIST_FIELD_FL_VAR) {
5686 		seq_printf(m, "      var.name: %s\n", field->var.name);
5687 		seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5688 			   field->var.idx);
5689 	}
5690 
5691 	if (field->flags & HIST_FIELD_FL_CONST)
5692 		seq_printf(m, "      constant: %llu\n", field->constant);
5693 
5694 	if (field->flags & HIST_FIELD_FL_ALIAS)
5695 		seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5696 			   field->var_ref_idx);
5697 
5698 	if (field->flags & HIST_FIELD_FL_VAR_REF) {
5699 		seq_printf(m, "      name: %s\n", field->name);
5700 		seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5701 			   field->var.idx);
5702 		seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
5703 		seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5704 			   field->var_ref_idx);
5705 		if (field->system)
5706 			seq_printf(m, "      system: %s\n", field->system);
5707 		if (field->event_name)
5708 			seq_printf(m, "      event_name: %s\n", field->event_name);
5709 	}
5710 
5711 	seq_printf(m, "      type: %s\n", field->type);
5712 	seq_printf(m, "      size: %u\n", field->size);
5713 	seq_printf(m, "      is_signed: %u\n", field->is_signed);
5714 
5715 	return 0;
5716 }
5717 
5718 static int field_var_debug_show(struct seq_file *m,
5719 				struct field_var *field_var, unsigned int i,
5720 				bool save_vars)
5721 {
5722 	const char *vars_name = save_vars ? "save_vars" : "field_vars";
5723 	struct hist_field *field;
5724 	int ret = 0;
5725 
5726 	seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
5727 
5728 	field = field_var->var;
5729 
5730 	seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
5731 
5732 	hist_field_debug_show_flags(m, field->flags);
5733 	seq_printf(m, "      var.name: %s\n", field->var.name);
5734 	seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5735 		   field->var.idx);
5736 
5737 	field = field_var->val;
5738 
5739 	seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
5740 	if (field->field)
5741 		seq_printf(m, "      ftrace_event_field name: %s\n",
5742 			   field->field->name);
5743 	else {
5744 		ret = -EINVAL;
5745 		goto out;
5746 	}
5747 
5748 	seq_printf(m, "      type: %s\n", field->type);
5749 	seq_printf(m, "      size: %u\n", field->size);
5750 	seq_printf(m, "      is_signed: %u\n", field->is_signed);
5751 out:
5752 	return ret;
5753 }
5754 
5755 static int hist_action_debug_show(struct seq_file *m,
5756 				  struct action_data *data, int i)
5757 {
5758 	int ret = 0;
5759 
5760 	if (data->handler == HANDLER_ONMAX ||
5761 	    data->handler == HANDLER_ONCHANGE) {
5762 		seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
5763 		ret = hist_field_debug_show(m, data->track_data.var_ref,
5764 					    HIST_FIELD_FL_VAR_REF);
5765 		if (ret)
5766 			goto out;
5767 
5768 		seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
5769 		ret = hist_field_debug_show(m, data->track_data.track_var,
5770 					    HIST_FIELD_FL_VAR);
5771 		if (ret)
5772 			goto out;
5773 	}
5774 
5775 	if (data->handler == HANDLER_ONMATCH) {
5776 		seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
5777 			   i, data->match_data.event_system);
5778 		seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
5779 			   i, data->match_data.event);
5780 	}
5781 out:
5782 	return ret;
5783 }
5784 
5785 static int hist_actions_debug_show(struct seq_file *m,
5786 				   struct hist_trigger_data *hist_data)
5787 {
5788 	int i, ret = 0;
5789 
5790 	if (hist_data->n_actions)
5791 		seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
5792 
5793 	for (i = 0; i < hist_data->n_actions; i++) {
5794 		struct action_data *action = hist_data->actions[i];
5795 
5796 		ret = hist_action_debug_show(m, action, i);
5797 		if (ret)
5798 			goto out;
5799 	}
5800 
5801 	if (hist_data->n_save_vars)
5802 		seq_puts(m, "\n  save action variables (save() params):\n");
5803 
5804 	for (i = 0; i < hist_data->n_save_vars; i++) {
5805 		ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
5806 		if (ret)
5807 			goto out;
5808 	}
5809 out:
5810 	return ret;
5811 }
5812 
5813 static void hist_trigger_debug_show(struct seq_file *m,
5814 				    struct event_trigger_data *data, int n)
5815 {
5816 	struct hist_trigger_data *hist_data;
5817 	int i, ret;
5818 
5819 	if (n > 0)
5820 		seq_puts(m, "\n\n");
5821 
5822 	seq_puts(m, "# event histogram\n#\n# trigger info: ");
5823 	data->ops->print(m, data);
5824 	seq_puts(m, "#\n\n");
5825 
5826 	hist_data = data->private_data;
5827 
5828 	seq_printf(m, "hist_data: %p\n\n", hist_data);
5829 	seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
5830 	seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
5831 	seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
5832 
5833 	seq_puts(m, "\n  val fields:\n\n");
5834 
5835 	seq_puts(m, "    hist_data->fields[0]:\n");
5836 	ret = hist_field_debug_show(m, hist_data->fields[0],
5837 				    HIST_FIELD_FL_HITCOUNT);
5838 	if (ret)
5839 		return;
5840 
5841 	for (i = 1; i < hist_data->n_vals; i++) {
5842 		seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5843 		ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5844 		if (ret)
5845 			return;
5846 	}
5847 
5848 	seq_puts(m, "\n  key fields:\n");
5849 
5850 	for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5851 		seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5852 		ret = hist_field_debug_show(m, hist_data->fields[i],
5853 					    HIST_FIELD_FL_KEY);
5854 		if (ret)
5855 			return;
5856 	}
5857 
5858 	if (hist_data->n_var_refs)
5859 		seq_puts(m, "\n  variable reference fields:\n");
5860 
5861 	for (i = 0; i < hist_data->n_var_refs; i++) {
5862 		seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5863 		ret = hist_field_debug_show(m, hist_data->var_refs[i],
5864 					    HIST_FIELD_FL_VAR_REF);
5865 		if (ret)
5866 			return;
5867 	}
5868 
5869 	if (hist_data->n_field_vars)
5870 		seq_puts(m, "\n  field variables:\n");
5871 
5872 	for (i = 0; i < hist_data->n_field_vars; i++) {
5873 		ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5874 		if (ret)
5875 			return;
5876 	}
5877 
5878 	ret = hist_actions_debug_show(m, hist_data);
5879 	if (ret)
5880 		return;
5881 }
5882 
5883 static int hist_debug_show(struct seq_file *m, void *v)
5884 {
5885 	struct event_trigger_data *data;
5886 	struct trace_event_file *event_file;
5887 	int n = 0, ret = 0;
5888 
5889 	mutex_lock(&event_mutex);
5890 
5891 	event_file = event_file_data(m->private);
5892 	if (unlikely(!event_file)) {
5893 		ret = -ENODEV;
5894 		goto out_unlock;
5895 	}
5896 
5897 	list_for_each_entry(data, &event_file->triggers, list) {
5898 		if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5899 			hist_trigger_debug_show(m, data, n++);
5900 	}
5901 
5902  out_unlock:
5903 	mutex_unlock(&event_mutex);
5904 
5905 	return ret;
5906 }
5907 
5908 static int event_hist_debug_open(struct inode *inode, struct file *file)
5909 {
5910 	int ret;
5911 
5912 	ret = tracing_open_file_tr(inode, file);
5913 	if (ret)
5914 		return ret;
5915 
5916 	/* Clear private_data to avoid warning in single_open() */
5917 	file->private_data = NULL;
5918 	return single_open(file, hist_debug_show, file);
5919 }
5920 
5921 const struct file_operations event_hist_debug_fops = {
5922 	.open = event_hist_debug_open,
5923 	.read = seq_read,
5924 	.llseek = seq_lseek,
5925 	.release = tracing_single_release_file_tr,
5926 };
5927 #endif
5928 
5929 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5930 {
5931 	const char *field_name = hist_field_name(hist_field, 0);
5932 
5933 	if (hist_field->var.name)
5934 		seq_printf(m, "%s=", hist_field->var.name);
5935 
5936 	if (hist_field->flags & HIST_FIELD_FL_CPU)
5937 		seq_puts(m, "common_cpu");
5938 	else if (hist_field->flags & HIST_FIELD_FL_CONST)
5939 		seq_printf(m, "%llu", hist_field->constant);
5940 	else if (field_name) {
5941 		if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5942 		    hist_field->flags & HIST_FIELD_FL_ALIAS)
5943 			seq_putc(m, '$');
5944 		seq_printf(m, "%s", field_name);
5945 	} else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5946 		seq_puts(m, "common_timestamp");
5947 
5948 	if (hist_field->flags) {
5949 		if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5950 		    !(hist_field->flags & HIST_FIELD_FL_EXPR) &&
5951 		    !(hist_field->flags & HIST_FIELD_FL_STACKTRACE)) {
5952 			const char *flags = get_hist_field_flags(hist_field);
5953 
5954 			if (flags)
5955 				seq_printf(m, ".%s", flags);
5956 		}
5957 	}
5958 	if (hist_field->buckets)
5959 		seq_printf(m, "=%ld", hist_field->buckets);
5960 }
5961 
5962 static int event_hist_trigger_print(struct seq_file *m,
5963 				    struct event_trigger_data *data)
5964 {
5965 	struct hist_trigger_data *hist_data = data->private_data;
5966 	struct hist_field *field;
5967 	bool have_var = false;
5968 	bool show_val = false;
5969 	unsigned int i;
5970 
5971 	seq_puts(m, HIST_PREFIX);
5972 
5973 	if (data->name)
5974 		seq_printf(m, "%s:", data->name);
5975 
5976 	seq_puts(m, "keys=");
5977 
5978 	for_each_hist_key_field(i, hist_data) {
5979 		field = hist_data->fields[i];
5980 
5981 		if (i > hist_data->n_vals)
5982 			seq_puts(m, ",");
5983 
5984 		if (field->flags & HIST_FIELD_FL_STACKTRACE) {
5985 			if (field->field)
5986 				seq_printf(m, "%s.stacktrace", field->field->name);
5987 			else
5988 				seq_puts(m, "common_stacktrace");
5989 		} else
5990 			hist_field_print(m, field);
5991 	}
5992 
5993 	seq_puts(m, ":vals=");
5994 
5995 	for_each_hist_val_field(i, hist_data) {
5996 		field = hist_data->fields[i];
5997 		if (field->flags & HIST_FIELD_FL_VAR) {
5998 			have_var = true;
5999 			continue;
6000 		}
6001 
6002 		if (i == HITCOUNT_IDX) {
6003 			if (hist_data->attrs->no_hitcount)
6004 				continue;
6005 			seq_puts(m, "hitcount");
6006 		} else {
6007 			if (show_val)
6008 				seq_puts(m, ",");
6009 			hist_field_print(m, field);
6010 		}
6011 		show_val = true;
6012 	}
6013 
6014 	if (have_var) {
6015 		unsigned int n = 0;
6016 
6017 		seq_puts(m, ":");
6018 
6019 		for_each_hist_val_field(i, hist_data) {
6020 			field = hist_data->fields[i];
6021 
6022 			if (field->flags & HIST_FIELD_FL_VAR) {
6023 				if (n++)
6024 					seq_puts(m, ",");
6025 				hist_field_print(m, field);
6026 			}
6027 		}
6028 	}
6029 
6030 	seq_puts(m, ":sort=");
6031 
6032 	for (i = 0; i < hist_data->n_sort_keys; i++) {
6033 		struct tracing_map_sort_key *sort_key;
6034 		unsigned int idx, first_key_idx;
6035 
6036 		/* skip VAR vals */
6037 		first_key_idx = hist_data->n_vals - hist_data->n_vars;
6038 
6039 		sort_key = &hist_data->sort_keys[i];
6040 		idx = sort_key->field_idx;
6041 
6042 		if (WARN_ON(idx >= HIST_FIELDS_MAX))
6043 			return -EINVAL;
6044 
6045 		if (i > 0)
6046 			seq_puts(m, ",");
6047 
6048 		if (idx == HITCOUNT_IDX)
6049 			seq_puts(m, "hitcount");
6050 		else {
6051 			if (idx >= first_key_idx)
6052 				idx += hist_data->n_vars;
6053 			hist_field_print(m, hist_data->fields[idx]);
6054 		}
6055 
6056 		if (sort_key->descending)
6057 			seq_puts(m, ".descending");
6058 	}
6059 	seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
6060 	if (hist_data->enable_timestamps)
6061 		seq_printf(m, ":clock=%s", hist_data->attrs->clock);
6062 	if (hist_data->attrs->no_hitcount)
6063 		seq_puts(m, ":nohitcount");
6064 
6065 	print_actions_spec(m, hist_data);
6066 
6067 	if (data->filter_str)
6068 		seq_printf(m, " if %s", data->filter_str);
6069 
6070 	if (data->paused)
6071 		seq_puts(m, " [paused]");
6072 	else
6073 		seq_puts(m, " [active]");
6074 
6075 	seq_putc(m, '\n');
6076 
6077 	return 0;
6078 }
6079 
6080 static int event_hist_trigger_init(struct event_trigger_data *data)
6081 {
6082 	struct hist_trigger_data *hist_data = data->private_data;
6083 
6084 	if (!data->ref && hist_data->attrs->name)
6085 		save_named_trigger(hist_data->attrs->name, data);
6086 
6087 	data->ref++;
6088 
6089 	return 0;
6090 }
6091 
6092 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
6093 {
6094 	struct trace_event_file *file;
6095 	unsigned int i;
6096 	char *cmd;
6097 	int ret;
6098 
6099 	for (i = 0; i < hist_data->n_field_var_hists; i++) {
6100 		file = hist_data->field_var_hists[i]->hist_data->event_file;
6101 		cmd = hist_data->field_var_hists[i]->cmd;
6102 		ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
6103 					       "!hist", "hist", cmd);
6104 		WARN_ON_ONCE(ret < 0);
6105 	}
6106 }
6107 
6108 static void event_hist_trigger_free(struct event_trigger_data *data)
6109 {
6110 	struct hist_trigger_data *hist_data = data->private_data;
6111 
6112 	if (WARN_ON_ONCE(data->ref <= 0))
6113 		return;
6114 
6115 	data->ref--;
6116 	if (!data->ref) {
6117 		if (data->name)
6118 			del_named_trigger(data);
6119 
6120 		trigger_data_free(data);
6121 
6122 		remove_hist_vars(hist_data);
6123 
6124 		unregister_field_var_hists(hist_data);
6125 
6126 		destroy_hist_data(hist_data);
6127 	}
6128 }
6129 
6130 static struct event_trigger_ops event_hist_trigger_ops = {
6131 	.trigger		= event_hist_trigger,
6132 	.print			= event_hist_trigger_print,
6133 	.init			= event_hist_trigger_init,
6134 	.free			= event_hist_trigger_free,
6135 };
6136 
6137 static int event_hist_trigger_named_init(struct event_trigger_data *data)
6138 {
6139 	data->ref++;
6140 
6141 	save_named_trigger(data->named_data->name, data);
6142 
6143 	event_hist_trigger_init(data->named_data);
6144 
6145 	return 0;
6146 }
6147 
6148 static void event_hist_trigger_named_free(struct event_trigger_data *data)
6149 {
6150 	if (WARN_ON_ONCE(data->ref <= 0))
6151 		return;
6152 
6153 	event_hist_trigger_free(data->named_data);
6154 
6155 	data->ref--;
6156 	if (!data->ref) {
6157 		del_named_trigger(data);
6158 		trigger_data_free(data);
6159 	}
6160 }
6161 
6162 static struct event_trigger_ops event_hist_trigger_named_ops = {
6163 	.trigger		= event_hist_trigger,
6164 	.print			= event_hist_trigger_print,
6165 	.init			= event_hist_trigger_named_init,
6166 	.free			= event_hist_trigger_named_free,
6167 };
6168 
6169 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
6170 							    char *param)
6171 {
6172 	return &event_hist_trigger_ops;
6173 }
6174 
6175 static void hist_clear(struct event_trigger_data *data)
6176 {
6177 	struct hist_trigger_data *hist_data = data->private_data;
6178 
6179 	if (data->name)
6180 		pause_named_trigger(data);
6181 
6182 	tracepoint_synchronize_unregister();
6183 
6184 	tracing_map_clear(hist_data->map);
6185 
6186 	if (data->name)
6187 		unpause_named_trigger(data);
6188 }
6189 
6190 static bool compatible_field(struct ftrace_event_field *field,
6191 			     struct ftrace_event_field *test_field)
6192 {
6193 	if (field == test_field)
6194 		return true;
6195 	if (field == NULL || test_field == NULL)
6196 		return false;
6197 	if (strcmp(field->name, test_field->name) != 0)
6198 		return false;
6199 	if (strcmp(field->type, test_field->type) != 0)
6200 		return false;
6201 	if (field->size != test_field->size)
6202 		return false;
6203 	if (field->is_signed != test_field->is_signed)
6204 		return false;
6205 
6206 	return true;
6207 }
6208 
6209 static bool hist_trigger_match(struct event_trigger_data *data,
6210 			       struct event_trigger_data *data_test,
6211 			       struct event_trigger_data *named_data,
6212 			       bool ignore_filter)
6213 {
6214 	struct tracing_map_sort_key *sort_key, *sort_key_test;
6215 	struct hist_trigger_data *hist_data, *hist_data_test;
6216 	struct hist_field *key_field, *key_field_test;
6217 	unsigned int i;
6218 
6219 	if (named_data && (named_data != data_test) &&
6220 	    (named_data != data_test->named_data))
6221 		return false;
6222 
6223 	if (!named_data && is_named_trigger(data_test))
6224 		return false;
6225 
6226 	hist_data = data->private_data;
6227 	hist_data_test = data_test->private_data;
6228 
6229 	if (hist_data->n_vals != hist_data_test->n_vals ||
6230 	    hist_data->n_fields != hist_data_test->n_fields ||
6231 	    hist_data->n_sort_keys != hist_data_test->n_sort_keys)
6232 		return false;
6233 
6234 	if (!ignore_filter) {
6235 		if ((data->filter_str && !data_test->filter_str) ||
6236 		   (!data->filter_str && data_test->filter_str))
6237 			return false;
6238 	}
6239 
6240 	for_each_hist_field(i, hist_data) {
6241 		key_field = hist_data->fields[i];
6242 		key_field_test = hist_data_test->fields[i];
6243 
6244 		if (key_field->flags != key_field_test->flags)
6245 			return false;
6246 		if (!compatible_field(key_field->field, key_field_test->field))
6247 			return false;
6248 		if (key_field->offset != key_field_test->offset)
6249 			return false;
6250 		if (key_field->size != key_field_test->size)
6251 			return false;
6252 		if (key_field->is_signed != key_field_test->is_signed)
6253 			return false;
6254 		if (!!key_field->var.name != !!key_field_test->var.name)
6255 			return false;
6256 		if (key_field->var.name &&
6257 		    strcmp(key_field->var.name, key_field_test->var.name) != 0)
6258 			return false;
6259 	}
6260 
6261 	for (i = 0; i < hist_data->n_sort_keys; i++) {
6262 		sort_key = &hist_data->sort_keys[i];
6263 		sort_key_test = &hist_data_test->sort_keys[i];
6264 
6265 		if (sort_key->field_idx != sort_key_test->field_idx ||
6266 		    sort_key->descending != sort_key_test->descending)
6267 			return false;
6268 	}
6269 
6270 	if (!ignore_filter && data->filter_str &&
6271 	    (strcmp(data->filter_str, data_test->filter_str) != 0))
6272 		return false;
6273 
6274 	if (!actions_match(hist_data, hist_data_test))
6275 		return false;
6276 
6277 	return true;
6278 }
6279 
6280 static bool existing_hist_update_only(char *glob,
6281 				      struct event_trigger_data *data,
6282 				      struct trace_event_file *file)
6283 {
6284 	struct hist_trigger_data *hist_data = data->private_data;
6285 	struct event_trigger_data *test, *named_data = NULL;
6286 	bool updated = false;
6287 
6288 	if (!hist_data->attrs->pause && !hist_data->attrs->cont &&
6289 	    !hist_data->attrs->clear)
6290 		goto out;
6291 
6292 	if (hist_data->attrs->name) {
6293 		named_data = find_named_trigger(hist_data->attrs->name);
6294 		if (named_data) {
6295 			if (!hist_trigger_match(data, named_data, named_data,
6296 						true))
6297 				goto out;
6298 		}
6299 	}
6300 
6301 	if (hist_data->attrs->name && !named_data)
6302 		goto out;
6303 
6304 	list_for_each_entry(test, &file->triggers, list) {
6305 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6306 			if (!hist_trigger_match(data, test, named_data, false))
6307 				continue;
6308 			if (hist_data->attrs->pause)
6309 				test->paused = true;
6310 			else if (hist_data->attrs->cont)
6311 				test->paused = false;
6312 			else if (hist_data->attrs->clear)
6313 				hist_clear(test);
6314 			updated = true;
6315 			goto out;
6316 		}
6317 	}
6318  out:
6319 	return updated;
6320 }
6321 
6322 static int hist_register_trigger(char *glob,
6323 				 struct event_trigger_data *data,
6324 				 struct trace_event_file *file)
6325 {
6326 	struct hist_trigger_data *hist_data = data->private_data;
6327 	struct event_trigger_data *test, *named_data = NULL;
6328 	struct trace_array *tr = file->tr;
6329 	int ret = 0;
6330 
6331 	if (hist_data->attrs->name) {
6332 		named_data = find_named_trigger(hist_data->attrs->name);
6333 		if (named_data) {
6334 			if (!hist_trigger_match(data, named_data, named_data,
6335 						true)) {
6336 				hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
6337 				ret = -EINVAL;
6338 				goto out;
6339 			}
6340 		}
6341 	}
6342 
6343 	if (hist_data->attrs->name && !named_data)
6344 		goto new;
6345 
6346 	lockdep_assert_held(&event_mutex);
6347 
6348 	list_for_each_entry(test, &file->triggers, list) {
6349 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6350 			if (hist_trigger_match(data, test, named_data, false)) {
6351 				hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
6352 				ret = -EEXIST;
6353 				goto out;
6354 			}
6355 		}
6356 	}
6357  new:
6358 	if (hist_data->attrs->cont || hist_data->attrs->clear) {
6359 		hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
6360 		ret = -ENOENT;
6361 		goto out;
6362 	}
6363 
6364 	if (hist_data->attrs->pause)
6365 		data->paused = true;
6366 
6367 	if (named_data) {
6368 		data->private_data = named_data->private_data;
6369 		set_named_trigger_data(data, named_data);
6370 		data->ops = &event_hist_trigger_named_ops;
6371 	}
6372 
6373 	if (data->ops->init) {
6374 		ret = data->ops->init(data);
6375 		if (ret < 0)
6376 			goto out;
6377 	}
6378 
6379 	if (hist_data->enable_timestamps) {
6380 		char *clock = hist_data->attrs->clock;
6381 
6382 		ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
6383 		if (ret) {
6384 			hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
6385 			goto out;
6386 		}
6387 
6388 		tracing_set_filter_buffering(file->tr, true);
6389 	}
6390 
6391 	if (named_data)
6392 		destroy_hist_data(hist_data);
6393  out:
6394 	return ret;
6395 }
6396 
6397 static int hist_trigger_enable(struct event_trigger_data *data,
6398 			       struct trace_event_file *file)
6399 {
6400 	int ret = 0;
6401 
6402 	list_add_tail_rcu(&data->list, &file->triggers);
6403 
6404 	update_cond_flag(file);
6405 
6406 	if (trace_event_trigger_enable_disable(file, 1) < 0) {
6407 		list_del_rcu(&data->list);
6408 		update_cond_flag(file);
6409 		ret--;
6410 	}
6411 
6412 	return ret;
6413 }
6414 
6415 static bool have_hist_trigger_match(struct event_trigger_data *data,
6416 				    struct trace_event_file *file)
6417 {
6418 	struct hist_trigger_data *hist_data = data->private_data;
6419 	struct event_trigger_data *test, *named_data = NULL;
6420 	bool match = false;
6421 
6422 	lockdep_assert_held(&event_mutex);
6423 
6424 	if (hist_data->attrs->name)
6425 		named_data = find_named_trigger(hist_data->attrs->name);
6426 
6427 	list_for_each_entry(test, &file->triggers, list) {
6428 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6429 			if (hist_trigger_match(data, test, named_data, false)) {
6430 				match = true;
6431 				break;
6432 			}
6433 		}
6434 	}
6435 
6436 	return match;
6437 }
6438 
6439 static bool hist_trigger_check_refs(struct event_trigger_data *data,
6440 				    struct trace_event_file *file)
6441 {
6442 	struct hist_trigger_data *hist_data = data->private_data;
6443 	struct event_trigger_data *test, *named_data = NULL;
6444 
6445 	lockdep_assert_held(&event_mutex);
6446 
6447 	if (hist_data->attrs->name)
6448 		named_data = find_named_trigger(hist_data->attrs->name);
6449 
6450 	list_for_each_entry(test, &file->triggers, list) {
6451 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6452 			if (!hist_trigger_match(data, test, named_data, false))
6453 				continue;
6454 			hist_data = test->private_data;
6455 			if (check_var_refs(hist_data))
6456 				return true;
6457 			break;
6458 		}
6459 	}
6460 
6461 	return false;
6462 }
6463 
6464 static void hist_unregister_trigger(char *glob,
6465 				    struct event_trigger_data *data,
6466 				    struct trace_event_file *file)
6467 {
6468 	struct event_trigger_data *test = NULL, *iter, *named_data = NULL;
6469 	struct hist_trigger_data *hist_data = data->private_data;
6470 
6471 	lockdep_assert_held(&event_mutex);
6472 
6473 	if (hist_data->attrs->name)
6474 		named_data = find_named_trigger(hist_data->attrs->name);
6475 
6476 	list_for_each_entry(iter, &file->triggers, list) {
6477 		if (iter->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6478 			if (!hist_trigger_match(data, iter, named_data, false))
6479 				continue;
6480 			test = iter;
6481 			list_del_rcu(&test->list);
6482 			trace_event_trigger_enable_disable(file, 0);
6483 			update_cond_flag(file);
6484 			break;
6485 		}
6486 	}
6487 
6488 	if (test && test->ops->free)
6489 		test->ops->free(test);
6490 
6491 	if (hist_data->enable_timestamps) {
6492 		if (!hist_data->remove || test)
6493 			tracing_set_filter_buffering(file->tr, false);
6494 	}
6495 }
6496 
6497 static bool hist_file_check_refs(struct trace_event_file *file)
6498 {
6499 	struct hist_trigger_data *hist_data;
6500 	struct event_trigger_data *test;
6501 
6502 	lockdep_assert_held(&event_mutex);
6503 
6504 	list_for_each_entry(test, &file->triggers, list) {
6505 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6506 			hist_data = test->private_data;
6507 			if (check_var_refs(hist_data))
6508 				return true;
6509 		}
6510 	}
6511 
6512 	return false;
6513 }
6514 
6515 static void hist_unreg_all(struct trace_event_file *file)
6516 {
6517 	struct event_trigger_data *test, *n;
6518 	struct hist_trigger_data *hist_data;
6519 	struct synth_event *se;
6520 	const char *se_name;
6521 
6522 	lockdep_assert_held(&event_mutex);
6523 
6524 	if (hist_file_check_refs(file))
6525 		return;
6526 
6527 	list_for_each_entry_safe(test, n, &file->triggers, list) {
6528 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6529 			hist_data = test->private_data;
6530 			list_del_rcu(&test->list);
6531 			trace_event_trigger_enable_disable(file, 0);
6532 
6533 			se_name = trace_event_name(file->event_call);
6534 			se = find_synth_event(se_name);
6535 			if (se)
6536 				se->ref--;
6537 
6538 			update_cond_flag(file);
6539 			if (hist_data->enable_timestamps)
6540 				tracing_set_filter_buffering(file->tr, false);
6541 			if (test->ops->free)
6542 				test->ops->free(test);
6543 		}
6544 	}
6545 }
6546 
6547 static int event_hist_trigger_parse(struct event_command *cmd_ops,
6548 				    struct trace_event_file *file,
6549 				    char *glob, char *cmd,
6550 				    char *param_and_filter)
6551 {
6552 	unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6553 	struct event_trigger_data *trigger_data;
6554 	struct hist_trigger_attrs *attrs;
6555 	struct hist_trigger_data *hist_data;
6556 	char *param, *filter, *p, *start;
6557 	struct synth_event *se;
6558 	const char *se_name;
6559 	bool remove;
6560 	int ret = 0;
6561 
6562 	lockdep_assert_held(&event_mutex);
6563 
6564 	if (WARN_ON(!glob))
6565 		return -EINVAL;
6566 
6567 	if (glob[0]) {
6568 		hist_err_clear();
6569 		last_cmd_set(file, param_and_filter);
6570 	}
6571 
6572 	remove = event_trigger_check_remove(glob);
6573 
6574 	if (event_trigger_empty_param(param_and_filter))
6575 		return -EINVAL;
6576 
6577 	/*
6578 	 * separate the trigger from the filter (k:v [if filter])
6579 	 * allowing for whitespace in the trigger
6580 	 */
6581 	p = param = param_and_filter;
6582 	do {
6583 		p = strstr(p, "if");
6584 		if (!p)
6585 			break;
6586 		if (p == param_and_filter)
6587 			return -EINVAL;
6588 		if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6589 			p++;
6590 			continue;
6591 		}
6592 		if (p >= param_and_filter + strlen(param_and_filter) - (sizeof("if") - 1) - 1)
6593 			return -EINVAL;
6594 		if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6595 			p++;
6596 			continue;
6597 		}
6598 		break;
6599 	} while (1);
6600 
6601 	if (!p)
6602 		filter = NULL;
6603 	else {
6604 		*(p - 1) = '\0';
6605 		filter = strstrip(p);
6606 		param = strstrip(param);
6607 	}
6608 
6609 	/*
6610 	 * To simplify arithmetic expression parsing, replace occurrences of
6611 	 * '.sym-offset' modifier with '.symXoffset'
6612 	 */
6613 	start = strstr(param, ".sym-offset");
6614 	while (start) {
6615 		*(start + 4) = 'X';
6616 		start = strstr(start + 11, ".sym-offset");
6617 	}
6618 
6619 	attrs = parse_hist_trigger_attrs(file->tr, param);
6620 	if (IS_ERR(attrs))
6621 		return PTR_ERR(attrs);
6622 
6623 	if (attrs->map_bits)
6624 		hist_trigger_bits = attrs->map_bits;
6625 
6626 	hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6627 	if (IS_ERR(hist_data)) {
6628 		destroy_hist_trigger_attrs(attrs);
6629 		return PTR_ERR(hist_data);
6630 	}
6631 
6632 	trigger_data = event_trigger_alloc(cmd_ops, cmd, param, hist_data);
6633 	if (!trigger_data) {
6634 		ret = -ENOMEM;
6635 		goto out_free;
6636 	}
6637 
6638 	ret = event_trigger_set_filter(cmd_ops, file, filter, trigger_data);
6639 	if (ret < 0)
6640 		goto out_free;
6641 
6642 	if (remove) {
6643 		if (!have_hist_trigger_match(trigger_data, file))
6644 			goto out_free;
6645 
6646 		if (hist_trigger_check_refs(trigger_data, file)) {
6647 			ret = -EBUSY;
6648 			goto out_free;
6649 		}
6650 
6651 		event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6652 		se_name = trace_event_name(file->event_call);
6653 		se = find_synth_event(se_name);
6654 		if (se)
6655 			se->ref--;
6656 		ret = 0;
6657 		goto out_free;
6658 	}
6659 
6660 	if (existing_hist_update_only(glob, trigger_data, file))
6661 		goto out_free;
6662 
6663 	ret = event_trigger_register(cmd_ops, file, glob, trigger_data);
6664 	if (ret < 0)
6665 		goto out_free;
6666 
6667 	if (get_named_trigger_data(trigger_data))
6668 		goto enable;
6669 
6670 	ret = create_actions(hist_data);
6671 	if (ret)
6672 		goto out_unreg;
6673 
6674 	if (has_hist_vars(hist_data) || hist_data->n_var_refs) {
6675 		ret = save_hist_vars(hist_data);
6676 		if (ret)
6677 			goto out_unreg;
6678 	}
6679 
6680 	ret = tracing_map_init(hist_data->map);
6681 	if (ret)
6682 		goto out_unreg;
6683 enable:
6684 	ret = hist_trigger_enable(trigger_data, file);
6685 	if (ret)
6686 		goto out_unreg;
6687 
6688 	se_name = trace_event_name(file->event_call);
6689 	se = find_synth_event(se_name);
6690 	if (se)
6691 		se->ref++;
6692  out:
6693 	if (ret == 0 && glob[0])
6694 		hist_err_clear();
6695 
6696 	return ret;
6697  out_unreg:
6698 	event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6699  out_free:
6700 	event_trigger_reset_filter(cmd_ops, trigger_data);
6701 
6702 	remove_hist_vars(hist_data);
6703 
6704 	kfree(trigger_data);
6705 
6706 	destroy_hist_data(hist_data);
6707 	goto out;
6708 }
6709 
6710 static struct event_command trigger_hist_cmd = {
6711 	.name			= "hist",
6712 	.trigger_type		= ETT_EVENT_HIST,
6713 	.flags			= EVENT_CMD_FL_NEEDS_REC,
6714 	.parse			= event_hist_trigger_parse,
6715 	.reg			= hist_register_trigger,
6716 	.unreg			= hist_unregister_trigger,
6717 	.unreg_all		= hist_unreg_all,
6718 	.get_trigger_ops	= event_hist_get_trigger_ops,
6719 	.set_filter		= set_trigger_filter,
6720 };
6721 
6722 __init int register_trigger_hist_cmd(void)
6723 {
6724 	int ret;
6725 
6726 	ret = register_event_command(&trigger_hist_cmd);
6727 	WARN_ON(ret < 0);
6728 
6729 	return ret;
6730 }
6731 
6732 static void
6733 hist_enable_trigger(struct event_trigger_data *data,
6734 		    struct trace_buffer *buffer,  void *rec,
6735 		    struct ring_buffer_event *event)
6736 {
6737 	struct enable_trigger_data *enable_data = data->private_data;
6738 	struct event_trigger_data *test;
6739 
6740 	list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6741 				lockdep_is_held(&event_mutex)) {
6742 		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6743 			if (enable_data->enable)
6744 				test->paused = false;
6745 			else
6746 				test->paused = true;
6747 		}
6748 	}
6749 }
6750 
6751 static void
6752 hist_enable_count_trigger(struct event_trigger_data *data,
6753 			  struct trace_buffer *buffer,  void *rec,
6754 			  struct ring_buffer_event *event)
6755 {
6756 	if (!data->count)
6757 		return;
6758 
6759 	if (data->count != -1)
6760 		(data->count)--;
6761 
6762 	hist_enable_trigger(data, buffer, rec, event);
6763 }
6764 
6765 static struct event_trigger_ops hist_enable_trigger_ops = {
6766 	.trigger		= hist_enable_trigger,
6767 	.print			= event_enable_trigger_print,
6768 	.init			= event_trigger_init,
6769 	.free			= event_enable_trigger_free,
6770 };
6771 
6772 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6773 	.trigger		= hist_enable_count_trigger,
6774 	.print			= event_enable_trigger_print,
6775 	.init			= event_trigger_init,
6776 	.free			= event_enable_trigger_free,
6777 };
6778 
6779 static struct event_trigger_ops hist_disable_trigger_ops = {
6780 	.trigger		= hist_enable_trigger,
6781 	.print			= event_enable_trigger_print,
6782 	.init			= event_trigger_init,
6783 	.free			= event_enable_trigger_free,
6784 };
6785 
6786 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6787 	.trigger		= hist_enable_count_trigger,
6788 	.print			= event_enable_trigger_print,
6789 	.init			= event_trigger_init,
6790 	.free			= event_enable_trigger_free,
6791 };
6792 
6793 static struct event_trigger_ops *
6794 hist_enable_get_trigger_ops(char *cmd, char *param)
6795 {
6796 	struct event_trigger_ops *ops;
6797 	bool enable;
6798 
6799 	enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6800 
6801 	if (enable)
6802 		ops = param ? &hist_enable_count_trigger_ops :
6803 			&hist_enable_trigger_ops;
6804 	else
6805 		ops = param ? &hist_disable_count_trigger_ops :
6806 			&hist_disable_trigger_ops;
6807 
6808 	return ops;
6809 }
6810 
6811 static void hist_enable_unreg_all(struct trace_event_file *file)
6812 {
6813 	struct event_trigger_data *test, *n;
6814 
6815 	list_for_each_entry_safe(test, n, &file->triggers, list) {
6816 		if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6817 			list_del_rcu(&test->list);
6818 			update_cond_flag(file);
6819 			trace_event_trigger_enable_disable(file, 0);
6820 			if (test->ops->free)
6821 				test->ops->free(test);
6822 		}
6823 	}
6824 }
6825 
6826 static struct event_command trigger_hist_enable_cmd = {
6827 	.name			= ENABLE_HIST_STR,
6828 	.trigger_type		= ETT_HIST_ENABLE,
6829 	.parse			= event_enable_trigger_parse,
6830 	.reg			= event_enable_register_trigger,
6831 	.unreg			= event_enable_unregister_trigger,
6832 	.unreg_all		= hist_enable_unreg_all,
6833 	.get_trigger_ops	= hist_enable_get_trigger_ops,
6834 	.set_filter		= set_trigger_filter,
6835 };
6836 
6837 static struct event_command trigger_hist_disable_cmd = {
6838 	.name			= DISABLE_HIST_STR,
6839 	.trigger_type		= ETT_HIST_ENABLE,
6840 	.parse			= event_enable_trigger_parse,
6841 	.reg			= event_enable_register_trigger,
6842 	.unreg			= event_enable_unregister_trigger,
6843 	.unreg_all		= hist_enable_unreg_all,
6844 	.get_trigger_ops	= hist_enable_get_trigger_ops,
6845 	.set_filter		= set_trigger_filter,
6846 };
6847 
6848 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6849 {
6850 	unregister_event_command(&trigger_hist_enable_cmd);
6851 	unregister_event_command(&trigger_hist_disable_cmd);
6852 }
6853 
6854 __init int register_trigger_hist_enable_disable_cmds(void)
6855 {
6856 	int ret;
6857 
6858 	ret = register_event_command(&trigger_hist_enable_cmd);
6859 	if (WARN_ON(ret < 0))
6860 		return ret;
6861 	ret = register_event_command(&trigger_hist_disable_cmd);
6862 	if (WARN_ON(ret < 0))
6863 		unregister_trigger_hist_enable_disable_cmds();
6864 
6865 	return ret;
6866 }
6867