1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * trace_events_hist - trace event hist triggers
   4 *
   5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6 */
   7
   8#include <linux/module.h>
   9#include <linux/kallsyms.h>
  10#include <linux/security.h>
  11#include <linux/mutex.h>
  12#include <linux/slab.h>
  13#include <linux/stacktrace.h>
  14#include <linux/rculist.h>
  15#include <linux/tracefs.h>
  16
  17/* for gfp flag names */
  18#include <linux/trace_events.h>
  19#include <trace/events/mmflags.h>
  20
  21#include "tracing_map.h"
  22#include "trace_synth.h"
  23
  24#define ERRORS								\
  25	C(NONE,			"No error"),				\
  26	C(DUPLICATE_VAR,	"Variable already defined"),		\
  27	C(VAR_NOT_UNIQUE,	"Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28	C(TOO_MANY_VARS,	"Too many variables defined"),		\
  29	C(MALFORMED_ASSIGNMENT,	"Malformed assignment"),		\
  30	C(NAMED_MISMATCH,	"Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31	C(TRIGGER_EEXIST,	"Hist trigger already exists"),		\
  32	C(TRIGGER_ENOENT_CLEAR,	"Can't clear or continue a nonexistent hist trigger"), \
  33	C(SET_CLOCK_FAIL,	"Couldn't set trace_clock"),		\
  34	C(BAD_FIELD_MODIFIER,	"Invalid field modifier"),		\
  35	C(TOO_MANY_SUBEXPR,	"Too many subexpressions (3 max)"),	\
  36	C(TIMESTAMP_MISMATCH,	"Timestamp units in expression don't match"), \
  37	C(TOO_MANY_FIELD_VARS,	"Too many field variables defined"),	\
  38	C(EVENT_FILE_NOT_FOUND,	"Event file not found"),		\
  39	C(HIST_NOT_FOUND,	"Matching event histogram not found"),	\
  40	C(HIST_CREATE_FAIL,	"Couldn't create histogram for field"),	\
  41	C(SYNTH_VAR_NOT_FOUND,	"Couldn't find synthetic variable"),	\
  42	C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),	\
  43	C(SYNTH_TYPE_MISMATCH,	"Param type doesn't match synthetic event field type"), \
  44	C(SYNTH_COUNT_MISMATCH,	"Param count doesn't match synthetic event field count"), \
  45	C(FIELD_VAR_PARSE_FAIL,	"Couldn't parse field variable"),	\
  46	C(VAR_CREATE_FIND_FAIL,	"Couldn't create or find variable"),	\
  47	C(ONX_NOT_VAR,		"For onmax(x) or onchange(x), x must be a variable"), \
  48	C(ONX_VAR_NOT_FOUND,	"Couldn't find onmax or onchange variable"), \
  49	C(ONX_VAR_CREATE_FAIL,	"Couldn't create onmax or onchange variable"), \
  50	C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),	\
  51	C(TOO_MANY_PARAMS,	"Too many action params"),		\
  52	C(PARAM_NOT_FOUND,	"Couldn't find param"),			\
  53	C(INVALID_PARAM,	"Invalid action param"),		\
  54	C(ACTION_NOT_FOUND,	"No action found"),			\
  55	C(NO_SAVE_PARAMS,	"No params found for save()"),		\
  56	C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57	C(ACTION_MISMATCH,	"Handler doesn't support action"),	\
  58	C(NO_CLOSING_PAREN,	"No closing paren found"),		\
  59	C(SUBSYS_NOT_FOUND,	"Missing subsystem"),			\
  60	C(INVALID_SUBSYS_EVENT,	"Invalid subsystem or event name"),	\
  61	C(INVALID_REF_KEY,	"Using variable references in keys not supported"), \
  62	C(VAR_NOT_FOUND,	"Couldn't find variable"),		\
  63	C(FIELD_NOT_FOUND,	"Couldn't find field"),			\
  64	C(EMPTY_ASSIGNMENT,	"Empty assignment"),			\
  65	C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),		\
  66	C(EMPTY_SORT_FIELD,	"Empty sort field"),			\
  67	C(TOO_MANY_SORT_FIELDS,	"Too many sort fields (Max = 2)"),	\
  68	C(INVALID_SORT_FIELD,	"Sort field must be a key or a val"),	\
  69	C(INVALID_STR_OPERAND,	"String type can not be an operand in expression"), \
  70	C(EXPECT_NUMBER,	"Expecting numeric literal"),		\
  71	C(UNARY_MINUS_SUBEXPR,	"Unary minus not supported in sub-expressions"), \
  72	C(DIVISION_BY_ZERO,	"Division by zero"),			\
  73	C(NEED_NOHC_VAL,	"Non-hitcount value is required for 'nohitcount'"),
  74
  75#undef C
  76#define C(a, b)		HIST_ERR_##a
  77
  78enum { ERRORS };
  79
  80#undef C
  81#define C(a, b)		b
  82
  83static const char *err_text[] = { ERRORS };
  84
  85struct hist_field;
  86
  87typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  88				struct tracing_map_elt *elt,
  89				struct trace_buffer *buffer,
  90				struct ring_buffer_event *rbe,
  91				void *event);
  92
  93#define HIST_FIELD_OPERANDS_MAX	2
  94#define HIST_FIELDS_MAX		(TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  95#define HIST_ACTIONS_MAX	8
  96#define HIST_CONST_DIGITS_MAX	21
  97#define HIST_DIV_SHIFT		20  /* For optimizing division by constants */
  98
  99enum field_op_id {
 100	FIELD_OP_NONE,
 101	FIELD_OP_PLUS,
 102	FIELD_OP_MINUS,
 103	FIELD_OP_UNARY_MINUS,
 104	FIELD_OP_DIV,
 105	FIELD_OP_MULT,
 106};
 107
 108enum hist_field_fn {
 109	HIST_FIELD_FN_NOP,
 110	HIST_FIELD_FN_VAR_REF,
 111	HIST_FIELD_FN_COUNTER,
 112	HIST_FIELD_FN_CONST,
 113	HIST_FIELD_FN_LOG2,
 114	HIST_FIELD_FN_BUCKET,
 115	HIST_FIELD_FN_TIMESTAMP,
 116	HIST_FIELD_FN_CPU,
 117	HIST_FIELD_FN_STRING,
 118	HIST_FIELD_FN_DYNSTRING,
 119	HIST_FIELD_FN_RELDYNSTRING,
 120	HIST_FIELD_FN_PSTRING,
 121	HIST_FIELD_FN_S64,
 122	HIST_FIELD_FN_U64,
 123	HIST_FIELD_FN_S32,
 124	HIST_FIELD_FN_U32,
 125	HIST_FIELD_FN_S16,
 126	HIST_FIELD_FN_U16,
 127	HIST_FIELD_FN_S8,
 128	HIST_FIELD_FN_U8,
 129	HIST_FIELD_FN_UMINUS,
 130	HIST_FIELD_FN_MINUS,
 131	HIST_FIELD_FN_PLUS,
 132	HIST_FIELD_FN_DIV,
 133	HIST_FIELD_FN_MULT,
 134	HIST_FIELD_FN_DIV_POWER2,
 135	HIST_FIELD_FN_DIV_NOT_POWER2,
 136	HIST_FIELD_FN_DIV_MULT_SHIFT,
 137	HIST_FIELD_FN_EXECNAME,
 138};
 139
 140/*
 141 * A hist_var (histogram variable) contains variable information for
 142 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 143 * flag set.  A hist_var has a variable name e.g. ts0, and is
 144 * associated with a given histogram trigger, as specified by
 145 * hist_data.  The hist_var idx is the unique index assigned to the
 146 * variable by the hist trigger's tracing_map.  The idx is what is
 147 * used to set a variable's value and, by a variable reference, to
 148 * retrieve it.
 149 */
 150struct hist_var {
 151	char				*name;
 152	struct hist_trigger_data	*hist_data;
 153	unsigned int			idx;
 154};
 155
 156struct hist_field {
 157	struct ftrace_event_field	*field;
 158	unsigned long			flags;
 159	unsigned long			buckets;
 160	const char			*type;
 161	struct hist_field		*operands[HIST_FIELD_OPERANDS_MAX];
 162	struct hist_trigger_data	*hist_data;
 163	enum hist_field_fn		fn_num;
 164	unsigned int			ref;
 165	unsigned int			size;
 166	unsigned int			offset;
 167	unsigned int                    is_signed;
 168
 169	/*
 170	 * Variable fields contain variable-specific info in var.
 171	 */
 172	struct hist_var			var;
 173	enum field_op_id		operator;
 174	char				*system;
 175	char				*event_name;
 176
 177	/*
 178	 * The name field is used for EXPR and VAR_REF fields.  VAR
 179	 * fields contain the variable name in var.name.
 180	 */
 181	char				*name;
 182
 183	/*
 184	 * When a histogram trigger is hit, if it has any references
 185	 * to variables, the values of those variables are collected
 186	 * into a var_ref_vals array by resolve_var_refs().  The
 187	 * current value of each variable is read from the tracing_map
 188	 * using the hist field's hist_var.idx and entered into the
 189	 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 190	 */
 191	unsigned int			var_ref_idx;
 192	bool                            read_once;
 193
 194	unsigned int			var_str_idx;
 195
 196	/* Numeric literals are represented as u64 */
 197	u64				constant;
 198	/* Used to optimize division by constants */
 199	u64				div_multiplier;
 200};
 201
 202static u64 hist_fn_call(struct hist_field *hist_field,
 203			struct tracing_map_elt *elt,
 204			struct trace_buffer *buffer,
 205			struct ring_buffer_event *rbe,
 206			void *event);
 207
 208static u64 hist_field_const(struct hist_field *field,
 209			   struct tracing_map_elt *elt,
 210			   struct trace_buffer *buffer,
 211			   struct ring_buffer_event *rbe,
 212			   void *event)
 213{
 214	return field->constant;
 215}
 216
 217static u64 hist_field_counter(struct hist_field *field,
 218			      struct tracing_map_elt *elt,
 219			      struct trace_buffer *buffer,
 220			      struct ring_buffer_event *rbe,
 221			      void *event)
 222{
 223	return 1;
 224}
 225
 226static u64 hist_field_string(struct hist_field *hist_field,
 227			     struct tracing_map_elt *elt,
 228			     struct trace_buffer *buffer,
 229			     struct ring_buffer_event *rbe,
 230			     void *event)
 231{
 232	char *addr = (char *)(event + hist_field->field->offset);
 233
 234	return (u64)(unsigned long)addr;
 235}
 236
 237static u64 hist_field_dynstring(struct hist_field *hist_field,
 238				struct tracing_map_elt *elt,
 239				struct trace_buffer *buffer,
 240				struct ring_buffer_event *rbe,
 241				void *event)
 242{
 243	u32 str_item = *(u32 *)(event + hist_field->field->offset);
 244	int str_loc = str_item & 0xffff;
 245	char *addr = (char *)(event + str_loc);
 246
 247	return (u64)(unsigned long)addr;
 248}
 249
 250static u64 hist_field_reldynstring(struct hist_field *hist_field,
 251				   struct tracing_map_elt *elt,
 252				   struct trace_buffer *buffer,
 253				   struct ring_buffer_event *rbe,
 254				   void *event)
 255{
 256	u32 *item = event + hist_field->field->offset;
 257	u32 str_item = *item;
 258	int str_loc = str_item & 0xffff;
 259	char *addr = (char *)&item[1] + str_loc;
 260
 261	return (u64)(unsigned long)addr;
 262}
 263
 264static u64 hist_field_pstring(struct hist_field *hist_field,
 265			      struct tracing_map_elt *elt,
 266			      struct trace_buffer *buffer,
 267			      struct ring_buffer_event *rbe,
 268			      void *event)
 269{
 270	char **addr = (char **)(event + hist_field->field->offset);
 271
 272	return (u64)(unsigned long)*addr;
 273}
 274
 275static u64 hist_field_log2(struct hist_field *hist_field,
 276			   struct tracing_map_elt *elt,
 277			   struct trace_buffer *buffer,
 278			   struct ring_buffer_event *rbe,
 279			   void *event)
 280{
 281	struct hist_field *operand = hist_field->operands[0];
 282
 283	u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 284
 285	return (u64) ilog2(roundup_pow_of_two(val));
 286}
 287
 288static u64 hist_field_bucket(struct hist_field *hist_field,
 289			     struct tracing_map_elt *elt,
 290			     struct trace_buffer *buffer,
 291			     struct ring_buffer_event *rbe,
 292			     void *event)
 293{
 294	struct hist_field *operand = hist_field->operands[0];
 295	unsigned long buckets = hist_field->buckets;
 296
 297	u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 298
 299	if (WARN_ON_ONCE(!buckets))
 300		return val;
 301
 302	if (val >= LONG_MAX)
 303		val = div64_ul(val, buckets);
 304	else
 305		val = (u64)((unsigned long)val / buckets);
 306	return val * buckets;
 307}
 308
 309static u64 hist_field_plus(struct hist_field *hist_field,
 310			   struct tracing_map_elt *elt,
 311			   struct trace_buffer *buffer,
 312			   struct ring_buffer_event *rbe,
 313			   void *event)
 314{
 315	struct hist_field *operand1 = hist_field->operands[0];
 316	struct hist_field *operand2 = hist_field->operands[1];
 317
 318	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 319	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 320
 321	return val1 + val2;
 322}
 323
 324static u64 hist_field_minus(struct hist_field *hist_field,
 325			    struct tracing_map_elt *elt,
 326			    struct trace_buffer *buffer,
 327			    struct ring_buffer_event *rbe,
 328			    void *event)
 329{
 330	struct hist_field *operand1 = hist_field->operands[0];
 331	struct hist_field *operand2 = hist_field->operands[1];
 332
 333	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 334	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 335
 336	return val1 - val2;
 337}
 338
 339static u64 hist_field_div(struct hist_field *hist_field,
 340			   struct tracing_map_elt *elt,
 341			   struct trace_buffer *buffer,
 342			   struct ring_buffer_event *rbe,
 343			   void *event)
 344{
 345	struct hist_field *operand1 = hist_field->operands[0];
 346	struct hist_field *operand2 = hist_field->operands[1];
 347
 348	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 349	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 350
 351	/* Return -1 for the undefined case */
 352	if (!val2)
 353		return -1;
 354
 355	/* Use shift if the divisor is a power of 2 */
 356	if (!(val2 & (val2 - 1)))
 357		return val1 >> __ffs64(val2);
 358
 359	return div64_u64(val1, val2);
 360}
 361
 362static u64 div_by_power_of_two(struct hist_field *hist_field,
 363				struct tracing_map_elt *elt,
 364				struct trace_buffer *buffer,
 365				struct ring_buffer_event *rbe,
 366				void *event)
 367{
 368	struct hist_field *operand1 = hist_field->operands[0];
 369	struct hist_field *operand2 = hist_field->operands[1];
 370
 371	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 372
 373	return val1 >> __ffs64(operand2->constant);
 374}
 375
 376static u64 div_by_not_power_of_two(struct hist_field *hist_field,
 377				struct tracing_map_elt *elt,
 378				struct trace_buffer *buffer,
 379				struct ring_buffer_event *rbe,
 380				void *event)
 381{
 382	struct hist_field *operand1 = hist_field->operands[0];
 383	struct hist_field *operand2 = hist_field->operands[1];
 384
 385	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 386
 387	return div64_u64(val1, operand2->constant);
 388}
 389
 390static u64 div_by_mult_and_shift(struct hist_field *hist_field,
 391				struct tracing_map_elt *elt,
 392				struct trace_buffer *buffer,
 393				struct ring_buffer_event *rbe,
 394				void *event)
 395{
 396	struct hist_field *operand1 = hist_field->operands[0];
 397	struct hist_field *operand2 = hist_field->operands[1];
 398
 399	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 400
 401	/*
 402	 * If the divisor is a constant, do a multiplication and shift instead.
 403	 *
 404	 * Choose Z = some power of 2. If Y <= Z, then:
 405	 *     X / Y = (X * (Z / Y)) / Z
 406	 *
 407	 * (Z / Y) is a constant (mult) which is calculated at parse time, so:
 408	 *     X / Y = (X * mult) / Z
 409	 *
 410	 * The division by Z can be replaced by a shift since Z is a power of 2:
 411	 *     X / Y = (X * mult) >> HIST_DIV_SHIFT
 412	 *
 413	 * As long, as X < Z the results will not be off by more than 1.
 414	 */
 415	if (val1 < (1 << HIST_DIV_SHIFT)) {
 416		u64 mult = operand2->div_multiplier;
 417
 418		return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT;
 419	}
 420
 421	return div64_u64(val1, operand2->constant);
 422}
 423
 424static u64 hist_field_mult(struct hist_field *hist_field,
 425			   struct tracing_map_elt *elt,
 426			   struct trace_buffer *buffer,
 427			   struct ring_buffer_event *rbe,
 428			   void *event)
 429{
 430	struct hist_field *operand1 = hist_field->operands[0];
 431	struct hist_field *operand2 = hist_field->operands[1];
 432
 433	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 434	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 435
 436	return val1 * val2;
 437}
 438
 439static u64 hist_field_unary_minus(struct hist_field *hist_field,
 440				  struct tracing_map_elt *elt,
 441				  struct trace_buffer *buffer,
 442				  struct ring_buffer_event *rbe,
 443				  void *event)
 444{
 445	struct hist_field *operand = hist_field->operands[0];
 446
 447	s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event);
 448	u64 val = (u64)-sval;
 449
 450	return val;
 451}
 452
 453#define DEFINE_HIST_FIELD_FN(type)					\
 454	static u64 hist_field_##type(struct hist_field *hist_field,	\
 455				     struct tracing_map_elt *elt,	\
 456				     struct trace_buffer *buffer,	\
 457				     struct ring_buffer_event *rbe,	\
 458				     void *event)			\
 459{									\
 460	type *addr = (type *)(event + hist_field->field->offset);	\
 461									\
 462	return (u64)(unsigned long)*addr;				\
 463}
 464
 465DEFINE_HIST_FIELD_FN(s64);
 466DEFINE_HIST_FIELD_FN(u64);
 467DEFINE_HIST_FIELD_FN(s32);
 468DEFINE_HIST_FIELD_FN(u32);
 469DEFINE_HIST_FIELD_FN(s16);
 470DEFINE_HIST_FIELD_FN(u16);
 471DEFINE_HIST_FIELD_FN(s8);
 472DEFINE_HIST_FIELD_FN(u8);
 473
 474#define for_each_hist_field(i, hist_data)	\
 475	for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 476
 477#define for_each_hist_val_field(i, hist_data)	\
 478	for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 479
 480#define for_each_hist_key_field(i, hist_data)	\
 481	for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 482
 483#define HIST_STACKTRACE_DEPTH	16
 484#define HIST_STACKTRACE_SIZE	(HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 485#define HIST_STACKTRACE_SKIP	5
 486
 487#define HITCOUNT_IDX		0
 488#define HIST_KEY_SIZE_MAX	(MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 489
 490enum hist_field_flags {
 491	HIST_FIELD_FL_HITCOUNT		= 1 << 0,
 492	HIST_FIELD_FL_KEY		= 1 << 1,
 493	HIST_FIELD_FL_STRING		= 1 << 2,
 494	HIST_FIELD_FL_HEX		= 1 << 3,
 495	HIST_FIELD_FL_SYM		= 1 << 4,
 496	HIST_FIELD_FL_SYM_OFFSET	= 1 << 5,
 497	HIST_FIELD_FL_EXECNAME		= 1 << 6,
 498	HIST_FIELD_FL_SYSCALL		= 1 << 7,
 499	HIST_FIELD_FL_STACKTRACE	= 1 << 8,
 500	HIST_FIELD_FL_LOG2		= 1 << 9,
 501	HIST_FIELD_FL_TIMESTAMP		= 1 << 10,
 502	HIST_FIELD_FL_TIMESTAMP_USECS	= 1 << 11,
 503	HIST_FIELD_FL_VAR		= 1 << 12,
 504	HIST_FIELD_FL_EXPR		= 1 << 13,
 505	HIST_FIELD_FL_VAR_REF		= 1 << 14,
 506	HIST_FIELD_FL_CPU		= 1 << 15,
 507	HIST_FIELD_FL_ALIAS		= 1 << 16,
 508	HIST_FIELD_FL_BUCKET		= 1 << 17,
 509	HIST_FIELD_FL_CONST		= 1 << 18,
 510	HIST_FIELD_FL_PERCENT		= 1 << 19,
 511	HIST_FIELD_FL_GRAPH		= 1 << 20,
 512};
 513
 514struct var_defs {
 515	unsigned int	n_vars;
 516	char		*name[TRACING_MAP_VARS_MAX];
 517	char		*expr[TRACING_MAP_VARS_MAX];
 518};
 519
 520struct hist_trigger_attrs {
 521	char		*keys_str;
 522	char		*vals_str;
 523	char		*sort_key_str;
 524	char		*name;
 525	char		*clock;
 526	bool		pause;
 527	bool		cont;
 528	bool		clear;
 529	bool		ts_in_usecs;
 530	bool		no_hitcount;
 531	unsigned int	map_bits;
 532
 533	char		*assignment_str[TRACING_MAP_VARS_MAX];
 534	unsigned int	n_assignments;
 535
 536	char		*action_str[HIST_ACTIONS_MAX];
 537	unsigned int	n_actions;
 538
 539	struct var_defs	var_defs;
 540};
 541
 542struct field_var {
 543	struct hist_field	*var;
 544	struct hist_field	*val;
 545};
 546
 547struct field_var_hist {
 548	struct hist_trigger_data	*hist_data;
 549	char				*cmd;
 550};
 551
 552struct hist_trigger_data {
 553	struct hist_field               *fields[HIST_FIELDS_MAX];
 554	unsigned int			n_vals;
 555	unsigned int			n_keys;
 556	unsigned int			n_fields;
 557	unsigned int			n_vars;
 558	unsigned int			n_var_str;
 559	unsigned int			key_size;
 560	struct tracing_map_sort_key	sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 561	unsigned int			n_sort_keys;
 562	struct trace_event_file		*event_file;
 563	struct hist_trigger_attrs	*attrs;
 564	struct tracing_map		*map;
 565	bool				enable_timestamps;
 566	bool				remove;
 567	struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 568	unsigned int			n_var_refs;
 569
 570	struct action_data		*actions[HIST_ACTIONS_MAX];
 571	unsigned int			n_actions;
 572
 573	struct field_var		*field_vars[SYNTH_FIELDS_MAX];
 574	unsigned int			n_field_vars;
 575	unsigned int			n_field_var_str;
 576	struct field_var_hist		*field_var_hists[SYNTH_FIELDS_MAX];
 577	unsigned int			n_field_var_hists;
 578
 579	struct field_var		*save_vars[SYNTH_FIELDS_MAX];
 580	unsigned int			n_save_vars;
 581	unsigned int			n_save_var_str;
 582};
 583
 584struct action_data;
 585
 586typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 587			     struct tracing_map_elt *elt,
 588			     struct trace_buffer *buffer, void *rec,
 589			     struct ring_buffer_event *rbe, void *key,
 590			     struct action_data *data, u64 *var_ref_vals);
 591
 592typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 593
 594enum handler_id {
 595	HANDLER_ONMATCH = 1,
 596	HANDLER_ONMAX,
 597	HANDLER_ONCHANGE,
 598};
 599
 600enum action_id {
 601	ACTION_SAVE = 1,
 602	ACTION_TRACE,
 603	ACTION_SNAPSHOT,
 604};
 605
 606struct action_data {
 607	enum handler_id		handler;
 608	enum action_id		action;
 609	char			*action_name;
 610	action_fn_t		fn;
 611
 612	unsigned int		n_params;
 613	char			*params[SYNTH_FIELDS_MAX];
 614
 615	/*
 616	 * When a histogram trigger is hit, the values of any
 617	 * references to variables, including variables being passed
 618	 * as parameters to synthetic events, are collected into a
 619	 * var_ref_vals array.  This var_ref_idx array is an array of
 620	 * indices into the var_ref_vals array, one for each synthetic
 621	 * event param, and is passed to the synthetic event
 622	 * invocation.
 623	 */
 624	unsigned int		var_ref_idx[SYNTH_FIELDS_MAX];
 625	struct synth_event	*synth_event;
 626	bool			use_trace_keyword;
 627	char			*synth_event_name;
 628
 629	union {
 630		struct {
 631			char			*event;
 632			char			*event_system;
 633		} match_data;
 634
 635		struct {
 636			/*
 637			 * var_str contains the $-unstripped variable
 638			 * name referenced by var_ref, and used when
 639			 * printing the action.  Because var_ref
 640			 * creation is deferred to create_actions(),
 641			 * we need a per-action way to save it until
 642			 * then, thus var_str.
 643			 */
 644			char			*var_str;
 645
 646			/*
 647			 * var_ref refers to the variable being
 648			 * tracked e.g onmax($var).
 649			 */
 650			struct hist_field	*var_ref;
 651
 652			/*
 653			 * track_var contains the 'invisible' tracking
 654			 * variable created to keep the current
 655			 * e.g. max value.
 656			 */
 657			struct hist_field	*track_var;
 658
 659			check_track_val_fn_t	check_val;
 660			action_fn_t		save_data;
 661		} track_data;
 662	};
 663};
 664
 665struct track_data {
 666	u64				track_val;
 667	bool				updated;
 668
 669	unsigned int			key_len;
 670	void				*key;
 671	struct tracing_map_elt		elt;
 672
 673	struct action_data		*action_data;
 674	struct hist_trigger_data	*hist_data;
 675};
 676
 677struct hist_elt_data {
 678	char *comm;
 679	u64 *var_ref_vals;
 680	char **field_var_str;
 681	int n_field_var_str;
 682};
 683
 684struct snapshot_context {
 685	struct tracing_map_elt	*elt;
 686	void			*key;
 687};
 688
 689/*
 690 * Returns the specific division function to use if the divisor
 691 * is constant. This avoids extra branches when the trigger is hit.
 692 */
 693static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor)
 694{
 695	u64 div = divisor->constant;
 696
 697	if (!(div & (div - 1)))
 698		return HIST_FIELD_FN_DIV_POWER2;
 699
 700	/* If the divisor is too large, do a regular division */
 701	if (div > (1 << HIST_DIV_SHIFT))
 702		return HIST_FIELD_FN_DIV_NOT_POWER2;
 703
 704	divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
 705	return HIST_FIELD_FN_DIV_MULT_SHIFT;
 706}
 707
 708static void track_data_free(struct track_data *track_data)
 709{
 710	struct hist_elt_data *elt_data;
 711
 712	if (!track_data)
 713		return;
 714
 715	kfree(track_data->key);
 716
 717	elt_data = track_data->elt.private_data;
 718	if (elt_data) {
 719		kfree(elt_data->comm);
 720		kfree(elt_data);
 721	}
 722
 723	kfree(track_data);
 724}
 725
 726static struct track_data *track_data_alloc(unsigned int key_len,
 727					   struct action_data *action_data,
 728					   struct hist_trigger_data *hist_data)
 729{
 730	struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 731	struct hist_elt_data *elt_data;
 732
 733	if (!data)
 734		return ERR_PTR(-ENOMEM);
 735
 736	data->key = kzalloc(key_len, GFP_KERNEL);
 737	if (!data->key) {
 738		track_data_free(data);
 739		return ERR_PTR(-ENOMEM);
 740	}
 741
 742	data->key_len = key_len;
 743	data->action_data = action_data;
 744	data->hist_data = hist_data;
 745
 746	elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 747	if (!elt_data) {
 748		track_data_free(data);
 749		return ERR_PTR(-ENOMEM);
 750	}
 751
 752	data->elt.private_data = elt_data;
 753
 754	elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 755	if (!elt_data->comm) {
 756		track_data_free(data);
 757		return ERR_PTR(-ENOMEM);
 758	}
 759
 760	return data;
 761}
 762
 763#define HIST_PREFIX "hist:"
 764
 765static char *last_cmd;
 766static char last_cmd_loc[MAX_FILTER_STR_VAL];
 767
 768static int errpos(char *str)
 769{
 770	if (!str || !last_cmd)
 771		return 0;
 772
 773	return err_pos(last_cmd, str);
 774}
 775
 776static void last_cmd_set(struct trace_event_file *file, char *str)
 777{
 778	const char *system = NULL, *name = NULL;
 779	struct trace_event_call *call;
 780	int len;
 781
 782	if (!str)
 783		return;
 784
 785	/* sizeof() contains the nul byte */
 786	len = sizeof(HIST_PREFIX) + strlen(str);
 787	kfree(last_cmd);
 788	last_cmd = kzalloc(len, GFP_KERNEL);
 789	if (!last_cmd)
 790		return;
 791
 792	strcpy(last_cmd, HIST_PREFIX);
 793	/* Again, sizeof() contains the nul byte */
 794	len -= sizeof(HIST_PREFIX);
 795	strncat(last_cmd, str, len);
 796
 797	if (file) {
 798		call = file->event_call;
 799		system = call->class->system;
 800		if (system) {
 801			name = trace_event_name(call);
 802			if (!name)
 803				system = NULL;
 804		}
 805	}
 806
 807	if (system)
 808		snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, HIST_PREFIX "%s:%s", system, name);
 809}
 810
 811static void hist_err(struct trace_array *tr, u8 err_type, u16 err_pos)
 812{
 813	if (!last_cmd)
 814		return;
 815
 816	tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 817			err_type, err_pos);
 818}
 819
 820static void hist_err_clear(void)
 821{
 822	if (last_cmd)
 823		last_cmd[0] = '\0';
 824	last_cmd_loc[0] = '\0';
 825}
 826
 827typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 828				    unsigned int *var_ref_idx);
 829
 830static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 831			       unsigned int *var_ref_idx)
 832{
 833	struct tracepoint *tp = event->tp;
 834
 835	if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 836		struct tracepoint_func *probe_func_ptr;
 837		synth_probe_func_t probe_func;
 838		void *__data;
 839
 840		if (!(cpu_online(raw_smp_processor_id())))
 841			return;
 842
 843		probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 844		if (probe_func_ptr) {
 845			do {
 846				probe_func = probe_func_ptr->func;
 847				__data = probe_func_ptr->data;
 848				probe_func(__data, var_ref_vals, var_ref_idx);
 849			} while ((++probe_func_ptr)->func);
 850		}
 851	}
 852}
 853
 854static void action_trace(struct hist_trigger_data *hist_data,
 855			 struct tracing_map_elt *elt,
 856			 struct trace_buffer *buffer, void *rec,
 857			 struct ring_buffer_event *rbe, void *key,
 858			 struct action_data *data, u64 *var_ref_vals)
 859{
 860	struct synth_event *event = data->synth_event;
 861
 862	trace_synth(event, var_ref_vals, data->var_ref_idx);
 863}
 864
 865struct hist_var_data {
 866	struct list_head list;
 867	struct hist_trigger_data *hist_data;
 868};
 869
 870static u64 hist_field_timestamp(struct hist_field *hist_field,
 871				struct tracing_map_elt *elt,
 872				struct trace_buffer *buffer,
 873				struct ring_buffer_event *rbe,
 874				void *event)
 875{
 876	struct hist_trigger_data *hist_data = hist_field->hist_data;
 877	struct trace_array *tr = hist_data->event_file->tr;
 878
 879	u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 880
 881	if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 882		ts = ns2usecs(ts);
 883
 884	return ts;
 885}
 886
 887static u64 hist_field_cpu(struct hist_field *hist_field,
 888			  struct tracing_map_elt *elt,
 889			  struct trace_buffer *buffer,
 890			  struct ring_buffer_event *rbe,
 891			  void *event)
 892{
 893	int cpu = smp_processor_id();
 894
 895	return cpu;
 896}
 897
 898/**
 899 * check_field_for_var_ref - Check if a VAR_REF field references a variable
 900 * @hist_field: The VAR_REF field to check
 901 * @var_data: The hist trigger that owns the variable
 902 * @var_idx: The trigger variable identifier
 903 *
 904 * Check the given VAR_REF field to see whether or not it references
 905 * the given variable associated with the given trigger.
 906 *
 907 * Return: The VAR_REF field if it does reference the variable, NULL if not
 908 */
 909static struct hist_field *
 910check_field_for_var_ref(struct hist_field *hist_field,
 911			struct hist_trigger_data *var_data,
 912			unsigned int var_idx)
 913{
 914	WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 915
 916	if (hist_field && hist_field->var.idx == var_idx &&
 917	    hist_field->var.hist_data == var_data)
 918		return hist_field;
 919
 920	return NULL;
 921}
 922
 923/**
 924 * find_var_ref - Check if a trigger has a reference to a trigger variable
 925 * @hist_data: The hist trigger that might have a reference to the variable
 926 * @var_data: The hist trigger that owns the variable
 927 * @var_idx: The trigger variable identifier
 928 *
 929 * Check the list of var_refs[] on the first hist trigger to see
 930 * whether any of them are references to the variable on the second
 931 * trigger.
 932 *
 933 * Return: The VAR_REF field referencing the variable if so, NULL if not
 934 */
 935static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 936				       struct hist_trigger_data *var_data,
 937				       unsigned int var_idx)
 938{
 939	struct hist_field *hist_field;
 940	unsigned int i;
 941
 942	for (i = 0; i < hist_data->n_var_refs; i++) {
 943		hist_field = hist_data->var_refs[i];
 944		if (check_field_for_var_ref(hist_field, var_data, var_idx))
 945			return hist_field;
 946	}
 947
 948	return NULL;
 949}
 950
 951/**
 952 * find_any_var_ref - Check if there is a reference to a given trigger variable
 953 * @hist_data: The hist trigger
 954 * @var_idx: The trigger variable identifier
 955 *
 956 * Check to see whether the given variable is currently referenced by
 957 * any other trigger.
 958 *
 959 * The trigger the variable is defined on is explicitly excluded - the
 960 * assumption being that a self-reference doesn't prevent a trigger
 961 * from being removed.
 962 *
 963 * Return: The VAR_REF field referencing the variable if so, NULL if not
 964 */
 965static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 966					   unsigned int var_idx)
 967{
 968	struct trace_array *tr = hist_data->event_file->tr;
 969	struct hist_field *found = NULL;
 970	struct hist_var_data *var_data;
 971
 972	list_for_each_entry(var_data, &tr->hist_vars, list) {
 973		if (var_data->hist_data == hist_data)
 974			continue;
 975		found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 976		if (found)
 977			break;
 978	}
 979
 980	return found;
 981}
 982
 983/**
 984 * check_var_refs - Check if there is a reference to any of trigger's variables
 985 * @hist_data: The hist trigger
 986 *
 987 * A trigger can define one or more variables.  If any one of them is
 988 * currently referenced by any other trigger, this function will
 989 * determine that.
 990 *
 991 * Typically used to determine whether or not a trigger can be removed
 992 * - if there are any references to a trigger's variables, it cannot.
 993 *
 994 * Return: True if there is a reference to any of trigger's variables
 995 */
 996static bool check_var_refs(struct hist_trigger_data *hist_data)
 997{
 998	struct hist_field *field;
 999	bool found = false;
1000	int i;
1001
1002	for_each_hist_field(i, hist_data) {
1003		field = hist_data->fields[i];
1004		if (field && field->flags & HIST_FIELD_FL_VAR) {
1005			if (find_any_var_ref(hist_data, field->var.idx)) {
1006				found = true;
1007				break;
1008			}
1009		}
1010	}
1011
1012	return found;
1013}
1014
1015static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1016{
1017	struct trace_array *tr = hist_data->event_file->tr;
1018	struct hist_var_data *var_data, *found = NULL;
1019
1020	list_for_each_entry(var_data, &tr->hist_vars, list) {
1021		if (var_data->hist_data == hist_data) {
1022			found = var_data;
1023			break;
1024		}
1025	}
1026
1027	return found;
1028}
1029
1030static bool field_has_hist_vars(struct hist_field *hist_field,
1031				unsigned int level)
1032{
1033	int i;
1034
1035	if (level > 3)
1036		return false;
1037
1038	if (!hist_field)
1039		return false;
1040
1041	if (hist_field->flags & HIST_FIELD_FL_VAR ||
1042	    hist_field->flags & HIST_FIELD_FL_VAR_REF)
1043		return true;
1044
1045	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1046		struct hist_field *operand;
1047
1048		operand = hist_field->operands[i];
1049		if (field_has_hist_vars(operand, level + 1))
1050			return true;
1051	}
1052
1053	return false;
1054}
1055
1056static bool has_hist_vars(struct hist_trigger_data *hist_data)
1057{
1058	struct hist_field *hist_field;
1059	int i;
1060
1061	for_each_hist_field(i, hist_data) {
1062		hist_field = hist_data->fields[i];
1063		if (field_has_hist_vars(hist_field, 0))
1064			return true;
1065	}
1066
1067	return false;
1068}
1069
1070static int save_hist_vars(struct hist_trigger_data *hist_data)
1071{
1072	struct trace_array *tr = hist_data->event_file->tr;
1073	struct hist_var_data *var_data;
1074
1075	var_data = find_hist_vars(hist_data);
1076	if (var_data)
1077		return 0;
1078
1079	if (tracing_check_open_get_tr(tr))
1080		return -ENODEV;
1081
1082	var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1083	if (!var_data) {
1084		trace_array_put(tr);
1085		return -ENOMEM;
1086	}
1087
1088	var_data->hist_data = hist_data;
1089	list_add(&var_data->list, &tr->hist_vars);
1090
1091	return 0;
1092}
1093
1094static void remove_hist_vars(struct hist_trigger_data *hist_data)
1095{
1096	struct trace_array *tr = hist_data->event_file->tr;
1097	struct hist_var_data *var_data;
1098
1099	var_data = find_hist_vars(hist_data);
1100	if (!var_data)
1101		return;
1102
1103	if (WARN_ON(check_var_refs(hist_data)))
1104		return;
1105
1106	list_del(&var_data->list);
1107
1108	kfree(var_data);
1109
1110	trace_array_put(tr);
1111}
1112
1113static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1114					 const char *var_name)
1115{
1116	struct hist_field *hist_field, *found = NULL;
1117	int i;
1118
1119	for_each_hist_field(i, hist_data) {
1120		hist_field = hist_data->fields[i];
1121		if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1122		    strcmp(hist_field->var.name, var_name) == 0) {
1123			found = hist_field;
1124			break;
1125		}
1126	}
1127
1128	return found;
1129}
1130
1131static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1132				   struct trace_event_file *file,
1133				   const char *var_name)
1134{
1135	struct hist_trigger_data *test_data;
1136	struct event_trigger_data *test;
1137	struct hist_field *hist_field;
1138
1139	lockdep_assert_held(&event_mutex);
1140
1141	hist_field = find_var_field(hist_data, var_name);
1142	if (hist_field)
1143		return hist_field;
1144
1145	list_for_each_entry(test, &file->triggers, list) {
1146		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1147			test_data = test->private_data;
1148			hist_field = find_var_field(test_data, var_name);
1149			if (hist_field)
1150				return hist_field;
1151		}
1152	}
1153
1154	return NULL;
1155}
1156
1157static struct trace_event_file *find_var_file(struct trace_array *tr,
1158					      char *system,
1159					      char *event_name,
1160					      char *var_name)
1161{
1162	struct hist_trigger_data *var_hist_data;
1163	struct hist_var_data *var_data;
1164	struct trace_event_file *file, *found = NULL;
1165
1166	if (system)
1167		return find_event_file(tr, system, event_name);
1168
1169	list_for_each_entry(var_data, &tr->hist_vars, list) {
1170		var_hist_data = var_data->hist_data;
1171		file = var_hist_data->event_file;
1172		if (file == found)
1173			continue;
1174
1175		if (find_var_field(var_hist_data, var_name)) {
1176			if (found) {
1177				hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1178				return NULL;
1179			}
1180
1181			found = file;
1182		}
1183	}
1184
1185	return found;
1186}
1187
1188static struct hist_field *find_file_var(struct trace_event_file *file,
1189					const char *var_name)
1190{
1191	struct hist_trigger_data *test_data;
1192	struct event_trigger_data *test;
1193	struct hist_field *hist_field;
1194
1195	lockdep_assert_held(&event_mutex);
1196
1197	list_for_each_entry(test, &file->triggers, list) {
1198		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1199			test_data = test->private_data;
1200			hist_field = find_var_field(test_data, var_name);
1201			if (hist_field)
1202				return hist_field;
1203		}
1204	}
1205
1206	return NULL;
1207}
1208
1209static struct hist_field *
1210find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1211{
1212	struct trace_array *tr = hist_data->event_file->tr;
1213	struct hist_field *hist_field, *found = NULL;
1214	struct trace_event_file *file;
1215	unsigned int i;
1216
1217	for (i = 0; i < hist_data->n_actions; i++) {
1218		struct action_data *data = hist_data->actions[i];
1219
1220		if (data->handler == HANDLER_ONMATCH) {
1221			char *system = data->match_data.event_system;
1222			char *event_name = data->match_data.event;
1223
1224			file = find_var_file(tr, system, event_name, var_name);
1225			if (!file)
1226				continue;
1227			hist_field = find_file_var(file, var_name);
1228			if (hist_field) {
1229				if (found) {
1230					hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1231						 errpos(var_name));
1232					return ERR_PTR(-EINVAL);
1233				}
1234
1235				found = hist_field;
1236			}
1237		}
1238	}
1239	return found;
1240}
1241
1242static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1243					 char *system,
1244					 char *event_name,
1245					 char *var_name)
1246{
1247	struct trace_array *tr = hist_data->event_file->tr;
1248	struct hist_field *hist_field = NULL;
1249	struct trace_event_file *file;
1250
1251	if (!system || !event_name) {
1252		hist_field = find_match_var(hist_data, var_name);
1253		if (IS_ERR(hist_field))
1254			return NULL;
1255		if (hist_field)
1256			return hist_field;
1257	}
1258
1259	file = find_var_file(tr, system, event_name, var_name);
1260	if (!file)
1261		return NULL;
1262
1263	hist_field = find_file_var(file, var_name);
1264
1265	return hist_field;
1266}
1267
1268static u64 hist_field_var_ref(struct hist_field *hist_field,
1269			      struct tracing_map_elt *elt,
1270			      struct trace_buffer *buffer,
1271			      struct ring_buffer_event *rbe,
1272			      void *event)
1273{
1274	struct hist_elt_data *elt_data;
1275	u64 var_val = 0;
1276
1277	if (WARN_ON_ONCE(!elt))
1278		return var_val;
1279
1280	elt_data = elt->private_data;
1281	var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1282
1283	return var_val;
1284}
1285
1286static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1287			     u64 *var_ref_vals, bool self)
1288{
1289	struct hist_trigger_data *var_data;
1290	struct tracing_map_elt *var_elt;
1291	struct hist_field *hist_field;
1292	unsigned int i, var_idx;
1293	bool resolved = true;
1294	u64 var_val = 0;
1295
1296	for (i = 0; i < hist_data->n_var_refs; i++) {
1297		hist_field = hist_data->var_refs[i];
1298		var_idx = hist_field->var.idx;
1299		var_data = hist_field->var.hist_data;
1300
1301		if (var_data == NULL) {
1302			resolved = false;
1303			break;
1304		}
1305
1306		if ((self && var_data != hist_data) ||
1307		    (!self && var_data == hist_data))
1308			continue;
1309
1310		var_elt = tracing_map_lookup(var_data->map, key);
1311		if (!var_elt) {
1312			resolved = false;
1313			break;
1314		}
1315
1316		if (!tracing_map_var_set(var_elt, var_idx)) {
1317			resolved = false;
1318			break;
1319		}
1320
1321		if (self || !hist_field->read_once)
1322			var_val = tracing_map_read_var(var_elt, var_idx);
1323		else
1324			var_val = tracing_map_read_var_once(var_elt, var_idx);
1325
1326		var_ref_vals[i] = var_val;
1327	}
1328
1329	return resolved;
1330}
1331
1332static const char *hist_field_name(struct hist_field *field,
1333				   unsigned int level)
1334{
1335	const char *field_name = "";
1336
1337	if (level > 1)
1338		return field_name;
1339
1340	if (field->field)
1341		field_name = field->field->name;
1342	else if (field->flags & HIST_FIELD_FL_LOG2 ||
1343		 field->flags & HIST_FIELD_FL_ALIAS ||
1344		 field->flags & HIST_FIELD_FL_BUCKET)
1345		field_name = hist_field_name(field->operands[0], ++level);
1346	else if (field->flags & HIST_FIELD_FL_CPU)
1347		field_name = "common_cpu";
1348	else if (field->flags & HIST_FIELD_FL_EXPR ||
1349		 field->flags & HIST_FIELD_FL_VAR_REF) {
1350		if (field->system) {
1351			static char full_name[MAX_FILTER_STR_VAL];
1352
1353			strcat(full_name, field->system);
1354			strcat(full_name, ".");
1355			strcat(full_name, field->event_name);
1356			strcat(full_name, ".");
1357			strcat(full_name, field->name);
1358			field_name = full_name;
1359		} else
1360			field_name = field->name;
1361	} else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1362		field_name = "common_timestamp";
1363	else if (field->flags & HIST_FIELD_FL_HITCOUNT)
1364		field_name = "hitcount";
1365
1366	if (field_name == NULL)
1367		field_name = "";
1368
1369	return field_name;
1370}
1371
1372static enum hist_field_fn select_value_fn(int field_size, int field_is_signed)
1373{
1374	switch (field_size) {
1375	case 8:
1376		if (field_is_signed)
1377			return HIST_FIELD_FN_S64;
1378		else
1379			return HIST_FIELD_FN_U64;
1380	case 4:
1381		if (field_is_signed)
1382			return HIST_FIELD_FN_S32;
1383		else
1384			return HIST_FIELD_FN_U32;
1385	case 2:
1386		if (field_is_signed)
1387			return HIST_FIELD_FN_S16;
1388		else
1389			return HIST_FIELD_FN_U16;
1390	case 1:
1391		if (field_is_signed)
1392			return HIST_FIELD_FN_S8;
1393		else
1394			return HIST_FIELD_FN_U8;
1395	}
1396
1397	return HIST_FIELD_FN_NOP;
1398}
1399
1400static int parse_map_size(char *str)
1401{
1402	unsigned long size, map_bits;
1403	int ret;
1404
1405	ret = kstrtoul(str, 0, &size);
1406	if (ret)
1407		goto out;
1408
1409	map_bits = ilog2(roundup_pow_of_two(size));
1410	if (map_bits < TRACING_MAP_BITS_MIN ||
1411	    map_bits > TRACING_MAP_BITS_MAX)
1412		ret = -EINVAL;
1413	else
1414		ret = map_bits;
1415 out:
1416	return ret;
1417}
1418
1419static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1420{
1421	unsigned int i;
1422
1423	if (!attrs)
1424		return;
1425
1426	for (i = 0; i < attrs->n_assignments; i++)
1427		kfree(attrs->assignment_str[i]);
1428
1429	for (i = 0; i < attrs->n_actions; i++)
1430		kfree(attrs->action_str[i]);
1431
1432	kfree(attrs->name);
1433	kfree(attrs->sort_key_str);
1434	kfree(attrs->keys_str);
1435	kfree(attrs->vals_str);
1436	kfree(attrs->clock);
1437	kfree(attrs);
1438}
1439
1440static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1441{
1442	int ret = -EINVAL;
1443
1444	if (attrs->n_actions >= HIST_ACTIONS_MAX)
1445		return ret;
1446
1447	if ((str_has_prefix(str, "onmatch(")) ||
1448	    (str_has_prefix(str, "onmax(")) ||
1449	    (str_has_prefix(str, "onchange("))) {
1450		attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1451		if (!attrs->action_str[attrs->n_actions]) {
1452			ret = -ENOMEM;
1453			return ret;
1454		}
1455		attrs->n_actions++;
1456		ret = 0;
1457	}
1458	return ret;
1459}
1460
1461static int parse_assignment(struct trace_array *tr,
1462			    char *str, struct hist_trigger_attrs *attrs)
1463{
1464	int len, ret = 0;
1465
1466	if ((len = str_has_prefix(str, "key=")) ||
1467	    (len = str_has_prefix(str, "keys="))) {
1468		attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1469		if (!attrs->keys_str) {
1470			ret = -ENOMEM;
1471			goto out;
1472		}
1473	} else if ((len = str_has_prefix(str, "val=")) ||
1474		   (len = str_has_prefix(str, "vals=")) ||
1475		   (len = str_has_prefix(str, "values="))) {
1476		attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1477		if (!attrs->vals_str) {
1478			ret = -ENOMEM;
1479			goto out;
1480		}
1481	} else if ((len = str_has_prefix(str, "sort="))) {
1482		attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1483		if (!attrs->sort_key_str) {
1484			ret = -ENOMEM;
1485			goto out;
1486		}
1487	} else if (str_has_prefix(str, "name=")) {
1488		attrs->name = kstrdup(str, GFP_KERNEL);
1489		if (!attrs->name) {
1490			ret = -ENOMEM;
1491			goto out;
1492		}
1493	} else if ((len = str_has_prefix(str, "clock="))) {
1494		str += len;
1495
1496		str = strstrip(str);
1497		attrs->clock = kstrdup(str, GFP_KERNEL);
1498		if (!attrs->clock) {
1499			ret = -ENOMEM;
1500			goto out;
1501		}
1502	} else if ((len = str_has_prefix(str, "size="))) {
1503		int map_bits = parse_map_size(str + len);
1504
1505		if (map_bits < 0) {
1506			ret = map_bits;
1507			goto out;
1508		}
1509		attrs->map_bits = map_bits;
1510	} else {
1511		char *assignment;
1512
1513		if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1514			hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1515			ret = -EINVAL;
1516			goto out;
1517		}
1518
1519		assignment = kstrdup(str, GFP_KERNEL);
1520		if (!assignment) {
1521			ret = -ENOMEM;
1522			goto out;
1523		}
1524
1525		attrs->assignment_str[attrs->n_assignments++] = assignment;
1526	}
1527 out:
1528	return ret;
1529}
1530
1531static struct hist_trigger_attrs *
1532parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1533{
1534	struct hist_trigger_attrs *attrs;
1535	int ret = 0;
1536
1537	attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1538	if (!attrs)
1539		return ERR_PTR(-ENOMEM);
1540
1541	while (trigger_str) {
1542		char *str = strsep(&trigger_str, ":");
1543		char *rhs;
1544
1545		rhs = strchr(str, '=');
1546		if (rhs) {
1547			if (!strlen(++rhs)) {
1548				ret = -EINVAL;
1549				hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1550				goto free;
1551			}
1552			ret = parse_assignment(tr, str, attrs);
1553			if (ret)
1554				goto free;
1555		} else if (strcmp(str, "nohitcount") == 0 ||
1556			   strcmp(str, "NOHC") == 0)
1557			attrs->no_hitcount = true;
1558		else if (strcmp(str, "pause") == 0)
1559			attrs->pause = true;
1560		else if ((strcmp(str, "cont") == 0) ||
1561			 (strcmp(str, "continue") == 0))
1562			attrs->cont = true;
1563		else if (strcmp(str, "clear") == 0)
1564			attrs->clear = true;
1565		else {
1566			ret = parse_action(str, attrs);
1567			if (ret)
1568				goto free;
1569		}
1570	}
1571
1572	if (!attrs->keys_str) {
1573		ret = -EINVAL;
1574		goto free;
1575	}
1576
1577	if (!attrs->clock) {
1578		attrs->clock = kstrdup("global", GFP_KERNEL);
1579		if (!attrs->clock) {
1580			ret = -ENOMEM;
1581			goto free;
1582		}
1583	}
1584
1585	return attrs;
1586 free:
1587	destroy_hist_trigger_attrs(attrs);
1588
1589	return ERR_PTR(ret);
1590}
1591
1592static inline void save_comm(char *comm, struct task_struct *task)
1593{
1594	if (!task->pid) {
1595		strcpy(comm, "<idle>");
1596		return;
1597	}
1598
1599	if (WARN_ON_ONCE(task->pid < 0)) {
1600		strcpy(comm, "<XXX>");
1601		return;
1602	}
1603
1604	strncpy(comm, task->comm, TASK_COMM_LEN);
1605}
1606
1607static void hist_elt_data_free(struct hist_elt_data *elt_data)
1608{
1609	unsigned int i;
1610
1611	for (i = 0; i < elt_data->n_field_var_str; i++)
1612		kfree(elt_data->field_var_str[i]);
1613
1614	kfree(elt_data->field_var_str);
1615
1616	kfree(elt_data->comm);
1617	kfree(elt_data);
1618}
1619
1620static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1621{
1622	struct hist_elt_data *elt_data = elt->private_data;
1623
1624	hist_elt_data_free(elt_data);
1625}
1626
1627static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1628{
1629	struct hist_trigger_data *hist_data = elt->map->private_data;
1630	unsigned int size = TASK_COMM_LEN;
1631	struct hist_elt_data *elt_data;
1632	struct hist_field *hist_field;
1633	unsigned int i, n_str;
1634
1635	elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1636	if (!elt_data)
1637		return -ENOMEM;
1638
1639	for_each_hist_field(i, hist_data) {
1640		hist_field = hist_data->fields[i];
1641
1642		if (hist_field->flags & HIST_FIELD_FL_EXECNAME) {
1643			elt_data->comm = kzalloc(size, GFP_KERNEL);
1644			if (!elt_data->comm) {
1645				kfree(elt_data);
1646				return -ENOMEM;
1647			}
1648			break;
1649		}
1650	}
1651
1652	n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1653		hist_data->n_var_str;
1654	if (n_str > SYNTH_FIELDS_MAX) {
1655		hist_elt_data_free(elt_data);
1656		return -EINVAL;
1657	}
1658
1659	BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1660
1661	size = STR_VAR_LEN_MAX;
1662
1663	elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL);
1664	if (!elt_data->field_var_str) {
1665		hist_elt_data_free(elt_data);
1666		return -EINVAL;
1667	}
1668	elt_data->n_field_var_str = n_str;
1669
1670	for (i = 0; i < n_str; i++) {
1671		elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1672		if (!elt_data->field_var_str[i]) {
1673			hist_elt_data_free(elt_data);
1674			return -ENOMEM;
1675		}
1676	}
1677
1678	elt->private_data = elt_data;
1679
1680	return 0;
1681}
1682
1683static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1684{
1685	struct hist_elt_data *elt_data = elt->private_data;
1686
1687	if (elt_data->comm)
1688		save_comm(elt_data->comm, current);
1689}
1690
1691static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1692	.elt_alloc	= hist_trigger_elt_data_alloc,
1693	.elt_free	= hist_trigger_elt_data_free,
1694	.elt_init	= hist_trigger_elt_data_init,
1695};
1696
1697static const char *get_hist_field_flags(struct hist_field *hist_field)
1698{
1699	const char *flags_str = NULL;
1700
1701	if (hist_field->flags & HIST_FIELD_FL_HEX)
1702		flags_str = "hex";
1703	else if (hist_field->flags & HIST_FIELD_FL_SYM)
1704		flags_str = "sym";
1705	else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1706		flags_str = "sym-offset";
1707	else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1708		flags_str = "execname";
1709	else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1710		flags_str = "syscall";
1711	else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1712		flags_str = "log2";
1713	else if (hist_field->flags & HIST_FIELD_FL_BUCKET)
1714		flags_str = "buckets";
1715	else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1716		flags_str = "usecs";
1717	else if (hist_field->flags & HIST_FIELD_FL_PERCENT)
1718		flags_str = "percent";
1719	else if (hist_field->flags & HIST_FIELD_FL_GRAPH)
1720		flags_str = "graph";
1721
1722	return flags_str;
1723}
1724
1725static void expr_field_str(struct hist_field *field, char *expr)
1726{
1727	if (field->flags & HIST_FIELD_FL_VAR_REF)
1728		strcat(expr, "$");
1729	else if (field->flags & HIST_FIELD_FL_CONST) {
1730		char str[HIST_CONST_DIGITS_MAX];
1731
1732		snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant);
1733		strcat(expr, str);
1734	}
1735
1736	strcat(expr, hist_field_name(field, 0));
1737
1738	if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1739		const char *flags_str = get_hist_field_flags(field);
1740
1741		if (flags_str) {
1742			strcat(expr, ".");
1743			strcat(expr, flags_str);
1744		}
1745	}
1746}
1747
1748static char *expr_str(struct hist_field *field, unsigned int level)
1749{
1750	char *expr;
1751
1752	if (level > 1)
1753		return NULL;
1754
1755	expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1756	if (!expr)
1757		return NULL;
1758
1759	if (!field->operands[0]) {
1760		expr_field_str(field, expr);
1761		return expr;
1762	}
1763
1764	if (field->operator == FIELD_OP_UNARY_MINUS) {
1765		char *subexpr;
1766
1767		strcat(expr, "-(");
1768		subexpr = expr_str(field->operands[0], ++level);
1769		if (!subexpr) {
1770			kfree(expr);
1771			return NULL;
1772		}
1773		strcat(expr, subexpr);
1774		strcat(expr, ")");
1775
1776		kfree(subexpr);
1777
1778		return expr;
1779	}
1780
1781	expr_field_str(field->operands[0], expr);
1782
1783	switch (field->operator) {
1784	case FIELD_OP_MINUS:
1785		strcat(expr, "-");
1786		break;
1787	case FIELD_OP_PLUS:
1788		strcat(expr, "+");
1789		break;
1790	case FIELD_OP_DIV:
1791		strcat(expr, "/");
1792		break;
1793	case FIELD_OP_MULT:
1794		strcat(expr, "*");
1795		break;
1796	default:
1797		kfree(expr);
1798		return NULL;
1799	}
1800
1801	expr_field_str(field->operands[1], expr);
1802
1803	return expr;
1804}
1805
1806/*
1807 * If field_op != FIELD_OP_NONE, *sep points to the root operator
1808 * of the expression tree to be evaluated.
1809 */
1810static int contains_operator(char *str, char **sep)
1811{
1812	enum field_op_id field_op = FIELD_OP_NONE;
1813	char *minus_op, *plus_op, *div_op, *mult_op;
1814
1815
1816	/*
1817	 * Report the last occurrence of the operators first, so that the
1818	 * expression is evaluated left to right. This is important since
1819	 * subtraction and division are not associative.
1820	 *
1821	 *	e.g
1822	 *		64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2
1823	 *		14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2
1824	 */
1825
1826	/*
1827	 * First, find lower precedence addition and subtraction
1828	 * since the expression will be evaluated recursively.
1829	 */
1830	minus_op = strrchr(str, '-');
1831	if (minus_op) {
1832		/*
1833		 * Unary minus is not supported in sub-expressions. If
1834		 * present, it is always the next root operator.
1835		 */
1836		if (minus_op == str) {
1837			field_op = FIELD_OP_UNARY_MINUS;
1838			goto out;
1839		}
1840
1841		field_op = FIELD_OP_MINUS;
1842	}
1843
1844	plus_op = strrchr(str, '+');
1845	if (plus_op || minus_op) {
1846		/*
1847		 * For operators of the same precedence use to rightmost as the
1848		 * root, so that the expression is evaluated left to right.
1849		 */
1850		if (plus_op > minus_op)
1851			field_op = FIELD_OP_PLUS;
1852		goto out;
1853	}
1854
1855	/*
1856	 * Multiplication and division have higher precedence than addition and
1857	 * subtraction.
1858	 */
1859	div_op = strrchr(str, '/');
1860	if (div_op)
1861		field_op = FIELD_OP_DIV;
1862
1863	mult_op = strrchr(str, '*');
1864	/*
1865	 * For operators of the same precedence use to rightmost as the
1866	 * root, so that the expression is evaluated left to right.
1867	 */
1868	if (mult_op > div_op)
1869		field_op = FIELD_OP_MULT;
1870
1871out:
1872	if (sep) {
1873		switch (field_op) {
1874		case FIELD_OP_UNARY_MINUS:
1875		case FIELD_OP_MINUS:
1876			*sep = minus_op;
1877			break;
1878		case FIELD_OP_PLUS:
1879			*sep = plus_op;
1880			break;
1881		case FIELD_OP_DIV:
1882			*sep = div_op;
1883			break;
1884		case FIELD_OP_MULT:
1885			*sep = mult_op;
1886			break;
1887		case FIELD_OP_NONE:
1888		default:
1889			*sep = NULL;
1890			break;
1891		}
1892	}
1893
1894	return field_op;
1895}
1896
1897static void get_hist_field(struct hist_field *hist_field)
1898{
1899	hist_field->ref++;
1900}
1901
1902static void __destroy_hist_field(struct hist_field *hist_field)
1903{
1904	if (--hist_field->ref > 1)
1905		return;
1906
1907	kfree(hist_field->var.name);
1908	kfree(hist_field->name);
1909
1910	/* Can likely be a const */
1911	kfree_const(hist_field->type);
1912
1913	kfree(hist_field->system);
1914	kfree(hist_field->event_name);
1915
1916	kfree(hist_field);
1917}
1918
1919static void destroy_hist_field(struct hist_field *hist_field,
1920			       unsigned int level)
1921{
1922	unsigned int i;
1923
1924	if (level > 3)
1925		return;
1926
1927	if (!hist_field)
1928		return;
1929
1930	if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1931		return; /* var refs will be destroyed separately */
1932
1933	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1934		destroy_hist_field(hist_field->operands[i], level + 1);
1935
1936	__destroy_hist_field(hist_field);
1937}
1938
1939static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1940					    struct ftrace_event_field *field,
1941					    unsigned long flags,
1942					    char *var_name)
1943{
1944	struct hist_field *hist_field;
1945
1946	if (field && is_function_field(field))
1947		return NULL;
1948
1949	hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1950	if (!hist_field)
1951		return NULL;
1952
1953	hist_field->ref = 1;
1954
1955	hist_field->hist_data = hist_data;
1956
1957	if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1958		goto out; /* caller will populate */
1959
1960	if (flags & HIST_FIELD_FL_VAR_REF) {
1961		hist_field->fn_num = HIST_FIELD_FN_VAR_REF;
1962		goto out;
1963	}
1964
1965	if (flags & HIST_FIELD_FL_HITCOUNT) {
1966		hist_field->fn_num = HIST_FIELD_FN_COUNTER;
1967		hist_field->size = sizeof(u64);
1968		hist_field->type = "u64";
1969		goto out;
1970	}
1971
1972	if (flags & HIST_FIELD_FL_CONST) {
1973		hist_field->fn_num = HIST_FIELD_FN_CONST;
1974		hist_field->size = sizeof(u64);
1975		hist_field->type = kstrdup("u64", GFP_KERNEL);
1976		if (!hist_field->type)
1977			goto free;
1978		goto out;
1979	}
1980
1981	if (flags & HIST_FIELD_FL_STACKTRACE) {
1982		hist_field->fn_num = HIST_FIELD_FN_NOP;
1983		goto out;
1984	}
1985
1986	if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
1987		unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
1988		hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 :
1989			HIST_FIELD_FN_BUCKET;
1990		hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1991		if (!hist_field->operands[0])
1992			goto free;
1993		hist_field->size = hist_field->operands[0]->size;
1994		hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
1995		if (!hist_field->type)
1996			goto free;
1997		goto out;
1998	}
1999
2000	if (flags & HIST_FIELD_FL_TIMESTAMP) {
2001		hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP;
2002		hist_field->size = sizeof(u64);
2003		hist_field->type = "u64";
2004		goto out;
2005	}
2006
2007	if (flags & HIST_FIELD_FL_CPU) {
2008		hist_field->fn_num = HIST_FIELD_FN_CPU;
2009		hist_field->size = sizeof(int);
2010		hist_field->type = "unsigned int";
2011		goto out;
2012	}
2013
2014	if (WARN_ON_ONCE(!field))
2015		goto out;
2016
2017	/* Pointers to strings are just pointers and dangerous to dereference */
2018	if (is_string_field(field) &&
2019	    (field->filter_type != FILTER_PTR_STRING)) {
2020		flags |= HIST_FIELD_FL_STRING;
2021
2022		hist_field->size = MAX_FILTER_STR_VAL;
2023		hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2024		if (!hist_field->type)
2025			goto free;
2026
2027		if (field->filter_type == FILTER_STATIC_STRING) {
2028			hist_field->fn_num = HIST_FIELD_FN_STRING;
2029			hist_field->size = field->size;
2030		} else if (field->filter_type == FILTER_DYN_STRING) {
2031			hist_field->fn_num = HIST_FIELD_FN_DYNSTRING;
2032		} else if (field->filter_type == FILTER_RDYN_STRING)
2033			hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING;
2034		else
2035			hist_field->fn_num = HIST_FIELD_FN_PSTRING;
2036	} else {
2037		hist_field->size = field->size;
2038		hist_field->is_signed = field->is_signed;
2039		hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2040		if (!hist_field->type)
2041			goto free;
2042
2043		hist_field->fn_num = select_value_fn(field->size,
2044						     field->is_signed);
2045		if (hist_field->fn_num == HIST_FIELD_FN_NOP) {
2046			destroy_hist_field(hist_field, 0);
2047			return NULL;
2048		}
2049	}
2050 out:
2051	hist_field->field = field;
2052	hist_field->flags = flags;
2053
2054	if (var_name) {
2055		hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2056		if (!hist_field->var.name)
2057			goto free;
2058	}
2059
2060	return hist_field;
2061 free:
2062	destroy_hist_field(hist_field, 0);
2063	return NULL;
2064}
2065
2066static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2067{
2068	unsigned int i;
2069
2070	for (i = 0; i < HIST_FIELDS_MAX; i++) {
2071		if (hist_data->fields[i]) {
2072			destroy_hist_field(hist_data->fields[i], 0);
2073			hist_data->fields[i] = NULL;
2074		}
2075	}
2076
2077	for (i = 0; i < hist_data->n_var_refs; i++) {
2078		WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2079		__destroy_hist_field(hist_data->var_refs[i]);
2080		hist_data->var_refs[i] = NULL;
2081	}
2082}
2083
2084static int init_var_ref(struct hist_field *ref_field,
2085			struct hist_field *var_field,
2086			char *system, char *event_name)
2087{
2088	int err = 0;
2089
2090	ref_field->var.idx = var_field->var.idx;
2091	ref_field->var.hist_data = var_field->hist_data;
2092	ref_field->size = var_field->size;
2093	ref_field->is_signed = var_field->is_signed;
2094	ref_field->flags |= var_field->flags &
2095		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2096
2097	if (system) {
2098		ref_field->system = kstrdup(system, GFP_KERNEL);
2099		if (!ref_field->system)
2100			return -ENOMEM;
2101	}
2102
2103	if (event_name) {
2104		ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2105		if (!ref_field->event_name) {
2106			err = -ENOMEM;
2107			goto free;
2108		}
2109	}
2110
2111	if (var_field->var.name) {
2112		ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2113		if (!ref_field->name) {
2114			err = -ENOMEM;
2115			goto free;
2116		}
2117	} else if (var_field->name) {
2118		ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2119		if (!ref_field->name) {
2120			err = -ENOMEM;
2121			goto free;
2122		}
2123	}
2124
2125	ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL);
2126	if (!ref_field->type) {
2127		err = -ENOMEM;
2128		goto free;
2129	}
2130 out:
2131	return err;
2132 free:
2133	kfree(ref_field->system);
2134	ref_field->system = NULL;
2135	kfree(ref_field->event_name);
2136	ref_field->event_name = NULL;
2137	kfree(ref_field->name);
2138	ref_field->name = NULL;
2139
2140	goto out;
2141}
2142
2143static int find_var_ref_idx(struct hist_trigger_data *hist_data,
2144			    struct hist_field *var_field)
2145{
2146	struct hist_field *ref_field;
2147	int i;
2148
2149	for (i = 0; i < hist_data->n_var_refs; i++) {
2150		ref_field = hist_data->var_refs[i];
2151		if (ref_field->var.idx == var_field->var.idx &&
2152		    ref_field->var.hist_data == var_field->hist_data)
2153			return i;
2154	}
2155
2156	return -ENOENT;
2157}
2158
2159/**
2160 * create_var_ref - Create a variable reference and attach it to trigger
2161 * @hist_data: The trigger that will be referencing the variable
2162 * @var_field: The VAR field to create a reference to
2163 * @system: The optional system string
2164 * @event_name: The optional event_name string
2165 *
2166 * Given a variable hist_field, create a VAR_REF hist_field that
2167 * represents a reference to it.
2168 *
2169 * This function also adds the reference to the trigger that
2170 * now references the variable.
2171 *
2172 * Return: The VAR_REF field if successful, NULL if not
2173 */
2174static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2175					 struct hist_field *var_field,
2176					 char *system, char *event_name)
2177{
2178	unsigned long flags = HIST_FIELD_FL_VAR_REF;
2179	struct hist_field *ref_field;
2180	int i;
2181
2182	/* Check if the variable already exists */
2183	for (i = 0; i < hist_data->n_var_refs; i++) {
2184		ref_field = hist_data->var_refs[i];
2185		if (ref_field->var.idx == var_field->var.idx &&
2186		    ref_field->var.hist_data == var_field->hist_data) {
2187			get_hist_field(ref_field);
2188			return ref_field;
2189		}
2190	}
2191	/* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */
2192	if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX)
2193		return NULL;
2194	ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2195	if (ref_field) {
2196		if (init_var_ref(ref_field, var_field, system, event_name)) {
2197			destroy_hist_field(ref_field, 0);
2198			return NULL;
2199		}
2200
2201		hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2202		ref_field->var_ref_idx = hist_data->n_var_refs++;
2203	}
2204
2205	return ref_field;
2206}
2207
2208static bool is_var_ref(char *var_name)
2209{
2210	if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2211		return false;
2212
2213	return true;
2214}
2215
2216static char *field_name_from_var(struct hist_trigger_data *hist_data,
2217				 char *var_name)
2218{
2219	char *name, *field;
2220	unsigned int i;
2221
2222	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2223		name = hist_data->attrs->var_defs.name[i];
2224
2225		if (strcmp(var_name, name) == 0) {
2226			field = hist_data->attrs->var_defs.expr[i];
2227			if (contains_operator(field, NULL) || is_var_ref(field))
2228				continue;
2229			return field;
2230		}
2231	}
2232
2233	return NULL;
2234}
2235
2236static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2237				 char *system, char *event_name,
2238				 char *var_name)
2239{
2240	struct trace_event_call *call;
2241
2242	if (system && event_name) {
2243		call = hist_data->event_file->event_call;
2244
2245		if (strcmp(system, call->class->system) != 0)
2246			return NULL;
2247
2248		if (strcmp(event_name, trace_event_name(call)) != 0)
2249			return NULL;
2250	}
2251
2252	if (!!system != !!event_name)
2253		return NULL;
2254
2255	if (!is_var_ref(var_name))
2256		return NULL;
2257
2258	var_name++;
2259
2260	return field_name_from_var(hist_data, var_name);
2261}
2262
2263static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2264					char *system, char *event_name,
2265					char *var_name)
2266{
2267	struct hist_field *var_field = NULL, *ref_field = NULL;
2268	struct trace_array *tr = hist_data->event_file->tr;
2269
2270	if (!is_var_ref(var_name))
2271		return NULL;
2272
2273	var_name++;
2274
2275	var_field = find_event_var(hist_data, system, event_name, var_name);
2276	if (var_field)
2277		ref_field = create_var_ref(hist_data, var_field,
2278					   system, event_name);
2279
2280	if (!ref_field)
2281		hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2282
2283	return ref_field;
2284}
2285
2286static struct ftrace_event_field *
2287parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2288	    char *field_str, unsigned long *flags, unsigned long *buckets)
2289{
2290	struct ftrace_event_field *field = NULL;
2291	char *field_name, *modifier, *str;
2292	struct trace_array *tr = file->tr;
2293
2294	modifier = str = kstrdup(field_str, GFP_KERNEL);
2295	if (!modifier)
2296		return ERR_PTR(-ENOMEM);
2297
2298	field_name = strsep(&modifier, ".");
2299	if (modifier) {
2300		if (strcmp(modifier, "hex") == 0)
2301			*flags |= HIST_FIELD_FL_HEX;
2302		else if (strcmp(modifier, "sym") == 0)
2303			*flags |= HIST_FIELD_FL_SYM;
2304		/*
2305		 * 'sym-offset' occurrences in the trigger string are modified
2306		 * to 'symXoffset' to simplify arithmetic expression parsing.
2307		 */
2308		else if (strcmp(modifier, "symXoffset") == 0)
2309			*flags |= HIST_FIELD_FL_SYM_OFFSET;
2310		else if ((strcmp(modifier, "execname") == 0) &&
2311			 (strcmp(field_name, "common_pid") == 0))
2312			*flags |= HIST_FIELD_FL_EXECNAME;
2313		else if (strcmp(modifier, "syscall") == 0)
2314			*flags |= HIST_FIELD_FL_SYSCALL;
2315		else if (strcmp(modifier, "log2") == 0)
2316			*flags |= HIST_FIELD_FL_LOG2;
2317		else if (strcmp(modifier, "usecs") == 0)
2318			*flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2319		else if (strncmp(modifier, "bucket", 6) == 0) {
2320			int ret;
2321
2322			modifier += 6;
2323
2324			if (*modifier == 's')
2325				modifier++;
2326			if (*modifier != '=')
2327				goto error;
2328			modifier++;
2329			ret = kstrtoul(modifier, 0, buckets);
2330			if (ret || !(*buckets))
2331				goto error;
2332			*flags |= HIST_FIELD_FL_BUCKET;
2333		} else if (strncmp(modifier, "percent", 7) == 0) {
2334			if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2335				goto error;
2336			*flags |= HIST_FIELD_FL_PERCENT;
2337		} else if (strncmp(modifier, "graph", 5) == 0) {
2338			if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2339				goto error;
2340			*flags |= HIST_FIELD_FL_GRAPH;
2341		} else {
2342 error:
2343			hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2344			field = ERR_PTR(-EINVAL);
2345			goto out;
2346		}
2347	}
2348
2349	if (strcmp(field_name, "common_timestamp") == 0) {
2350		*flags |= HIST_FIELD_FL_TIMESTAMP;
2351		hist_data->enable_timestamps = true;
2352		if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2353			hist_data->attrs->ts_in_usecs = true;
2354	} else if (strcmp(field_name, "common_cpu") == 0)
2355		*flags |= HIST_FIELD_FL_CPU;
2356	else if (strcmp(field_name, "hitcount") == 0)
2357		*flags |= HIST_FIELD_FL_HITCOUNT;
2358	else {
2359		field = trace_find_event_field(file->event_call, field_name);
2360		if (!field || !field->size) {
2361			/*
2362			 * For backward compatibility, if field_name
2363			 * was "cpu", then we treat this the same as
2364			 * common_cpu. This also works for "CPU".
2365			 */
2366			if (field && field->filter_type == FILTER_CPU) {
2367				*flags |= HIST_FIELD_FL_CPU;
2368			} else {
2369				hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2370					 errpos(field_name));
2371				field = ERR_PTR(-EINVAL);
2372				goto out;
2373			}
2374		}
2375	}
2376 out:
2377	kfree(str);
2378
2379	return field;
2380}
2381
2382static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2383				       struct hist_field *var_ref,
2384				       char *var_name)
2385{
2386	struct hist_field *alias = NULL;
2387	unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2388
2389	alias = create_hist_field(hist_data, NULL, flags, var_name);
2390	if (!alias)
2391		return NULL;
2392
2393	alias->fn_num = var_ref->fn_num;
2394	alias->operands[0] = var_ref;
2395
2396	if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2397		destroy_hist_field(alias, 0);
2398		return NULL;
2399	}
2400
2401	alias->var_ref_idx = var_ref->var_ref_idx;
2402
2403	return alias;
2404}
2405
2406static struct hist_field *parse_const(struct hist_trigger_data *hist_data,
2407				      char *str, char *var_name,
2408				      unsigned long *flags)
2409{
2410	struct trace_array *tr = hist_data->event_file->tr;
2411	struct hist_field *field = NULL;
2412	u64 constant;
2413
2414	if (kstrtoull(str, 0, &constant)) {
2415		hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str));
2416		return NULL;
2417	}
2418
2419	*flags |= HIST_FIELD_FL_CONST;
2420	field = create_hist_field(hist_data, NULL, *flags, var_name);
2421	if (!field)
2422		return NULL;
2423
2424	field->constant = constant;
2425
2426	return field;
2427}
2428
2429static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2430				     struct trace_event_file *file, char *str,
2431				     unsigned long *flags, char *var_name)
2432{
2433	char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2434	struct ftrace_event_field *field = NULL;
2435	struct hist_field *hist_field = NULL;
2436	unsigned long buckets = 0;
2437	int ret = 0;
2438
2439	if (isdigit(str[0])) {
2440		hist_field = parse_const(hist_data, str, var_name, flags);
2441		if (!hist_field) {
2442			ret = -EINVAL;
2443			goto out;
2444		}
2445		return hist_field;
2446	}
2447
2448	s = strchr(str, '.');
2449	if (s) {
2450		s = strchr(++s, '.');
2451		if (s) {
2452			ref_system = strsep(&str, ".");
2453			if (!str) {
2454				ret = -EINVAL;
2455				goto out;
2456			}
2457			ref_event = strsep(&str, ".");
2458			if (!str) {
2459				ret = -EINVAL;
2460				goto out;
2461			}
2462			ref_var = str;
2463		}
2464	}
2465
2466	s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2467	if (!s) {
2468		hist_field = parse_var_ref(hist_data, ref_system,
2469					   ref_event, ref_var);
2470		if (hist_field) {
2471			if (var_name) {
2472				hist_field = create_alias(hist_data, hist_field, var_name);
2473				if (!hist_field) {
2474					ret = -ENOMEM;
2475					goto out;
2476				}
2477			}
2478			return hist_field;
2479		}
2480	} else
2481		str = s;
2482
2483	field = parse_field(hist_data, file, str, flags, &buckets);
2484	if (IS_ERR(field)) {
2485		ret = PTR_ERR(field);
2486		goto out;
2487	}
2488
2489	hist_field = create_hist_field(hist_data, field, *flags, var_name);
2490	if (!hist_field) {
2491		ret = -ENOMEM;
2492		goto out;
2493	}
2494	hist_field->buckets = buckets;
2495
2496	return hist_field;
2497 out:
2498	return ERR_PTR(ret);
2499}
2500
2501static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2502				     struct trace_event_file *file,
2503				     char *str, unsigned long flags,
2504				     char *var_name, unsigned int *n_subexprs);
2505
2506static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2507				      struct trace_event_file *file,
2508				      char *str, unsigned long flags,
2509				      char *var_name, unsigned int *n_subexprs)
2510{
2511	struct hist_field *operand1, *expr = NULL;
2512	unsigned long operand_flags;
2513	int ret = 0;
2514	char *s;
2515
2516	/* Unary minus operator, increment n_subexprs */
2517	++*n_subexprs;
2518
2519	/* we support only -(xxx) i.e. explicit parens required */
2520
2521	if (*n_subexprs > 3) {
2522		hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2523		ret = -EINVAL;
2524		goto free;
2525	}
2526
2527	str++; /* skip leading '-' */
2528
2529	s = strchr(str, '(');
2530	if (s)
2531		str++;
2532	else {
2533		ret = -EINVAL;
2534		goto free;
2535	}
2536
2537	s = strrchr(str, ')');
2538	if (s) {
2539		 /* unary minus not supported in sub-expressions */
2540		if (*(s+1) != '\0') {
2541			hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR,
2542				 errpos(str));
2543			ret = -EINVAL;
2544			goto free;
2545		}
2546		*s = '\0';
2547	}
2548	else {
2549		ret = -EINVAL; /* no closing ')' */
2550		goto free;
2551	}
2552
2553	flags |= HIST_FIELD_FL_EXPR;
2554	expr = create_hist_field(hist_data, NULL, flags, var_name);
2555	if (!expr) {
2556		ret = -ENOMEM;
2557		goto free;
2558	}
2559
2560	operand_flags = 0;
2561	operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2562	if (IS_ERR(operand1)) {
2563		ret = PTR_ERR(operand1);
2564		goto free;
2565	}
2566	if (operand1->flags & HIST_FIELD_FL_STRING) {
2567		/* String type can not be the operand of unary operator. */
2568		hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2569		destroy_hist_field(operand1, 0);
2570		ret = -EINVAL;
2571		goto free;
2572	}
2573
2574	expr->flags |= operand1->flags &
2575		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2576	expr->fn_num = HIST_FIELD_FN_UMINUS;
2577	expr->operands[0] = operand1;
2578	expr->size = operand1->size;
2579	expr->is_signed = operand1->is_signed;
2580	expr->operator = FIELD_OP_UNARY_MINUS;
2581	expr->name = expr_str(expr, 0);
2582	expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2583	if (!expr->type) {
2584		ret = -ENOMEM;
2585		goto free;
2586	}
2587
2588	return expr;
2589 free:
2590	destroy_hist_field(expr, 0);
2591	return ERR_PTR(ret);
2592}
2593
2594/*
2595 * If the operands are var refs, return pointers the
2596 * variable(s) referenced in var1 and var2, else NULL.
2597 */
2598static int check_expr_operands(struct trace_array *tr,
2599			       struct hist_field *operand1,
2600			       struct hist_field *operand2,
2601			       struct hist_field **var1,
2602			       struct hist_field **var2)
2603{
2604	unsigned long operand1_flags = operand1->flags;
2605	unsigned long operand2_flags = operand2->flags;
2606
2607	if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2608	    (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2609		struct hist_field *var;
2610
2611		var = find_var_field(operand1->var.hist_data, operand1->name);
2612		if (!var)
2613			return -EINVAL;
2614		operand1_flags = var->flags;
2615		*var1 = var;
2616	}
2617
2618	if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2619	    (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2620		struct hist_field *var;
2621
2622		var = find_var_field(operand2->var.hist_data, operand2->name);
2623		if (!var)
2624			return -EINVAL;
2625		operand2_flags = var->flags;
2626		*var2 = var;
2627	}
2628
2629	if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2630	    (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2631		hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2632		return -EINVAL;
2633	}
2634
2635	return 0;
2636}
2637
2638static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2639				     struct trace_event_file *file,
2640				     char *str, unsigned long flags,
2641				     char *var_name, unsigned int *n_subexprs)
2642{
2643	struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2644	struct hist_field *var1 = NULL, *var2 = NULL;
2645	unsigned long operand_flags, operand2_flags;
2646	int field_op, ret = -EINVAL;
2647	char *sep, *operand1_str;
2648	enum hist_field_fn op_fn;
2649	bool combine_consts;
2650
2651	if (*n_subexprs > 3) {
2652		hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2653		return ERR_PTR(-EINVAL);
2654	}
2655
2656	field_op = contains_operator(str, &sep);
2657
2658	if (field_op == FIELD_OP_NONE)
2659		return parse_atom(hist_data, file, str, &flags, var_name);
2660
2661	if (field_op == FIELD_OP_UNARY_MINUS)
2662		return parse_unary(hist_data, file, str, flags, var_name, n_subexprs);
2663
2664	/* Binary operator found, increment n_subexprs */
2665	++*n_subexprs;
2666
2667	/* Split the expression string at the root operator */
2668	if (!sep)
2669		return ERR_PTR(-EINVAL);
2670
2671	*sep = '\0';
2672	operand1_str = str;
2673	str = sep+1;
2674
2675	/* Binary operator requires both operands */
2676	if (*operand1_str == '\0' || *str == '\0')
2677		return ERR_PTR(-EINVAL);
2678
2679	operand_flags = 0;
2680
2681	/* LHS of string is an expression e.g. a+b in a+b+c */
2682	operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs);
2683	if (IS_ERR(operand1))
2684		return ERR_CAST(operand1);
2685
2686	if (operand1->flags & HIST_FIELD_FL_STRING) {
2687		hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2688		ret = -EINVAL;
2689		goto free_op1;
2690	}
2691
2692	/* RHS of string is another expression e.g. c in a+b+c */
2693	operand_flags = 0;
2694	operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2695	if (IS_ERR(operand2)) {
2696		ret = PTR_ERR(operand2);
2697		goto free_op1;
2698	}
2699	if (operand2->flags & HIST_FIELD_FL_STRING) {
2700		hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2701		ret = -EINVAL;
2702		goto free_operands;
2703	}
2704
2705	switch (field_op) {
2706	case FIELD_OP_MINUS:
2707		op_fn = HIST_FIELD_FN_MINUS;
2708		break;
2709	case FIELD_OP_PLUS:
2710		op_fn = HIST_FIELD_FN_PLUS;
2711		break;
2712	case FIELD_OP_DIV:
2713		op_fn = HIST_FIELD_FN_DIV;
2714		break;
2715	case FIELD_OP_MULT:
2716		op_fn = HIST_FIELD_FN_MULT;
2717		break;
2718	default:
2719		ret = -EINVAL;
2720		goto free_operands;
2721	}
2722
2723	ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2);
2724	if (ret)
2725		goto free_operands;
2726
2727	operand_flags = var1 ? var1->flags : operand1->flags;
2728	operand2_flags = var2 ? var2->flags : operand2->flags;
2729
2730	/*
2731	 * If both operands are constant, the expression can be
2732	 * collapsed to a single constant.
2733	 */
2734	combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST;
2735
2736	flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR;
2737
2738	flags |= operand1->flags &
2739		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2740
2741	expr = create_hist_field(hist_data, NULL, flags, var_name);
2742	if (!expr) {
2743		ret = -ENOMEM;
2744		goto free_operands;
2745	}
2746
2747	operand1->read_once = true;
2748	operand2->read_once = true;
2749
2750	/* The operands are now owned and free'd by 'expr' */
2751	expr->operands[0] = operand1;
2752	expr->operands[1] = operand2;
2753
2754	if (field_op == FIELD_OP_DIV &&
2755			operand2_flags & HIST_FIELD_FL_CONST) {
2756		u64 divisor = var2 ? var2->constant : operand2->constant;
2757
2758		if (!divisor) {
2759			hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str));
2760			ret = -EDOM;
2761			goto free_expr;
2762		}
2763
2764		/*
2765		 * Copy the divisor here so we don't have to look it up
2766		 * later if this is a var ref
2767		 */
2768		operand2->constant = divisor;
2769		op_fn = hist_field_get_div_fn(operand2);
2770	}
2771
2772	expr->fn_num = op_fn;
2773
2774	if (combine_consts) {
2775		if (var1)
2776			expr->operands[0] = var1;
2777		if (var2)
2778			expr->operands[1] = var2;
2779
2780		expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL);
2781		expr->fn_num = HIST_FIELD_FN_CONST;
2782
2783		expr->operands[0] = NULL;
2784		expr->operands[1] = NULL;
2785
2786		/*
2787		 * var refs won't be destroyed immediately
2788		 * See: destroy_hist_field()
2789		 */
2790		destroy_hist_field(operand2, 0);
2791		destroy_hist_field(operand1, 0);
2792
2793		expr->name = expr_str(expr, 0);
2794	} else {
2795		/* The operand sizes should be the same, so just pick one */
2796		expr->size = operand1->size;
2797		expr->is_signed = operand1->is_signed;
2798
2799		expr->operator = field_op;
2800		expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2801		if (!expr->type) {
2802			ret = -ENOMEM;
2803			goto free_expr;
2804		}
2805
2806		expr->name = expr_str(expr, 0);
2807	}
2808
2809	return expr;
2810
2811free_operands:
2812	destroy_hist_field(operand2, 0);
2813free_op1:
2814	destroy_hist_field(operand1, 0);
2815	return ERR_PTR(ret);
2816
2817free_expr:
2818	destroy_hist_field(expr, 0);
2819	return ERR_PTR(ret);
2820}
2821
2822static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2823				 struct trace_event_file *file)
2824{
2825	struct event_trigger_data *test;
2826
2827	lockdep_assert_held(&event_mutex);
2828
2829	list_for_each_entry(test, &file->triggers, list) {
2830		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2831			if (test->private_data == hist_data)
2832				return test->filter_str;
2833		}
2834	}
2835
2836	return NULL;
2837}
2838
2839static struct event_command trigger_hist_cmd;
2840static int event_hist_trigger_parse(struct event_command *cmd_ops,
2841				    struct trace_event_file *file,
2842				    char *glob, char *cmd,
2843				    char *param_and_filter);
2844
2845static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2846			    struct hist_trigger_data *hist_data,
2847			    unsigned int n_keys)
2848{
2849	struct hist_field *target_hist_field, *hist_field;
2850	unsigned int n, i, j;
2851
2852	if (hist_data->n_fields - hist_data->n_vals != n_keys)
2853		return false;
2854
2855	i = hist_data->n_vals;
2856	j = target_hist_data->n_vals;
2857
2858	for (n = 0; n < n_keys; n++) {
2859		hist_field = hist_data->fields[i + n];
2860		target_hist_field = target_hist_data->fields[j + n];
2861
2862		if (strcmp(hist_field->type, target_hist_field->type) != 0)
2863			return false;
2864		if (hist_field->size != target_hist_field->size)
2865			return false;
2866		if (hist_field->is_signed != target_hist_field->is_signed)
2867			return false;
2868	}
2869
2870	return true;
2871}
2872
2873static struct hist_trigger_data *
2874find_compatible_hist(struct hist_trigger_data *target_hist_data,
2875		     struct trace_event_file *file)
2876{
2877	struct hist_trigger_data *hist_data;
2878	struct event_trigger_data *test;
2879	unsigned int n_keys;
2880
2881	lockdep_assert_held(&event_mutex);
2882
2883	n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2884
2885	list_for_each_entry(test, &file->triggers, list) {
2886		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2887			hist_data = test->private_data;
2888
2889			if (compatible_keys(target_hist_data, hist_data, n_keys))
2890				return hist_data;
2891		}
2892	}
2893
2894	return NULL;
2895}
2896
2897static struct trace_event_file *event_file(struct trace_array *tr,
2898					   char *system, char *event_name)
2899{
2900	struct trace_event_file *file;
2901
2902	file = __find_event_file(tr, system, event_name);
2903	if (!file)
2904		return ERR_PTR(-EINVAL);
2905
2906	return file;
2907}
2908
2909static struct hist_field *
2910find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2911			 char *system, char *event_name, char *field_name)
2912{
2913	struct hist_field *event_var;
2914	char *synthetic_name;
2915
2916	synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2917	if (!synthetic_name)
2918		return ERR_PTR(-ENOMEM);
2919
2920	strcpy(synthetic_name, "synthetic_");
2921	strcat(synthetic_name, field_name);
2922
2923	event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2924
2925	kfree(synthetic_name);
2926
2927	return event_var;
2928}
2929
2930/**
2931 * create_field_var_hist - Automatically create a histogram and var for a field
2932 * @target_hist_data: The target hist trigger
2933 * @subsys_name: Optional subsystem name
2934 * @event_name: Optional event name
2935 * @field_name: The name of the field (and the resulting variable)
2936 *
2937 * Hist trigger actions fetch data from variables, not directly from
2938 * events.  However, for convenience, users are allowed to directly
2939 * specify an event field in an action, which will be automatically
2940 * converted into a variable on their behalf.
2941 *
2942 * If a user specifies a field on an event that isn't the event the
2943 * histogram currently being defined (the target event histogram), the
2944 * only way that can be accomplished is if a new hist trigger is
2945 * created and the field variable defined on that.
2946 *
2947 * This function creates a new histogram compatible with the target
2948 * event (meaning a histogram with the same key as the target
2949 * histogram), and creates a variable for the specified field, but
2950 * with 'synthetic_' prepended to the variable name in order to avoid
2951 * collision with normal field variables.
2952 *
2953 * Return: The variable created for the field.
2954 */
2955static struct hist_field *
2956create_field_var_hist(struct hist_trigger_data *target_hist_data,
2957		      char *subsys_name, char *event_name, char *field_name)
2958{
2959	struct trace_array *tr = target_hist_data->event_file->tr;
2960	struct hist_trigger_data *hist_data;
2961	unsigned int i, n, first = true;
2962	struct field_var_hist *var_hist;
2963	struct trace_event_file *file;
2964	struct hist_field *key_field;
2965	struct hist_field *event_var;
2966	char *saved_filter;
2967	char *cmd;
2968	int ret;
2969
2970	if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2971		hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2972		return ERR_PTR(-EINVAL);
2973	}
2974
2975	file = event_file(tr, subsys_name, event_name);
2976
2977	if (IS_ERR(file)) {
2978		hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2979		ret = PTR_ERR(file);
2980		return ERR_PTR(ret);
2981	}
2982
2983	/*
2984	 * Look for a histogram compatible with target.  We'll use the
2985	 * found histogram specification to create a new matching
2986	 * histogram with our variable on it.  target_hist_data is not
2987	 * yet a registered histogram so we can't use that.
2988	 */
2989	hist_data = find_compatible_hist(target_hist_data, file);
2990	if (!hist_data) {
2991		hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2992		return ERR_PTR(-EINVAL);
2993	}
2994
2995	/* See if a synthetic field variable has already been created */
2996	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2997					     event_name, field_name);
2998	if (!IS_ERR_OR_NULL(event_var))
2999		return event_var;
3000
3001	var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3002	if (!var_hist)
3003		return ERR_PTR(-ENOMEM);
3004
3005	cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3006	if (!cmd) {
3007		kfree(var_hist);
3008		return ERR_PTR(-ENOMEM);
3009	}
3010
3011	/* Use the same keys as the compatible histogram */
3012	strcat(cmd, "keys=");
3013
3014	for_each_hist_key_field(i, hist_data) {
3015		key_field = hist_data->fields[i];
3016		if (!first)
3017			strcat(cmd, ",");
3018		strcat(cmd, key_field->field->name);
3019		first = false;
3020	}
3021
3022	/* Create the synthetic field variable specification */
3023	strcat(cmd, ":synthetic_");
3024	strcat(cmd, field_name);
3025	strcat(cmd, "=");
3026	strcat(cmd, field_name);
3027
3028	/* Use the same filter as the compatible histogram */
3029	saved_filter = find_trigger_filter(hist_data, file);
3030	if (saved_filter) {
3031		strcat(cmd, " if ");
3032		strcat(cmd, saved_filter);
3033	}
3034
3035	var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3036	if (!var_hist->cmd) {
3037		kfree(cmd);
3038		kfree(var_hist);
3039		return ERR_PTR(-ENOMEM);
3040	}
3041
3042	/* Save the compatible histogram information */
3043	var_hist->hist_data = hist_data;
3044
3045	/* Create the new histogram with our variable */
3046	ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
3047				       "", "hist", cmd);
3048	if (ret) {
3049		kfree(cmd);
3050		kfree(var_hist->cmd);
3051		kfree(var_hist);
3052		hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
3053		return ERR_PTR(ret);
3054	}
3055
3056	kfree(cmd);
3057
3058	/* If we can't find the variable, something went wrong */
3059	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3060					     event_name, field_name);
3061	if (IS_ERR_OR_NULL(event_var)) {
3062		kfree(var_hist->cmd);
3063		kfree(var_hist);
3064		hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
3065		return ERR_PTR(-EINVAL);
3066	}
3067
3068	n = target_hist_data->n_field_var_hists;
3069	target_hist_data->field_var_hists[n] = var_hist;
3070	target_hist_data->n_field_var_hists++;
3071
3072	return event_var;
3073}
3074
3075static struct hist_field *
3076find_target_event_var(struct hist_trigger_data *hist_data,
3077		      char *subsys_name, char *event_name, char *var_name)
3078{
3079	struct trace_event_file *file = hist_data->event_file;
3080	struct hist_field *hist_field = NULL;
3081
3082	if (subsys_name) {
3083		struct trace_event_call *call;
3084
3085		if (!event_name)
3086			return NULL;
3087
3088		call = file->event_call;
3089
3090		if (strcmp(subsys_name, call->class->system) != 0)
3091			return NULL;
3092
3093		if (strcmp(event_name, trace_event_name(call)) != 0)
3094			return NULL;
3095	}
3096
3097	hist_field = find_var_field(hist_data, var_name);
3098
3099	return hist_field;
3100}
3101
3102static inline void __update_field_vars(struct tracing_map_elt *elt,
3103				       struct trace_buffer *buffer,
3104				       struct ring_buffer_event *rbe,
3105				       void *rec,
3106				       struct field_var **field_vars,
3107				       unsigned int n_field_vars,
3108				       unsigned int field_var_str_start)
3109{
3110	struct hist_elt_data *elt_data = elt->private_data;
3111	unsigned int i, j, var_idx;
3112	u64 var_val;
3113
3114	for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3115		struct field_var *field_var = field_vars[i];
3116		struct hist_field *var = field_var->var;
3117		struct hist_field *val = field_var->val;
3118
3119		var_val = hist_fn_call(val, elt, buffer, rbe, rec);
3120		var_idx = var->var.idx;
3121
3122		if (val->flags & HIST_FIELD_FL_STRING) {
3123			char *str = elt_data->field_var_str[j++];
3124			char *val_str = (char *)(uintptr_t)var_val;
3125			unsigned int size;
3126
3127			size = min(val->size, STR_VAR_LEN_MAX);
3128			strscpy(str, val_str, size);
3129			var_val = (u64)(uintptr_t)str;
3130		}
3131		tracing_map_set_var(elt, var_idx, var_val);
3132	}
3133}
3134
3135static void update_field_vars(struct hist_trigger_data *hist_data,
3136			      struct tracing_map_elt *elt,
3137			      struct trace_buffer *buffer,
3138			      struct ring_buffer_event *rbe,
3139			      void *rec)
3140{
3141	__update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
3142			    hist_data->n_field_vars, 0);
3143}
3144
3145static void save_track_data_vars(struct hist_trigger_data *hist_data,
3146				 struct tracing_map_elt *elt,
3147				 struct trace_buffer *buffer,  void *rec,
3148				 struct ring_buffer_event *rbe, void *key,
3149				 struct action_data *data, u64 *var_ref_vals)
3150{
3151	__update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
3152			    hist_data->n_save_vars, hist_data->n_field_var_str);
3153}
3154
3155static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3156				     struct trace_event_file *file,
3157				     char *name, int size, const char *type)
3158{
3159	struct hist_field *var;
3160	int idx;
3161
3162	if (find_var(hist_data, file, name) && !hist_data->remove) {
3163		var = ERR_PTR(-EINVAL);
3164		goto out;
3165	}
3166
3167	var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3168	if (!var) {
3169		var = ERR_PTR(-ENOMEM);
3170		goto out;
3171	}
3172
3173	idx = tracing_map_add_var(hist_data->map);
3174	if (idx < 0) {
3175		kfree(var);
3176		var = ERR_PTR(-EINVAL);
3177		goto out;
3178	}
3179
3180	var->ref = 1;
3181	var->flags = HIST_FIELD_FL_VAR;
3182	var->var.idx = idx;
3183	var->var.hist_data = var->hist_data = hist_data;
3184	var->size = size;
3185	var->var.name = kstrdup(name, GFP_KERNEL);
3186	var->type = kstrdup_const(type, GFP_KERNEL);
3187	if (!var->var.name || !var->type) {
3188		kfree_const(var->type);
3189		kfree(var->var.name);
3190		kfree(var);
3191		var = ERR_PTR(-ENOMEM);
3192	}
3193 out:
3194	return var;
3195}
3196
3197static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3198					  struct trace_event_file *file,
3199					  char *field_name)
3200{
3201	struct hist_field *val = NULL, *var = NULL;
3202	unsigned long flags = HIST_FIELD_FL_VAR;
3203	struct trace_array *tr = file->tr;
3204	struct field_var *field_var;
3205	int ret = 0;
3206
3207	if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3208		hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3209		ret = -EINVAL;
3210		goto err;
3211	}
3212
3213	val = parse_atom(hist_data, file, field_name, &flags, NULL);
3214	if (IS_ERR(val)) {
3215		hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3216		ret = PTR_ERR(val);
3217		goto err;
3218	}
3219
3220	var = create_var(hist_data, file, field_name, val->size, val->type);
3221	if (IS_ERR(var)) {
3222		hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3223		kfree(val);
3224		ret = PTR_ERR(var);
3225		goto err;
3226	}
3227
3228	field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3229	if (!field_var) {
3230		kfree(val);
3231		kfree(var);
3232		ret =  -ENOMEM;
3233		goto err;
3234	}
3235
3236	field_var->var = var;
3237	field_var->val = val;
3238 out:
3239	return field_var;
3240 err:
3241	field_var = ERR_PTR(ret);
3242	goto out;
3243}
3244
3245/**
3246 * create_target_field_var - Automatically create a variable for a field
3247 * @target_hist_data: The target hist trigger
3248 * @subsys_name: Optional subsystem name
3249 * @event_name: Optional event name
3250 * @var_name: The name of the field (and the resulting variable)
3251 *
3252 * Hist trigger actions fetch data from variables, not directly from
3253 * events.  However, for convenience, users are allowed to directly
3254 * specify an event field in an action, which will be automatically
3255 * converted into a variable on their behalf.
3256 *
3257 * This function creates a field variable with the name var_name on
3258 * the hist trigger currently being defined on the target event.  If
3259 * subsys_name and event_name are specified, this function simply
3260 * verifies that they do in fact match the target event subsystem and
3261 * event name.
3262 *
3263 * Return: The variable created for the field.
3264 */
3265static struct field_var *
3266create_target_field_var(struct hist_trigger_data *target_hist_data,
3267			char *subsys_name, char *event_name, char *var_name)
3268{
3269	struct trace_event_file *file = target_hist_data->event_file;
3270
3271	if (subsys_name) {
3272		struct trace_event_call *call;
3273
3274		if (!event_name)
3275			return NULL;
3276
3277		call = file->event_call;
3278
3279		if (strcmp(subsys_name, call->class->system) != 0)
3280			return NULL;
3281
3282		if (strcmp(event_name, trace_event_name(call)) != 0)
3283			return NULL;
3284	}
3285
3286	return create_field_var(target_hist_data, file, var_name);
3287}
3288
3289static bool check_track_val_max(u64 track_val, u64 var_val)
3290{
3291	if (var_val <= track_val)
3292		return false;
3293
3294	return true;
3295}
3296
3297static bool check_track_val_changed(u64 track_val, u64 var_val)
3298{
3299	if (var_val == track_val)
3300		return false;
3301
3302	return true;
3303}
3304
3305static u64 get_track_val(struct hist_trigger_data *hist_data,
3306			 struct tracing_map_elt *elt,
3307			 struct action_data *data)
3308{
3309	unsigned int track_var_idx = data->track_data.track_var->var.idx;
3310	u64 track_val;
3311
3312	track_val = tracing_map_read_var(elt, track_var_idx);
3313
3314	return track_val;
3315}
3316
3317static void save_track_val(struct hist_trigger_data *hist_data,
3318			   struct tracing_map_elt *elt,
3319			   struct action_data *data, u64 var_val)
3320{
3321	unsigned int track_var_idx = data->track_data.track_var->var.idx;
3322
3323	tracing_map_set_var(elt, track_var_idx, var_val);
3324}
3325
3326static void save_track_data(struct hist_trigger_data *hist_data,
3327			    struct tracing_map_elt *elt,
3328			    struct trace_buffer *buffer, void *rec,
3329			    struct ring_buffer_event *rbe, void *key,
3330			    struct action_data *data, u64 *var_ref_vals)
3331{
3332	if (data->track_data.save_data)
3333		data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
3334					   key, data, var_ref_vals);
3335}
3336
3337static bool check_track_val(struct tracing_map_elt *elt,
3338			    struct action_data *data,
3339			    u64 var_val)
3340{
3341	struct hist_trigger_data *hist_data;
3342	u64 track_val;
3343
3344	hist_data = data->track_data.track_var->hist_data;
3345	track_val = get_track_val(hist_data, elt, data);
3346
3347	return data->track_data.check_val(track_val, var_val);
3348}
3349
3350#ifdef CONFIG_TRACER_SNAPSHOT
3351static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3352{
3353	/* called with tr->max_lock held */
3354	struct track_data *track_data = tr->cond_snapshot->cond_data;
3355	struct hist_elt_data *elt_data, *track_elt_data;
3356	struct snapshot_context *context = cond_data;
3357	struct action_data *action;
3358	u64 track_val;
3359
3360	if (!track_data)
3361		return false;
3362
3363	action = track_data->action_data;
3364
3365	track_val = get_track_val(track_data->hist_data, context->elt,
3366				  track_data->action_data);
3367
3368	if (!action->track_data.check_val(track_data->track_val, track_val))
3369		return false;
3370
3371	track_data->track_val = track_val;
3372	memcpy(track_data->key, context->key, track_data->key_len);
3373
3374	elt_data = context->elt->private_data;
3375	track_elt_data = track_data->elt.private_data;
3376	if (elt_data->comm)
3377		strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3378
3379	track_data->updated = true;
3380
3381	return true;
3382}
3383
3384static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3385				     struct tracing_map_elt *elt,
3386				     struct trace_buffer *buffer, void *rec,
3387				     struct ring_buffer_event *rbe, void *key,
3388				     struct action_data *data,
3389				     u64 *var_ref_vals)
3390{
3391	struct trace_event_file *file = hist_data->event_file;
3392	struct snapshot_context context;
3393
3394	context.elt = elt;
3395	context.key = key;
3396
3397	tracing_snapshot_cond(file->tr, &context);
3398}
3399
3400static void hist_trigger_print_key(struct seq_file *m,
3401				   struct hist_trigger_data *hist_data,
3402				   void *key,
3403				   struct tracing_map_elt *elt);
3404
3405static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3406{
3407	unsigned int i;
3408
3409	if (!hist_data->n_actions)
3410		return NULL;
3411
3412	for (i = 0; i < hist_data->n_actions; i++) {
3413		struct action_data *data = hist_data->actions[i];
3414
3415		if (data->action == ACTION_SNAPSHOT)
3416			return data;
3417	}
3418
3419	return NULL;
3420}
3421
3422static void track_data_snapshot_print(struct seq_file *m,
3423				      struct hist_trigger_data *hist_data)
3424{
3425	struct trace_event_file *file = hist_data->event_file;
3426	struct track_data *track_data;
3427	struct action_data *action;
3428
3429	track_data = tracing_cond_snapshot_data(file->tr);
3430	if (!track_data)
3431		return;
3432
3433	if (!track_data->updated)
3434		return;
3435
3436	action = snapshot_action(hist_data);
3437	if (!action)
3438		return;
3439
3440	seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3441	seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3442		   action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3443		   action->track_data.var_str, track_data->track_val);
3444
3445	seq_puts(m, "\ttriggered by event with key: ");
3446	hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3447	seq_putc(m, '\n');
3448}
3449#else
3450static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3451{
3452	return false;
3453}
3454static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3455				     struct tracing_map_elt *elt,
3456				     struct trace_buffer *buffer, void *rec,
3457				     struct ring_buffer_event *rbe, void *key,
3458				     struct action_data *data,
3459				     u64 *var_ref_vals) {}
3460static void track_data_snapshot_print(struct seq_file *m,
3461				      struct hist_trigger_data *hist_data) {}
3462#endif /* CONFIG_TRACER_SNAPSHOT */
3463
3464static void track_data_print(struct seq_file *m,
3465			     struct hist_trigger_data *hist_data,
3466			     struct tracing_map_elt *elt,
3467			     struct action_data *data)
3468{
3469	u64 track_val = get_track_val(hist_data, elt, data);
3470	unsigned int i, save_var_idx;
3471
3472	if (data->handler == HANDLER_ONMAX)
3473		seq_printf(m, "\n\tmax: %10llu", track_val);
3474	else if (data->handler == HANDLER_ONCHANGE)
3475		seq_printf(m, "\n\tchanged: %10llu", track_val);
3476
3477	if (data->action == ACTION_SNAPSHOT)
3478		return;
3479
3480	for (i = 0; i < hist_data->n_save_vars; i++) {
3481		struct hist_field *save_val = hist_data->save_vars[i]->val;
3482		struct hist_field *save_var = hist_data->save_vars[i]->var;
3483		u64 val;
3484
3485		save_var_idx = save_var->var.idx;
3486
3487		val = tracing_map_read_var(elt, save_var_idx);
3488
3489		if (save_val->flags & HIST_FIELD_FL_STRING) {
3490			seq_printf(m, "  %s: %-32s", save_var->var.name,
3491				   (char *)(uintptr_t)(val));
3492		} else
3493			seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3494	}
3495}
3496
3497static void ontrack_action(struct hist_trigger_data *hist_data,
3498			   struct tracing_map_elt *elt,
3499			   struct trace_buffer *buffer, void *rec,
3500			   struct ring_buffer_event *rbe, void *key,
3501			   struct action_data *data, u64 *var_ref_vals)
3502{
3503	u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3504
3505	if (check_track_val(elt, data, var_val)) {
3506		save_track_val(hist_data, elt, data, var_val);
3507		save_track_data(hist_data, elt, buffer, rec, rbe,
3508				key, data, var_ref_vals);
3509	}
3510}
3511
3512static void action_data_destroy(struct action_data *data)
3513{
3514	unsigned int i;
3515
3516	lockdep_assert_held(&event_mutex);
3517
3518	kfree(data->action_name);
3519
3520	for (i = 0; i < data->n_params; i++)
3521		kfree(data->params[i]);
3522
3523	if (data->synth_event)
3524		data->synth_event->ref--;
3525
3526	kfree(data->synth_event_name);
3527
3528	kfree(data);
3529}
3530
3531static void track_data_destroy(struct hist_trigger_data *hist_data,
3532			       struct action_data *data)
3533{
3534	struct trace_event_file *file = hist_data->event_file;
3535
3536	destroy_hist_field(data->track_data.track_var, 0);
3537
3538	if (data->action == ACTION_SNAPSHOT) {
3539		struct track_data *track_data;
3540
3541		track_data = tracing_cond_snapshot_data(file->tr);
3542		if (track_data && track_data->hist_data == hist_data) {
3543			tracing_snapshot_cond_disable(file->tr);
3544			track_data_free(track_data);
3545		}
3546	}
3547
3548	kfree(data->track_data.var_str);
3549
3550	action_data_destroy(data);
3551}
3552
3553static int action_create(struct hist_trigger_data *hist_data,
3554			 struct action_data *data);
3555
3556static int track_data_create(struct hist_trigger_data *hist_data,
3557			     struct action_data *data)
3558{
3559	struct hist_field *var_field, *ref_field, *track_var = NULL;
3560	struct trace_event_file *file = hist_data->event_file;
3561	struct trace_array *tr = file->tr;
3562	char *track_data_var_str;
3563	int ret = 0;
3564
3565	track_data_var_str = data->track_data.var_str;
3566	if (track_data_var_str[0] != '$') {
3567		hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3568		return -EINVAL;
3569	}
3570	track_data_var_str++;
3571
3572	var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3573	if (!var_field) {
3574		hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3575		return -EINVAL;
3576	}
3577
3578	ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3579	if (!ref_field)
3580		return -ENOMEM;
3581
3582	data->track_data.var_ref = ref_field;
3583
3584	if (data->handler == HANDLER_ONMAX)
3585		track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3586	if (IS_ERR(track_var)) {
3587		hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3588		ret = PTR_ERR(track_var);
3589		goto out;
3590	}
3591
3592	if (data->handler == HANDLER_ONCHANGE)
3593		track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3594	if (IS_ERR(track_var)) {
3595		hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3596		ret = PTR_ERR(track_var);
3597		goto out;
3598	}
3599	data->track_data.track_var = track_var;
3600
3601	ret = action_create(hist_data, data);
3602 out:
3603	return ret;
3604}
3605
3606static int parse_action_params(struct trace_array *tr, char *params,
3607			       struct action_data *data)
3608{
3609	char *param, *saved_param;
3610	bool first_param = true;
3611	int ret = 0;
3612
3613	while (params) {
3614		if (data->n_params >= SYNTH_FIELDS_MAX) {
3615			hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3616			ret = -EINVAL;
3617			goto out;
3618		}
3619
3620		param = strsep(&params, ",");
3621		if (!param) {
3622			hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3623			ret = -EINVAL;
3624			goto out;
3625		}
3626
3627		param = strstrip(param);
3628		if (strlen(param) < 2) {
3629			hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3630			ret = -EINVAL;
3631			goto out;
3632		}
3633
3634		saved_param = kstrdup(param, GFP_KERNEL);
3635		if (!saved_param) {
3636			ret = -ENOMEM;
3637			goto out;
3638		}
3639
3640		if (first_param && data->use_trace_keyword) {
3641			data->synth_event_name = saved_param;
3642			first_param = false;
3643			continue;
3644		}
3645		first_param = false;
3646
3647		data->params[data->n_params++] = saved_param;
3648	}
3649 out:
3650	return ret;
3651}
3652
3653static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3654			enum handler_id handler)
3655{
3656	char *action_name;
3657	int ret = 0;
3658
3659	strsep(&str, ".");
3660	if (!str) {
3661		hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3662		ret = -EINVAL;
3663		goto out;
3664	}
3665
3666	action_name = strsep(&str, "(");
3667	if (!action_name || !str) {
3668		hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3669		ret = -EINVAL;
3670		goto out;
3671	}
3672
3673	if (str_has_prefix(action_name, "save")) {
3674		char *params = strsep(&str, ")");
3675
3676		if (!params) {
3677			hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3678			ret = -EINVAL;
3679			goto out;
3680		}
3681
3682		ret = parse_action_params(tr, params, data);
3683		if (ret)
3684			goto out;
3685
3686		if (handler == HANDLER_ONMAX)
3687			data->track_data.check_val = check_track_val_max;
3688		else if (handler == HANDLER_ONCHANGE)
3689			data->track_data.check_val = check_track_val_changed;
3690		else {
3691			hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3692			ret = -EINVAL;
3693			goto out;
3694		}
3695
3696		data->track_data.save_data = save_track_data_vars;
3697		data->fn = ontrack_action;
3698		data->action = ACTION_SAVE;
3699	} else if (str_has_prefix(action_name, "snapshot")) {
3700		char *params = strsep(&str, ")");
3701
3702		if (!str) {
3703			hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3704			ret = -EINVAL;
3705			goto out;
3706		}
3707
3708		if (handler == HANDLER_ONMAX)
3709			data->track_data.check_val = check_track_val_max;
3710		else if (handler == HANDLER_ONCHANGE)
3711			data->track_data.check_val = check_track_val_changed;
3712		else {
3713			hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3714			ret = -EINVAL;
3715			goto out;
3716		}
3717
3718		data->track_data.save_data = save_track_data_snapshot;
3719		data->fn = ontrack_action;
3720		data->action = ACTION_SNAPSHOT;
3721	} else {
3722		char *params = strsep(&str, ")");
3723
3724		if (str_has_prefix(action_name, "trace"))
3725			data->use_trace_keyword = true;
3726
3727		if (params) {
3728			ret = parse_action_params(tr, params, data);
3729			if (ret)
3730				goto out;
3731		}
3732
3733		if (handler == HANDLER_ONMAX)
3734			data->track_data.check_val = check_track_val_max;
3735		else if (handler == HANDLER_ONCHANGE)
3736			data->track_data.check_val = check_track_val_changed;
3737
3738		if (handler != HANDLER_ONMATCH) {
3739			data->track_data.save_data = action_trace;
3740			data->fn = ontrack_action;
3741		} else
3742			data->fn = action_trace;
3743
3744		data->action = ACTION_TRACE;
3745	}
3746
3747	data->action_name = kstrdup(action_name, GFP_KERNEL);
3748	if (!data->action_name) {
3749		ret = -ENOMEM;
3750		goto out;
3751	}
3752
3753	data->handler = handler;
3754 out:
3755	return ret;
3756}
3757
3758static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3759					    char *str, enum handler_id handler)
3760{
3761	struct action_data *data;
3762	int ret = -EINVAL;
3763	char *var_str;
3764
3765	data = kzalloc(sizeof(*data), GFP_KERNEL);
3766	if (!data)
3767		return ERR_PTR(-ENOMEM);
3768
3769	var_str = strsep(&str, ")");
3770	if (!var_str || !str) {
3771		ret = -EINVAL;
3772		goto free;
3773	}
3774
3775	data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3776	if (!data->track_data.var_str) {
3777		ret = -ENOMEM;
3778		goto free;
3779	}
3780
3781	ret = action_parse(hist_data->event_file->tr, str, data, handler);
3782	if (ret)
3783		goto free;
3784 out:
3785	return data;
3786 free:
3787	track_data_destroy(hist_data, data);
3788	data = ERR_PTR(ret);
3789	goto out;
3790}
3791
3792static void onmatch_destroy(struct action_data *data)
3793{
3794	kfree(data->match_data.event);
3795	kfree(data->match_data.event_system);
3796
3797	action_data_destroy(data);
3798}
3799
3800static void destroy_field_var(struct field_var *field_var)
3801{
3802	if (!field_var)
3803		return;
3804
3805	destroy_hist_field(field_var->var, 0);
3806	destroy_hist_field(field_var->val, 0);
3807
3808	kfree(field_var);
3809}
3810
3811static void destroy_field_vars(struct hist_trigger_data *hist_data)
3812{
3813	unsigned int i;
3814
3815	for (i = 0; i < hist_data->n_field_vars; i++)
3816		destroy_field_var(hist_data->field_vars[i]);
3817
3818	for (i = 0; i < hist_data->n_save_vars; i++)
3819		destroy_field_var(hist_data->save_vars[i]);
3820}
3821
3822static void save_field_var(struct hist_trigger_data *hist_data,
3823			   struct field_var *field_var)
3824{
3825	hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3826
3827	if (field_var->val->flags & HIST_FIELD_FL_STRING)
3828		hist_data->n_field_var_str++;
3829}
3830
3831
3832static int check_synth_field(struct synth_event *event,
3833			     struct hist_field *hist_field,
3834			     unsigned int field_pos)
3835{
3836	struct synth_field *field;
3837
3838	if (field_pos >= event->n_fields)
3839		return -EINVAL;
3840
3841	field = event->fields[field_pos];
3842
3843	/*
3844	 * A dynamic string synth field can accept static or
3845	 * dynamic. A static string synth field can only accept a
3846	 * same-sized static string, which is checked for later.
3847	 */
3848	if (strstr(hist_field->type, "char[") && field->is_string
3849	    && field->is_dynamic)
3850		return 0;
3851
3852	if (strcmp(field->type, hist_field->type) != 0) {
3853		if (field->size != hist_field->size ||
3854		    (!field->is_string && field->is_signed != hist_field->is_signed))
3855			return -EINVAL;
3856	}
3857
3858	return 0;
3859}
3860
3861static struct hist_field *
3862trace_action_find_var(struct hist_trigger_data *hist_data,
3863		      struct action_data *data,
3864		      char *system, char *event, char *var)
3865{
3866	struct trace_array *tr = hist_data->event_file->tr;
3867	struct hist_field *hist_field;
3868
3869	var++; /* skip '$' */
3870
3871	hist_field = find_target_event_var(hist_data, system, event, var);
3872	if (!hist_field) {
3873		if (!system && data->handler == HANDLER_ONMATCH) {
3874			system = data->match_data.event_system;
3875			event = data->match_data.event;
3876		}
3877
3878		hist_field = find_event_var(hist_data, system, event, var);
3879	}
3880
3881	if (!hist_field)
3882		hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3883
3884	return hist_field;
3885}
3886
3887static struct hist_field *
3888trace_action_create_field_var(struct hist_trigger_data *hist_data,
3889			      struct action_data *data, char *system,
3890			      char *event, char *var)
3891{
3892	struct hist_field *hist_field = NULL;
3893	struct field_var *field_var;
3894
3895	/*
3896	 * First try to create a field var on the target event (the
3897	 * currently being defined).  This will create a variable for
3898	 * unqualified fields on the target event, or if qualified,
3899	 * target fields that have qualified names matching the target.
3900	 */
3901	field_var = create_target_field_var(hist_data, system, event, var);
3902
3903	if (field_var && !IS_ERR(field_var)) {
3904		save_field_var(hist_data, field_var);
3905		hist_field = field_var->var;
3906	} else {
3907		field_var = NULL;
3908		/*
3909		 * If no explicit system.event is specified, default to
3910		 * looking for fields on the onmatch(system.event.xxx)
3911		 * event.
3912		 */
3913		if (!system && data->handler == HANDLER_ONMATCH) {
3914			system = data->match_data.event_system;
3915			event = data->match_data.event;
3916		}
3917
3918		if (!event)
3919			goto free;
3920		/*
3921		 * At this point, we're looking at a field on another
3922		 * event.  Because we can't modify a hist trigger on
3923		 * another event to add a variable for a field, we need
3924		 * to create a new trigger on that event and create the
3925		 * variable at the same time.
3926		 */
3927		hist_field = create_field_var_hist(hist_data, system, event, var);
3928		if (IS_ERR(hist_field))
3929			goto free;
3930	}
3931 out:
3932	return hist_field;
3933 free:
3934	destroy_field_var(field_var);
3935	hist_field = NULL;
3936	goto out;
3937}
3938
3939static int trace_action_create(struct hist_trigger_data *hist_data,
3940			       struct action_data *data)
3941{
3942	struct trace_array *tr = hist_data->event_file->tr;
3943	char *event_name, *param, *system = NULL;
3944	struct hist_field *hist_field, *var_ref;
3945	unsigned int i;
3946	unsigned int field_pos = 0;
3947	struct synth_event *event;
3948	char *synth_event_name;
3949	int var_ref_idx, ret = 0;
3950
3951	lockdep_assert_held(&event_mutex);
3952
3953	/* Sanity check to avoid out-of-bound write on 'data->var_ref_idx' */
3954	if (data->n_params > SYNTH_FIELDS_MAX)
3955		return -EINVAL;
3956
3957	if (data->use_trace_keyword)
3958		synth_event_name = data->synth_event_name;
3959	else
3960		synth_event_name = data->action_name;
3961
3962	event = find_synth_event(synth_event_name);
3963	if (!event) {
3964		hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3965		return -EINVAL;
3966	}
3967
3968	event->ref++;
3969
3970	for (i = 0; i < data->n_params; i++) {
3971		char *p;
3972
3973		p = param = kstrdup(data->params[i], GFP_KERNEL);
3974		if (!param) {
3975			ret = -ENOMEM;
3976			goto err;
3977		}
3978
3979		system = strsep(&param, ".");
3980		if (!param) {
3981			param = (char *)system;
3982			system = event_name = NULL;
3983		} else {
3984			event_name = strsep(&param, ".");
3985			if (!param) {
3986				kfree(p);
3987				ret = -EINVAL;
3988				goto err;
3989			}
3990		}
3991
3992		if (param[0] == '$')
3993			hist_field = trace_action_find_var(hist_data, data,
3994							   system, event_name,
3995							   param);
3996		else
3997			hist_field = trace_action_create_field_var(hist_data,
3998								   data,
3999								   system,
4000								   event_name,
4001								   param);
4002
4003		if (!hist_field) {
4004			kfree(p);
4005			ret = -EINVAL;
4006			goto err;
4007		}
4008
4009		if (check_synth_field(event, hist_field, field_pos) == 0) {
4010			var_ref = create_var_ref(hist_data, hist_field,
4011						 system, event_name);
4012			if (!var_ref) {
4013				kfree(p);
4014				ret = -ENOMEM;
4015				goto err;
4016			}
4017
4018			var_ref_idx = find_var_ref_idx(hist_data, var_ref);
4019			if (WARN_ON(var_ref_idx < 0)) {
4020				kfree(p);
4021				ret = var_ref_idx;
4022				goto err;
4023			}
4024
4025			data->var_ref_idx[i] = var_ref_idx;
4026
4027			field_pos++;
4028			kfree(p);
4029			continue;
4030		}
4031
4032		hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
4033		kfree(p);
4034		ret = -EINVAL;
4035		goto err;
4036	}
4037
4038	if (field_pos != event->n_fields) {
4039		hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
4040		ret = -EINVAL;
4041		goto err;
4042	}
4043
4044	data->synth_event = event;
4045 out:
4046	return ret;
4047 err:
4048	event->ref--;
4049
4050	goto out;
4051}
4052
4053static int action_create(struct hist_trigger_data *hist_data,
4054			 struct action_data *data)
4055{
4056	struct trace_event_file *file = hist_data->event_file;
4057	struct trace_array *tr = file->tr;
4058	struct track_data *track_data;
4059	struct field_var *field_var;
4060	unsigned int i;
4061	char *param;
4062	int ret = 0;
4063
4064	if (data->action == ACTION_TRACE)
4065		return trace_action_create(hist_data, data);
4066
4067	if (data->action == ACTION_SNAPSHOT) {
4068		track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4069		if (IS_ERR(track_data)) {
4070			ret = PTR_ERR(track_data);
4071			goto out;
4072		}
4073
4074		ret = tracing_snapshot_cond_enable(file->tr, track_data,
4075						   cond_snapshot_update);
4076		if (ret)
4077			track_data_free(track_data);
4078
4079		goto out;
4080	}
4081
4082	if (data->action == ACTION_SAVE) {
4083		if (hist_data->n_save_vars) {
4084			ret = -EEXIST;
4085			hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4086			goto out;
4087		}
4088
4089		for (i = 0; i < data->n_params; i++) {
4090			param = kstrdup(data->params[i], GFP_KERNEL);
4091			if (!param) {
4092				ret = -ENOMEM;
4093				goto out;
4094			}
4095
4096			field_var = create_target_field_var(hist_data, NULL, NULL, param);
4097			if (IS_ERR(field_var)) {
4098				hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4099					 errpos(param));
4100				ret = PTR_ERR(field_var);
4101				kfree(param);
4102				goto out;
4103			}
4104
4105			hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4106			if (field_var->val->flags & HIST_FIELD_FL_STRING)
4107				hist_data->n_save_var_str++;
4108			kfree(param);
4109		}
4110	}
4111 out:
4112	return ret;
4113}
4114
4115static int onmatch_create(struct hist_trigger_data *hist_data,
4116			  struct action_data *data)
4117{
4118	return action_create(hist_data, data);
4119}
4120
4121static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4122{
4123	char *match_event, *match_event_system;
4124	struct action_data *data;
4125	int ret = -EINVAL;
4126
4127	data = kzalloc(sizeof(*data), GFP_KERNEL);
4128	if (!data)
4129		return ERR_PTR(-ENOMEM);
4130
4131	match_event = strsep(&str, ")");
4132	if (!match_event || !str) {
4133		hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4134		goto free;
4135	}
4136
4137	match_event_system = strsep(&match_event, ".");
4138	if (!match_event) {
4139		hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4140		goto free;
4141	}
4142
4143	if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4144		hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4145		goto free;
4146	}
4147
4148	data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4149	if (!data->match_data.event) {
4150		ret = -ENOMEM;
4151		goto free;
4152	}
4153
4154	data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4155	if (!data->match_data.event_system) {
4156		ret = -ENOMEM;
4157		goto free;
4158	}
4159
4160	ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4161	if (ret)
4162		goto free;
4163 out:
4164	return data;
4165 free:
4166	onmatch_destroy(data);
4167	data = ERR_PTR(ret);
4168	goto out;
4169}
4170
4171static int create_hitcount_val(struct hist_trigger_data *hist_data)
4172{
4173	hist_data->fields[HITCOUNT_IDX] =
4174		create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4175	if (!hist_data->fields[HITCOUNT_IDX])
4176		return -ENOMEM;
4177
4178	hist_data->n_vals++;
4179	hist_data->n_fields++;
4180
4181	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4182		return -EINVAL;
4183
4184	return 0;
4185}
4186
4187static int __create_val_field(struct hist_trigger_data *hist_data,
4188			      unsigned int val_idx,
4189			      struct trace_event_file *file,
4190			      char *var_name, char *field_str,
4191			      unsigned long flags)
4192{
4193	struct hist_field *hist_field;
4194	int ret = 0, n_subexprs = 0;
4195
4196	hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs);
4197	if (IS_ERR(hist_field)) {
4198		ret = PTR_ERR(hist_field);
4199		goto out;
4200	}
4201
4202	hist_data->fields[val_idx] = hist_field;
4203
4204	++hist_data->n_vals;
4205	++hist_data->n_fields;
4206
4207	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4208		ret = -EINVAL;
4209 out:
4210	return ret;
4211}
4212
4213static int create_val_field(struct hist_trigger_data *hist_data,
4214			    unsigned int val_idx,
4215			    struct trace_event_file *file,
4216			    char *field_str)
4217{
4218	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4219		return -EINVAL;
4220
4221	return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4222}
4223
4224static const char no_comm[] = "(no comm)";
4225
4226static u64 hist_field_execname(struct hist_field *hist_field,
4227			       struct tracing_map_elt *elt,
4228			       struct trace_buffer *buffer,
4229			       struct ring_buffer_event *rbe,
4230			       void *event)
4231{
4232	struct hist_elt_data *elt_data;
4233
4234	if (WARN_ON_ONCE(!elt))
4235		return (u64)(unsigned long)no_comm;
4236
4237	elt_data = elt->private_data;
4238
4239	if (WARN_ON_ONCE(!elt_data->comm))
4240		return (u64)(unsigned long)no_comm;
4241
4242	return (u64)(unsigned long)(elt_data->comm);
4243}
4244
4245static u64 hist_fn_call(struct hist_field *hist_field,
4246			struct tracing_map_elt *elt,
4247			struct trace_buffer *buffer,
4248			struct ring_buffer_event *rbe,
4249			void *event)
4250{
4251	switch (hist_field->fn_num) {
4252	case HIST_FIELD_FN_VAR_REF:
4253		return hist_field_var_ref(hist_field, elt, buffer, rbe, event);
4254	case HIST_FIELD_FN_COUNTER:
4255		return hist_field_counter(hist_field, elt, buffer, rbe, event);
4256	case HIST_FIELD_FN_CONST:
4257		return hist_field_const(hist_field, elt, buffer, rbe, event);
4258	case HIST_FIELD_FN_LOG2:
4259		return hist_field_log2(hist_field, elt, buffer, rbe, event);
4260	case HIST_FIELD_FN_BUCKET:
4261		return hist_field_bucket(hist_field, elt, buffer, rbe, event);
4262	case HIST_FIELD_FN_TIMESTAMP:
4263		return hist_field_timestamp(hist_field, elt, buffer, rbe, event);
4264	case HIST_FIELD_FN_CPU:
4265		return hist_field_cpu(hist_field, elt, buffer, rbe, event);
4266	case HIST_FIELD_FN_STRING:
4267		return hist_field_string(hist_field, elt, buffer, rbe, event);
4268	case HIST_FIELD_FN_DYNSTRING:
4269		return hist_field_dynstring(hist_field, elt, buffer, rbe, event);
4270	case HIST_FIELD_FN_RELDYNSTRING:
4271		return hist_field_reldynstring(hist_field, elt, buffer, rbe, event);
4272	case HIST_FIELD_FN_PSTRING:
4273		return hist_field_pstring(hist_field, elt, buffer, rbe, event);
4274	case HIST_FIELD_FN_S64:
4275		return hist_field_s64(hist_field, elt, buffer, rbe, event);
4276	case HIST_FIELD_FN_U64:
4277		return hist_field_u64(hist_field, elt, buffer, rbe, event);
4278	case HIST_FIELD_FN_S32:
4279		return hist_field_s32(hist_field, elt, buffer, rbe, event);
4280	case HIST_FIELD_FN_U32:
4281		return hist_field_u32(hist_field, elt, buffer, rbe, event);
4282	case HIST_FIELD_FN_S16:
4283		return hist_field_s16(hist_field, elt, buffer, rbe, event);
4284	case HIST_FIELD_FN_U16:
4285		return hist_field_u16(hist_field, elt, buffer, rbe, event);
4286	case HIST_FIELD_FN_S8:
4287		return hist_field_s8(hist_field, elt, buffer, rbe, event);
4288	case HIST_FIELD_FN_U8:
4289		return hist_field_u8(hist_field, elt, buffer, rbe, event);
4290	case HIST_FIELD_FN_UMINUS:
4291		return hist_field_unary_minus(hist_field, elt, buffer, rbe, event);
4292	case HIST_FIELD_FN_MINUS:
4293		return hist_field_minus(hist_field, elt, buffer, rbe, event);
4294	case HIST_FIELD_FN_PLUS:
4295		return hist_field_plus(hist_field, elt, buffer, rbe, event);
4296	case HIST_FIELD_FN_DIV:
4297		return hist_field_div(hist_field, elt, buffer, rbe, event);
4298	case HIST_FIELD_FN_MULT:
4299		return hist_field_mult(hist_field, elt, buffer, rbe, event);
4300	case HIST_FIELD_FN_DIV_POWER2:
4301		return div_by_power_of_two(hist_field, elt, buffer, rbe, event);
4302	case HIST_FIELD_FN_DIV_NOT_POWER2:
4303		return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event);
4304	case HIST_FIELD_FN_DIV_MULT_SHIFT:
4305		return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event);
4306	case HIST_FIELD_FN_EXECNAME:
4307		return hist_field_execname(hist_field, elt, buffer, rbe, event);
4308	default:
4309		return 0;
4310	}
4311}
4312
4313/* Convert a var that points to common_pid.execname to a string */
4314static void update_var_execname(struct hist_field *hist_field)
4315{
4316	hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR |
4317		HIST_FIELD_FL_EXECNAME;
4318	hist_field->size = MAX_FILTER_STR_VAL;
4319	hist_field->is_signed = 0;
4320
4321	kfree_const(hist_field->type);
4322	hist_field->type = "char[]";
4323
4324	hist_field->fn_num = HIST_FIELD_FN_EXECNAME;
4325}
4326
4327static int create_var_field(struct hist_trigger_data *hist_data,
4328			    unsigned int val_idx,
4329			    struct trace_event_file *file,
4330			    char *var_name, char *expr_str)
4331{
4332	struct trace_array *tr = hist_data->event_file->tr;
4333	unsigned long flags = 0;
4334	int ret;
4335
4336	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4337		return -EINVAL;
4338
4339	if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4340		hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4341		return -EINVAL;
4342	}
4343
4344	flags |= HIST_FIELD_FL_VAR;
4345	hist_data->n_vars++;
4346	if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4347		return -EINVAL;
4348
4349	ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4350
4351	if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME)
4352		update_var_execname(hist_data->fields[val_idx]);
4353
4354	if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_STRING)
4355		hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
4356
4357	return ret;
4358}
4359
4360static int create_val_fields(struct hist_trigger_data *hist_data,
4361			     struct trace_event_file *file)
4362{
4363	unsigned int i, j = 1, n_hitcount = 0;
4364	char *fields_str, *field_str;
4365	int ret;
4366
4367	ret = create_hitcount_val(hist_data);
4368	if (ret)
4369		goto out;
4370
4371	fields_str = hist_data->attrs->vals_str;
4372	if (!fields_str)
4373		goto out;
4374
4375	for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4376		     j < TRACING_MAP_VALS_MAX; i++) {
4377		field_str = strsep(&fields_str, ",");
4378		if (!field_str)
4379			break;
4380
4381		if (strcmp(field_str, "hitcount") == 0) {
4382			if (!n_hitcount++)
4383				continue;
4384		}
4385
4386		ret = create_val_field(hist_data, j++, file, field_str);
4387		if (ret)
4388			goto out;
4389	}
4390
4391	if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4392		ret = -EINVAL;
4393 out:
4394	/* There is only raw hitcount but nohitcount suppresses it. */
4395	if (j == 1 && hist_data->attrs->no_hitcount) {
4396		hist_err(hist_data->event_file->tr, HIST_ERR_NEED_NOHC_VAL, 0);
4397		ret = -ENOENT;
4398	}
4399
4400	return ret;
4401}
4402
4403static int create_key_field(struct hist_trigger_data *hist_data,
4404			    unsigned int key_idx,
4405			    unsigned int key_offset,
4406			    struct trace_event_file *file,
4407			    char *field_str)
4408{
4409	struct trace_array *tr = hist_data->event_file->tr;
4410	struct hist_field *hist_field = NULL;
4411	unsigned long flags = 0;
4412	unsigned int key_size;
4413	int ret = 0, n_subexprs = 0;
4414
4415	if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4416		return -EINVAL;
4417
4418	flags |= HIST_FIELD_FL_KEY;
4419
4420	if (strcmp(field_str, "stacktrace") == 0) {
4421		flags |= HIST_FIELD_FL_STACKTRACE;
4422		key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4423		hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4424	} else {
4425		hist_field = parse_expr(hist_data, file, field_str, flags,
4426					NULL, &n_subexprs);
4427		if (IS_ERR(hist_field)) {
4428			ret = PTR_ERR(hist_field);
4429			goto out;
4430		}
4431
4432		if (field_has_hist_vars(hist_field, 0))	{
4433			hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4434			destroy_hist_field(hist_field, 0);
4435			ret = -EINVAL;
4436			goto out;
4437		}
4438
4439		key_size = hist_field->size;
4440	}
4441
4442	hist_data->fields[key_idx] = hist_field;
4443
4444	key_size = ALIGN(key_size, sizeof(u64));
4445	hist_data->fields[key_idx]->size = key_size;
4446	hist_data->fields[key_idx]->offset = key_offset;
4447
4448	hist_data->key_size += key_size;
4449
4450	if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4451		ret = -EINVAL;
4452		goto out;
4453	}
4454
4455	hist_data->n_keys++;
4456	hist_data->n_fields++;
4457
4458	if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4459		return -EINVAL;
4460
4461	ret = key_size;
4462 out:
4463	return ret;
4464}
4465
4466static int create_key_fields(struct hist_trigger_data *hist_data,
4467			     struct trace_event_file *file)
4468{
4469	unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4470	char *fields_str, *field_str;
4471	int ret = -EINVAL;
4472
4473	fields_str = hist_data->attrs->keys_str;
4474	if (!fields_str)
4475		goto out;
4476
4477	for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4478		field_str = strsep(&fields_str, ",");
4479		if (!field_str)
4480			break;
4481		ret = create_key_field(hist_data, i, key_offset,
4482				       file, field_str);
4483		if (ret < 0)
4484			goto out;
4485		key_offset += ret;
4486	}
4487	if (fields_str) {
4488		ret = -EINVAL;
4489		goto out;
4490	}
4491	ret = 0;
4492 out:
4493	return ret;
4494}
4495
4496static int create_var_fields(struct hist_trigger_data *hist_data,
4497			     struct trace_event_file *file)
4498{
4499	unsigned int i, j = hist_data->n_vals;
4500	int ret = 0;
4501
4502	unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4503
4504	for (i = 0; i < n_vars; i++) {
4505		char *var_name = hist_data->attrs->var_defs.name[i];
4506		char *expr = hist_data->attrs->var_defs.expr[i];
4507
4508		ret = create_var_field(hist_data, j++, file, var_name, expr);
4509		if (ret)
4510			goto out;
4511	}
4512 out:
4513	return ret;
4514}
4515
4516static void free_var_defs(struct hist_trigger_data *hist_data)
4517{
4518	unsigned int i;
4519
4520	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4521		kfree(hist_data->attrs->var_defs.name[i]);
4522		kfree(hist_data->attrs->var_defs.expr[i]);
4523	}
4524
4525	hist_data->attrs->var_defs.n_vars = 0;
4526}
4527
4528static int parse_var_defs(struct hist_trigger_data *hist_data)
4529{
4530	struct trace_array *tr = hist_data->event_file->tr;
4531	char *s, *str, *var_name, *field_str;
4532	unsigned int i, j, n_vars = 0;
4533	int ret = 0;
4534
4535	for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4536		str = hist_data->attrs->assignment_str[i];
4537		for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4538			field_str = strsep(&str, ",");
4539			if (!field_str)
4540				break;
4541
4542			var_name = strsep(&field_str, "=");
4543			if (!var_name || !field_str) {
4544				hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4545					 errpos(var_name));
4546				ret = -EINVAL;
4547				goto free;
4548			}
4549
4550			if (n_vars == TRACING_MAP_VARS_MAX) {
4551				hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4552				ret = -EINVAL;
4553				goto free;
4554			}
4555
4556			s = kstrdup(var_name, GFP_KERNEL);
4557			if (!s) {
4558				ret = -ENOMEM;
4559				goto free;
4560			}
4561			hist_data->attrs->var_defs.name[n_vars] = s;
4562
4563			s = kstrdup(field_str, GFP_KERNEL);
4564			if (!s) {
4565				kfree(hist_data->attrs->var_defs.name[n_vars]);
4566				hist_data->attrs->var_defs.name[n_vars] = NULL;
4567				ret = -ENOMEM;
4568				goto free;
4569			}
4570			hist_data->attrs->var_defs.expr[n_vars++] = s;
4571
4572			hist_data->attrs->var_defs.n_vars = n_vars;
4573		}
4574	}
4575
4576	return ret;
4577 free:
4578	free_var_defs(hist_data);
4579
4580	return ret;
4581}
4582
4583static int create_hist_fields(struct hist_trigger_data *hist_data,
4584			      struct trace_event_file *file)
4585{
4586	int ret;
4587
4588	ret = parse_var_defs(hist_data);
4589	if (ret)
4590		return ret;
4591
4592	ret = create_val_fields(hist_data, file);
4593	if (ret)
4594		goto out;
4595
4596	ret = create_var_fields(hist_data, file);
4597	if (ret)
4598		goto out;
4599
4600	ret = create_key_fields(hist_data, file);
4601
4602 out:
4603	free_var_defs(hist_data);
4604
4605	return ret;
4606}
4607
4608static int is_descending(struct trace_array *tr, const char *str)
4609{
4610	if (!str)
4611		return 0;
4612
4613	if (strcmp(str, "descending") == 0)
4614		return 1;
4615
4616	if (strcmp(str, "ascending") == 0)
4617		return 0;
4618
4619	hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4620
4621	return -EINVAL;
4622}
4623
4624static int create_sort_keys(struct hist_trigger_data *hist_data)
4625{
4626	struct trace_array *tr = hist_data->event_file->tr;
4627	char *fields_str = hist_data->attrs->sort_key_str;
4628	struct tracing_map_sort_key *sort_key;
4629	int descending, ret = 0;
4630	unsigned int i, j, k;
4631
4632	hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4633
4634	if (!fields_str)
4635		goto out;
4636
4637	for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4638		struct hist_field *hist_field;
4639		char *field_str, *field_name;
4640		const char *test_name;
4641
4642		sort_key = &hist_data->sort_keys[i];
4643
4644		field_str = strsep(&fields_str, ",");
4645		if (!field_str)
4646			break;
4647
4648		if (!*field_str) {
4649			ret = -EINVAL;
4650			hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4651			break;
4652		}
4653
4654		if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4655			hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4656			ret = -EINVAL;
4657			break;
4658		}
4659
4660		field_name = strsep(&field_str, ".");
4661		if (!field_name || !*field_name) {
4662			ret = -EINVAL;
4663			hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4664			break;
4665		}
4666
4667		if (strcmp(field_name, "hitcount") == 0) {
4668			descending = is_descending(tr, field_str);
4669			if (descending < 0) {
4670				ret = descending;
4671				break;
4672			}
4673			sort_key->descending = descending;
4674			continue;
4675		}
4676
4677		for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4678			unsigned int idx;
4679
4680			hist_field = hist_data->fields[j];
4681			if (hist_field->flags & HIST_FIELD_FL_VAR)
4682				continue;
4683
4684			idx = k++;
4685
4686			test_name = hist_field_name(hist_field, 0);
4687
4688			if (strcmp(field_name, test_name) == 0) {
4689				sort_key->field_idx = idx;
4690				descending = is_descending(tr, field_str);
4691				if (descending < 0) {
4692					ret = descending;
4693					goto out;
4694				}
4695				sort_key->descending = descending;
4696				break;
4697			}
4698		}
4699		if (j == hist_data->n_fields) {
4700			ret = -EINVAL;
4701			hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4702			break;
4703		}
4704	}
4705
4706	hist_data->n_sort_keys = i;
4707 out:
4708	return ret;
4709}
4710
4711static void destroy_actions(struct hist_trigger_data *hist_data)
4712{
4713	unsigned int i;
4714
4715	for (i = 0; i < hist_data->n_actions; i++) {
4716		struct action_data *data = hist_data->actions[i];
4717
4718		if (data->handler == HANDLER_ONMATCH)
4719			onmatch_destroy(data);
4720		else if (data->handler == HANDLER_ONMAX ||
4721			 data->handler == HANDLER_ONCHANGE)
4722			track_data_destroy(hist_data, data);
4723		else
4724			kfree(data);
4725	}
4726}
4727
4728static int parse_actions(struct hist_trigger_data *hist_data)
4729{
4730	struct trace_array *tr = hist_data->event_file->tr;
4731	struct action_data *data;
4732	unsigned int i;
4733	int ret = 0;
4734	char *str;
4735	int len;
4736
4737	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4738		str = hist_data->attrs->action_str[i];
4739
4740		if ((len = str_has_prefix(str, "onmatch("))) {
4741			char *action_str = str + len;
4742
4743			data = onmatch_parse(tr, action_str);
4744			if (IS_ERR(data)) {
4745				ret = PTR_ERR(data);
4746				break;
4747			}
4748		} else if ((len = str_has_prefix(str, "onmax("))) {
4749			char *action_str = str + len;
4750
4751			data = track_data_parse(hist_data, action_str,
4752						HANDLER_ONMAX);
4753			if (IS_ERR(data)) {
4754				ret = PTR_ERR(data);
4755				break;
4756			}
4757		} else if ((len = str_has_prefix(str, "onchange("))) {
4758			char *action_str = str + len;
4759
4760			data = track_data_parse(hist_data, action_str,
4761						HANDLER_ONCHANGE);
4762			if (IS_ERR(data)) {
4763				ret = PTR_ERR(data);
4764				break;
4765			}
4766		} else {
4767			ret = -EINVAL;
4768			break;
4769		}
4770
4771		hist_data->actions[hist_data->n_actions++] = data;
4772	}
4773
4774	return ret;
4775}
4776
4777static int create_actions(struct hist_trigger_data *hist_data)
4778{
4779	struct action_data *data;
4780	unsigned int i;
4781	int ret = 0;
4782
4783	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4784		data = hist_data->actions[i];
4785
4786		if (data->handler == HANDLER_ONMATCH) {
4787			ret = onmatch_create(hist_data, data);
4788			if (ret)
4789				break;
4790		} else if (data->handler == HANDLER_ONMAX ||
4791			   data->handler == HANDLER_ONCHANGE) {
4792			ret = track_data_create(hist_data, data);
4793			if (ret)
4794				break;
4795		} else {
4796			ret = -EINVAL;
4797			break;
4798		}
4799	}
4800
4801	return ret;
4802}
4803
4804static void print_actions(struct seq_file *m,
4805			  struct hist_trigger_data *hist_data,
4806			  struct tracing_map_elt *elt)
4807{
4808	unsigned int i;
4809
4810	for (i = 0; i < hist_data->n_actions; i++) {
4811		struct action_data *data = hist_data->actions[i];
4812
4813		if (data->action == ACTION_SNAPSHOT)
4814			continue;
4815
4816		if (data->handler == HANDLER_ONMAX ||
4817		    data->handler == HANDLER_ONCHANGE)
4818			track_data_print(m, hist_data, elt, data);
4819	}
4820}
4821
4822static void print_action_spec(struct seq_file *m,
4823			      struct hist_trigger_data *hist_data,
4824			      struct action_data *data)
4825{
4826	unsigned int i;
4827
4828	if (data->action == ACTION_SAVE) {
4829		for (i = 0; i < hist_data->n_save_vars; i++) {
4830			seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4831			if (i < hist_data->n_save_vars - 1)
4832				seq_puts(m, ",");
4833		}
4834	} else if (data->action == ACTION_TRACE) {
4835		if (data->use_trace_keyword)
4836			seq_printf(m, "%s", data->synth_event_name);
4837		for (i = 0; i < data->n_params; i++) {
4838			if (i || data->use_trace_keyword)
4839				seq_puts(m, ",");
4840			seq_printf(m, "%s", data->params[i]);
4841		}
4842	}
4843}
4844
4845static void print_track_data_spec(struct seq_file *m,
4846				  struct hist_trigger_data *hist_data,
4847				  struct action_data *data)
4848{
4849	if (data->handler == HANDLER_ONMAX)
4850		seq_puts(m, ":onmax(");
4851	else if (data->handler == HANDLER_ONCHANGE)
4852		seq_puts(m, ":onchange(");
4853	seq_printf(m, "%s", data->track_data.var_str);
4854	seq_printf(m, ").%s(", data->action_name);
4855
4856	print_action_spec(m, hist_data, data);
4857
4858	seq_puts(m, ")");
4859}
4860
4861static void print_onmatch_spec(struct seq_file *m,
4862			       struct hist_trigger_data *hist_data,
4863			       struct action_data *data)
4864{
4865	seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4866		   data->match_data.event);
4867
4868	seq_printf(m, "%s(", data->action_name);
4869
4870	print_action_spec(m, hist_data, data);
4871
4872	seq_puts(m, ")");
4873}
4874
4875static bool actions_match(struct hist_trigger_data *hist_data,
4876			  struct hist_trigger_data *hist_data_test)
4877{
4878	unsigned int i, j;
4879
4880	if (hist_data->n_actions != hist_data_test->n_actions)
4881		return false;
4882
4883	for (i = 0; i < hist_data->n_actions; i++) {
4884		struct action_data *data = hist_data->actions[i];
4885		struct action_data *data_test = hist_data_test->actions[i];
4886		char *action_name, *action_name_test;
4887
4888		if (data->handler != data_test->handler)
4889			return false;
4890		if (data->action != data_test->action)
4891			return false;
4892
4893		if (data->n_params != data_test->n_params)
4894			return false;
4895
4896		for (j = 0; j < data->n_params; j++) {
4897			if (strcmp(data->params[j], data_test->params[j]) != 0)
4898				return false;
4899		}
4900
4901		if (data->use_trace_keyword)
4902			action_name = data->synth_event_name;
4903		else
4904			action_name = data->action_name;
4905
4906		if (data_test->use_trace_keyword)
4907			action_name_test = data_test->synth_event_name;
4908		else
4909			action_name_test = data_test->action_name;
4910
4911		if (strcmp(action_name, action_name_test) != 0)
4912			return false;
4913
4914		if (data->handler == HANDLER_ONMATCH) {
4915			if (strcmp(data->match_data.event_system,
4916				   data_test->match_data.event_system) != 0)
4917				return false;
4918			if (strcmp(data->match_data.event,
4919				   data_test->match_data.event) != 0)
4920				return false;
4921		} else if (data->handler == HANDLER_ONMAX ||
4922			   data->handler == HANDLER_ONCHANGE) {
4923			if (strcmp(data->track_data.var_str,
4924				   data_test->track_data.var_str) != 0)
4925				return false;
4926		}
4927	}
4928
4929	return true;
4930}
4931
4932
4933static void print_actions_spec(struct seq_file *m,
4934			       struct hist_trigger_data *hist_data)
4935{
4936	unsigned int i;
4937
4938	for (i = 0; i < hist_data->n_actions; i++) {
4939		struct action_data *data = hist_data->actions[i];
4940
4941		if (data->handler == HANDLER_ONMATCH)
4942			print_onmatch_spec(m, hist_data, data);
4943		else if (data->handler == HANDLER_ONMAX ||
4944			 data->handler == HANDLER_ONCHANGE)
4945			print_track_data_spec(m, hist_data, data);
4946	}
4947}
4948
4949static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4950{
4951	unsigned int i;
4952
4953	for (i = 0; i < hist_data->n_field_var_hists; i++) {
4954		kfree(hist_data->field_var_hists[i]->cmd);
4955		kfree(hist_data->field_var_hists[i]);
4956	}
4957}
4958
4959static void destroy_hist_data(struct hist_trigger_data *hist_data)
4960{
4961	if (!hist_data)
4962		return;
4963
4964	destroy_hist_trigger_attrs(hist_data->attrs);
4965	destroy_hist_fields(hist_data);
4966	tracing_map_destroy(hist_data->map);
4967
4968	destroy_actions(hist_data);
4969	destroy_field_vars(hist_data);
4970	destroy_field_var_hists(hist_data);
4971
4972	kfree(hist_data);
4973}
4974
4975static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4976{
4977	struct tracing_map *map = hist_data->map;
4978	struct ftrace_event_field *field;
4979	struct hist_field *hist_field;
4980	int i, idx = 0;
4981
4982	for_each_hist_field(i, hist_data) {
4983		hist_field = hist_data->fields[i];
4984		if (hist_field->flags & HIST_FIELD_FL_KEY) {
4985			tracing_map_cmp_fn_t cmp_fn;
4986
4987			field = hist_field->field;
4988
4989			if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4990				cmp_fn = tracing_map_cmp_none;
4991			else if (!field || hist_field->flags & HIST_FIELD_FL_CPU)
4992				cmp_fn = tracing_map_cmp_num(hist_field->size,
4993							     hist_field->is_signed);
4994			else if (is_string_field(field))
4995				cmp_fn = tracing_map_cmp_string;
4996			else
4997				cmp_fn = tracing_map_cmp_num(field->size,
4998							     field->is_signed);
4999			idx = tracing_map_add_key_field(map,
5000							hist_field->offset,
5001							cmp_fn);
5002		} else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5003			idx = tracing_map_add_sum_field(map);
5004
5005		if (idx < 0)
5006			return idx;
5007
5008		if (hist_field->flags & HIST_FIELD_FL_VAR) {
5009			idx = tracing_map_add_var(map);
5010			if (idx < 0)
5011				return idx;
5012			hist_field->var.idx = idx;
5013			hist_field->var.hist_data = hist_data;
5014		}
5015	}
5016
5017	return 0;
5018}
5019
5020static struct hist_trigger_data *
5021create_hist_data(unsigned int map_bits,
5022		 struct hist_trigger_attrs *attrs,
5023		 struct trace_event_file *file,
5024		 bool remove)
5025{
5026	const struct tracing_map_ops *map_ops = NULL;
5027	struct hist_trigger_data *hist_data;
5028	int ret = 0;
5029
5030	hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5031	if (!hist_data)
5032		return ERR_PTR(-ENOMEM);
5033
5034	hist_data->attrs = attrs;
5035	hist_data->remove = remove;
5036	hist_data->event_file = file;
5037
5038	ret = parse_actions(hist_data);
5039	if (ret)
5040		goto free;
5041
5042	ret = create_hist_fields(hist_data, file);
5043	if (ret)
5044		goto free;
5045
5046	ret = create_sort_keys(hist_data);
5047	if (ret)
5048		goto free;
5049
5050	map_ops = &hist_trigger_elt_data_ops;
5051
5052	hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5053					    map_ops, hist_data);
5054	if (IS_ERR(hist_data->map)) {
5055		ret = PTR_ERR(hist_data->map);
5056		hist_data->map = NULL;
5057		goto free;
5058	}
5059
5060	ret = create_tracing_map_fields(hist_data);
5061	if (ret)
5062		goto free;
5063 out:
5064	return hist_data;
5065 free:
5066	hist_data->attrs = NULL;
5067
5068	destroy_hist_data(hist_data);
5069
5070	hist_data = ERR_PTR(ret);
5071
5072	goto out;
5073}
5074
5075static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5076				    struct tracing_map_elt *elt,
5077				    struct trace_buffer *buffer, void *rec,
5078				    struct ring_buffer_event *rbe,
5079				    u64 *var_ref_vals)
5080{
5081	struct hist_elt_data *elt_data;
5082	struct hist_field *hist_field;
5083	unsigned int i, var_idx;
5084	u64 hist_val;
5085
5086	elt_data = elt->private_data;
5087	elt_data->var_ref_vals = var_ref_vals;
5088
5089	for_each_hist_val_field(i, hist_data) {
5090		hist_field = hist_data->fields[i];
5091		hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5092		if (hist_field->flags & HIST_FIELD_FL_VAR) {
5093			var_idx = hist_field->var.idx;
5094
5095			if (hist_field->flags & HIST_FIELD_FL_STRING) {
5096				unsigned int str_start, var_str_idx, idx;
5097				char *str, *val_str;
5098				unsigned int size;
5099
5100				str_start = hist_data->n_field_var_str +
5101					hist_data->n_save_var_str;
5102				var_str_idx = hist_field->var_str_idx;
5103				idx = str_start + var_str_idx;
5104
5105				str = elt_data->field_var_str[idx];
5106				val_str = (char *)(uintptr_t)hist_val;
5107
5108				size = min(hist_field->size, STR_VAR_LEN_MAX);
5109				strscpy(str, val_str, size);
5110
5111				hist_val = (u64)(uintptr_t)str;
5112			}
5113			tracing_map_set_var(elt, var_idx, hist_val);
5114			continue;
5115		}
5116		tracing_map_update_sum(elt, i, hist_val);
5117	}
5118
5119	for_each_hist_key_field(i, hist_data) {
5120		hist_field = hist_data->fields[i];
5121		if (hist_field->flags & HIST_FIELD_FL_VAR) {
5122			hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5123			var_idx = hist_field->var.idx;
5124			tracing_map_set_var(elt, var_idx, hist_val);
5125		}
5126	}
5127
5128	update_field_vars(hist_data, elt, buffer, rbe, rec);
5129}
5130
5131static inline void add_to_key(char *compound_key, void *key,
5132			      struct hist_field *key_field, void *rec)
5133{
5134	size_t size = key_field->size;
5135
5136	if (key_field->flags & HIST_FIELD_FL_STRING) {
5137		struct ftrace_event_field *field;
5138
5139		field = key_field->field;
5140		if (field->filter_type == FILTER_DYN_STRING ||
5141		    field->filter_type == FILTER_RDYN_STRING)
5142			size = *(u32 *)(rec + field->offset) >> 16;
5143		else if (field->filter_type == FILTER_STATIC_STRING)
5144			size = field->size;
5145
5146		/* ensure NULL-termination */
5147		if (size > key_field->size - 1)
5148			size = key_field->size - 1;
5149
5150		strncpy(compound_key + key_field->offset, (char *)key, size);
5151	} else
5152		memcpy(compound_key + key_field->offset, key, size);
5153}
5154
5155static void
5156hist_trigger_actions(struct hist_trigger_data *hist_data,
5157		     struct tracing_map_elt *elt,
5158		     struct trace_buffer *buffer, void *rec,
5159		     struct ring_buffer_event *rbe, void *key,
5160		     u64 *var_ref_vals)
5161{
5162	struct action_data *data;
5163	unsigned int i;
5164
5165	for (i = 0; i < hist_data->n_actions; i++) {
5166		data = hist_data->actions[i];
5167		data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
5168	}
5169}
5170
5171static void event_hist_trigger(struct event_trigger_data *data,
5172			       struct trace_buffer *buffer, void *rec,
5173			       struct ring_buffer_event *rbe)
5174{
5175	struct hist_trigger_data *hist_data = data->private_data;
5176	bool use_compound_key = (hist_data->n_keys > 1);
5177	unsigned long entries[HIST_STACKTRACE_DEPTH];
5178	u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5179	char compound_key[HIST_KEY_SIZE_MAX];
5180	struct tracing_map_elt *elt = NULL;
5181	struct hist_field *key_field;
5182	u64 field_contents;
5183	void *key = NULL;
5184	unsigned int i;
5185
5186	if (unlikely(!rbe))
5187		return;
5188
5189	memset(compound_key, 0, hist_data->key_size);
5190
5191	for_each_hist_key_field(i, hist_data) {
5192		key_field = hist_data->fields[i];
5193
5194		if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5195			memset(entries, 0, HIST_STACKTRACE_SIZE);
5196			stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5197					 HIST_STACKTRACE_SKIP);
5198			key = entries;
5199		} else {
5200			field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
5201			if (key_field->flags & HIST_FIELD_FL_STRING) {
5202				key = (void *)(unsigned long)field_contents;
5203				use_compound_key = true;
5204			} else
5205				key = (void *)&field_contents;
5206		}
5207
5208		if (use_compound_key)
5209			add_to_key(compound_key, key, key_field, rec);
5210	}
5211
5212	if (use_compound_key)
5213		key = compound_key;
5214
5215	if (hist_data->n_var_refs &&
5216	    !resolve_var_refs(hist_data, key, var_ref_vals, false))
5217		return;
5218
5219	elt = tracing_map_insert(hist_data->map, key);
5220	if (!elt)
5221		return;
5222
5223	hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
5224
5225	if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5226		hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
5227}
5228
5229static void hist_trigger_stacktrace_print(struct seq_file *m,
5230					  unsigned long *stacktrace_entries,
5231					  unsigned int max_entries)
5232{
5233	unsigned int spaces = 8;
5234	unsigned int i;
5235
5236	for (i = 0; i < max_entries; i++) {
5237		if (!stacktrace_entries[i])
5238			return;
5239
5240		seq_printf(m, "%*c", 1 + spaces, ' ');
5241		seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]);
5242	}
5243}
5244
5245static void hist_trigger_print_key(struct seq_file *m,
5246				   struct hist_trigger_data *hist_data,
5247				   void *key,
5248				   struct tracing_map_elt *elt)
5249{
5250	struct hist_field *key_field;
5251	bool multiline = false;
5252	const char *field_name;
5253	unsigned int i;
5254	u64 uval;
5255
5256	seq_puts(m, "{ ");
5257
5258	for_each_hist_key_field(i, hist_data) {
5259		key_field = hist_data->fields[i];
5260
5261		if (i > hist_data->n_vals)
5262			seq_puts(m, ", ");
5263
5264		field_name = hist_field_name(key_field, 0);
5265
5266		if (key_field->flags & HIST_FIELD_FL_HEX) {
5267			uval = *(u64 *)(key + key_field->offset);
5268			seq_printf(m, "%s: %llx", field_name, uval);
5269		} else if (key_field->flags & HIST_FIELD_FL_SYM) {
5270			uval = *(u64 *)(key + key_field->offset);
5271			seq_printf(m, "%s: [%llx] %-45ps", field_name,
5272				   uval, (void *)(uintptr_t)uval);
5273		} else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5274			uval = *(u64 *)(key + key_field->offset);
5275			seq_printf(m, "%s: [%llx] %-55pS", field_name,
5276				   uval, (void *)(uintptr_t)uval);
5277		} else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5278			struct hist_elt_data *elt_data = elt->private_data;
5279			char *comm;
5280
5281			if (WARN_ON_ONCE(!elt_data))
5282				return;
5283
5284			comm = elt_data->comm;
5285
5286			uval = *(u64 *)(key + key_field->offset);
5287			seq_printf(m, "%s: %-16s[%10llu]", field_name,
5288				   comm, uval);
5289		} else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5290			const char *syscall_name;
5291
5292			uval = *(u64 *)(key + key_field->offset);
5293			syscall_name = get_syscall_name(uval);
5294			if (!syscall_name)
5295				syscall_name = "unknown_syscall";
5296
5297			seq_printf(m, "%s: %-30s[%3llu]", field_name,
5298				   syscall_name, uval);
5299		} else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5300			seq_puts(m, "stacktrace:\n");
5301			hist_trigger_stacktrace_print(m,
5302						      key + key_field->offset,
5303						      HIST_STACKTRACE_DEPTH);
5304			multiline = true;
5305		} else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5306			seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5307				   *(u64 *)(key + key_field->offset));
5308		} else if (key_field->flags & HIST_FIELD_FL_BUCKET) {
5309			unsigned long buckets = key_field->buckets;
5310			uval = *(u64 *)(key + key_field->offset);
5311			seq_printf(m, "%s: ~ %llu-%llu", field_name,
5312				   uval, uval + buckets -1);
5313		} else if (key_field->flags & HIST_FIELD_FL_STRING) {
5314			seq_printf(m, "%s: %-50s", field_name,
5315				   (char *)(key + key_field->offset));
5316		} else {
5317			uval = *(u64 *)(key + key_field->offset);
5318			seq_printf(m, "%s: %10llu", field_name, uval);
5319		}
5320	}
5321
5322	if (!multiline)
5323		seq_puts(m, " ");
5324
5325	seq_puts(m, "}");
5326}
5327
5328/* Get the 100 times of the percentage of @val in @total */
5329static inline unsigned int __get_percentage(u64 val, u64 total)
5330{
5331	if (!total)
5332		goto div0;
5333
5334	if (val < (U64_MAX / 10000))
5335		return (unsigned int)div64_ul(val * 10000, total);
5336
5337	total = div64_u64(total, 10000);
5338	if (!total)
5339		goto div0;
5340
5341	return (unsigned int)div64_ul(val, total);
5342div0:
5343	return val ? UINT_MAX : 0;
5344}
5345
5346#define BAR_CHAR '#'
5347
5348static inline const char *__fill_bar_str(char *buf, int size, u64 val, u64 max)
5349{
5350	unsigned int len = __get_percentage(val, max);
5351	int i;
5352
5353	if (len == UINT_MAX) {
5354		snprintf(buf, size, "[ERROR]");
5355		return buf;
5356	}
5357
5358	len = len * size / 10000;
5359	for (i = 0; i < len && i < size; i++)
5360		buf[i] = BAR_CHAR;
5361	while (i < size)
5362		buf[i++] = ' ';
5363	buf[size] = '\0';
5364
5365	return buf;
5366}
5367
5368struct hist_val_stat {
5369	u64 max;
5370	u64 total;
5371};
5372
5373static void hist_trigger_print_val(struct seq_file *m, unsigned int idx,
5374				   const char *field_name, unsigned long flags,
5375				   struct hist_val_stat *stats,
5376				   struct tracing_map_elt *elt)
5377{
5378	u64 val = tracing_map_read_sum(elt, idx);
5379	unsigned int pc;
5380	char bar[21];
5381
5382	if (flags & HIST_FIELD_FL_PERCENT) {
5383		pc = __get_percentage(val, stats[idx].total);
5384		if (pc == UINT_MAX)
5385			seq_printf(m, " %s (%%):[ERROR]", field_name);
5386		else
5387			seq_printf(m, " %s (%%): %3u.%02u", field_name,
5388					pc / 100, pc % 100);
5389	} else if (flags & HIST_FIELD_FL_GRAPH) {
5390		seq_printf(m, " %s: %20s", field_name,
5391			   __fill_bar_str(bar, 20, val, stats[idx].max));
5392	} else if (flags & HIST_FIELD_FL_HEX) {
5393		seq_printf(m, " %s: %10llx", field_name, val);
5394	} else {
5395		seq_printf(m, " %s: %10llu", field_name, val);
5396	}
5397}
5398
5399static void hist_trigger_entry_print(struct seq_file *m,
5400				     struct hist_trigger_data *hist_data,
5401				     struct hist_val_stat *stats,
5402				     void *key,
5403				     struct tracing_map_elt *elt)
5404{
5405	const char *field_name;
5406	unsigned int i = HITCOUNT_IDX;
5407	unsigned long flags;
5408
5409	hist_trigger_print_key(m, hist_data, key, elt);
5410
5411	/* At first, show the raw hitcount if !nohitcount */
5412	if (!hist_data->attrs->no_hitcount)
5413		hist_trigger_print_val(m, i, "hitcount", 0, stats, elt);
5414
5415	for (i = 1; i < hist_data->n_vals; i++) {
5416		field_name = hist_field_name(hist_data->fields[i], 0);
5417		flags = hist_data->fields[i]->flags;
5418		if (flags & HIST_FIELD_FL_VAR || flags & HIST_FIELD_FL_EXPR)
5419			continue;
5420
5421		seq_puts(m, " ");
5422		hist_trigger_print_val(m, i, field_name, flags, stats, elt);
5423	}
5424
5425	print_actions(m, hist_data, elt);
5426
5427	seq_puts(m, "\n");
5428}
5429
5430static int print_entries(struct seq_file *m,
5431			 struct hist_trigger_data *hist_data)
5432{
5433	struct tracing_map_sort_entry **sort_entries = NULL;
5434	struct tracing_map *map = hist_data->map;
5435	int i, j, n_entries;
5436	struct hist_val_stat *stats = NULL;
5437	u64 val;
5438
5439	n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5440					     hist_data->n_sort_keys,
5441					     &sort_entries);
5442	if (n_entries < 0)
5443		return n_entries;
5444
5445	/* Calculate the max and the total for each field if needed. */
5446	for (j = 0; j < hist_data->n_vals; j++) {
5447		if (!(hist_data->fields[j]->flags &
5448			(HIST_FIELD_FL_PERCENT | HIST_FIELD_FL_GRAPH)))
5449			continue;
5450		if (!stats) {
5451			stats = kcalloc(hist_data->n_vals, sizeof(*stats),
5452				       GFP_KERNEL);
5453			if (!stats) {
5454				n_entries = -ENOMEM;
5455				goto out;
5456			}
5457		}
5458		for (i = 0; i < n_entries; i++) {
5459			val = tracing_map_read_sum(sort_entries[i]->elt, j);
5460			stats[j].total += val;
5461			if (stats[j].max < val)
5462				stats[j].max = val;
5463		}
5464	}
5465
5466	for (i = 0; i < n_entries; i++)
5467		hist_trigger_entry_print(m, hist_data, stats,
5468					 sort_entries[i]->key,
5469					 sort_entries[i]->elt);
5470
5471	kfree(stats);
5472out:
5473	tracing_map_destroy_sort_entries(sort_entries, n_entries);
5474
5475	return n_entries;
5476}
5477
5478static void hist_trigger_show(struct seq_file *m,
5479			      struct event_trigger_data *data, int n)
5480{
5481	struct hist_trigger_data *hist_data;
5482	int n_entries;
5483
5484	if (n > 0)
5485		seq_puts(m, "\n\n");
5486
5487	seq_puts(m, "# event histogram\n#\n# trigger info: ");
5488	data->ops->print(m, data);
5489	seq_puts(m, "#\n\n");
5490
5491	hist_data = data->private_data;
5492	n_entries = print_entries(m, hist_data);
5493	if (n_entries < 0)
5494		n_entries = 0;
5495
5496	track_data_snapshot_print(m, hist_data);
5497
5498	seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5499		   (u64)atomic64_read(&hist_data->map->hits),
5500		   n_entries, (u64)atomic64_read(&hist_data->map->drops));
5501}
5502
5503static int hist_show(struct seq_file *m, void *v)
5504{
5505	struct event_trigger_data *data;
5506	struct trace_event_file *event_file;
5507	int n = 0, ret = 0;
5508
5509	mutex_lock(&event_mutex);
5510
5511	event_file = event_file_data(m->private);
5512	if (unlikely(!event_file)) {
5513		ret = -ENODEV;
5514		goto out_unlock;
5515	}
5516
5517	list_for_each_entry(data, &event_file->triggers, list) {
5518		if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5519			hist_trigger_show(m, data, n++);
5520	}
5521
5522 out_unlock:
5523	mutex_unlock(&event_mutex);
5524
5525	return ret;
5526}
5527
5528static int event_hist_open(struct inode *inode, struct file *file)
5529{
5530	int ret;
5531
5532	ret = security_locked_down(LOCKDOWN_TRACEFS);
5533	if (ret)
5534		return ret;
5535
5536	return single_open(file, hist_show, file);
5537}
5538
5539const struct file_operations event_hist_fops = {
5540	.open = event_hist_open,
5541	.read = seq_read,
5542	.llseek = seq_lseek,
5543	.release = single_release,
5544};
5545
5546#ifdef CONFIG_HIST_TRIGGERS_DEBUG
5547static void hist_field_debug_show_flags(struct seq_file *m,
5548					unsigned long flags)
5549{
5550	seq_puts(m, "      flags:\n");
5551
5552	if (flags & HIST_FIELD_FL_KEY)
5553		seq_puts(m, "        HIST_FIELD_FL_KEY\n");
5554	else if (flags & HIST_FIELD_FL_HITCOUNT)
5555		seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
5556	else if (flags & HIST_FIELD_FL_VAR)
5557		seq_puts(m, "        HIST_FIELD_FL_VAR\n");
5558	else if (flags & HIST_FIELD_FL_VAR_REF)
5559		seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
5560	else
5561		seq_puts(m, "        VAL: normal u64 value\n");
5562
5563	if (flags & HIST_FIELD_FL_ALIAS)
5564		seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
5565	else if (flags & HIST_FIELD_FL_CONST)
5566		seq_puts(m, "        HIST_FIELD_FL_CONST\n");
5567}
5568
5569static int hist_field_debug_show(struct seq_file *m,
5570				 struct hist_field *field, unsigned long flags)
5571{
5572	if ((field->flags & flags) != flags) {
5573		seq_printf(m, "ERROR: bad flags - %lx\n", flags);
5574		return -EINVAL;
5575	}
5576
5577	hist_field_debug_show_flags(m, field->flags);
5578	if (field->field)
5579		seq_printf(m, "      ftrace_event_field name: %s\n",
5580			   field->field->name);
5581
5582	if (field->flags & HIST_FIELD_FL_VAR) {
5583		seq_printf(m, "      var.name: %s\n", field->var.name);
5584		seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5585			   field->var.idx);
5586	}
5587
5588	if (field->flags & HIST_FIELD_FL_CONST)
5589		seq_printf(m, "      constant: %llu\n", field->constant);
5590
5591	if (field->flags & HIST_FIELD_FL_ALIAS)
5592		seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5593			   field->var_ref_idx);
5594
5595	if (field->flags & HIST_FIELD_FL_VAR_REF) {
5596		seq_printf(m, "      name: %s\n", field->name);
5597		seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5598			   field->var.idx);
5599		seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
5600		seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5601			   field->var_ref_idx);
5602		if (field->system)
5603			seq_printf(m, "      system: %s\n", field->system);
5604		if (field->event_name)
5605			seq_printf(m, "      event_name: %s\n", field->event_name);
5606	}
5607
5608	seq_printf(m, "      type: %s\n", field->type);
5609	seq_printf(m, "      size: %u\n", field->size);
5610	seq_printf(m, "      is_signed: %u\n", field->is_signed);
5611
5612	return 0;
5613}
5614
5615static int field_var_debug_show(struct seq_file *m,
5616				struct field_var *field_var, unsigned int i,
5617				bool save_vars)
5618{
5619	const char *vars_name = save_vars ? "save_vars" : "field_vars";
5620	struct hist_field *field;
5621	int ret = 0;
5622
5623	seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
5624
5625	field = field_var->var;
5626
5627	seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
5628
5629	hist_field_debug_show_flags(m, field->flags);
5630	seq_printf(m, "      var.name: %s\n", field->var.name);
5631	seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5632		   field->var.idx);
5633
5634	field = field_var->val;
5635
5636	seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
5637	if (field->field)
5638		seq_printf(m, "      ftrace_event_field name: %s\n",
5639			   field->field->name);
5640	else {
5641		ret = -EINVAL;
5642		goto out;
5643	}
5644
5645	seq_printf(m, "      type: %s\n", field->type);
5646	seq_printf(m, "      size: %u\n", field->size);
5647	seq_printf(m, "      is_signed: %u\n", field->is_signed);
5648out:
5649	return ret;
5650}
5651
5652static int hist_action_debug_show(struct seq_file *m,
5653				  struct action_data *data, int i)
5654{
5655	int ret = 0;
5656
5657	if (data->handler == HANDLER_ONMAX ||
5658	    data->handler == HANDLER_ONCHANGE) {
5659		seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
5660		ret = hist_field_debug_show(m, data->track_data.var_ref,
5661					    HIST_FIELD_FL_VAR_REF);
5662		if (ret)
5663			goto out;
5664
5665		seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
5666		ret = hist_field_debug_show(m, data->track_data.track_var,
5667					    HIST_FIELD_FL_VAR);
5668		if (ret)
5669			goto out;
5670	}
5671
5672	if (data->handler == HANDLER_ONMATCH) {
5673		seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
5674			   i, data->match_data.event_system);
5675		seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
5676			   i, data->match_data.event);
5677	}
5678out:
5679	return ret;
5680}
5681
5682static int hist_actions_debug_show(struct seq_file *m,
5683				   struct hist_trigger_data *hist_data)
5684{
5685	int i, ret = 0;
5686
5687	if (hist_data->n_actions)
5688		seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
5689
5690	for (i = 0; i < hist_data->n_actions; i++) {
5691		struct action_data *action = hist_data->actions[i];
5692
5693		ret = hist_action_debug_show(m, action, i);
5694		if (ret)
5695			goto out;
5696	}
5697
5698	if (hist_data->n_save_vars)
5699		seq_puts(m, "\n  save action variables (save() params):\n");
5700
5701	for (i = 0; i < hist_data->n_save_vars; i++) {
5702		ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
5703		if (ret)
5704			goto out;
5705	}
5706out:
5707	return ret;
5708}
5709
5710static void hist_trigger_debug_show(struct seq_file *m,
5711				    struct event_trigger_data *data, int n)
5712{
5713	struct hist_trigger_data *hist_data;
5714	int i, ret;
5715
5716	if (n > 0)
5717		seq_puts(m, "\n\n");
5718
5719	seq_puts(m, "# event histogram\n#\n# trigger info: ");
5720	data->ops->print(m, data);
5721	seq_puts(m, "#\n\n");
5722
5723	hist_data = data->private_data;
5724
5725	seq_printf(m, "hist_data: %p\n\n", hist_data);
5726	seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
5727	seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
5728	seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
5729
5730	seq_puts(m, "\n  val fields:\n\n");
5731
5732	seq_puts(m, "    hist_data->fields[0]:\n");
5733	ret = hist_field_debug_show(m, hist_data->fields[0],
5734				    HIST_FIELD_FL_HITCOUNT);
5735	if (ret)
5736		return;
5737
5738	for (i = 1; i < hist_data->n_vals; i++) {
5739		seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5740		ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5741		if (ret)
5742			return;
5743	}
5744
5745	seq_puts(m, "\n  key fields:\n");
5746
5747	for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5748		seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5749		ret = hist_field_debug_show(m, hist_data->fields[i],
5750					    HIST_FIELD_FL_KEY);
5751		if (ret)
5752			return;
5753	}
5754
5755	if (hist_data->n_var_refs)
5756		seq_puts(m, "\n  variable reference fields:\n");
5757
5758	for (i = 0; i < hist_data->n_var_refs; i++) {
5759		seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5760		ret = hist_field_debug_show(m, hist_data->var_refs[i],
5761					    HIST_FIELD_FL_VAR_REF);
5762		if (ret)
5763			return;
5764	}
5765
5766	if (hist_data->n_field_vars)
5767		seq_puts(m, "\n  field variables:\n");
5768
5769	for (i = 0; i < hist_data->n_field_vars; i++) {
5770		ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5771		if (ret)
5772			return;
5773	}
5774
5775	ret = hist_actions_debug_show(m, hist_data);
5776	if (ret)
5777		return;
5778}
5779
5780static int hist_debug_show(struct seq_file *m, void *v)
5781{
5782	struct event_trigger_data *data;
5783	struct trace_event_file *event_file;
5784	int n = 0, ret = 0;
5785
5786	mutex_lock(&event_mutex);
5787
5788	event_file = event_file_data(m->private);
5789	if (unlikely(!event_file)) {
5790		ret = -ENODEV;
5791		goto out_unlock;
5792	}
5793
5794	list_for_each_entry(data, &event_file->triggers, list) {
5795		if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5796			hist_trigger_debug_show(m, data, n++);
5797	}
5798
5799 out_unlock:
5800	mutex_unlock(&event_mutex);
5801
5802	return ret;
5803}
5804
5805static int event_hist_debug_open(struct inode *inode, struct file *file)
5806{
5807	int ret;
5808
5809	ret = security_locked_down(LOCKDOWN_TRACEFS);
5810	if (ret)
5811		return ret;
5812
5813	return single_open(file, hist_debug_show, file);
5814}
5815
5816const struct file_operations event_hist_debug_fops = {
5817	.open = event_hist_debug_open,
5818	.read = seq_read,
5819	.llseek = seq_lseek,
5820	.release = single_release,
5821};
5822#endif
5823
5824static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5825{
5826	const char *field_name = hist_field_name(hist_field, 0);
5827
5828	if (hist_field->var.name)
5829		seq_printf(m, "%s=", hist_field->var.name);
5830
5831	if (hist_field->flags & HIST_FIELD_FL_CPU)
5832		seq_puts(m, "common_cpu");
5833	else if (hist_field->flags & HIST_FIELD_FL_CONST)
5834		seq_printf(m, "%llu", hist_field->constant);
5835	else if (field_name) {
5836		if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5837		    hist_field->flags & HIST_FIELD_FL_ALIAS)
5838			seq_putc(m, '$');
5839		seq_printf(m, "%s", field_name);
5840	} else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5841		seq_puts(m, "common_timestamp");
5842
5843	if (hist_field->flags) {
5844		if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5845		    !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5846			const char *flags = get_hist_field_flags(hist_field);
5847
5848			if (flags)
5849				seq_printf(m, ".%s", flags);
5850		}
5851	}
5852	if (hist_field->buckets)
5853		seq_printf(m, "=%ld", hist_field->buckets);
5854}
5855
5856static int event_hist_trigger_print(struct seq_file *m,
5857				    struct event_trigger_data *data)
5858{
5859	struct hist_trigger_data *hist_data = data->private_data;
5860	struct hist_field *field;
5861	bool have_var = false;
5862	bool show_val = false;
5863	unsigned int i;
5864
5865	seq_puts(m, HIST_PREFIX);
5866
5867	if (data->name)
5868		seq_printf(m, "%s:", data->name);
5869
5870	seq_puts(m, "keys=");
5871
5872	for_each_hist_key_field(i, hist_data) {
5873		field = hist_data->fields[i];
5874
5875		if (i > hist_data->n_vals)
5876			seq_puts(m, ",");
5877
5878		if (field->flags & HIST_FIELD_FL_STACKTRACE)
5879			seq_puts(m, "stacktrace");
5880		else
5881			hist_field_print(m, field);
5882	}
5883
5884	seq_puts(m, ":vals=");
5885
5886	for_each_hist_val_field(i, hist_data) {
5887		field = hist_data->fields[i];
5888		if (field->flags & HIST_FIELD_FL_VAR) {
5889			have_var = true;
5890			continue;
5891		}
5892
5893		if (i == HITCOUNT_IDX) {
5894			if (hist_data->attrs->no_hitcount)
5895				continue;
5896			seq_puts(m, "hitcount");
5897		} else {
5898			if (show_val)
5899				seq_puts(m, ",");
5900			hist_field_print(m, field);
5901		}
5902		show_val = true;
5903	}
5904
5905	if (have_var) {
5906		unsigned int n = 0;
5907
5908		seq_puts(m, ":");
5909
5910		for_each_hist_val_field(i, hist_data) {
5911			field = hist_data->fields[i];
5912
5913			if (field->flags & HIST_FIELD_FL_VAR) {
5914				if (n++)
5915					seq_puts(m, ",");
5916				hist_field_print(m, field);
5917			}
5918		}
5919	}
5920
5921	seq_puts(m, ":sort=");
5922
5923	for (i = 0; i < hist_data->n_sort_keys; i++) {
5924		struct tracing_map_sort_key *sort_key;
5925		unsigned int idx, first_key_idx;
5926
5927		/* skip VAR vals */
5928		first_key_idx = hist_data->n_vals - hist_data->n_vars;
5929
5930		sort_key = &hist_data->sort_keys[i];
5931		idx = sort_key->field_idx;
5932
5933		if (WARN_ON(idx >= HIST_FIELDS_MAX))
5934			return -EINVAL;
5935
5936		if (i > 0)
5937			seq_puts(m, ",");
5938
5939		if (idx == HITCOUNT_IDX)
5940			seq_puts(m, "hitcount");
5941		else {
5942			if (idx >= first_key_idx)
5943				idx += hist_data->n_vars;
5944			hist_field_print(m, hist_data->fields[idx]);
5945		}
5946
5947		if (sort_key->descending)
5948			seq_puts(m, ".descending");
5949	}
5950	seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5951	if (hist_data->enable_timestamps)
5952		seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5953	if (hist_data->attrs->no_hitcount)
5954		seq_puts(m, ":nohitcount");
5955
5956	print_actions_spec(m, hist_data);
5957
5958	if (data->filter_str)
5959		seq_printf(m, " if %s", data->filter_str);
5960
5961	if (data->paused)
5962		seq_puts(m, " [paused]");
5963	else
5964		seq_puts(m, " [active]");
5965
5966	seq_putc(m, '\n');
5967
5968	return 0;
5969}
5970
5971static int event_hist_trigger_init(struct event_trigger_data *data)
5972{
5973	struct hist_trigger_data *hist_data = data->private_data;
5974
5975	if (!data->ref && hist_data->attrs->name)
5976		save_named_trigger(hist_data->attrs->name, data);
5977
5978	data->ref++;
5979
5980	return 0;
5981}
5982
5983static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5984{
5985	struct trace_event_file *file;
5986	unsigned int i;
5987	char *cmd;
5988	int ret;
5989
5990	for (i = 0; i < hist_data->n_field_var_hists; i++) {
5991		file = hist_data->field_var_hists[i]->hist_data->event_file;
5992		cmd = hist_data->field_var_hists[i]->cmd;
5993		ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
5994					       "!hist", "hist", cmd);
5995		WARN_ON_ONCE(ret < 0);
5996	}
5997}
5998
5999static void event_hist_trigger_free(struct event_trigger_data *data)
6000{
6001	struct hist_trigger_data *hist_data = data->private_data;
6002
6003	if (WARN_ON_ONCE(data->ref <= 0))
6004		return;
6005
6006	data->ref--;
6007	if (!data->ref) {
6008		if (data->name)
6009			del_named_trigger(data);
6010
6011		trigger_data_free(data);
6012
6013		remove_hist_vars(hist_data);
6014
6015		unregister_field_var_hists(hist_data);
6016
6017		destroy_hist_data(hist_data);
6018	}
6019}
6020
6021static struct event_trigger_ops event_hist_trigger_ops = {
6022	.trigger		= event_hist_trigger,
6023	.print			= event_hist_trigger_print,
6024	.init			= event_hist_trigger_init,
6025	.free			= event_hist_trigger_free,
6026};
6027
6028static int event_hist_trigger_named_init(struct event_trigger_data *data)
6029{
6030	data->ref++;
6031
6032	save_named_trigger(data->named_data->name, data);
6033
6034	event_hist_trigger_init(data->named_data);
6035
6036	return 0;
6037}
6038
6039static void event_hist_trigger_named_free(struct event_trigger_data *data)
6040{
6041	if (WARN_ON_ONCE(data->ref <= 0))
6042		return;
6043
6044	event_hist_trigger_free(data->named_data);
6045
6046	data->ref--;
6047	if (!data->ref) {
6048		del_named_trigger(data);
6049		trigger_data_free(data);
6050	}
6051}
6052
6053static struct event_trigger_ops event_hist_trigger_named_ops = {
6054	.trigger		= event_hist_trigger,
6055	.print			= event_hist_trigger_print,
6056	.init			= event_hist_trigger_named_init,
6057	.free			= event_hist_trigger_named_free,
6058};
6059
6060static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
6061							    char *param)
6062{
6063	return &event_hist_trigger_ops;
6064}
6065
6066static void hist_clear(struct event_trigger_data *data)
6067{
6068	struct hist_trigger_data *hist_data = data->private_data;
6069
6070	if (data->name)
6071		pause_named_trigger(data);
6072
6073	tracepoint_synchronize_unregister();
6074
6075	tracing_map_clear(hist_data->map);
6076
6077	if (data->name)
6078		unpause_named_trigger(data);
6079}
6080
6081static bool compatible_field(struct ftrace_event_field *field,
6082			     struct ftrace_event_field *test_field)
6083{
6084	if (field == test_field)
6085		return true;
6086	if (field == NULL || test_field == NULL)
6087		return false;
6088	if (strcmp(field->name, test_field->name) != 0)
6089		return false;
6090	if (strcmp(field->type, test_field->type) != 0)
6091		return false;
6092	if (field->size != test_field->size)
6093		return false;
6094	if (field->is_signed != test_field->is_signed)
6095		return false;
6096
6097	return true;
6098}
6099
6100static bool hist_trigger_match(struct event_trigger_data *data,
6101			       struct event_trigger_data *data_test,
6102			       struct event_trigger_data *named_data,
6103			       bool ignore_filter)
6104{
6105	struct tracing_map_sort_key *sort_key, *sort_key_test;
6106	struct hist_trigger_data *hist_data, *hist_data_test;
6107	struct hist_field *key_field, *key_field_test;
6108	unsigned int i;
6109
6110	if (named_data && (named_data != data_test) &&
6111	    (named_data != data_test->named_data))
6112		return false;
6113
6114	if (!named_data && is_named_trigger(data_test))
6115		return false;
6116
6117	hist_data = data->private_data;
6118	hist_data_test = data_test->private_data;
6119
6120	if (hist_data->n_vals != hist_data_test->n_vals ||
6121	    hist_data->n_fields != hist_data_test->n_fields ||
6122	    hist_data->n_sort_keys != hist_data_test->n_sort_keys)
6123		return false;
6124
6125	if (!ignore_filter) {
6126		if ((data->filter_str && !data_test->filter_str) ||
6127		   (!data->filter_str && data_test->filter_str))
6128			return false;
6129	}
6130
6131	for_each_hist_field(i, hist_data) {
6132		key_field = hist_data->fields[i];
6133		key_field_test = hist_data_test->fields[i];
6134
6135		if (key_field->flags != key_field_test->flags)
6136			return false;
6137		if (!compatible_field(key_field->field, key_field_test->field))
6138			return false;
6139		if (key_field->offset != key_field_test->offset)
6140			return false;
6141		if (key_field->size != key_field_test->size)
6142			return false;
6143		if (key_field->is_signed != key_field_test->is_signed)
6144			return false;
6145		if (!!key_field->var.name != !!key_field_test->var.name)
6146			return false;
6147		if (key_field->var.name &&
6148		    strcmp(key_field->var.name, key_field_test->var.name) != 0)
6149			return false;
6150	}
6151
6152	for (i = 0; i < hist_data->n_sort_keys; i++) {
6153		sort_key = &hist_data->sort_keys[i];
6154		sort_key_test = &hist_data_test->sort_keys[i];
6155
6156		if (sort_key->field_idx != sort_key_test->field_idx ||
6157		    sort_key->descending != sort_key_test->descending)
6158			return false;
6159	}
6160
6161	if (!ignore_filter && data->filter_str &&
6162	    (strcmp(data->filter_str, data_test->filter_str) != 0))
6163		return false;
6164
6165	if (!actions_match(hist_data, hist_data_test))
6166		return false;
6167
6168	return true;
6169}
6170
6171static bool existing_hist_update_only(char *glob,
6172				      struct event_trigger_data *data,
6173				      struct trace_event_file *file)
6174{
6175	struct hist_trigger_data *hist_data = data->private_data;
6176	struct event_trigger_data *test, *named_data = NULL;
6177	bool updated = false;
6178
6179	if (!hist_data->attrs->pause && !hist_data->attrs->cont &&
6180	    !hist_data->attrs->clear)
6181		goto out;
6182
6183	if (hist_data->attrs->name) {
6184		named_data = find_named_trigger(hist_data->attrs->name);
6185		if (named_data) {
6186			if (!hist_trigger_match(data, named_data, named_data,
6187						true))
6188				goto out;
6189		}
6190	}
6191
6192	if (hist_data->attrs->name && !named_data)
6193		goto out;
6194
6195	list_for_each_entry(test, &file->triggers, list) {
6196		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6197			if (!hist_trigger_match(data, test, named_data, false))
6198				continue;
6199			if (hist_data->attrs->pause)
6200				test->paused = true;
6201			else if (hist_data->attrs->cont)
6202				test->paused = false;
6203			else if (hist_data->attrs->clear)
6204				hist_clear(test);
6205			updated = true;
6206			goto out;
6207		}
6208	}
6209 out:
6210	return updated;
6211}
6212
6213static int hist_register_trigger(char *glob,
6214				 struct event_trigger_data *data,
6215				 struct trace_event_file *file)
6216{
6217	struct hist_trigger_data *hist_data = data->private_data;
6218	struct event_trigger_data *test, *named_data = NULL;
6219	struct trace_array *tr = file->tr;
6220	int ret = 0;
6221
6222	if (hist_data->attrs->name) {
6223		named_data = find_named_trigger(hist_data->attrs->name);
6224		if (named_data) {
6225			if (!hist_trigger_match(data, named_data, named_data,
6226						true)) {
6227				hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
6228				ret = -EINVAL;
6229				goto out;
6230			}
6231		}
6232	}
6233
6234	if (hist_data->attrs->name && !named_data)
6235		goto new;
6236
6237	lockdep_assert_held(&event_mutex);
6238
6239	list_for_each_entry(test, &file->triggers, list) {
6240		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6241			if (hist_trigger_match(data, test, named_data, false)) {
6242				hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
6243				ret = -EEXIST;
6244				goto out;
6245			}
6246		}
6247	}
6248 new:
6249	if (hist_data->attrs->cont || hist_data->attrs->clear) {
6250		hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
6251		ret = -ENOENT;
6252		goto out;
6253	}
6254
6255	if (hist_data->attrs->pause)
6256		data->paused = true;
6257
6258	if (named_data) {
6259		data->private_data = named_data->private_data;
6260		set_named_trigger_data(data, named_data);
6261		data->ops = &event_hist_trigger_named_ops;
6262	}
6263
6264	if (data->ops->init) {
6265		ret = data->ops->init(data);
6266		if (ret < 0)
6267			goto out;
6268	}
6269
6270	if (hist_data->enable_timestamps) {
6271		char *clock = hist_data->attrs->clock;
6272
6273		ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
6274		if (ret) {
6275			hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
6276			goto out;
6277		}
6278
6279		tracing_set_filter_buffering(file->tr, true);
6280	}
6281
6282	if (named_data)
6283		destroy_hist_data(hist_data);
6284 out:
6285	return ret;
6286}
6287
6288static int hist_trigger_enable(struct event_trigger_data *data,
6289			       struct trace_event_file *file)
6290{
6291	int ret = 0;
6292
6293	list_add_tail_rcu(&data->list, &file->triggers);
6294
6295	update_cond_flag(file);
6296
6297	if (trace_event_trigger_enable_disable(file, 1) < 0) {
6298		list_del_rcu(&data->list);
6299		update_cond_flag(file);
6300		ret--;
6301	}
6302
6303	return ret;
6304}
6305
6306static bool have_hist_trigger_match(struct event_trigger_data *data,
6307				    struct trace_event_file *file)
6308{
6309	struct hist_trigger_data *hist_data = data->private_data;
6310	struct event_trigger_data *test, *named_data = NULL;
6311	bool match = false;
6312
6313	lockdep_assert_held(&event_mutex);
6314
6315	if (hist_data->attrs->name)
6316		named_data = find_named_trigger(hist_data->attrs->name);
6317
6318	list_for_each_entry(test, &file->triggers, list) {
6319		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6320			if (hist_trigger_match(data, test, named_data, false)) {
6321				match = true;
6322				break;
6323			}
6324		}
6325	}
6326
6327	return match;
6328}
6329
6330static bool hist_trigger_check_refs(struct event_trigger_data *data,
6331				    struct trace_event_file *file)
6332{
6333	struct hist_trigger_data *hist_data = data->private_data;
6334	struct event_trigger_data *test, *named_data = NULL;
6335
6336	lockdep_assert_held(&event_mutex);
6337
6338	if (hist_data->attrs->name)
6339		named_data = find_named_trigger(hist_data->attrs->name);
6340
6341	list_for_each_entry(test, &file->triggers, list) {
6342		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6343			if (!hist_trigger_match(data, test, named_data, false))
6344				continue;
6345			hist_data = test->private_data;
6346			if (check_var_refs(hist_data))
6347				return true;
6348			break;
6349		}
6350	}
6351
6352	return false;
6353}
6354
6355static void hist_unregister_trigger(char *glob,
6356				    struct event_trigger_data *data,
6357				    struct trace_event_file *file)
6358{
6359	struct event_trigger_data *test = NULL, *iter, *named_data = NULL;
6360	struct hist_trigger_data *hist_data = data->private_data;
6361
6362	lockdep_assert_held(&event_mutex);
6363
6364	if (hist_data->attrs->name)
6365		named_data = find_named_trigger(hist_data->attrs->name);
6366
6367	list_for_each_entry(iter, &file->triggers, list) {
6368		if (iter->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6369			if (!hist_trigger_match(data, iter, named_data, false))
6370				continue;
6371			test = iter;
6372			list_del_rcu(&test->list);
6373			trace_event_trigger_enable_disable(file, 0);
6374			update_cond_flag(file);
6375			break;
6376		}
6377	}
6378
6379	if (test && test->ops->free)
6380		test->ops->free(test);
6381
6382	if (hist_data->enable_timestamps) {
6383		if (!hist_data->remove || test)
6384			tracing_set_filter_buffering(file->tr, false);
6385	}
6386}
6387
6388static bool hist_file_check_refs(struct trace_event_file *file)
6389{
6390	struct hist_trigger_data *hist_data;
6391	struct event_trigger_data *test;
6392
6393	lockdep_assert_held(&event_mutex);
6394
6395	list_for_each_entry(test, &file->triggers, list) {
6396		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6397			hist_data = test->private_data;
6398			if (check_var_refs(hist_data))
6399				return true;
6400		}
6401	}
6402
6403	return false;
6404}
6405
6406static void hist_unreg_all(struct trace_event_file *file)
6407{
6408	struct event_trigger_data *test, *n;
6409	struct hist_trigger_data *hist_data;
6410	struct synth_event *se;
6411	const char *se_name;
6412
6413	lockdep_assert_held(&event_mutex);
6414
6415	if (hist_file_check_refs(file))
6416		return;
6417
6418	list_for_each_entry_safe(test, n, &file->triggers, list) {
6419		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6420			hist_data = test->private_data;
6421			list_del_rcu(&test->list);
6422			trace_event_trigger_enable_disable(file, 0);
6423
6424			se_name = trace_event_name(file->event_call);
6425			se = find_synth_event(se_name);
6426			if (se)
6427				se->ref--;
6428
6429			update_cond_flag(file);
6430			if (hist_data->enable_timestamps)
6431				tracing_set_filter_buffering(file->tr, false);
6432			if (test->ops->free)
6433				test->ops->free(test);
6434		}
6435	}
6436}
6437
6438static int event_hist_trigger_parse(struct event_command *cmd_ops,
6439				    struct trace_event_file *file,
6440				    char *glob, char *cmd,
6441				    char *param_and_filter)
6442{
6443	unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6444	struct event_trigger_data *trigger_data;
6445	struct hist_trigger_attrs *attrs;
6446	struct hist_trigger_data *hist_data;
6447	char *param, *filter, *p, *start;
6448	struct synth_event *se;
6449	const char *se_name;
6450	bool remove;
6451	int ret = 0;
6452
6453	lockdep_assert_held(&event_mutex);
6454
6455	if (WARN_ON(!glob))
6456		return -EINVAL;
6457
6458	if (glob[0]) {
6459		hist_err_clear();
6460		last_cmd_set(file, param_and_filter);
6461	}
6462
6463	remove = event_trigger_check_remove(glob);
6464
6465	if (event_trigger_empty_param(param_and_filter))
6466		return -EINVAL;
6467
6468	/*
6469	 * separate the trigger from the filter (k:v [if filter])
6470	 * allowing for whitespace in the trigger
6471	 */
6472	p = param = param_and_filter;
6473	do {
6474		p = strstr(p, "if");
6475		if (!p)
6476			break;
6477		if (p == param_and_filter)
6478			return -EINVAL;
6479		if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6480			p++;
6481			continue;
6482		}
6483		if (p >= param_and_filter + strlen(param_and_filter) - (sizeof("if") - 1) - 1)
6484			return -EINVAL;
6485		if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6486			p++;
6487			continue;
6488		}
6489		break;
6490	} while (1);
6491
6492	if (!p)
6493		filter = NULL;
6494	else {
6495		*(p - 1) = '\0';
6496		filter = strstrip(p);
6497		param = strstrip(param);
6498	}
6499
6500	/*
6501	 * To simplify arithmetic expression parsing, replace occurrences of
6502	 * '.sym-offset' modifier with '.symXoffset'
6503	 */
6504	start = strstr(param, ".sym-offset");
6505	while (start) {
6506		*(start + 4) = 'X';
6507		start = strstr(start + 11, ".sym-offset");
6508	}
6509
6510	attrs = parse_hist_trigger_attrs(file->tr, param);
6511	if (IS_ERR(attrs))
6512		return PTR_ERR(attrs);
6513
6514	if (attrs->map_bits)
6515		hist_trigger_bits = attrs->map_bits;
6516
6517	hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6518	if (IS_ERR(hist_data)) {
6519		destroy_hist_trigger_attrs(attrs);
6520		return PTR_ERR(hist_data);
6521	}
6522
6523	trigger_data = event_trigger_alloc(cmd_ops, cmd, param, hist_data);
6524	if (!trigger_data) {
6525		ret = -ENOMEM;
6526		goto out_free;
6527	}
6528
6529	ret = event_trigger_set_filter(cmd_ops, file, filter, trigger_data);
6530	if (ret < 0)
6531		goto out_free;
6532
6533	if (remove) {
6534		if (!have_hist_trigger_match(trigger_data, file))
6535			goto out_free;
6536
6537		if (hist_trigger_check_refs(trigger_data, file)) {
6538			ret = -EBUSY;
6539			goto out_free;
6540		}
6541
6542		event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6543		se_name = trace_event_name(file->event_call);
6544		se = find_synth_event(se_name);
6545		if (se)
6546			se->ref--;
6547		ret = 0;
6548		goto out_free;
6549	}
6550
6551	if (existing_hist_update_only(glob, trigger_data, file))
6552		goto out_free;
6553
6554	ret = event_trigger_register(cmd_ops, file, glob, trigger_data);
6555	if (ret < 0)
6556		goto out_free;
6557
6558	if (get_named_trigger_data(trigger_data))
6559		goto enable;
6560
6561	if (has_hist_vars(hist_data))
6562		save_hist_vars(hist_data);
6563
6564	ret = create_actions(hist_data);
6565	if (ret)
6566		goto out_unreg;
6567
6568	ret = tracing_map_init(hist_data->map);
6569	if (ret)
6570		goto out_unreg;
6571enable:
6572	ret = hist_trigger_enable(trigger_data, file);
6573	if (ret)
6574		goto out_unreg;
6575
6576	se_name = trace_event_name(file->event_call);
6577	se = find_synth_event(se_name);
6578	if (se)
6579		se->ref++;
6580 out:
6581	if (ret == 0 && glob[0])
6582		hist_err_clear();
6583
6584	return ret;
6585 out_unreg:
6586	event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6587 out_free:
6588	event_trigger_reset_filter(cmd_ops, trigger_data);
6589
6590	remove_hist_vars(hist_data);
6591
6592	kfree(trigger_data);
6593
6594	destroy_hist_data(hist_data);
6595	goto out;
6596}
6597
6598static struct event_command trigger_hist_cmd = {
6599	.name			= "hist",
6600	.trigger_type		= ETT_EVENT_HIST,
6601	.flags			= EVENT_CMD_FL_NEEDS_REC,
6602	.parse			= event_hist_trigger_parse,
6603	.reg			= hist_register_trigger,
6604	.unreg			= hist_unregister_trigger,
6605	.unreg_all		= hist_unreg_all,
6606	.get_trigger_ops	= event_hist_get_trigger_ops,
6607	.set_filter		= set_trigger_filter,
6608};
6609
6610__init int register_trigger_hist_cmd(void)
6611{
6612	int ret;
6613
6614	ret = register_event_command(&trigger_hist_cmd);
6615	WARN_ON(ret < 0);
6616
6617	return ret;
6618}
6619
6620static void
6621hist_enable_trigger(struct event_trigger_data *data,
6622		    struct trace_buffer *buffer,  void *rec,
6623		    struct ring_buffer_event *event)
6624{
6625	struct enable_trigger_data *enable_data = data->private_data;
6626	struct event_trigger_data *test;
6627
6628	list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6629				lockdep_is_held(&event_mutex)) {
6630		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6631			if (enable_data->enable)
6632				test->paused = false;
6633			else
6634				test->paused = true;
6635		}
6636	}
6637}
6638
6639static void
6640hist_enable_count_trigger(struct event_trigger_data *data,
6641			  struct trace_buffer *buffer,  void *rec,
6642			  struct ring_buffer_event *event)
6643{
6644	if (!data->count)
6645		return;
6646
6647	if (data->count != -1)
6648		(data->count)--;
6649
6650	hist_enable_trigger(data, buffer, rec, event);
6651}
6652
6653static struct event_trigger_ops hist_enable_trigger_ops = {
6654	.trigger		= hist_enable_trigger,
6655	.print			= event_enable_trigger_print,
6656	.init			= event_trigger_init,
6657	.free			= event_enable_trigger_free,
6658};
6659
6660static struct event_trigger_ops hist_enable_count_trigger_ops = {
6661	.trigger		= hist_enable_count_trigger,
6662	.print			= event_enable_trigger_print,
6663	.init			= event_trigger_init,
6664	.free			= event_enable_trigger_free,
6665};
6666
6667static struct event_trigger_ops hist_disable_trigger_ops = {
6668	.trigger		= hist_enable_trigger,
6669	.print			= event_enable_trigger_print,
6670	.init			= event_trigger_init,
6671	.free			= event_enable_trigger_free,
6672};
6673
6674static struct event_trigger_ops hist_disable_count_trigger_ops = {
6675	.trigger		= hist_enable_count_trigger,
6676	.print			= event_enable_trigger_print,
6677	.init			= event_trigger_init,
6678	.free			= event_enable_trigger_free,
6679};
6680
6681static struct event_trigger_ops *
6682hist_enable_get_trigger_ops(char *cmd, char *param)
6683{
6684	struct event_trigger_ops *ops;
6685	bool enable;
6686
6687	enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6688
6689	if (enable)
6690		ops = param ? &hist_enable_count_trigger_ops :
6691			&hist_enable_trigger_ops;
6692	else
6693		ops = param ? &hist_disable_count_trigger_ops :
6694			&hist_disable_trigger_ops;
6695
6696	return ops;
6697}
6698
6699static void hist_enable_unreg_all(struct trace_event_file *file)
6700{
6701	struct event_trigger_data *test, *n;
6702
6703	list_for_each_entry_safe(test, n, &file->triggers, list) {
6704		if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6705			list_del_rcu(&test->list);
6706			update_cond_flag(file);
6707			trace_event_trigger_enable_disable(file, 0);
6708			if (test->ops->free)
6709				test->ops->free(test);
6710		}
6711	}
6712}
6713
6714static struct event_command trigger_hist_enable_cmd = {
6715	.name			= ENABLE_HIST_STR,
6716	.trigger_type		= ETT_HIST_ENABLE,
6717	.parse			= event_enable_trigger_parse,
6718	.reg			= event_enable_register_trigger,
6719	.unreg			= event_enable_unregister_trigger,
6720	.unreg_all		= hist_enable_unreg_all,
6721	.get_trigger_ops	= hist_enable_get_trigger_ops,
6722	.set_filter		= set_trigger_filter,
6723};
6724
6725static struct event_command trigger_hist_disable_cmd = {
6726	.name			= DISABLE_HIST_STR,
6727	.trigger_type		= ETT_HIST_ENABLE,
6728	.parse			= event_enable_trigger_parse,
6729	.reg			= event_enable_register_trigger,
6730	.unreg			= event_enable_unregister_trigger,
6731	.unreg_all		= hist_enable_unreg_all,
6732	.get_trigger_ops	= hist_enable_get_trigger_ops,
6733	.set_filter		= set_trigger_filter,
6734};
6735
6736static __init void unregister_trigger_hist_enable_disable_cmds(void)
6737{
6738	unregister_event_command(&trigger_hist_enable_cmd);
6739	unregister_event_command(&trigger_hist_disable_cmd);
6740}
6741
6742__init int register_trigger_hist_enable_disable_cmds(void)
6743{
6744	int ret;
6745
6746	ret = register_event_command(&trigger_hist_enable_cmd);
6747	if (WARN_ON(ret < 0))
6748		return ret;
6749	ret = register_event_command(&trigger_hist_disable_cmd);
6750	if (WARN_ON(ret < 0))
6751		unregister_trigger_hist_enable_disable_cmds();
6752
6753	return ret;
6754}