1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * trace_events_hist - trace event hist triggers
   4 *
   5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6 */
   7
   8#include <linux/module.h>
   9#include <linux/kallsyms.h>
  10#include <linux/security.h>
  11#include <linux/mutex.h>
  12#include <linux/slab.h>
  13#include <linux/stacktrace.h>
  14#include <linux/rculist.h>
  15#include <linux/tracefs.h>
  16
  17/* for gfp flag names */
  18#include <linux/trace_events.h>
  19#include <trace/events/mmflags.h>
  20
  21#include "tracing_map.h"
  22#include "trace_synth.h"
  23
  24#define ERRORS								\
  25	C(NONE,			"No error"),				\
  26	C(DUPLICATE_VAR,	"Variable already defined"),		\
  27	C(VAR_NOT_UNIQUE,	"Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28	C(TOO_MANY_VARS,	"Too many variables defined"),		\
  29	C(MALFORMED_ASSIGNMENT,	"Malformed assignment"),		\
  30	C(NAMED_MISMATCH,	"Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31	C(TRIGGER_EEXIST,	"Hist trigger already exists"),		\
  32	C(TRIGGER_ENOENT_CLEAR,	"Can't clear or continue a nonexistent hist trigger"), \
  33	C(SET_CLOCK_FAIL,	"Couldn't set trace_clock"),		\
  34	C(BAD_FIELD_MODIFIER,	"Invalid field modifier"),		\
  35	C(TOO_MANY_SUBEXPR,	"Too many subexpressions (3 max)"),	\
  36	C(TIMESTAMP_MISMATCH,	"Timestamp units in expression don't match"), \
  37	C(TOO_MANY_FIELD_VARS,	"Too many field variables defined"),	\
  38	C(EVENT_FILE_NOT_FOUND,	"Event file not found"),		\
  39	C(HIST_NOT_FOUND,	"Matching event histogram not found"),	\
  40	C(HIST_CREATE_FAIL,	"Couldn't create histogram for field"),	\
  41	C(SYNTH_VAR_NOT_FOUND,	"Couldn't find synthetic variable"),	\
  42	C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),	\
  43	C(SYNTH_TYPE_MISMATCH,	"Param type doesn't match synthetic event field type"), \
  44	C(SYNTH_COUNT_MISMATCH,	"Param count doesn't match synthetic event field count"), \
  45	C(FIELD_VAR_PARSE_FAIL,	"Couldn't parse field variable"),	\
  46	C(VAR_CREATE_FIND_FAIL,	"Couldn't create or find variable"),	\
  47	C(ONX_NOT_VAR,		"For onmax(x) or onchange(x), x must be a variable"), \
  48	C(ONX_VAR_NOT_FOUND,	"Couldn't find onmax or onchange variable"), \
  49	C(ONX_VAR_CREATE_FAIL,	"Couldn't create onmax or onchange variable"), \
  50	C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),	\
  51	C(TOO_MANY_PARAMS,	"Too many action params"),		\
  52	C(PARAM_NOT_FOUND,	"Couldn't find param"),			\
  53	C(INVALID_PARAM,	"Invalid action param"),		\
  54	C(ACTION_NOT_FOUND,	"No action found"),			\
  55	C(NO_SAVE_PARAMS,	"No params found for save()"),		\
  56	C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57	C(ACTION_MISMATCH,	"Handler doesn't support action"),	\
  58	C(NO_CLOSING_PAREN,	"No closing paren found"),		\
  59	C(SUBSYS_NOT_FOUND,	"Missing subsystem"),			\
  60	C(INVALID_SUBSYS_EVENT,	"Invalid subsystem or event name"),	\
  61	C(INVALID_REF_KEY,	"Using variable references in keys not supported"), \
  62	C(VAR_NOT_FOUND,	"Couldn't find variable"),		\
  63	C(FIELD_NOT_FOUND,	"Couldn't find field"),			\
  64	C(EMPTY_ASSIGNMENT,	"Empty assignment"),			\
  65	C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),		\
  66	C(EMPTY_SORT_FIELD,	"Empty sort field"),			\
  67	C(TOO_MANY_SORT_FIELDS,	"Too many sort fields (Max = 2)"),	\
  68	C(INVALID_SORT_FIELD,	"Sort field must be a key or a val"),
  69
  70#undef C
  71#define C(a, b)		HIST_ERR_##a
  72
  73enum { ERRORS };
  74
  75#undef C
  76#define C(a, b)		b
  77
  78static const char *err_text[] = { ERRORS };
  79
  80struct hist_field;
  81
  82typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  83				struct tracing_map_elt *elt,
  84				struct ring_buffer_event *rbe,
  85				void *event);
  86
  87#define HIST_FIELD_OPERANDS_MAX	2
  88#define HIST_FIELDS_MAX		(TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  89#define HIST_ACTIONS_MAX	8
  90
  91enum field_op_id {
  92	FIELD_OP_NONE,
  93	FIELD_OP_PLUS,
  94	FIELD_OP_MINUS,
  95	FIELD_OP_UNARY_MINUS,
  96};
  97
  98/*
  99 * A hist_var (histogram variable) contains variable information for
 100 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 101 * flag set.  A hist_var has a variable name e.g. ts0, and is
 102 * associated with a given histogram trigger, as specified by
 103 * hist_data.  The hist_var idx is the unique index assigned to the
 104 * variable by the hist trigger's tracing_map.  The idx is what is
 105 * used to set a variable's value and, by a variable reference, to
 106 * retrieve it.
 107 */
 108struct hist_var {
 109	char				*name;
 110	struct hist_trigger_data	*hist_data;
 111	unsigned int			idx;
 112};
 113
 114struct hist_field {
 115	struct ftrace_event_field	*field;
 116	unsigned long			flags;
 117	hist_field_fn_t			fn;
 118	unsigned int			ref;
 119	unsigned int			size;
 120	unsigned int			offset;
 121	unsigned int                    is_signed;
 122	const char			*type;
 123	struct hist_field		*operands[HIST_FIELD_OPERANDS_MAX];
 124	struct hist_trigger_data	*hist_data;
 125
 126	/*
 127	 * Variable fields contain variable-specific info in var.
 128	 */
 129	struct hist_var			var;
 130	enum field_op_id		operator;
 131	char				*system;
 132	char				*event_name;
 133
 134	/*
 135	 * The name field is used for EXPR and VAR_REF fields.  VAR
 136	 * fields contain the variable name in var.name.
 137	 */
 138	char				*name;
 139
 140	/*
 141	 * When a histogram trigger is hit, if it has any references
 142	 * to variables, the values of those variables are collected
 143	 * into a var_ref_vals array by resolve_var_refs().  The
 144	 * current value of each variable is read from the tracing_map
 145	 * using the hist field's hist_var.idx and entered into the
 146	 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 147	 */
 148	unsigned int			var_ref_idx;
 149	bool                            read_once;
 150};
 151
 152static u64 hist_field_none(struct hist_field *field,
 153			   struct tracing_map_elt *elt,
 154			   struct ring_buffer_event *rbe,
 155			   void *event)
 156{
 157	return 0;
 158}
 159
 160static u64 hist_field_counter(struct hist_field *field,
 161			      struct tracing_map_elt *elt,
 162			      struct ring_buffer_event *rbe,
 163			      void *event)
 164{
 165	return 1;
 166}
 167
 168static u64 hist_field_string(struct hist_field *hist_field,
 169			     struct tracing_map_elt *elt,
 170			     struct ring_buffer_event *rbe,
 171			     void *event)
 172{
 173	char *addr = (char *)(event + hist_field->field->offset);
 174
 175	return (u64)(unsigned long)addr;
 176}
 177
 178static u64 hist_field_dynstring(struct hist_field *hist_field,
 179				struct tracing_map_elt *elt,
 180				struct ring_buffer_event *rbe,
 181				void *event)
 182{
 183	u32 str_item = *(u32 *)(event + hist_field->field->offset);
 184	int str_loc = str_item & 0xffff;
 185	char *addr = (char *)(event + str_loc);
 186
 187	return (u64)(unsigned long)addr;
 188}
 189
 190static u64 hist_field_pstring(struct hist_field *hist_field,
 191			      struct tracing_map_elt *elt,
 192			      struct ring_buffer_event *rbe,
 193			      void *event)
 194{
 195	char **addr = (char **)(event + hist_field->field->offset);
 196
 197	return (u64)(unsigned long)*addr;
 198}
 199
 200static u64 hist_field_log2(struct hist_field *hist_field,
 201			   struct tracing_map_elt *elt,
 202			   struct ring_buffer_event *rbe,
 203			   void *event)
 204{
 205	struct hist_field *operand = hist_field->operands[0];
 206
 207	u64 val = operand->fn(operand, elt, rbe, event);
 208
 209	return (u64) ilog2(roundup_pow_of_two(val));
 210}
 211
 212static u64 hist_field_plus(struct hist_field *hist_field,
 213			   struct tracing_map_elt *elt,
 214			   struct ring_buffer_event *rbe,
 215			   void *event)
 216{
 217	struct hist_field *operand1 = hist_field->operands[0];
 218	struct hist_field *operand2 = hist_field->operands[1];
 219
 220	u64 val1 = operand1->fn(operand1, elt, rbe, event);
 221	u64 val2 = operand2->fn(operand2, elt, rbe, event);
 222
 223	return val1 + val2;
 224}
 225
 226static u64 hist_field_minus(struct hist_field *hist_field,
 227			    struct tracing_map_elt *elt,
 228			    struct ring_buffer_event *rbe,
 229			    void *event)
 230{
 231	struct hist_field *operand1 = hist_field->operands[0];
 232	struct hist_field *operand2 = hist_field->operands[1];
 233
 234	u64 val1 = operand1->fn(operand1, elt, rbe, event);
 235	u64 val2 = operand2->fn(operand2, elt, rbe, event);
 236
 237	return val1 - val2;
 238}
 239
 240static u64 hist_field_unary_minus(struct hist_field *hist_field,
 241				  struct tracing_map_elt *elt,
 242				  struct ring_buffer_event *rbe,
 243				  void *event)
 244{
 245	struct hist_field *operand = hist_field->operands[0];
 246
 247	s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 248	u64 val = (u64)-sval;
 249
 250	return val;
 251}
 252
 253#define DEFINE_HIST_FIELD_FN(type)					\
 254	static u64 hist_field_##type(struct hist_field *hist_field,	\
 255				     struct tracing_map_elt *elt,	\
 256				     struct ring_buffer_event *rbe,	\
 257				     void *event)			\
 258{									\
 259	type *addr = (type *)(event + hist_field->field->offset);	\
 260									\
 261	return (u64)(unsigned long)*addr;				\
 262}
 263
 264DEFINE_HIST_FIELD_FN(s64);
 265DEFINE_HIST_FIELD_FN(u64);
 266DEFINE_HIST_FIELD_FN(s32);
 267DEFINE_HIST_FIELD_FN(u32);
 268DEFINE_HIST_FIELD_FN(s16);
 269DEFINE_HIST_FIELD_FN(u16);
 270DEFINE_HIST_FIELD_FN(s8);
 271DEFINE_HIST_FIELD_FN(u8);
 272
 273#define for_each_hist_field(i, hist_data)	\
 274	for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 275
 276#define for_each_hist_val_field(i, hist_data)	\
 277	for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 278
 279#define for_each_hist_key_field(i, hist_data)	\
 280	for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 281
 282#define HIST_STACKTRACE_DEPTH	16
 283#define HIST_STACKTRACE_SIZE	(HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 284#define HIST_STACKTRACE_SKIP	5
 285
 286#define HITCOUNT_IDX		0
 287#define HIST_KEY_SIZE_MAX	(MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 288
 289enum hist_field_flags {
 290	HIST_FIELD_FL_HITCOUNT		= 1 << 0,
 291	HIST_FIELD_FL_KEY		= 1 << 1,
 292	HIST_FIELD_FL_STRING		= 1 << 2,
 293	HIST_FIELD_FL_HEX		= 1 << 3,
 294	HIST_FIELD_FL_SYM		= 1 << 4,
 295	HIST_FIELD_FL_SYM_OFFSET	= 1 << 5,
 296	HIST_FIELD_FL_EXECNAME		= 1 << 6,
 297	HIST_FIELD_FL_SYSCALL		= 1 << 7,
 298	HIST_FIELD_FL_STACKTRACE	= 1 << 8,
 299	HIST_FIELD_FL_LOG2		= 1 << 9,
 300	HIST_FIELD_FL_TIMESTAMP		= 1 << 10,
 301	HIST_FIELD_FL_TIMESTAMP_USECS	= 1 << 11,
 302	HIST_FIELD_FL_VAR		= 1 << 12,
 303	HIST_FIELD_FL_EXPR		= 1 << 13,
 304	HIST_FIELD_FL_VAR_REF		= 1 << 14,
 305	HIST_FIELD_FL_CPU		= 1 << 15,
 306	HIST_FIELD_FL_ALIAS		= 1 << 16,
 307};
 308
 309struct var_defs {
 310	unsigned int	n_vars;
 311	char		*name[TRACING_MAP_VARS_MAX];
 312	char		*expr[TRACING_MAP_VARS_MAX];
 313};
 314
 315struct hist_trigger_attrs {
 316	char		*keys_str;
 317	char		*vals_str;
 318	char		*sort_key_str;
 319	char		*name;
 320	char		*clock;
 321	bool		pause;
 322	bool		cont;
 323	bool		clear;
 324	bool		ts_in_usecs;
 325	unsigned int	map_bits;
 326
 327	char		*assignment_str[TRACING_MAP_VARS_MAX];
 328	unsigned int	n_assignments;
 329
 330	char		*action_str[HIST_ACTIONS_MAX];
 331	unsigned int	n_actions;
 332
 333	struct var_defs	var_defs;
 334};
 335
 336struct field_var {
 337	struct hist_field	*var;
 338	struct hist_field	*val;
 339};
 340
 341struct field_var_hist {
 342	struct hist_trigger_data	*hist_data;
 343	char				*cmd;
 344};
 345
 346struct hist_trigger_data {
 347	struct hist_field               *fields[HIST_FIELDS_MAX];
 348	unsigned int			n_vals;
 349	unsigned int			n_keys;
 350	unsigned int			n_fields;
 351	unsigned int			n_vars;
 352	unsigned int			key_size;
 353	struct tracing_map_sort_key	sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 354	unsigned int			n_sort_keys;
 355	struct trace_event_file		*event_file;
 356	struct hist_trigger_attrs	*attrs;
 357	struct tracing_map		*map;
 358	bool				enable_timestamps;
 359	bool				remove;
 360	struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 361	unsigned int			n_var_refs;
 362
 363	struct action_data		*actions[HIST_ACTIONS_MAX];
 364	unsigned int			n_actions;
 365
 366	struct field_var		*field_vars[SYNTH_FIELDS_MAX];
 367	unsigned int			n_field_vars;
 368	unsigned int			n_field_var_str;
 369	struct field_var_hist		*field_var_hists[SYNTH_FIELDS_MAX];
 370	unsigned int			n_field_var_hists;
 371
 372	struct field_var		*save_vars[SYNTH_FIELDS_MAX];
 373	unsigned int			n_save_vars;
 374	unsigned int			n_save_var_str;
 375};
 376
 377struct action_data;
 378
 379typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 380			     struct tracing_map_elt *elt, void *rec,
 381			     struct ring_buffer_event *rbe, void *key,
 382			     struct action_data *data, u64 *var_ref_vals);
 383
 384typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 385
 386enum handler_id {
 387	HANDLER_ONMATCH = 1,
 388	HANDLER_ONMAX,
 389	HANDLER_ONCHANGE,
 390};
 391
 392enum action_id {
 393	ACTION_SAVE = 1,
 394	ACTION_TRACE,
 395	ACTION_SNAPSHOT,
 396};
 397
 398struct action_data {
 399	enum handler_id		handler;
 400	enum action_id		action;
 401	char			*action_name;
 402	action_fn_t		fn;
 403
 404	unsigned int		n_params;
 405	char			*params[SYNTH_FIELDS_MAX];
 406
 407	/*
 408	 * When a histogram trigger is hit, the values of any
 409	 * references to variables, including variables being passed
 410	 * as parameters to synthetic events, are collected into a
 411	 * var_ref_vals array.  This var_ref_idx array is an array of
 412	 * indices into the var_ref_vals array, one for each synthetic
 413	 * event param, and is passed to the synthetic event
 414	 * invocation.
 415	 */
 416	unsigned int		var_ref_idx[TRACING_MAP_VARS_MAX];
 417	struct synth_event	*synth_event;
 418	bool			use_trace_keyword;
 419	char			*synth_event_name;
 420
 421	union {
 422		struct {
 423			char			*event;
 424			char			*event_system;
 425		} match_data;
 426
 427		struct {
 428			/*
 429			 * var_str contains the $-unstripped variable
 430			 * name referenced by var_ref, and used when
 431			 * printing the action.  Because var_ref
 432			 * creation is deferred to create_actions(),
 433			 * we need a per-action way to save it until
 434			 * then, thus var_str.
 435			 */
 436			char			*var_str;
 437
 438			/*
 439			 * var_ref refers to the variable being
 440			 * tracked e.g onmax($var).
 441			 */
 442			struct hist_field	*var_ref;
 443
 444			/*
 445			 * track_var contains the 'invisible' tracking
 446			 * variable created to keep the current
 447			 * e.g. max value.
 448			 */
 449			struct hist_field	*track_var;
 450
 451			check_track_val_fn_t	check_val;
 452			action_fn_t		save_data;
 453		} track_data;
 454	};
 455};
 456
 457struct track_data {
 458	u64				track_val;
 459	bool				updated;
 460
 461	unsigned int			key_len;
 462	void				*key;
 463	struct tracing_map_elt		elt;
 464
 465	struct action_data		*action_data;
 466	struct hist_trigger_data	*hist_data;
 467};
 468
 469struct hist_elt_data {
 470	char *comm;
 471	u64 *var_ref_vals;
 472	char *field_var_str[SYNTH_FIELDS_MAX];
 473};
 474
 475struct snapshot_context {
 476	struct tracing_map_elt	*elt;
 477	void			*key;
 478};
 479
 480static void track_data_free(struct track_data *track_data)
 481{
 482	struct hist_elt_data *elt_data;
 483
 484	if (!track_data)
 485		return;
 486
 487	kfree(track_data->key);
 488
 489	elt_data = track_data->elt.private_data;
 490	if (elt_data) {
 491		kfree(elt_data->comm);
 492		kfree(elt_data);
 493	}
 494
 495	kfree(track_data);
 496}
 497
 498static struct track_data *track_data_alloc(unsigned int key_len,
 499					   struct action_data *action_data,
 500					   struct hist_trigger_data *hist_data)
 501{
 502	struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 503	struct hist_elt_data *elt_data;
 504
 505	if (!data)
 506		return ERR_PTR(-ENOMEM);
 507
 508	data->key = kzalloc(key_len, GFP_KERNEL);
 509	if (!data->key) {
 510		track_data_free(data);
 511		return ERR_PTR(-ENOMEM);
 512	}
 513
 514	data->key_len = key_len;
 515	data->action_data = action_data;
 516	data->hist_data = hist_data;
 517
 518	elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 519	if (!elt_data) {
 520		track_data_free(data);
 521		return ERR_PTR(-ENOMEM);
 522	}
 523
 524	data->elt.private_data = elt_data;
 525
 526	elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 527	if (!elt_data->comm) {
 528		track_data_free(data);
 529		return ERR_PTR(-ENOMEM);
 530	}
 531
 532	return data;
 533}
 534
 535static char last_cmd[MAX_FILTER_STR_VAL];
 536static char last_cmd_loc[MAX_FILTER_STR_VAL];
 537
 538static int errpos(char *str)
 539{
 540	return err_pos(last_cmd, str);
 541}
 542
 543static void last_cmd_set(struct trace_event_file *file, char *str)
 544{
 545	const char *system = NULL, *name = NULL;
 546	struct trace_event_call *call;
 547
 548	if (!str)
 549		return;
 550
 551	strcpy(last_cmd, "hist:");
 552	strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
 553
 554	if (file) {
 555		call = file->event_call;
 556		system = call->class->system;
 557		if (system) {
 558			name = trace_event_name(call);
 559			if (!name)
 560				system = NULL;
 561		}
 562	}
 563
 564	if (system)
 565		snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 566}
 567
 568static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 569{
 570	tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 571			err_type, err_pos);
 572}
 573
 574static void hist_err_clear(void)
 575{
 576	last_cmd[0] = '\0';
 577	last_cmd_loc[0] = '\0';
 578}
 579
 580typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 581				    unsigned int *var_ref_idx);
 582
 583static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 584			       unsigned int *var_ref_idx)
 585{
 586	struct tracepoint *tp = event->tp;
 587
 588	if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 589		struct tracepoint_func *probe_func_ptr;
 590		synth_probe_func_t probe_func;
 591		void *__data;
 592
 593		if (!(cpu_online(raw_smp_processor_id())))
 594			return;
 595
 596		probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 597		if (probe_func_ptr) {
 598			do {
 599				probe_func = probe_func_ptr->func;
 600				__data = probe_func_ptr->data;
 601				probe_func(__data, var_ref_vals, var_ref_idx);
 602			} while ((++probe_func_ptr)->func);
 603		}
 604	}
 605}
 606
 607static void action_trace(struct hist_trigger_data *hist_data,
 608			 struct tracing_map_elt *elt, void *rec,
 609			 struct ring_buffer_event *rbe, void *key,
 610			 struct action_data *data, u64 *var_ref_vals)
 611{
 612	struct synth_event *event = data->synth_event;
 613
 614	trace_synth(event, var_ref_vals, data->var_ref_idx);
 615}
 616
 617struct hist_var_data {
 618	struct list_head list;
 619	struct hist_trigger_data *hist_data;
 620};
 621
 622static u64 hist_field_timestamp(struct hist_field *hist_field,
 623				struct tracing_map_elt *elt,
 624				struct ring_buffer_event *rbe,
 625				void *event)
 626{
 627	struct hist_trigger_data *hist_data = hist_field->hist_data;
 628	struct trace_array *tr = hist_data->event_file->tr;
 629
 630	u64 ts = ring_buffer_event_time_stamp(rbe);
 631
 632	if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 633		ts = ns2usecs(ts);
 634
 635	return ts;
 636}
 637
 638static u64 hist_field_cpu(struct hist_field *hist_field,
 639			  struct tracing_map_elt *elt,
 640			  struct ring_buffer_event *rbe,
 641			  void *event)
 642{
 643	int cpu = smp_processor_id();
 644
 645	return cpu;
 646}
 647
 648/**
 649 * check_field_for_var_ref - Check if a VAR_REF field references a variable
 650 * @hist_field: The VAR_REF field to check
 651 * @var_data: The hist trigger that owns the variable
 652 * @var_idx: The trigger variable identifier
 653 *
 654 * Check the given VAR_REF field to see whether or not it references
 655 * the given variable associated with the given trigger.
 656 *
 657 * Return: The VAR_REF field if it does reference the variable, NULL if not
 658 */
 659static struct hist_field *
 660check_field_for_var_ref(struct hist_field *hist_field,
 661			struct hist_trigger_data *var_data,
 662			unsigned int var_idx)
 663{
 664	WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 665
 666	if (hist_field && hist_field->var.idx == var_idx &&
 667	    hist_field->var.hist_data == var_data)
 668		return hist_field;
 669
 670	return NULL;
 671}
 672
 673/**
 674 * find_var_ref - Check if a trigger has a reference to a trigger variable
 675 * @hist_data: The hist trigger that might have a reference to the variable
 676 * @var_data: The hist trigger that owns the variable
 677 * @var_idx: The trigger variable identifier
 678 *
 679 * Check the list of var_refs[] on the first hist trigger to see
 680 * whether any of them are references to the variable on the second
 681 * trigger.
 682 *
 683 * Return: The VAR_REF field referencing the variable if so, NULL if not
 684 */
 685static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 686				       struct hist_trigger_data *var_data,
 687				       unsigned int var_idx)
 688{
 689	struct hist_field *hist_field;
 690	unsigned int i;
 691
 692	for (i = 0; i < hist_data->n_var_refs; i++) {
 693		hist_field = hist_data->var_refs[i];
 694		if (check_field_for_var_ref(hist_field, var_data, var_idx))
 695			return hist_field;
 696	}
 697
 698	return NULL;
 699}
 700
 701/**
 702 * find_any_var_ref - Check if there is a reference to a given trigger variable
 703 * @hist_data: The hist trigger
 704 * @var_idx: The trigger variable identifier
 705 *
 706 * Check to see whether the given variable is currently referenced by
 707 * any other trigger.
 708 *
 709 * The trigger the variable is defined on is explicitly excluded - the
 710 * assumption being that a self-reference doesn't prevent a trigger
 711 * from being removed.
 712 *
 713 * Return: The VAR_REF field referencing the variable if so, NULL if not
 714 */
 715static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 716					   unsigned int var_idx)
 717{
 718	struct trace_array *tr = hist_data->event_file->tr;
 719	struct hist_field *found = NULL;
 720	struct hist_var_data *var_data;
 721
 722	list_for_each_entry(var_data, &tr->hist_vars, list) {
 723		if (var_data->hist_data == hist_data)
 724			continue;
 725		found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 726		if (found)
 727			break;
 728	}
 729
 730	return found;
 731}
 732
 733/**
 734 * check_var_refs - Check if there is a reference to any of trigger's variables
 735 * @hist_data: The hist trigger
 736 *
 737 * A trigger can define one or more variables.  If any one of them is
 738 * currently referenced by any other trigger, this function will
 739 * determine that.
 740
 741 * Typically used to determine whether or not a trigger can be removed
 742 * - if there are any references to a trigger's variables, it cannot.
 743 *
 744 * Return: True if there is a reference to any of trigger's variables
 745 */
 746static bool check_var_refs(struct hist_trigger_data *hist_data)
 747{
 748	struct hist_field *field;
 749	bool found = false;
 750	int i;
 751
 752	for_each_hist_field(i, hist_data) {
 753		field = hist_data->fields[i];
 754		if (field && field->flags & HIST_FIELD_FL_VAR) {
 755			if (find_any_var_ref(hist_data, field->var.idx)) {
 756				found = true;
 757				break;
 758			}
 759		}
 760	}
 761
 762	return found;
 763}
 764
 765static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
 766{
 767	struct trace_array *tr = hist_data->event_file->tr;
 768	struct hist_var_data *var_data, *found = NULL;
 769
 770	list_for_each_entry(var_data, &tr->hist_vars, list) {
 771		if (var_data->hist_data == hist_data) {
 772			found = var_data;
 773			break;
 774		}
 775	}
 776
 777	return found;
 778}
 779
 780static bool field_has_hist_vars(struct hist_field *hist_field,
 781				unsigned int level)
 782{
 783	int i;
 784
 785	if (level > 3)
 786		return false;
 787
 788	if (!hist_field)
 789		return false;
 790
 791	if (hist_field->flags & HIST_FIELD_FL_VAR ||
 792	    hist_field->flags & HIST_FIELD_FL_VAR_REF)
 793		return true;
 794
 795	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
 796		struct hist_field *operand;
 797
 798		operand = hist_field->operands[i];
 799		if (field_has_hist_vars(operand, level + 1))
 800			return true;
 801	}
 802
 803	return false;
 804}
 805
 806static bool has_hist_vars(struct hist_trigger_data *hist_data)
 807{
 808	struct hist_field *hist_field;
 809	int i;
 810
 811	for_each_hist_field(i, hist_data) {
 812		hist_field = hist_data->fields[i];
 813		if (field_has_hist_vars(hist_field, 0))
 814			return true;
 815	}
 816
 817	return false;
 818}
 819
 820static int save_hist_vars(struct hist_trigger_data *hist_data)
 821{
 822	struct trace_array *tr = hist_data->event_file->tr;
 823	struct hist_var_data *var_data;
 824
 825	var_data = find_hist_vars(hist_data);
 826	if (var_data)
 827		return 0;
 828
 829	if (tracing_check_open_get_tr(tr))
 830		return -ENODEV;
 831
 832	var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
 833	if (!var_data) {
 834		trace_array_put(tr);
 835		return -ENOMEM;
 836	}
 837
 838	var_data->hist_data = hist_data;
 839	list_add(&var_data->list, &tr->hist_vars);
 840
 841	return 0;
 842}
 843
 844static void remove_hist_vars(struct hist_trigger_data *hist_data)
 845{
 846	struct trace_array *tr = hist_data->event_file->tr;
 847	struct hist_var_data *var_data;
 848
 849	var_data = find_hist_vars(hist_data);
 850	if (!var_data)
 851		return;
 852
 853	if (WARN_ON(check_var_refs(hist_data)))
 854		return;
 855
 856	list_del(&var_data->list);
 857
 858	kfree(var_data);
 859
 860	trace_array_put(tr);
 861}
 862
 863static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
 864					 const char *var_name)
 865{
 866	struct hist_field *hist_field, *found = NULL;
 867	int i;
 868
 869	for_each_hist_field(i, hist_data) {
 870		hist_field = hist_data->fields[i];
 871		if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
 872		    strcmp(hist_field->var.name, var_name) == 0) {
 873			found = hist_field;
 874			break;
 875		}
 876	}
 877
 878	return found;
 879}
 880
 881static struct hist_field *find_var(struct hist_trigger_data *hist_data,
 882				   struct trace_event_file *file,
 883				   const char *var_name)
 884{
 885	struct hist_trigger_data *test_data;
 886	struct event_trigger_data *test;
 887	struct hist_field *hist_field;
 888
 889	lockdep_assert_held(&event_mutex);
 890
 891	hist_field = find_var_field(hist_data, var_name);
 892	if (hist_field)
 893		return hist_field;
 894
 895	list_for_each_entry(test, &file->triggers, list) {
 896		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 897			test_data = test->private_data;
 898			hist_field = find_var_field(test_data, var_name);
 899			if (hist_field)
 900				return hist_field;
 901		}
 902	}
 903
 904	return NULL;
 905}
 906
 907static struct trace_event_file *find_var_file(struct trace_array *tr,
 908					      char *system,
 909					      char *event_name,
 910					      char *var_name)
 911{
 912	struct hist_trigger_data *var_hist_data;
 913	struct hist_var_data *var_data;
 914	struct trace_event_file *file, *found = NULL;
 915
 916	if (system)
 917		return find_event_file(tr, system, event_name);
 918
 919	list_for_each_entry(var_data, &tr->hist_vars, list) {
 920		var_hist_data = var_data->hist_data;
 921		file = var_hist_data->event_file;
 922		if (file == found)
 923			continue;
 924
 925		if (find_var_field(var_hist_data, var_name)) {
 926			if (found) {
 927				hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
 928				return NULL;
 929			}
 930
 931			found = file;
 932		}
 933	}
 934
 935	return found;
 936}
 937
 938static struct hist_field *find_file_var(struct trace_event_file *file,
 939					const char *var_name)
 940{
 941	struct hist_trigger_data *test_data;
 942	struct event_trigger_data *test;
 943	struct hist_field *hist_field;
 944
 945	lockdep_assert_held(&event_mutex);
 946
 947	list_for_each_entry(test, &file->triggers, list) {
 948		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 949			test_data = test->private_data;
 950			hist_field = find_var_field(test_data, var_name);
 951			if (hist_field)
 952				return hist_field;
 953		}
 954	}
 955
 956	return NULL;
 957}
 958
 959static struct hist_field *
 960find_match_var(struct hist_trigger_data *hist_data, char *var_name)
 961{
 962	struct trace_array *tr = hist_data->event_file->tr;
 963	struct hist_field *hist_field, *found = NULL;
 964	struct trace_event_file *file;
 965	unsigned int i;
 966
 967	for (i = 0; i < hist_data->n_actions; i++) {
 968		struct action_data *data = hist_data->actions[i];
 969
 970		if (data->handler == HANDLER_ONMATCH) {
 971			char *system = data->match_data.event_system;
 972			char *event_name = data->match_data.event;
 973
 974			file = find_var_file(tr, system, event_name, var_name);
 975			if (!file)
 976				continue;
 977			hist_field = find_file_var(file, var_name);
 978			if (hist_field) {
 979				if (found) {
 980					hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
 981						 errpos(var_name));
 982					return ERR_PTR(-EINVAL);
 983				}
 984
 985				found = hist_field;
 986			}
 987		}
 988	}
 989	return found;
 990}
 991
 992static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
 993					 char *system,
 994					 char *event_name,
 995					 char *var_name)
 996{
 997	struct trace_array *tr = hist_data->event_file->tr;
 998	struct hist_field *hist_field = NULL;
 999	struct trace_event_file *file;
1000
1001	if (!system || !event_name) {
1002		hist_field = find_match_var(hist_data, var_name);
1003		if (IS_ERR(hist_field))
1004			return NULL;
1005		if (hist_field)
1006			return hist_field;
1007	}
1008
1009	file = find_var_file(tr, system, event_name, var_name);
1010	if (!file)
1011		return NULL;
1012
1013	hist_field = find_file_var(file, var_name);
1014
1015	return hist_field;
1016}
1017
1018static u64 hist_field_var_ref(struct hist_field *hist_field,
1019			      struct tracing_map_elt *elt,
1020			      struct ring_buffer_event *rbe,
1021			      void *event)
1022{
1023	struct hist_elt_data *elt_data;
1024	u64 var_val = 0;
1025
1026	if (WARN_ON_ONCE(!elt))
1027		return var_val;
1028
1029	elt_data = elt->private_data;
1030	var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1031
1032	return var_val;
1033}
1034
1035static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1036			     u64 *var_ref_vals, bool self)
1037{
1038	struct hist_trigger_data *var_data;
1039	struct tracing_map_elt *var_elt;
1040	struct hist_field *hist_field;
1041	unsigned int i, var_idx;
1042	bool resolved = true;
1043	u64 var_val = 0;
1044
1045	for (i = 0; i < hist_data->n_var_refs; i++) {
1046		hist_field = hist_data->var_refs[i];
1047		var_idx = hist_field->var.idx;
1048		var_data = hist_field->var.hist_data;
1049
1050		if (var_data == NULL) {
1051			resolved = false;
1052			break;
1053		}
1054
1055		if ((self && var_data != hist_data) ||
1056		    (!self && var_data == hist_data))
1057			continue;
1058
1059		var_elt = tracing_map_lookup(var_data->map, key);
1060		if (!var_elt) {
1061			resolved = false;
1062			break;
1063		}
1064
1065		if (!tracing_map_var_set(var_elt, var_idx)) {
1066			resolved = false;
1067			break;
1068		}
1069
1070		if (self || !hist_field->read_once)
1071			var_val = tracing_map_read_var(var_elt, var_idx);
1072		else
1073			var_val = tracing_map_read_var_once(var_elt, var_idx);
1074
1075		var_ref_vals[i] = var_val;
1076	}
1077
1078	return resolved;
1079}
1080
1081static const char *hist_field_name(struct hist_field *field,
1082				   unsigned int level)
1083{
1084	const char *field_name = "";
1085
1086	if (level > 1)
1087		return field_name;
1088
1089	if (field->field)
1090		field_name = field->field->name;
1091	else if (field->flags & HIST_FIELD_FL_LOG2 ||
1092		 field->flags & HIST_FIELD_FL_ALIAS)
1093		field_name = hist_field_name(field->operands[0], ++level);
1094	else if (field->flags & HIST_FIELD_FL_CPU)
1095		field_name = "cpu";
1096	else if (field->flags & HIST_FIELD_FL_EXPR ||
1097		 field->flags & HIST_FIELD_FL_VAR_REF) {
1098		if (field->system) {
1099			static char full_name[MAX_FILTER_STR_VAL];
1100
1101			strcat(full_name, field->system);
1102			strcat(full_name, ".");
1103			strcat(full_name, field->event_name);
1104			strcat(full_name, ".");
1105			strcat(full_name, field->name);
1106			field_name = full_name;
1107		} else
1108			field_name = field->name;
1109	} else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1110		field_name = "common_timestamp";
1111
1112	if (field_name == NULL)
1113		field_name = "";
1114
1115	return field_name;
1116}
1117
1118static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1119{
1120	hist_field_fn_t fn = NULL;
1121
1122	switch (field_size) {
1123	case 8:
1124		if (field_is_signed)
1125			fn = hist_field_s64;
1126		else
1127			fn = hist_field_u64;
1128		break;
1129	case 4:
1130		if (field_is_signed)
1131			fn = hist_field_s32;
1132		else
1133			fn = hist_field_u32;
1134		break;
1135	case 2:
1136		if (field_is_signed)
1137			fn = hist_field_s16;
1138		else
1139			fn = hist_field_u16;
1140		break;
1141	case 1:
1142		if (field_is_signed)
1143			fn = hist_field_s8;
1144		else
1145			fn = hist_field_u8;
1146		break;
1147	}
1148
1149	return fn;
1150}
1151
1152static int parse_map_size(char *str)
1153{
1154	unsigned long size, map_bits;
1155	int ret;
1156
1157	ret = kstrtoul(str, 0, &size);
1158	if (ret)
1159		goto out;
1160
1161	map_bits = ilog2(roundup_pow_of_two(size));
1162	if (map_bits < TRACING_MAP_BITS_MIN ||
1163	    map_bits > TRACING_MAP_BITS_MAX)
1164		ret = -EINVAL;
1165	else
1166		ret = map_bits;
1167 out:
1168	return ret;
1169}
1170
1171static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1172{
1173	unsigned int i;
1174
1175	if (!attrs)
1176		return;
1177
1178	for (i = 0; i < attrs->n_assignments; i++)
1179		kfree(attrs->assignment_str[i]);
1180
1181	for (i = 0; i < attrs->n_actions; i++)
1182		kfree(attrs->action_str[i]);
1183
1184	kfree(attrs->name);
1185	kfree(attrs->sort_key_str);
1186	kfree(attrs->keys_str);
1187	kfree(attrs->vals_str);
1188	kfree(attrs->clock);
1189	kfree(attrs);
1190}
1191
1192static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1193{
1194	int ret = -EINVAL;
1195
1196	if (attrs->n_actions >= HIST_ACTIONS_MAX)
1197		return ret;
1198
1199	if ((str_has_prefix(str, "onmatch(")) ||
1200	    (str_has_prefix(str, "onmax(")) ||
1201	    (str_has_prefix(str, "onchange("))) {
1202		attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1203		if (!attrs->action_str[attrs->n_actions]) {
1204			ret = -ENOMEM;
1205			return ret;
1206		}
1207		attrs->n_actions++;
1208		ret = 0;
1209	}
1210	return ret;
1211}
1212
1213static int parse_assignment(struct trace_array *tr,
1214			    char *str, struct hist_trigger_attrs *attrs)
1215{
1216	int len, ret = 0;
1217
1218	if ((len = str_has_prefix(str, "key=")) ||
1219	    (len = str_has_prefix(str, "keys="))) {
1220		attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1221		if (!attrs->keys_str) {
1222			ret = -ENOMEM;
1223			goto out;
1224		}
1225	} else if ((len = str_has_prefix(str, "val=")) ||
1226		   (len = str_has_prefix(str, "vals=")) ||
1227		   (len = str_has_prefix(str, "values="))) {
1228		attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1229		if (!attrs->vals_str) {
1230			ret = -ENOMEM;
1231			goto out;
1232		}
1233	} else if ((len = str_has_prefix(str, "sort="))) {
1234		attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1235		if (!attrs->sort_key_str) {
1236			ret = -ENOMEM;
1237			goto out;
1238		}
1239	} else if (str_has_prefix(str, "name=")) {
1240		attrs->name = kstrdup(str, GFP_KERNEL);
1241		if (!attrs->name) {
1242			ret = -ENOMEM;
1243			goto out;
1244		}
1245	} else if ((len = str_has_prefix(str, "clock="))) {
1246		str += len;
1247
1248		str = strstrip(str);
1249		attrs->clock = kstrdup(str, GFP_KERNEL);
1250		if (!attrs->clock) {
1251			ret = -ENOMEM;
1252			goto out;
1253		}
1254	} else if ((len = str_has_prefix(str, "size="))) {
1255		int map_bits = parse_map_size(str + len);
1256
1257		if (map_bits < 0) {
1258			ret = map_bits;
1259			goto out;
1260		}
1261		attrs->map_bits = map_bits;
1262	} else {
1263		char *assignment;
1264
1265		if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1266			hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1267			ret = -EINVAL;
1268			goto out;
1269		}
1270
1271		assignment = kstrdup(str, GFP_KERNEL);
1272		if (!assignment) {
1273			ret = -ENOMEM;
1274			goto out;
1275		}
1276
1277		attrs->assignment_str[attrs->n_assignments++] = assignment;
1278	}
1279 out:
1280	return ret;
1281}
1282
1283static struct hist_trigger_attrs *
1284parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1285{
1286	struct hist_trigger_attrs *attrs;
1287	int ret = 0;
1288
1289	attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1290	if (!attrs)
1291		return ERR_PTR(-ENOMEM);
1292
1293	while (trigger_str) {
1294		char *str = strsep(&trigger_str, ":");
1295		char *rhs;
1296
1297		rhs = strchr(str, '=');
1298		if (rhs) {
1299			if (!strlen(++rhs)) {
1300				ret = -EINVAL;
1301				hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1302				goto free;
1303			}
1304			ret = parse_assignment(tr, str, attrs);
1305			if (ret)
1306				goto free;
1307		} else if (strcmp(str, "pause") == 0)
1308			attrs->pause = true;
1309		else if ((strcmp(str, "cont") == 0) ||
1310			 (strcmp(str, "continue") == 0))
1311			attrs->cont = true;
1312		else if (strcmp(str, "clear") == 0)
1313			attrs->clear = true;
1314		else {
1315			ret = parse_action(str, attrs);
1316			if (ret)
1317				goto free;
1318		}
1319	}
1320
1321	if (!attrs->keys_str) {
1322		ret = -EINVAL;
1323		goto free;
1324	}
1325
1326	if (!attrs->clock) {
1327		attrs->clock = kstrdup("global", GFP_KERNEL);
1328		if (!attrs->clock) {
1329			ret = -ENOMEM;
1330			goto free;
1331		}
1332	}
1333
1334	return attrs;
1335 free:
1336	destroy_hist_trigger_attrs(attrs);
1337
1338	return ERR_PTR(ret);
1339}
1340
1341static inline void save_comm(char *comm, struct task_struct *task)
1342{
1343	if (!task->pid) {
1344		strcpy(comm, "<idle>");
1345		return;
1346	}
1347
1348	if (WARN_ON_ONCE(task->pid < 0)) {
1349		strcpy(comm, "<XXX>");
1350		return;
1351	}
1352
1353	strncpy(comm, task->comm, TASK_COMM_LEN);
1354}
1355
1356static void hist_elt_data_free(struct hist_elt_data *elt_data)
1357{
1358	unsigned int i;
1359
1360	for (i = 0; i < SYNTH_FIELDS_MAX; i++)
1361		kfree(elt_data->field_var_str[i]);
1362
1363	kfree(elt_data->comm);
1364	kfree(elt_data);
1365}
1366
1367static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1368{
1369	struct hist_elt_data *elt_data = elt->private_data;
1370
1371	hist_elt_data_free(elt_data);
1372}
1373
1374static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1375{
1376	struct hist_trigger_data *hist_data = elt->map->private_data;
1377	unsigned int size = TASK_COMM_LEN;
1378	struct hist_elt_data *elt_data;
1379	struct hist_field *key_field;
1380	unsigned int i, n_str;
1381
1382	elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1383	if (!elt_data)
1384		return -ENOMEM;
1385
1386	for_each_hist_key_field(i, hist_data) {
1387		key_field = hist_data->fields[i];
1388
1389		if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
1390			elt_data->comm = kzalloc(size, GFP_KERNEL);
1391			if (!elt_data->comm) {
1392				kfree(elt_data);
1393				return -ENOMEM;
1394			}
1395			break;
1396		}
1397	}
1398
1399	n_str = hist_data->n_field_var_str + hist_data->n_save_var_str;
1400
1401	size = STR_VAR_LEN_MAX;
1402
1403	for (i = 0; i < n_str; i++) {
1404		elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1405		if (!elt_data->field_var_str[i]) {
1406			hist_elt_data_free(elt_data);
1407			return -ENOMEM;
1408		}
1409	}
1410
1411	elt->private_data = elt_data;
1412
1413	return 0;
1414}
1415
1416static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1417{
1418	struct hist_elt_data *elt_data = elt->private_data;
1419
1420	if (elt_data->comm)
1421		save_comm(elt_data->comm, current);
1422}
1423
1424static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1425	.elt_alloc	= hist_trigger_elt_data_alloc,
1426	.elt_free	= hist_trigger_elt_data_free,
1427	.elt_init	= hist_trigger_elt_data_init,
1428};
1429
1430static const char *get_hist_field_flags(struct hist_field *hist_field)
1431{
1432	const char *flags_str = NULL;
1433
1434	if (hist_field->flags & HIST_FIELD_FL_HEX)
1435		flags_str = "hex";
1436	else if (hist_field->flags & HIST_FIELD_FL_SYM)
1437		flags_str = "sym";
1438	else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1439		flags_str = "sym-offset";
1440	else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1441		flags_str = "execname";
1442	else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1443		flags_str = "syscall";
1444	else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1445		flags_str = "log2";
1446	else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1447		flags_str = "usecs";
1448
1449	return flags_str;
1450}
1451
1452static void expr_field_str(struct hist_field *field, char *expr)
1453{
1454	if (field->flags & HIST_FIELD_FL_VAR_REF)
1455		strcat(expr, "$");
1456
1457	strcat(expr, hist_field_name(field, 0));
1458
1459	if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1460		const char *flags_str = get_hist_field_flags(field);
1461
1462		if (flags_str) {
1463			strcat(expr, ".");
1464			strcat(expr, flags_str);
1465		}
1466	}
1467}
1468
1469static char *expr_str(struct hist_field *field, unsigned int level)
1470{
1471	char *expr;
1472
1473	if (level > 1)
1474		return NULL;
1475
1476	expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1477	if (!expr)
1478		return NULL;
1479
1480	if (!field->operands[0]) {
1481		expr_field_str(field, expr);
1482		return expr;
1483	}
1484
1485	if (field->operator == FIELD_OP_UNARY_MINUS) {
1486		char *subexpr;
1487
1488		strcat(expr, "-(");
1489		subexpr = expr_str(field->operands[0], ++level);
1490		if (!subexpr) {
1491			kfree(expr);
1492			return NULL;
1493		}
1494		strcat(expr, subexpr);
1495		strcat(expr, ")");
1496
1497		kfree(subexpr);
1498
1499		return expr;
1500	}
1501
1502	expr_field_str(field->operands[0], expr);
1503
1504	switch (field->operator) {
1505	case FIELD_OP_MINUS:
1506		strcat(expr, "-");
1507		break;
1508	case FIELD_OP_PLUS:
1509		strcat(expr, "+");
1510		break;
1511	default:
1512		kfree(expr);
1513		return NULL;
1514	}
1515
1516	expr_field_str(field->operands[1], expr);
1517
1518	return expr;
1519}
1520
1521static int contains_operator(char *str)
1522{
1523	enum field_op_id field_op = FIELD_OP_NONE;
1524	char *op;
1525
1526	op = strpbrk(str, "+-");
1527	if (!op)
1528		return FIELD_OP_NONE;
1529
1530	switch (*op) {
1531	case '-':
1532		if (*str == '-')
1533			field_op = FIELD_OP_UNARY_MINUS;
1534		else
1535			field_op = FIELD_OP_MINUS;
1536		break;
1537	case '+':
1538		field_op = FIELD_OP_PLUS;
1539		break;
1540	default:
1541		break;
1542	}
1543
1544	return field_op;
1545}
1546
1547static void get_hist_field(struct hist_field *hist_field)
1548{
1549	hist_field->ref++;
1550}
1551
1552static void __destroy_hist_field(struct hist_field *hist_field)
1553{
1554	if (--hist_field->ref > 1)
1555		return;
1556
1557	kfree(hist_field->var.name);
1558	kfree(hist_field->name);
1559	kfree(hist_field->type);
1560
1561	kfree(hist_field->system);
1562	kfree(hist_field->event_name);
1563
1564	kfree(hist_field);
1565}
1566
1567static void destroy_hist_field(struct hist_field *hist_field,
1568			       unsigned int level)
1569{
1570	unsigned int i;
1571
1572	if (level > 3)
1573		return;
1574
1575	if (!hist_field)
1576		return;
1577
1578	if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1579		return; /* var refs will be destroyed separately */
1580
1581	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1582		destroy_hist_field(hist_field->operands[i], level + 1);
1583
1584	__destroy_hist_field(hist_field);
1585}
1586
1587static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1588					    struct ftrace_event_field *field,
1589					    unsigned long flags,
1590					    char *var_name)
1591{
1592	struct hist_field *hist_field;
1593
1594	if (field && is_function_field(field))
1595		return NULL;
1596
1597	hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1598	if (!hist_field)
1599		return NULL;
1600
1601	hist_field->ref = 1;
1602
1603	hist_field->hist_data = hist_data;
1604
1605	if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1606		goto out; /* caller will populate */
1607
1608	if (flags & HIST_FIELD_FL_VAR_REF) {
1609		hist_field->fn = hist_field_var_ref;
1610		goto out;
1611	}
1612
1613	if (flags & HIST_FIELD_FL_HITCOUNT) {
1614		hist_field->fn = hist_field_counter;
1615		hist_field->size = sizeof(u64);
1616		hist_field->type = kstrdup("u64", GFP_KERNEL);
1617		if (!hist_field->type)
1618			goto free;
1619		goto out;
1620	}
1621
1622	if (flags & HIST_FIELD_FL_STACKTRACE) {
1623		hist_field->fn = hist_field_none;
1624		goto out;
1625	}
1626
1627	if (flags & HIST_FIELD_FL_LOG2) {
1628		unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
1629		hist_field->fn = hist_field_log2;
1630		hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1631		hist_field->size = hist_field->operands[0]->size;
1632		hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
1633		if (!hist_field->type)
1634			goto free;
1635		goto out;
1636	}
1637
1638	if (flags & HIST_FIELD_FL_TIMESTAMP) {
1639		hist_field->fn = hist_field_timestamp;
1640		hist_field->size = sizeof(u64);
1641		hist_field->type = kstrdup("u64", GFP_KERNEL);
1642		if (!hist_field->type)
1643			goto free;
1644		goto out;
1645	}
1646
1647	if (flags & HIST_FIELD_FL_CPU) {
1648		hist_field->fn = hist_field_cpu;
1649		hist_field->size = sizeof(int);
1650		hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
1651		if (!hist_field->type)
1652			goto free;
1653		goto out;
1654	}
1655
1656	if (WARN_ON_ONCE(!field))
1657		goto out;
1658
1659	if (is_string_field(field)) {
1660		flags |= HIST_FIELD_FL_STRING;
1661
1662		hist_field->size = MAX_FILTER_STR_VAL;
1663		hist_field->type = kstrdup(field->type, GFP_KERNEL);
1664		if (!hist_field->type)
1665			goto free;
1666
1667		if (field->filter_type == FILTER_STATIC_STRING)
1668			hist_field->fn = hist_field_string;
1669		else if (field->filter_type == FILTER_DYN_STRING)
1670			hist_field->fn = hist_field_dynstring;
1671		else
1672			hist_field->fn = hist_field_pstring;
1673	} else {
1674		hist_field->size = field->size;
1675		hist_field->is_signed = field->is_signed;
1676		hist_field->type = kstrdup(field->type, GFP_KERNEL);
1677		if (!hist_field->type)
1678			goto free;
1679
1680		hist_field->fn = select_value_fn(field->size,
1681						 field->is_signed);
1682		if (!hist_field->fn) {
1683			destroy_hist_field(hist_field, 0);
1684			return NULL;
1685		}
1686	}
1687 out:
1688	hist_field->field = field;
1689	hist_field->flags = flags;
1690
1691	if (var_name) {
1692		hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1693		if (!hist_field->var.name)
1694			goto free;
1695	}
1696
1697	return hist_field;
1698 free:
1699	destroy_hist_field(hist_field, 0);
1700	return NULL;
1701}
1702
1703static void destroy_hist_fields(struct hist_trigger_data *hist_data)
1704{
1705	unsigned int i;
1706
1707	for (i = 0; i < HIST_FIELDS_MAX; i++) {
1708		if (hist_data->fields[i]) {
1709			destroy_hist_field(hist_data->fields[i], 0);
1710			hist_data->fields[i] = NULL;
1711		}
1712	}
1713
1714	for (i = 0; i < hist_data->n_var_refs; i++) {
1715		WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
1716		__destroy_hist_field(hist_data->var_refs[i]);
1717		hist_data->var_refs[i] = NULL;
1718	}
1719}
1720
1721static int init_var_ref(struct hist_field *ref_field,
1722			struct hist_field *var_field,
1723			char *system, char *event_name)
1724{
1725	int err = 0;
1726
1727	ref_field->var.idx = var_field->var.idx;
1728	ref_field->var.hist_data = var_field->hist_data;
1729	ref_field->size = var_field->size;
1730	ref_field->is_signed = var_field->is_signed;
1731	ref_field->flags |= var_field->flags &
1732		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
1733
1734	if (system) {
1735		ref_field->system = kstrdup(system, GFP_KERNEL);
1736		if (!ref_field->system)
1737			return -ENOMEM;
1738	}
1739
1740	if (event_name) {
1741		ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
1742		if (!ref_field->event_name) {
1743			err = -ENOMEM;
1744			goto free;
1745		}
1746	}
1747
1748	if (var_field->var.name) {
1749		ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
1750		if (!ref_field->name) {
1751			err = -ENOMEM;
1752			goto free;
1753		}
1754	} else if (var_field->name) {
1755		ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
1756		if (!ref_field->name) {
1757			err = -ENOMEM;
1758			goto free;
1759		}
1760	}
1761
1762	ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
1763	if (!ref_field->type) {
1764		err = -ENOMEM;
1765		goto free;
1766	}
1767 out:
1768	return err;
1769 free:
1770	kfree(ref_field->system);
1771	kfree(ref_field->event_name);
1772	kfree(ref_field->name);
1773
1774	goto out;
1775}
1776
1777static int find_var_ref_idx(struct hist_trigger_data *hist_data,
1778			    struct hist_field *var_field)
1779{
1780	struct hist_field *ref_field;
1781	int i;
1782
1783	for (i = 0; i < hist_data->n_var_refs; i++) {
1784		ref_field = hist_data->var_refs[i];
1785		if (ref_field->var.idx == var_field->var.idx &&
1786		    ref_field->var.hist_data == var_field->hist_data)
1787			return i;
1788	}
1789
1790	return -ENOENT;
1791}
1792
1793/**
1794 * create_var_ref - Create a variable reference and attach it to trigger
1795 * @hist_data: The trigger that will be referencing the variable
1796 * @var_field: The VAR field to create a reference to
1797 * @system: The optional system string
1798 * @event_name: The optional event_name string
1799 *
1800 * Given a variable hist_field, create a VAR_REF hist_field that
1801 * represents a reference to it.
1802 *
1803 * This function also adds the reference to the trigger that
1804 * now references the variable.
1805 *
1806 * Return: The VAR_REF field if successful, NULL if not
1807 */
1808static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
1809					 struct hist_field *var_field,
1810					 char *system, char *event_name)
1811{
1812	unsigned long flags = HIST_FIELD_FL_VAR_REF;
1813	struct hist_field *ref_field;
1814	int i;
1815
1816	/* Check if the variable already exists */
1817	for (i = 0; i < hist_data->n_var_refs; i++) {
1818		ref_field = hist_data->var_refs[i];
1819		if (ref_field->var.idx == var_field->var.idx &&
1820		    ref_field->var.hist_data == var_field->hist_data) {
1821			get_hist_field(ref_field);
1822			return ref_field;
1823		}
1824	}
1825
1826	ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
1827	if (ref_field) {
1828		if (init_var_ref(ref_field, var_field, system, event_name)) {
1829			destroy_hist_field(ref_field, 0);
1830			return NULL;
1831		}
1832
1833		hist_data->var_refs[hist_data->n_var_refs] = ref_field;
1834		ref_field->var_ref_idx = hist_data->n_var_refs++;
1835	}
1836
1837	return ref_field;
1838}
1839
1840static bool is_var_ref(char *var_name)
1841{
1842	if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
1843		return false;
1844
1845	return true;
1846}
1847
1848static char *field_name_from_var(struct hist_trigger_data *hist_data,
1849				 char *var_name)
1850{
1851	char *name, *field;
1852	unsigned int i;
1853
1854	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
1855		name = hist_data->attrs->var_defs.name[i];
1856
1857		if (strcmp(var_name, name) == 0) {
1858			field = hist_data->attrs->var_defs.expr[i];
1859			if (contains_operator(field) || is_var_ref(field))
1860				continue;
1861			return field;
1862		}
1863	}
1864
1865	return NULL;
1866}
1867
1868static char *local_field_var_ref(struct hist_trigger_data *hist_data,
1869				 char *system, char *event_name,
1870				 char *var_name)
1871{
1872	struct trace_event_call *call;
1873
1874	if (system && event_name) {
1875		call = hist_data->event_file->event_call;
1876
1877		if (strcmp(system, call->class->system) != 0)
1878			return NULL;
1879
1880		if (strcmp(event_name, trace_event_name(call)) != 0)
1881			return NULL;
1882	}
1883
1884	if (!!system != !!event_name)
1885		return NULL;
1886
1887	if (!is_var_ref(var_name))
1888		return NULL;
1889
1890	var_name++;
1891
1892	return field_name_from_var(hist_data, var_name);
1893}
1894
1895static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
1896					char *system, char *event_name,
1897					char *var_name)
1898{
1899	struct hist_field *var_field = NULL, *ref_field = NULL;
1900	struct trace_array *tr = hist_data->event_file->tr;
1901
1902	if (!is_var_ref(var_name))
1903		return NULL;
1904
1905	var_name++;
1906
1907	var_field = find_event_var(hist_data, system, event_name, var_name);
1908	if (var_field)
1909		ref_field = create_var_ref(hist_data, var_field,
1910					   system, event_name);
1911
1912	if (!ref_field)
1913		hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
1914
1915	return ref_field;
1916}
1917
1918static struct ftrace_event_field *
1919parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
1920	    char *field_str, unsigned long *flags)
1921{
1922	struct ftrace_event_field *field = NULL;
1923	char *field_name, *modifier, *str;
1924	struct trace_array *tr = file->tr;
1925
1926	modifier = str = kstrdup(field_str, GFP_KERNEL);
1927	if (!modifier)
1928		return ERR_PTR(-ENOMEM);
1929
1930	field_name = strsep(&modifier, ".");
1931	if (modifier) {
1932		if (strcmp(modifier, "hex") == 0)
1933			*flags |= HIST_FIELD_FL_HEX;
1934		else if (strcmp(modifier, "sym") == 0)
1935			*flags |= HIST_FIELD_FL_SYM;
1936		else if (strcmp(modifier, "sym-offset") == 0)
1937			*flags |= HIST_FIELD_FL_SYM_OFFSET;
1938		else if ((strcmp(modifier, "execname") == 0) &&
1939			 (strcmp(field_name, "common_pid") == 0))
1940			*flags |= HIST_FIELD_FL_EXECNAME;
1941		else if (strcmp(modifier, "syscall") == 0)
1942			*flags |= HIST_FIELD_FL_SYSCALL;
1943		else if (strcmp(modifier, "log2") == 0)
1944			*flags |= HIST_FIELD_FL_LOG2;
1945		else if (strcmp(modifier, "usecs") == 0)
1946			*flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
1947		else {
1948			hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
1949			field = ERR_PTR(-EINVAL);
1950			goto out;
1951		}
1952	}
1953
1954	if (strcmp(field_name, "common_timestamp") == 0) {
1955		*flags |= HIST_FIELD_FL_TIMESTAMP;
1956		hist_data->enable_timestamps = true;
1957		if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1958			hist_data->attrs->ts_in_usecs = true;
1959	} else if (strcmp(field_name, "cpu") == 0)
1960		*flags |= HIST_FIELD_FL_CPU;
1961	else {
1962		field = trace_find_event_field(file->event_call, field_name);
1963		if (!field || !field->size) {
1964			hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name));
1965			field = ERR_PTR(-EINVAL);
1966			goto out;
1967		}
1968	}
1969 out:
1970	kfree(str);
1971
1972	return field;
1973}
1974
1975static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
1976				       struct hist_field *var_ref,
1977				       char *var_name)
1978{
1979	struct hist_field *alias = NULL;
1980	unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
1981
1982	alias = create_hist_field(hist_data, NULL, flags, var_name);
1983	if (!alias)
1984		return NULL;
1985
1986	alias->fn = var_ref->fn;
1987	alias->operands[0] = var_ref;
1988
1989	if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
1990		destroy_hist_field(alias, 0);
1991		return NULL;
1992	}
1993
1994	alias->var_ref_idx = var_ref->var_ref_idx;
1995
1996	return alias;
1997}
1998
1999static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2000				     struct trace_event_file *file, char *str,
2001				     unsigned long *flags, char *var_name)
2002{
2003	char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2004	struct ftrace_event_field *field = NULL;
2005	struct hist_field *hist_field = NULL;
2006	int ret = 0;
2007
2008	s = strchr(str, '.');
2009	if (s) {
2010		s = strchr(++s, '.');
2011		if (s) {
2012			ref_system = strsep(&str, ".");
2013			if (!str) {
2014				ret = -EINVAL;
2015				goto out;
2016			}
2017			ref_event = strsep(&str, ".");
2018			if (!str) {
2019				ret = -EINVAL;
2020				goto out;
2021			}
2022			ref_var = str;
2023		}
2024	}
2025
2026	s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2027	if (!s) {
2028		hist_field = parse_var_ref(hist_data, ref_system,
2029					   ref_event, ref_var);
2030		if (hist_field) {
2031			if (var_name) {
2032				hist_field = create_alias(hist_data, hist_field, var_name);
2033				if (!hist_field) {
2034					ret = -ENOMEM;
2035					goto out;
2036				}
2037			}
2038			return hist_field;
2039		}
2040	} else
2041		str = s;
2042
2043	field = parse_field(hist_data, file, str, flags);
2044	if (IS_ERR(field)) {
2045		ret = PTR_ERR(field);
2046		goto out;
2047	}
2048
2049	hist_field = create_hist_field(hist_data, field, *flags, var_name);
2050	if (!hist_field) {
2051		ret = -ENOMEM;
2052		goto out;
2053	}
2054
2055	return hist_field;
2056 out:
2057	return ERR_PTR(ret);
2058}
2059
2060static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2061				     struct trace_event_file *file,
2062				     char *str, unsigned long flags,
2063				     char *var_name, unsigned int level);
2064
2065static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2066				      struct trace_event_file *file,
2067				      char *str, unsigned long flags,
2068				      char *var_name, unsigned int level)
2069{
2070	struct hist_field *operand1, *expr = NULL;
2071	unsigned long operand_flags;
2072	int ret = 0;
2073	char *s;
2074
2075	/* we support only -(xxx) i.e. explicit parens required */
2076
2077	if (level > 3) {
2078		hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2079		ret = -EINVAL;
2080		goto free;
2081	}
2082
2083	str++; /* skip leading '-' */
2084
2085	s = strchr(str, '(');
2086	if (s)
2087		str++;
2088	else {
2089		ret = -EINVAL;
2090		goto free;
2091	}
2092
2093	s = strrchr(str, ')');
2094	if (s)
2095		*s = '\0';
2096	else {
2097		ret = -EINVAL; /* no closing ')' */
2098		goto free;
2099	}
2100
2101	flags |= HIST_FIELD_FL_EXPR;
2102	expr = create_hist_field(hist_data, NULL, flags, var_name);
2103	if (!expr) {
2104		ret = -ENOMEM;
2105		goto free;
2106	}
2107
2108	operand_flags = 0;
2109	operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2110	if (IS_ERR(operand1)) {
2111		ret = PTR_ERR(operand1);
2112		goto free;
2113	}
2114
2115	expr->flags |= operand1->flags &
2116		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2117	expr->fn = hist_field_unary_minus;
2118	expr->operands[0] = operand1;
2119	expr->operator = FIELD_OP_UNARY_MINUS;
2120	expr->name = expr_str(expr, 0);
2121	expr->type = kstrdup(operand1->type, GFP_KERNEL);
2122	if (!expr->type) {
2123		ret = -ENOMEM;
2124		goto free;
2125	}
2126
2127	return expr;
2128 free:
2129	destroy_hist_field(expr, 0);
2130	return ERR_PTR(ret);
2131}
2132
2133static int check_expr_operands(struct trace_array *tr,
2134			       struct hist_field *operand1,
2135			       struct hist_field *operand2)
2136{
2137	unsigned long operand1_flags = operand1->flags;
2138	unsigned long operand2_flags = operand2->flags;
2139
2140	if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2141	    (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2142		struct hist_field *var;
2143
2144		var = find_var_field(operand1->var.hist_data, operand1->name);
2145		if (!var)
2146			return -EINVAL;
2147		operand1_flags = var->flags;
2148	}
2149
2150	if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2151	    (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2152		struct hist_field *var;
2153
2154		var = find_var_field(operand2->var.hist_data, operand2->name);
2155		if (!var)
2156			return -EINVAL;
2157		operand2_flags = var->flags;
2158	}
2159
2160	if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2161	    (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2162		hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2163		return -EINVAL;
2164	}
2165
2166	return 0;
2167}
2168
2169static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2170				     struct trace_event_file *file,
2171				     char *str, unsigned long flags,
2172				     char *var_name, unsigned int level)
2173{
2174	struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2175	unsigned long operand_flags;
2176	int field_op, ret = -EINVAL;
2177	char *sep, *operand1_str;
2178
2179	if (level > 3) {
2180		hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2181		return ERR_PTR(-EINVAL);
2182	}
2183
2184	field_op = contains_operator(str);
2185
2186	if (field_op == FIELD_OP_NONE)
2187		return parse_atom(hist_data, file, str, &flags, var_name);
2188
2189	if (field_op == FIELD_OP_UNARY_MINUS)
2190		return parse_unary(hist_data, file, str, flags, var_name, ++level);
2191
2192	switch (field_op) {
2193	case FIELD_OP_MINUS:
2194		sep = "-";
2195		break;
2196	case FIELD_OP_PLUS:
2197		sep = "+";
2198		break;
2199	default:
2200		goto free;
2201	}
2202
2203	operand1_str = strsep(&str, sep);
2204	if (!operand1_str || !str)
2205		goto free;
2206
2207	operand_flags = 0;
2208	operand1 = parse_atom(hist_data, file, operand1_str,
2209			      &operand_flags, NULL);
2210	if (IS_ERR(operand1)) {
2211		ret = PTR_ERR(operand1);
2212		operand1 = NULL;
2213		goto free;
2214	}
2215
2216	/* rest of string could be another expression e.g. b+c in a+b+c */
2217	operand_flags = 0;
2218	operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2219	if (IS_ERR(operand2)) {
2220		ret = PTR_ERR(operand2);
2221		operand2 = NULL;
2222		goto free;
2223	}
2224
2225	ret = check_expr_operands(file->tr, operand1, operand2);
2226	if (ret)
2227		goto free;
2228
2229	flags |= HIST_FIELD_FL_EXPR;
2230
2231	flags |= operand1->flags &
2232		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2233
2234	expr = create_hist_field(hist_data, NULL, flags, var_name);
2235	if (!expr) {
2236		ret = -ENOMEM;
2237		goto free;
2238	}
2239
2240	operand1->read_once = true;
2241	operand2->read_once = true;
2242
2243	expr->operands[0] = operand1;
2244	expr->operands[1] = operand2;
2245	expr->operator = field_op;
2246	expr->name = expr_str(expr, 0);
2247	expr->type = kstrdup(operand1->type, GFP_KERNEL);
2248	if (!expr->type) {
2249		ret = -ENOMEM;
2250		goto free;
2251	}
2252
2253	switch (field_op) {
2254	case FIELD_OP_MINUS:
2255		expr->fn = hist_field_minus;
2256		break;
2257	case FIELD_OP_PLUS:
2258		expr->fn = hist_field_plus;
2259		break;
2260	default:
2261		ret = -EINVAL;
2262		goto free;
2263	}
2264
2265	return expr;
2266 free:
2267	destroy_hist_field(operand1, 0);
2268	destroy_hist_field(operand2, 0);
2269	destroy_hist_field(expr, 0);
2270
2271	return ERR_PTR(ret);
2272}
2273
2274static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2275				 struct trace_event_file *file)
2276{
2277	struct event_trigger_data *test;
2278
2279	lockdep_assert_held(&event_mutex);
2280
2281	list_for_each_entry(test, &file->triggers, list) {
2282		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2283			if (test->private_data == hist_data)
2284				return test->filter_str;
2285		}
2286	}
2287
2288	return NULL;
2289}
2290
2291static struct event_command trigger_hist_cmd;
2292static int event_hist_trigger_func(struct event_command *cmd_ops,
2293				   struct trace_event_file *file,
2294				   char *glob, char *cmd, char *param);
2295
2296static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2297			    struct hist_trigger_data *hist_data,
2298			    unsigned int n_keys)
2299{
2300	struct hist_field *target_hist_field, *hist_field;
2301	unsigned int n, i, j;
2302
2303	if (hist_data->n_fields - hist_data->n_vals != n_keys)
2304		return false;
2305
2306	i = hist_data->n_vals;
2307	j = target_hist_data->n_vals;
2308
2309	for (n = 0; n < n_keys; n++) {
2310		hist_field = hist_data->fields[i + n];
2311		target_hist_field = target_hist_data->fields[j + n];
2312
2313		if (strcmp(hist_field->type, target_hist_field->type) != 0)
2314			return false;
2315		if (hist_field->size != target_hist_field->size)
2316			return false;
2317		if (hist_field->is_signed != target_hist_field->is_signed)
2318			return false;
2319	}
2320
2321	return true;
2322}
2323
2324static struct hist_trigger_data *
2325find_compatible_hist(struct hist_trigger_data *target_hist_data,
2326		     struct trace_event_file *file)
2327{
2328	struct hist_trigger_data *hist_data;
2329	struct event_trigger_data *test;
2330	unsigned int n_keys;
2331
2332	lockdep_assert_held(&event_mutex);
2333
2334	n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2335
2336	list_for_each_entry(test, &file->triggers, list) {
2337		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2338			hist_data = test->private_data;
2339
2340			if (compatible_keys(target_hist_data, hist_data, n_keys))
2341				return hist_data;
2342		}
2343	}
2344
2345	return NULL;
2346}
2347
2348static struct trace_event_file *event_file(struct trace_array *tr,
2349					   char *system, char *event_name)
2350{
2351	struct trace_event_file *file;
2352
2353	file = __find_event_file(tr, system, event_name);
2354	if (!file)
2355		return ERR_PTR(-EINVAL);
2356
2357	return file;
2358}
2359
2360static struct hist_field *
2361find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2362			 char *system, char *event_name, char *field_name)
2363{
2364	struct hist_field *event_var;
2365	char *synthetic_name;
2366
2367	synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2368	if (!synthetic_name)
2369		return ERR_PTR(-ENOMEM);
2370
2371	strcpy(synthetic_name, "synthetic_");
2372	strcat(synthetic_name, field_name);
2373
2374	event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2375
2376	kfree(synthetic_name);
2377
2378	return event_var;
2379}
2380
2381/**
2382 * create_field_var_hist - Automatically create a histogram and var for a field
2383 * @target_hist_data: The target hist trigger
2384 * @subsys_name: Optional subsystem name
2385 * @event_name: Optional event name
2386 * @field_name: The name of the field (and the resulting variable)
2387 *
2388 * Hist trigger actions fetch data from variables, not directly from
2389 * events.  However, for convenience, users are allowed to directly
2390 * specify an event field in an action, which will be automatically
2391 * converted into a variable on their behalf.
2392
2393 * If a user specifies a field on an event that isn't the event the
2394 * histogram currently being defined (the target event histogram), the
2395 * only way that can be accomplished is if a new hist trigger is
2396 * created and the field variable defined on that.
2397 *
2398 * This function creates a new histogram compatible with the target
2399 * event (meaning a histogram with the same key as the target
2400 * histogram), and creates a variable for the specified field, but
2401 * with 'synthetic_' prepended to the variable name in order to avoid
2402 * collision with normal field variables.
2403 *
2404 * Return: The variable created for the field.
2405 */
2406static struct hist_field *
2407create_field_var_hist(struct hist_trigger_data *target_hist_data,
2408		      char *subsys_name, char *event_name, char *field_name)
2409{
2410	struct trace_array *tr = target_hist_data->event_file->tr;
2411	struct hist_field *event_var = ERR_PTR(-EINVAL);
2412	struct hist_trigger_data *hist_data;
2413	unsigned int i, n, first = true;
2414	struct field_var_hist *var_hist;
2415	struct trace_event_file *file;
2416	struct hist_field *key_field;
2417	char *saved_filter;
2418	char *cmd;
2419	int ret;
2420
2421	if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2422		hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2423		return ERR_PTR(-EINVAL);
2424	}
2425
2426	file = event_file(tr, subsys_name, event_name);
2427
2428	if (IS_ERR(file)) {
2429		hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2430		ret = PTR_ERR(file);
2431		return ERR_PTR(ret);
2432	}
2433
2434	/*
2435	 * Look for a histogram compatible with target.  We'll use the
2436	 * found histogram specification to create a new matching
2437	 * histogram with our variable on it.  target_hist_data is not
2438	 * yet a registered histogram so we can't use that.
2439	 */
2440	hist_data = find_compatible_hist(target_hist_data, file);
2441	if (!hist_data) {
2442		hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2443		return ERR_PTR(-EINVAL);
2444	}
2445
2446	/* See if a synthetic field variable has already been created */
2447	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2448					     event_name, field_name);
2449	if (!IS_ERR_OR_NULL(event_var))
2450		return event_var;
2451
2452	var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2453	if (!var_hist)
2454		return ERR_PTR(-ENOMEM);
2455
2456	cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2457	if (!cmd) {
2458		kfree(var_hist);
2459		return ERR_PTR(-ENOMEM);
2460	}
2461
2462	/* Use the same keys as the compatible histogram */
2463	strcat(cmd, "keys=");
2464
2465	for_each_hist_key_field(i, hist_data) {
2466		key_field = hist_data->fields[i];
2467		if (!first)
2468			strcat(cmd, ",");
2469		strcat(cmd, key_field->field->name);
2470		first = false;
2471	}
2472
2473	/* Create the synthetic field variable specification */
2474	strcat(cmd, ":synthetic_");
2475	strcat(cmd, field_name);
2476	strcat(cmd, "=");
2477	strcat(cmd, field_name);
2478
2479	/* Use the same filter as the compatible histogram */
2480	saved_filter = find_trigger_filter(hist_data, file);
2481	if (saved_filter) {
2482		strcat(cmd, " if ");
2483		strcat(cmd, saved_filter);
2484	}
2485
2486	var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2487	if (!var_hist->cmd) {
2488		kfree(cmd);
2489		kfree(var_hist);
2490		return ERR_PTR(-ENOMEM);
2491	}
2492
2493	/* Save the compatible histogram information */
2494	var_hist->hist_data = hist_data;
2495
2496	/* Create the new histogram with our variable */
2497	ret = event_hist_trigger_func(&trigger_hist_cmd, file,
2498				      "", "hist", cmd);
2499	if (ret) {
2500		kfree(cmd);
2501		kfree(var_hist->cmd);
2502		kfree(var_hist);
2503		hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2504		return ERR_PTR(ret);
2505	}
2506
2507	kfree(cmd);
2508
2509	/* If we can't find the variable, something went wrong */
2510	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2511					     event_name, field_name);
2512	if (IS_ERR_OR_NULL(event_var)) {
2513		kfree(var_hist->cmd);
2514		kfree(var_hist);
2515		hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
2516		return ERR_PTR(-EINVAL);
2517	}
2518
2519	n = target_hist_data->n_field_var_hists;
2520	target_hist_data->field_var_hists[n] = var_hist;
2521	target_hist_data->n_field_var_hists++;
2522
2523	return event_var;
2524}
2525
2526static struct hist_field *
2527find_target_event_var(struct hist_trigger_data *hist_data,
2528		      char *subsys_name, char *event_name, char *var_name)
2529{
2530	struct trace_event_file *file = hist_data->event_file;
2531	struct hist_field *hist_field = NULL;
2532
2533	if (subsys_name) {
2534		struct trace_event_call *call;
2535
2536		if (!event_name)
2537			return NULL;
2538
2539		call = file->event_call;
2540
2541		if (strcmp(subsys_name, call->class->system) != 0)
2542			return NULL;
2543
2544		if (strcmp(event_name, trace_event_name(call)) != 0)
2545			return NULL;
2546	}
2547
2548	hist_field = find_var_field(hist_data, var_name);
2549
2550	return hist_field;
2551}
2552
2553static inline void __update_field_vars(struct tracing_map_elt *elt,
2554				       struct ring_buffer_event *rbe,
2555				       void *rec,
2556				       struct field_var **field_vars,
2557				       unsigned int n_field_vars,
2558				       unsigned int field_var_str_start)
2559{
2560	struct hist_elt_data *elt_data = elt->private_data;
2561	unsigned int i, j, var_idx;
2562	u64 var_val;
2563
2564	for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
2565		struct field_var *field_var = field_vars[i];
2566		struct hist_field *var = field_var->var;
2567		struct hist_field *val = field_var->val;
2568
2569		var_val = val->fn(val, elt, rbe, rec);
2570		var_idx = var->var.idx;
2571
2572		if (val->flags & HIST_FIELD_FL_STRING) {
2573			char *str = elt_data->field_var_str[j++];
2574			char *val_str = (char *)(uintptr_t)var_val;
2575
2576			strscpy(str, val_str, STR_VAR_LEN_MAX);
2577			var_val = (u64)(uintptr_t)str;
2578		}
2579		tracing_map_set_var(elt, var_idx, var_val);
2580	}
2581}
2582
2583static void update_field_vars(struct hist_trigger_data *hist_data,
2584			      struct tracing_map_elt *elt,
2585			      struct ring_buffer_event *rbe,
2586			      void *rec)
2587{
2588	__update_field_vars(elt, rbe, rec, hist_data->field_vars,
2589			    hist_data->n_field_vars, 0);
2590}
2591
2592static void save_track_data_vars(struct hist_trigger_data *hist_data,
2593				 struct tracing_map_elt *elt, void *rec,
2594				 struct ring_buffer_event *rbe, void *key,
2595				 struct action_data *data, u64 *var_ref_vals)
2596{
2597	__update_field_vars(elt, rbe, rec, hist_data->save_vars,
2598			    hist_data->n_save_vars, hist_data->n_field_var_str);
2599}
2600
2601static struct hist_field *create_var(struct hist_trigger_data *hist_data,
2602				     struct trace_event_file *file,
2603				     char *name, int size, const char *type)
2604{
2605	struct hist_field *var;
2606	int idx;
2607
2608	if (find_var(hist_data, file, name) && !hist_data->remove) {
2609		var = ERR_PTR(-EINVAL);
2610		goto out;
2611	}
2612
2613	var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2614	if (!var) {
2615		var = ERR_PTR(-ENOMEM);
2616		goto out;
2617	}
2618
2619	idx = tracing_map_add_var(hist_data->map);
2620	if (idx < 0) {
2621		kfree(var);
2622		var = ERR_PTR(-EINVAL);
2623		goto out;
2624	}
2625
2626	var->ref = 1;
2627	var->flags = HIST_FIELD_FL_VAR;
2628	var->var.idx = idx;
2629	var->var.hist_data = var->hist_data = hist_data;
2630	var->size = size;
2631	var->var.name = kstrdup(name, GFP_KERNEL);
2632	var->type = kstrdup(type, GFP_KERNEL);
2633	if (!var->var.name || !var->type) {
2634		kfree(var->var.name);
2635		kfree(var->type);
2636		kfree(var);
2637		var = ERR_PTR(-ENOMEM);
2638	}
2639 out:
2640	return var;
2641}
2642
2643static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
2644					  struct trace_event_file *file,
2645					  char *field_name)
2646{
2647	struct hist_field *val = NULL, *var = NULL;
2648	unsigned long flags = HIST_FIELD_FL_VAR;
2649	struct trace_array *tr = file->tr;
2650	struct field_var *field_var;
2651	int ret = 0;
2652
2653	if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
2654		hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2655		ret = -EINVAL;
2656		goto err;
2657	}
2658
2659	val = parse_atom(hist_data, file, field_name, &flags, NULL);
2660	if (IS_ERR(val)) {
2661		hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
2662		ret = PTR_ERR(val);
2663		goto err;
2664	}
2665
2666	var = create_var(hist_data, file, field_name, val->size, val->type);
2667	if (IS_ERR(var)) {
2668		hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
2669		kfree(val);
2670		ret = PTR_ERR(var);
2671		goto err;
2672	}
2673
2674	field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
2675	if (!field_var) {
2676		kfree(val);
2677		kfree(var);
2678		ret =  -ENOMEM;
2679		goto err;
2680	}
2681
2682	field_var->var = var;
2683	field_var->val = val;
2684 out:
2685	return field_var;
2686 err:
2687	field_var = ERR_PTR(ret);
2688	goto out;
2689}
2690
2691/**
2692 * create_target_field_var - Automatically create a variable for a field
2693 * @target_hist_data: The target hist trigger
2694 * @subsys_name: Optional subsystem name
2695 * @event_name: Optional event name
2696 * @var_name: The name of the field (and the resulting variable)
2697 *
2698 * Hist trigger actions fetch data from variables, not directly from
2699 * events.  However, for convenience, users are allowed to directly
2700 * specify an event field in an action, which will be automatically
2701 * converted into a variable on their behalf.
2702
2703 * This function creates a field variable with the name var_name on
2704 * the hist trigger currently being defined on the target event.  If
2705 * subsys_name and event_name are specified, this function simply
2706 * verifies that they do in fact match the target event subsystem and
2707 * event name.
2708 *
2709 * Return: The variable created for the field.
2710 */
2711static struct field_var *
2712create_target_field_var(struct hist_trigger_data *target_hist_data,
2713			char *subsys_name, char *event_name, char *var_name)
2714{
2715	struct trace_event_file *file = target_hist_data->event_file;
2716
2717	if (subsys_name) {
2718		struct trace_event_call *call;
2719
2720		if (!event_name)
2721			return NULL;
2722
2723		call = file->event_call;
2724
2725		if (strcmp(subsys_name, call->class->system) != 0)
2726			return NULL;
2727
2728		if (strcmp(event_name, trace_event_name(call)) != 0)
2729			return NULL;
2730	}
2731
2732	return create_field_var(target_hist_data, file, var_name);
2733}
2734
2735static bool check_track_val_max(u64 track_val, u64 var_val)
2736{
2737	if (var_val <= track_val)
2738		return false;
2739
2740	return true;
2741}
2742
2743static bool check_track_val_changed(u64 track_val, u64 var_val)
2744{
2745	if (var_val == track_val)
2746		return false;
2747
2748	return true;
2749}
2750
2751static u64 get_track_val(struct hist_trigger_data *hist_data,
2752			 struct tracing_map_elt *elt,
2753			 struct action_data *data)
2754{
2755	unsigned int track_var_idx = data->track_data.track_var->var.idx;
2756	u64 track_val;
2757
2758	track_val = tracing_map_read_var(elt, track_var_idx);
2759
2760	return track_val;
2761}
2762
2763static void save_track_val(struct hist_trigger_data *hist_data,
2764			   struct tracing_map_elt *elt,
2765			   struct action_data *data, u64 var_val)
2766{
2767	unsigned int track_var_idx = data->track_data.track_var->var.idx;
2768
2769	tracing_map_set_var(elt, track_var_idx, var_val);
2770}
2771
2772static void save_track_data(struct hist_trigger_data *hist_data,
2773			    struct tracing_map_elt *elt, void *rec,
2774			    struct ring_buffer_event *rbe, void *key,
2775			    struct action_data *data, u64 *var_ref_vals)
2776{
2777	if (data->track_data.save_data)
2778		data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
2779}
2780
2781static bool check_track_val(struct tracing_map_elt *elt,
2782			    struct action_data *data,
2783			    u64 var_val)
2784{
2785	struct hist_trigger_data *hist_data;
2786	u64 track_val;
2787
2788	hist_data = data->track_data.track_var->hist_data;
2789	track_val = get_track_val(hist_data, elt, data);
2790
2791	return data->track_data.check_val(track_val, var_val);
2792}
2793
2794#ifdef CONFIG_TRACER_SNAPSHOT
2795static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2796{
2797	/* called with tr->max_lock held */
2798	struct track_data *track_data = tr->cond_snapshot->cond_data;
2799	struct hist_elt_data *elt_data, *track_elt_data;
2800	struct snapshot_context *context = cond_data;
2801	struct action_data *action;
2802	u64 track_val;
2803
2804	if (!track_data)
2805		return false;
2806
2807	action = track_data->action_data;
2808
2809	track_val = get_track_val(track_data->hist_data, context->elt,
2810				  track_data->action_data);
2811
2812	if (!action->track_data.check_val(track_data->track_val, track_val))
2813		return false;
2814
2815	track_data->track_val = track_val;
2816	memcpy(track_data->key, context->key, track_data->key_len);
2817
2818	elt_data = context->elt->private_data;
2819	track_elt_data = track_data->elt.private_data;
2820	if (elt_data->comm)
2821		strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
2822
2823	track_data->updated = true;
2824
2825	return true;
2826}
2827
2828static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2829				     struct tracing_map_elt *elt, void *rec,
2830				     struct ring_buffer_event *rbe, void *key,
2831				     struct action_data *data,
2832				     u64 *var_ref_vals)
2833{
2834	struct trace_event_file *file = hist_data->event_file;
2835	struct snapshot_context context;
2836
2837	context.elt = elt;
2838	context.key = key;
2839
2840	tracing_snapshot_cond(file->tr, &context);
2841}
2842
2843static void hist_trigger_print_key(struct seq_file *m,
2844				   struct hist_trigger_data *hist_data,
2845				   void *key,
2846				   struct tracing_map_elt *elt);
2847
2848static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
2849{
2850	unsigned int i;
2851
2852	if (!hist_data->n_actions)
2853		return NULL;
2854
2855	for (i = 0; i < hist_data->n_actions; i++) {
2856		struct action_data *data = hist_data->actions[i];
2857
2858		if (data->action == ACTION_SNAPSHOT)
2859			return data;
2860	}
2861
2862	return NULL;
2863}
2864
2865static void track_data_snapshot_print(struct seq_file *m,
2866				      struct hist_trigger_data *hist_data)
2867{
2868	struct trace_event_file *file = hist_data->event_file;
2869	struct track_data *track_data;
2870	struct action_data *action;
2871
2872	track_data = tracing_cond_snapshot_data(file->tr);
2873	if (!track_data)
2874		return;
2875
2876	if (!track_data->updated)
2877		return;
2878
2879	action = snapshot_action(hist_data);
2880	if (!action)
2881		return;
2882
2883	seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
2884	seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
2885		   action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
2886		   action->track_data.var_str, track_data->track_val);
2887
2888	seq_puts(m, "\ttriggered by event with key: ");
2889	hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
2890	seq_putc(m, '\n');
2891}
2892#else
2893static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2894{
2895	return false;
2896}
2897static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2898				     struct tracing_map_elt *elt, void *rec,
2899				     struct ring_buffer_event *rbe, void *key,
2900				     struct action_data *data,
2901				     u64 *var_ref_vals) {}
2902static void track_data_snapshot_print(struct seq_file *m,
2903				      struct hist_trigger_data *hist_data) {}
2904#endif /* CONFIG_TRACER_SNAPSHOT */
2905
2906static void track_data_print(struct seq_file *m,
2907			     struct hist_trigger_data *hist_data,
2908			     struct tracing_map_elt *elt,
2909			     struct action_data *data)
2910{
2911	u64 track_val = get_track_val(hist_data, elt, data);
2912	unsigned int i, save_var_idx;
2913
2914	if (data->handler == HANDLER_ONMAX)
2915		seq_printf(m, "\n\tmax: %10llu", track_val);
2916	else if (data->handler == HANDLER_ONCHANGE)
2917		seq_printf(m, "\n\tchanged: %10llu", track_val);
2918
2919	if (data->action == ACTION_SNAPSHOT)
2920		return;
2921
2922	for (i = 0; i < hist_data->n_save_vars; i++) {
2923		struct hist_field *save_val = hist_data->save_vars[i]->val;
2924		struct hist_field *save_var = hist_data->save_vars[i]->var;
2925		u64 val;
2926
2927		save_var_idx = save_var->var.idx;
2928
2929		val = tracing_map_read_var(elt, save_var_idx);
2930
2931		if (save_val->flags & HIST_FIELD_FL_STRING) {
2932			seq_printf(m, "  %s: %-32s", save_var->var.name,
2933				   (char *)(uintptr_t)(val));
2934		} else
2935			seq_printf(m, "  %s: %10llu", save_var->var.name, val);
2936	}
2937}
2938
2939static void ontrack_action(struct hist_trigger_data *hist_data,
2940			   struct tracing_map_elt *elt, void *rec,
2941			   struct ring_buffer_event *rbe, void *key,
2942			   struct action_data *data, u64 *var_ref_vals)
2943{
2944	u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
2945
2946	if (check_track_val(elt, data, var_val)) {
2947		save_track_val(hist_data, elt, data, var_val);
2948		save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
2949	}
2950}
2951
2952static void action_data_destroy(struct action_data *data)
2953{
2954	unsigned int i;
2955
2956	lockdep_assert_held(&event_mutex);
2957
2958	kfree(data->action_name);
2959
2960	for (i = 0; i < data->n_params; i++)
2961		kfree(data->params[i]);
2962
2963	if (data->synth_event)
2964		data->synth_event->ref--;
2965
2966	kfree(data->synth_event_name);
2967
2968	kfree(data);
2969}
2970
2971static void track_data_destroy(struct hist_trigger_data *hist_data,
2972			       struct action_data *data)
2973{
2974	struct trace_event_file *file = hist_data->event_file;
2975
2976	destroy_hist_field(data->track_data.track_var, 0);
2977
2978	if (data->action == ACTION_SNAPSHOT) {
2979		struct track_data *track_data;
2980
2981		track_data = tracing_cond_snapshot_data(file->tr);
2982		if (track_data && track_data->hist_data == hist_data) {
2983			tracing_snapshot_cond_disable(file->tr);
2984			track_data_free(track_data);
2985		}
2986	}
2987
2988	kfree(data->track_data.var_str);
2989
2990	action_data_destroy(data);
2991}
2992
2993static int action_create(struct hist_trigger_data *hist_data,
2994			 struct action_data *data);
2995
2996static int track_data_create(struct hist_trigger_data *hist_data,
2997			     struct action_data *data)
2998{
2999	struct hist_field *var_field, *ref_field, *track_var = NULL;
3000	struct trace_event_file *file = hist_data->event_file;
3001	struct trace_array *tr = file->tr;
3002	char *track_data_var_str;
3003	int ret = 0;
3004
3005	track_data_var_str = data->track_data.var_str;
3006	if (track_data_var_str[0] != '$') {
3007		hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3008		return -EINVAL;
3009	}
3010	track_data_var_str++;
3011
3012	var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3013	if (!var_field) {
3014		hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3015		return -EINVAL;
3016	}
3017
3018	ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3019	if (!ref_field)
3020		return -ENOMEM;
3021
3022	data->track_data.var_ref = ref_field;
3023
3024	if (data->handler == HANDLER_ONMAX)
3025		track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3026	if (IS_ERR(track_var)) {
3027		hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3028		ret = PTR_ERR(track_var);
3029		goto out;
3030	}
3031
3032	if (data->handler == HANDLER_ONCHANGE)
3033		track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3034	if (IS_ERR(track_var)) {
3035		hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3036		ret = PTR_ERR(track_var);
3037		goto out;
3038	}
3039	data->track_data.track_var = track_var;
3040
3041	ret = action_create(hist_data, data);
3042 out:
3043	return ret;
3044}
3045
3046static int parse_action_params(struct trace_array *tr, char *params,
3047			       struct action_data *data)
3048{
3049	char *param, *saved_param;
3050	bool first_param = true;
3051	int ret = 0;
3052
3053	while (params) {
3054		if (data->n_params >= SYNTH_FIELDS_MAX) {
3055			hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3056			goto out;
3057		}
3058
3059		param = strsep(&params, ",");
3060		if (!param) {
3061			hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3062			ret = -EINVAL;
3063			goto out;
3064		}
3065
3066		param = strstrip(param);
3067		if (strlen(param) < 2) {
3068			hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3069			ret = -EINVAL;
3070			goto out;
3071		}
3072
3073		saved_param = kstrdup(param, GFP_KERNEL);
3074		if (!saved_param) {
3075			ret = -ENOMEM;
3076			goto out;
3077		}
3078
3079		if (first_param && data->use_trace_keyword) {
3080			data->synth_event_name = saved_param;
3081			first_param = false;
3082			continue;
3083		}
3084		first_param = false;
3085
3086		data->params[data->n_params++] = saved_param;
3087	}
3088 out:
3089	return ret;
3090}
3091
3092static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3093			enum handler_id handler)
3094{
3095	char *action_name;
3096	int ret = 0;
3097
3098	strsep(&str, ".");
3099	if (!str) {
3100		hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3101		ret = -EINVAL;
3102		goto out;
3103	}
3104
3105	action_name = strsep(&str, "(");
3106	if (!action_name || !str) {
3107		hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3108		ret = -EINVAL;
3109		goto out;
3110	}
3111
3112	if (str_has_prefix(action_name, "save")) {
3113		char *params = strsep(&str, ")");
3114
3115		if (!params) {
3116			hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3117			ret = -EINVAL;
3118			goto out;
3119		}
3120
3121		ret = parse_action_params(tr, params, data);
3122		if (ret)
3123			goto out;
3124
3125		if (handler == HANDLER_ONMAX)
3126			data->track_data.check_val = check_track_val_max;
3127		else if (handler == HANDLER_ONCHANGE)
3128			data->track_data.check_val = check_track_val_changed;
3129		else {
3130			hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3131			ret = -EINVAL;
3132			goto out;
3133		}
3134
3135		data->track_data.save_data = save_track_data_vars;
3136		data->fn = ontrack_action;
3137		data->action = ACTION_SAVE;
3138	} else if (str_has_prefix(action_name, "snapshot")) {
3139		char *params = strsep(&str, ")");
3140
3141		if (!str) {
3142			hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3143			ret = -EINVAL;
3144			goto out;
3145		}
3146
3147		if (handler == HANDLER_ONMAX)
3148			data->track_data.check_val = check_track_val_max;
3149		else if (handler == HANDLER_ONCHANGE)
3150			data->track_data.check_val = check_track_val_changed;
3151		else {
3152			hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3153			ret = -EINVAL;
3154			goto out;
3155		}
3156
3157		data->track_data.save_data = save_track_data_snapshot;
3158		data->fn = ontrack_action;
3159		data->action = ACTION_SNAPSHOT;
3160	} else {
3161		char *params = strsep(&str, ")");
3162
3163		if (str_has_prefix(action_name, "trace"))
3164			data->use_trace_keyword = true;
3165
3166		if (params) {
3167			ret = parse_action_params(tr, params, data);
3168			if (ret)
3169				goto out;
3170		}
3171
3172		if (handler == HANDLER_ONMAX)
3173			data->track_data.check_val = check_track_val_max;
3174		else if (handler == HANDLER_ONCHANGE)
3175			data->track_data.check_val = check_track_val_changed;
3176
3177		if (handler != HANDLER_ONMATCH) {
3178			data->track_data.save_data = action_trace;
3179			data->fn = ontrack_action;
3180		} else
3181			data->fn = action_trace;
3182
3183		data->action = ACTION_TRACE;
3184	}
3185
3186	data->action_name = kstrdup(action_name, GFP_KERNEL);
3187	if (!data->action_name) {
3188		ret = -ENOMEM;
3189		goto out;
3190	}
3191
3192	data->handler = handler;
3193 out:
3194	return ret;
3195}
3196
3197static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3198					    char *str, enum handler_id handler)
3199{
3200	struct action_data *data;
3201	int ret = -EINVAL;
3202	char *var_str;
3203
3204	data = kzalloc(sizeof(*data), GFP_KERNEL);
3205	if (!data)
3206		return ERR_PTR(-ENOMEM);
3207
3208	var_str = strsep(&str, ")");
3209	if (!var_str || !str) {
3210		ret = -EINVAL;
3211		goto free;
3212	}
3213
3214	data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3215	if (!data->track_data.var_str) {
3216		ret = -ENOMEM;
3217		goto free;
3218	}
3219
3220	ret = action_parse(hist_data->event_file->tr, str, data, handler);
3221	if (ret)
3222		goto free;
3223 out:
3224	return data;
3225 free:
3226	track_data_destroy(hist_data, data);
3227	data = ERR_PTR(ret);
3228	goto out;
3229}
3230
3231static void onmatch_destroy(struct action_data *data)
3232{
3233	kfree(data->match_data.event);
3234	kfree(data->match_data.event_system);
3235
3236	action_data_destroy(data);
3237}
3238
3239static void destroy_field_var(struct field_var *field_var)
3240{
3241	if (!field_var)
3242		return;
3243
3244	destroy_hist_field(field_var->var, 0);
3245	destroy_hist_field(field_var->val, 0);
3246
3247	kfree(field_var);
3248}
3249
3250static void destroy_field_vars(struct hist_trigger_data *hist_data)
3251{
3252	unsigned int i;
3253
3254	for (i = 0; i < hist_data->n_field_vars; i++)
3255		destroy_field_var(hist_data->field_vars[i]);
3256
3257	for (i = 0; i < hist_data->n_save_vars; i++)
3258		destroy_field_var(hist_data->save_vars[i]);
3259}
3260
3261static void save_field_var(struct hist_trigger_data *hist_data,
3262			   struct field_var *field_var)
3263{
3264	hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3265
3266	if (field_var->val->flags & HIST_FIELD_FL_STRING)
3267		hist_data->n_field_var_str++;
3268}
3269
3270
3271static int check_synth_field(struct synth_event *event,
3272			     struct hist_field *hist_field,
3273			     unsigned int field_pos)
3274{
3275	struct synth_field *field;
3276
3277	if (field_pos >= event->n_fields)
3278		return -EINVAL;
3279
3280	field = event->fields[field_pos];
3281
3282	if (strcmp(field->type, hist_field->type) != 0) {
3283		if (field->size != hist_field->size ||
3284		    field->is_signed != hist_field->is_signed)
3285			return -EINVAL;
3286	}
3287
3288	return 0;
3289}
3290
3291static struct hist_field *
3292trace_action_find_var(struct hist_trigger_data *hist_data,
3293		      struct action_data *data,
3294		      char *system, char *event, char *var)
3295{
3296	struct trace_array *tr = hist_data->event_file->tr;
3297	struct hist_field *hist_field;
3298
3299	var++; /* skip '$' */
3300
3301	hist_field = find_target_event_var(hist_data, system, event, var);
3302	if (!hist_field) {
3303		if (!system && data->handler == HANDLER_ONMATCH) {
3304			system = data->match_data.event_system;
3305			event = data->match_data.event;
3306		}
3307
3308		hist_field = find_event_var(hist_data, system, event, var);
3309	}
3310
3311	if (!hist_field)
3312		hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3313
3314	return hist_field;
3315}
3316
3317static struct hist_field *
3318trace_action_create_field_var(struct hist_trigger_data *hist_data,
3319			      struct action_data *data, char *system,
3320			      char *event, char *var)
3321{
3322	struct hist_field *hist_field = NULL;
3323	struct field_var *field_var;
3324
3325	/*
3326	 * First try to create a field var on the target event (the
3327	 * currently being defined).  This will create a variable for
3328	 * unqualified fields on the target event, or if qualified,
3329	 * target fields that have qualified names matching the target.
3330	 */
3331	field_var = create_target_field_var(hist_data, system, event, var);
3332
3333	if (field_var && !IS_ERR(field_var)) {
3334		save_field_var(hist_data, field_var);
3335		hist_field = field_var->var;
3336	} else {
3337		field_var = NULL;
3338		/*
3339		 * If no explicit system.event is specfied, default to
3340		 * looking for fields on the onmatch(system.event.xxx)
3341		 * event.
3342		 */
3343		if (!system && data->handler == HANDLER_ONMATCH) {
3344			system = data->match_data.event_system;
3345			event = data->match_data.event;
3346		}
3347
3348		/*
3349		 * At this point, we're looking at a field on another
3350		 * event.  Because we can't modify a hist trigger on
3351		 * another event to add a variable for a field, we need
3352		 * to create a new trigger on that event and create the
3353		 * variable at the same time.
3354		 */
3355		hist_field = create_field_var_hist(hist_data, system, event, var);
3356		if (IS_ERR(hist_field))
3357			goto free;
3358	}
3359 out:
3360	return hist_field;
3361 free:
3362	destroy_field_var(field_var);
3363	hist_field = NULL;
3364	goto out;
3365}
3366
3367static int trace_action_create(struct hist_trigger_data *hist_data,
3368			       struct action_data *data)
3369{
3370	struct trace_array *tr = hist_data->event_file->tr;
3371	char *event_name, *param, *system = NULL;
3372	struct hist_field *hist_field, *var_ref;
3373	unsigned int i;
3374	unsigned int field_pos = 0;
3375	struct synth_event *event;
3376	char *synth_event_name;
3377	int var_ref_idx, ret = 0;
3378
3379	lockdep_assert_held(&event_mutex);
3380
3381	if (data->use_trace_keyword)
3382		synth_event_name = data->synth_event_name;
3383	else
3384		synth_event_name = data->action_name;
3385
3386	event = find_synth_event(synth_event_name);
3387	if (!event) {
3388		hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3389		return -EINVAL;
3390	}
3391
3392	event->ref++;
3393
3394	for (i = 0; i < data->n_params; i++) {
3395		char *p;
3396
3397		p = param = kstrdup(data->params[i], GFP_KERNEL);
3398		if (!param) {
3399			ret = -ENOMEM;
3400			goto err;
3401		}
3402
3403		system = strsep(&param, ".");
3404		if (!param) {
3405			param = (char *)system;
3406			system = event_name = NULL;
3407		} else {
3408			event_name = strsep(&param, ".");
3409			if (!param) {
3410				kfree(p);
3411				ret = -EINVAL;
3412				goto err;
3413			}
3414		}
3415
3416		if (param[0] == '$')
3417			hist_field = trace_action_find_var(hist_data, data,
3418							   system, event_name,
3419							   param);
3420		else
3421			hist_field = trace_action_create_field_var(hist_data,
3422								   data,
3423								   system,
3424								   event_name,
3425								   param);
3426
3427		if (!hist_field) {
3428			kfree(p);
3429			ret = -EINVAL;
3430			goto err;
3431		}
3432
3433		if (check_synth_field(event, hist_field, field_pos) == 0) {
3434			var_ref = create_var_ref(hist_data, hist_field,
3435						 system, event_name);
3436			if (!var_ref) {
3437				kfree(p);
3438				ret = -ENOMEM;
3439				goto err;
3440			}
3441
3442			var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3443			if (WARN_ON(var_ref_idx < 0)) {
3444				ret = var_ref_idx;
3445				goto err;
3446			}
3447
3448			data->var_ref_idx[i] = var_ref_idx;
3449
3450			field_pos++;
3451			kfree(p);
3452			continue;
3453		}
3454
3455		hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3456		kfree(p);
3457		ret = -EINVAL;
3458		goto err;
3459	}
3460
3461	if (field_pos != event->n_fields) {
3462		hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3463		ret = -EINVAL;
3464		goto err;
3465	}
3466
3467	data->synth_event = event;
3468 out:
3469	return ret;
3470 err:
3471	event->ref--;
3472
3473	goto out;
3474}
3475
3476static int action_create(struct hist_trigger_data *hist_data,
3477			 struct action_data *data)
3478{
3479	struct trace_event_file *file = hist_data->event_file;
3480	struct trace_array *tr = file->tr;
3481	struct track_data *track_data;
3482	struct field_var *field_var;
3483	unsigned int i;
3484	char *param;
3485	int ret = 0;
3486
3487	if (data->action == ACTION_TRACE)
3488		return trace_action_create(hist_data, data);
3489
3490	if (data->action == ACTION_SNAPSHOT) {
3491		track_data = track_data_alloc(hist_data->key_size, data, hist_data);
3492		if (IS_ERR(track_data)) {
3493			ret = PTR_ERR(track_data);
3494			goto out;
3495		}
3496
3497		ret = tracing_snapshot_cond_enable(file->tr, track_data,
3498						   cond_snapshot_update);
3499		if (ret)
3500			track_data_free(track_data);
3501
3502		goto out;
3503	}
3504
3505	if (data->action == ACTION_SAVE) {
3506		if (hist_data->n_save_vars) {
3507			ret = -EEXIST;
3508			hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
3509			goto out;
3510		}
3511
3512		for (i = 0; i < data->n_params; i++) {
3513			param = kstrdup(data->params[i], GFP_KERNEL);
3514			if (!param) {
3515				ret = -ENOMEM;
3516				goto out;
3517			}
3518
3519			field_var = create_target_field_var(hist_data, NULL, NULL, param);
3520			if (IS_ERR(field_var)) {
3521				hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
3522					 errpos(param));
3523				ret = PTR_ERR(field_var);
3524				kfree(param);
3525				goto out;
3526			}
3527
3528			hist_data->save_vars[hist_data->n_save_vars++] = field_var;
3529			if (field_var->val->flags & HIST_FIELD_FL_STRING)
3530				hist_data->n_save_var_str++;
3531			kfree(param);
3532		}
3533	}
3534 out:
3535	return ret;
3536}
3537
3538static int onmatch_create(struct hist_trigger_data *hist_data,
3539			  struct action_data *data)
3540{
3541	return action_create(hist_data, data);
3542}
3543
3544static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3545{
3546	char *match_event, *match_event_system;
3547	struct action_data *data;
3548	int ret = -EINVAL;
3549
3550	data = kzalloc(sizeof(*data), GFP_KERNEL);
3551	if (!data)
3552		return ERR_PTR(-ENOMEM);
3553
3554	match_event = strsep(&str, ")");
3555	if (!match_event || !str) {
3556		hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
3557		goto free;
3558	}
3559
3560	match_event_system = strsep(&match_event, ".");
3561	if (!match_event) {
3562		hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
3563		goto free;
3564	}
3565
3566	if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3567		hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
3568		goto free;
3569	}
3570
3571	data->match_data.event = kstrdup(match_event, GFP_KERNEL);
3572	if (!data->match_data.event) {
3573		ret = -ENOMEM;
3574		goto free;
3575	}
3576
3577	data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
3578	if (!data->match_data.event_system) {
3579		ret = -ENOMEM;
3580		goto free;
3581	}
3582
3583	ret = action_parse(tr, str, data, HANDLER_ONMATCH);
3584	if (ret)
3585		goto free;
3586 out:
3587	return data;
3588 free:
3589	onmatch_destroy(data);
3590	data = ERR_PTR(ret);
3591	goto out;
3592}
3593
3594static int create_hitcount_val(struct hist_trigger_data *hist_data)
3595{
3596	hist_data->fields[HITCOUNT_IDX] =
3597		create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
3598	if (!hist_data->fields[HITCOUNT_IDX])
3599		return -ENOMEM;
3600
3601	hist_data->n_vals++;
3602	hist_data->n_fields++;
3603
3604	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3605		return -EINVAL;
3606
3607	return 0;
3608}
3609
3610static int __create_val_field(struct hist_trigger_data *hist_data,
3611			      unsigned int val_idx,
3612			      struct trace_event_file *file,
3613			      char *var_name, char *field_str,
3614			      unsigned long flags)
3615{
3616	struct hist_field *hist_field;
3617	int ret = 0;
3618
3619	hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3620	if (IS_ERR(hist_field)) {
3621		ret = PTR_ERR(hist_field);
3622		goto out;
3623	}
3624
3625	hist_data->fields[val_idx] = hist_field;
3626
3627	++hist_data->n_vals;
3628	++hist_data->n_fields;
3629
3630	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3631		ret = -EINVAL;
3632 out:
3633	return ret;
3634}
3635
3636static int create_val_field(struct hist_trigger_data *hist_data,
3637			    unsigned int val_idx,
3638			    struct trace_event_file *file,
3639			    char *field_str)
3640{
3641	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3642		return -EINVAL;
3643
3644	return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3645}
3646
3647static int create_var_field(struct hist_trigger_data *hist_data,
3648			    unsigned int val_idx,
3649			    struct trace_event_file *file,
3650			    char *var_name, char *expr_str)
3651{
3652	struct trace_array *tr = hist_data->event_file->tr;
3653	unsigned long flags = 0;
3654
3655	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3656		return -EINVAL;
3657
3658	if (find_var(hist_data, file, var_name) && !hist_data->remove) {
3659		hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
3660		return -EINVAL;
3661	}
3662
3663	flags |= HIST_FIELD_FL_VAR;
3664	hist_data->n_vars++;
3665	if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3666		return -EINVAL;
3667
3668	return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3669}
3670
3671static int create_val_fields(struct hist_trigger_data *hist_data,
3672			     struct trace_event_file *file)
3673{
3674	char *fields_str, *field_str;
3675	unsigned int i, j = 1;
3676	int ret;
3677
3678	ret = create_hitcount_val(hist_data);
3679	if (ret)
3680		goto out;
3681
3682	fields_str = hist_data->attrs->vals_str;
3683	if (!fields_str)
3684		goto out;
3685
3686	for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3687		     j < TRACING_MAP_VALS_MAX; i++) {
3688		field_str = strsep(&fields_str, ",");
3689		if (!field_str)
3690			break;
3691
3692		if (strcmp(field_str, "hitcount") == 0)
3693			continue;
3694
3695		ret = create_val_field(hist_data, j++, file, field_str);
3696		if (ret)
3697			goto out;
3698	}
3699
3700	if (fields_str && (strcmp(fields_str, "hitcount") != 0))
3701		ret = -EINVAL;
3702 out:
3703	return ret;
3704}
3705
3706static int create_key_field(struct hist_trigger_data *hist_data,
3707			    unsigned int key_idx,
3708			    unsigned int key_offset,
3709			    struct trace_event_file *file,
3710			    char *field_str)
3711{
3712	struct trace_array *tr = hist_data->event_file->tr;
3713	struct hist_field *hist_field = NULL;
3714	unsigned long flags = 0;
3715	unsigned int key_size;
3716	int ret = 0;
3717
3718	if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
3719		return -EINVAL;
3720
3721	flags |= HIST_FIELD_FL_KEY;
3722
3723	if (strcmp(field_str, "stacktrace") == 0) {
3724		flags |= HIST_FIELD_FL_STACKTRACE;
3725		key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
3726		hist_field = create_hist_field(hist_data, NULL, flags, NULL);
3727	} else {
3728		hist_field = parse_expr(hist_data, file, field_str, flags,
3729					NULL, 0);
3730		if (IS_ERR(hist_field)) {
3731			ret = PTR_ERR(hist_field);
3732			goto out;
3733		}
3734
3735		if (field_has_hist_vars(hist_field, 0))	{
3736			hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
3737			destroy_hist_field(hist_field, 0);
3738			ret = -EINVAL;
3739			goto out;
3740		}
3741
3742		key_size = hist_field->size;
3743	}
3744
3745	hist_data->fields[key_idx] = hist_field;
3746
3747	key_size = ALIGN(key_size, sizeof(u64));
3748	hist_data->fields[key_idx]->size = key_size;
3749	hist_data->fields[key_idx]->offset = key_offset;
3750
3751	hist_data->key_size += key_size;
3752
3753	if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
3754		ret = -EINVAL;
3755		goto out;
3756	}
3757
3758	hist_data->n_keys++;
3759	hist_data->n_fields++;
3760
3761	if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
3762		return -EINVAL;
3763
3764	ret = key_size;
3765 out:
3766	return ret;
3767}
3768
3769static int create_key_fields(struct hist_trigger_data *hist_data,
3770			     struct trace_event_file *file)
3771{
3772	unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
3773	char *fields_str, *field_str;
3774	int ret = -EINVAL;
3775
3776	fields_str = hist_data->attrs->keys_str;
3777	if (!fields_str)
3778		goto out;
3779
3780	for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
3781		field_str = strsep(&fields_str, ",");
3782		if (!field_str)
3783			break;
3784		ret = create_key_field(hist_data, i, key_offset,
3785				       file, field_str);
3786		if (ret < 0)
3787			goto out;
3788		key_offset += ret;
3789	}
3790	if (fields_str) {
3791		ret = -EINVAL;
3792		goto out;
3793	}
3794	ret = 0;
3795 out:
3796	return ret;
3797}
3798
3799static int create_var_fields(struct hist_trigger_data *hist_data,
3800			     struct trace_event_file *file)
3801{
3802	unsigned int i, j = hist_data->n_vals;
3803	int ret = 0;
3804
3805	unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
3806
3807	for (i = 0; i < n_vars; i++) {
3808		char *var_name = hist_data->attrs->var_defs.name[i];
3809		char *expr = hist_data->attrs->var_defs.expr[i];
3810
3811		ret = create_var_field(hist_data, j++, file, var_name, expr);
3812		if (ret)
3813			goto out;
3814	}
3815 out:
3816	return ret;
3817}
3818
3819static void free_var_defs(struct hist_trigger_data *hist_data)
3820{
3821	unsigned int i;
3822
3823	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
3824		kfree(hist_data->attrs->var_defs.name[i]);
3825		kfree(hist_data->attrs->var_defs.expr[i]);
3826	}
3827
3828	hist_data->attrs->var_defs.n_vars = 0;
3829}
3830
3831static int parse_var_defs(struct hist_trigger_data *hist_data)
3832{
3833	struct trace_array *tr = hist_data->event_file->tr;
3834	char *s, *str, *var_name, *field_str;
3835	unsigned int i, j, n_vars = 0;
3836	int ret = 0;
3837
3838	for (i = 0; i < hist_data->attrs->n_assignments; i++) {
3839		str = hist_data->attrs->assignment_str[i];
3840		for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
3841			field_str = strsep(&str, ",");
3842			if (!field_str)
3843				break;
3844
3845			var_name = strsep(&field_str, "=");
3846			if (!var_name || !field_str) {
3847				hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
3848					 errpos(var_name));
3849				ret = -EINVAL;
3850				goto free;
3851			}
3852
3853			if (n_vars == TRACING_MAP_VARS_MAX) {
3854				hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
3855				ret = -EINVAL;
3856				goto free;
3857			}
3858
3859			s = kstrdup(var_name, GFP_KERNEL);
3860			if (!s) {
3861				ret = -ENOMEM;
3862				goto free;
3863			}
3864			hist_data->attrs->var_defs.name[n_vars] = s;
3865
3866			s = kstrdup(field_str, GFP_KERNEL);
3867			if (!s) {
3868				ret = -ENOMEM;
3869				goto free;
3870			}
3871			hist_data->attrs->var_defs.expr[n_vars++] = s;
3872
3873			hist_data->attrs->var_defs.n_vars = n_vars;
3874		}
3875	}
3876
3877	return ret;
3878 free:
3879	free_var_defs(hist_data);
3880
3881	return ret;
3882}
3883
3884static int create_hist_fields(struct hist_trigger_data *hist_data,
3885			      struct trace_event_file *file)
3886{
3887	int ret;
3888
3889	ret = parse_var_defs(hist_data);
3890	if (ret)
3891		goto out;
3892
3893	ret = create_val_fields(hist_data, file);
3894	if (ret)
3895		goto out;
3896
3897	ret = create_var_fields(hist_data, file);
3898	if (ret)
3899		goto out;
3900
3901	ret = create_key_fields(hist_data, file);
3902	if (ret)
3903		goto out;
3904 out:
3905	free_var_defs(hist_data);
3906
3907	return ret;
3908}
3909
3910static int is_descending(struct trace_array *tr, const char *str)
3911{
3912	if (!str)
3913		return 0;
3914
3915	if (strcmp(str, "descending") == 0)
3916		return 1;
3917
3918	if (strcmp(str, "ascending") == 0)
3919		return 0;
3920
3921	hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
3922
3923	return -EINVAL;
3924}
3925
3926static int create_sort_keys(struct hist_trigger_data *hist_data)
3927{
3928	struct trace_array *tr = hist_data->event_file->tr;
3929	char *fields_str = hist_data->attrs->sort_key_str;
3930	struct tracing_map_sort_key *sort_key;
3931	int descending, ret = 0;
3932	unsigned int i, j, k;
3933
3934	hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
3935
3936	if (!fields_str)
3937		goto out;
3938
3939	for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
3940		struct hist_field *hist_field;
3941		char *field_str, *field_name;
3942		const char *test_name;
3943
3944		sort_key = &hist_data->sort_keys[i];
3945
3946		field_str = strsep(&fields_str, ",");
3947		if (!field_str)
3948			break;
3949
3950		if (!*field_str) {
3951			ret = -EINVAL;
3952			hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
3953			break;
3954		}
3955
3956		if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
3957			hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
3958			ret = -EINVAL;
3959			break;
3960		}
3961
3962		field_name = strsep(&field_str, ".");
3963		if (!field_name || !*field_name) {
3964			ret = -EINVAL;
3965			hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
3966			break;
3967		}
3968
3969		if (strcmp(field_name, "hitcount") == 0) {
3970			descending = is_descending(tr, field_str);
3971			if (descending < 0) {
3972				ret = descending;
3973				break;
3974			}
3975			sort_key->descending = descending;
3976			continue;
3977		}
3978
3979		for (j = 1, k = 1; j < hist_data->n_fields; j++) {
3980			unsigned int idx;
3981
3982			hist_field = hist_data->fields[j];
3983			if (hist_field->flags & HIST_FIELD_FL_VAR)
3984				continue;
3985
3986			idx = k++;
3987
3988			test_name = hist_field_name(hist_field, 0);
3989
3990			if (strcmp(field_name, test_name) == 0) {
3991				sort_key->field_idx = idx;
3992				descending = is_descending(tr, field_str);
3993				if (descending < 0) {
3994					ret = descending;
3995					goto out;
3996				}
3997				sort_key->descending = descending;
3998				break;
3999			}
4000		}
4001		if (j == hist_data->n_fields) {
4002			ret = -EINVAL;
4003			hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4004			break;
4005		}
4006	}
4007
4008	hist_data->n_sort_keys = i;
4009 out:
4010	return ret;
4011}
4012
4013static void destroy_actions(struct hist_trigger_data *hist_data)
4014{
4015	unsigned int i;
4016
4017	for (i = 0; i < hist_data->n_actions; i++) {
4018		struct action_data *data = hist_data->actions[i];
4019
4020		if (data->handler == HANDLER_ONMATCH)
4021			onmatch_destroy(data);
4022		else if (data->handler == HANDLER_ONMAX ||
4023			 data->handler == HANDLER_ONCHANGE)
4024			track_data_destroy(hist_data, data);
4025		else
4026			kfree(data);
4027	}
4028}
4029
4030static int parse_actions(struct hist_trigger_data *hist_data)
4031{
4032	struct trace_array *tr = hist_data->event_file->tr;
4033	struct action_data *data;
4034	unsigned int i;
4035	int ret = 0;
4036	char *str;
4037	int len;
4038
4039	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4040		str = hist_data->attrs->action_str[i];
4041
4042		if ((len = str_has_prefix(str, "onmatch("))) {
4043			char *action_str = str + len;
4044
4045			data = onmatch_parse(tr, action_str);
4046			if (IS_ERR(data)) {
4047				ret = PTR_ERR(data);
4048				break;
4049			}
4050		} else if ((len = str_has_prefix(str, "onmax("))) {
4051			char *action_str = str + len;
4052
4053			data = track_data_parse(hist_data, action_str,
4054						HANDLER_ONMAX);
4055			if (IS_ERR(data)) {
4056				ret = PTR_ERR(data);
4057				break;
4058			}
4059		} else if ((len = str_has_prefix(str, "onchange("))) {
4060			char *action_str = str + len;
4061
4062			data = track_data_parse(hist_data, action_str,
4063						HANDLER_ONCHANGE);
4064			if (IS_ERR(data)) {
4065				ret = PTR_ERR(data);
4066				break;
4067			}
4068		} else {
4069			ret = -EINVAL;
4070			break;
4071		}
4072
4073		hist_data->actions[hist_data->n_actions++] = data;
4074	}
4075
4076	return ret;
4077}
4078
4079static int create_actions(struct hist_trigger_data *hist_data)
4080{
4081	struct action_data *data;
4082	unsigned int i;
4083	int ret = 0;
4084
4085	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4086		data = hist_data->actions[i];
4087
4088		if (data->handler == HANDLER_ONMATCH) {
4089			ret = onmatch_create(hist_data, data);
4090			if (ret)
4091				break;
4092		} else if (data->handler == HANDLER_ONMAX ||
4093			   data->handler == HANDLER_ONCHANGE) {
4094			ret = track_data_create(hist_data, data);
4095			if (ret)
4096				break;
4097		} else {
4098			ret = -EINVAL;
4099			break;
4100		}
4101	}
4102
4103	return ret;
4104}
4105
4106static void print_actions(struct seq_file *m,
4107			  struct hist_trigger_data *hist_data,
4108			  struct tracing_map_elt *elt)
4109{
4110	unsigned int i;
4111
4112	for (i = 0; i < hist_data->n_actions; i++) {
4113		struct action_data *data = hist_data->actions[i];
4114
4115		if (data->action == ACTION_SNAPSHOT)
4116			continue;
4117
4118		if (data->handler == HANDLER_ONMAX ||
4119		    data->handler == HANDLER_ONCHANGE)
4120			track_data_print(m, hist_data, elt, data);
4121	}
4122}
4123
4124static void print_action_spec(struct seq_file *m,
4125			      struct hist_trigger_data *hist_data,
4126			      struct action_data *data)
4127{
4128	unsigned int i;
4129
4130	if (data->action == ACTION_SAVE) {
4131		for (i = 0; i < hist_data->n_save_vars; i++) {
4132			seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4133			if (i < hist_data->n_save_vars - 1)
4134				seq_puts(m, ",");
4135		}
4136	} else if (data->action == ACTION_TRACE) {
4137		if (data->use_trace_keyword)
4138			seq_printf(m, "%s", data->synth_event_name);
4139		for (i = 0; i < data->n_params; i++) {
4140			if (i || data->use_trace_keyword)
4141				seq_puts(m, ",");
4142			seq_printf(m, "%s", data->params[i]);
4143		}
4144	}
4145}
4146
4147static void print_track_data_spec(struct seq_file *m,
4148				  struct hist_trigger_data *hist_data,
4149				  struct action_data *data)
4150{
4151	if (data->handler == HANDLER_ONMAX)
4152		seq_puts(m, ":onmax(");
4153	else if (data->handler == HANDLER_ONCHANGE)
4154		seq_puts(m, ":onchange(");
4155	seq_printf(m, "%s", data->track_data.var_str);
4156	seq_printf(m, ").%s(", data->action_name);
4157
4158	print_action_spec(m, hist_data, data);
4159
4160	seq_puts(m, ")");
4161}
4162
4163static void print_onmatch_spec(struct seq_file *m,
4164			       struct hist_trigger_data *hist_data,
4165			       struct action_data *data)
4166{
4167	seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4168		   data->match_data.event);
4169
4170	seq_printf(m, "%s(", data->action_name);
4171
4172	print_action_spec(m, hist_data, data);
4173
4174	seq_puts(m, ")");
4175}
4176
4177static bool actions_match(struct hist_trigger_data *hist_data,
4178			  struct hist_trigger_data *hist_data_test)
4179{
4180	unsigned int i, j;
4181
4182	if (hist_data->n_actions != hist_data_test->n_actions)
4183		return false;
4184
4185	for (i = 0; i < hist_data->n_actions; i++) {
4186		struct action_data *data = hist_data->actions[i];
4187		struct action_data *data_test = hist_data_test->actions[i];
4188		char *action_name, *action_name_test;
4189
4190		if (data->handler != data_test->handler)
4191			return false;
4192		if (data->action != data_test->action)
4193			return false;
4194
4195		if (data->n_params != data_test->n_params)
4196			return false;
4197
4198		for (j = 0; j < data->n_params; j++) {
4199			if (strcmp(data->params[j], data_test->params[j]) != 0)
4200				return false;
4201		}
4202
4203		if (data->use_trace_keyword)
4204			action_name = data->synth_event_name;
4205		else
4206			action_name = data->action_name;
4207
4208		if (data_test->use_trace_keyword)
4209			action_name_test = data_test->synth_event_name;
4210		else
4211			action_name_test = data_test->action_name;
4212
4213		if (strcmp(action_name, action_name_test) != 0)
4214			return false;
4215
4216		if (data->handler == HANDLER_ONMATCH) {
4217			if (strcmp(data->match_data.event_system,
4218				   data_test->match_data.event_system) != 0)
4219				return false;
4220			if (strcmp(data->match_data.event,
4221				   data_test->match_data.event) != 0)
4222				return false;
4223		} else if (data->handler == HANDLER_ONMAX ||
4224			   data->handler == HANDLER_ONCHANGE) {
4225			if (strcmp(data->track_data.var_str,
4226				   data_test->track_data.var_str) != 0)
4227				return false;
4228		}
4229	}
4230
4231	return true;
4232}
4233
4234
4235static void print_actions_spec(struct seq_file *m,
4236			       struct hist_trigger_data *hist_data)
4237{
4238	unsigned int i;
4239
4240	for (i = 0; i < hist_data->n_actions; i++) {
4241		struct action_data *data = hist_data->actions[i];
4242
4243		if (data->handler == HANDLER_ONMATCH)
4244			print_onmatch_spec(m, hist_data, data);
4245		else if (data->handler == HANDLER_ONMAX ||
4246			 data->handler == HANDLER_ONCHANGE)
4247			print_track_data_spec(m, hist_data, data);
4248	}
4249}
4250
4251static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4252{
4253	unsigned int i;
4254
4255	for (i = 0; i < hist_data->n_field_var_hists; i++) {
4256		kfree(hist_data->field_var_hists[i]->cmd);
4257		kfree(hist_data->field_var_hists[i]);
4258	}
4259}
4260
4261static void destroy_hist_data(struct hist_trigger_data *hist_data)
4262{
4263	if (!hist_data)
4264		return;
4265
4266	destroy_hist_trigger_attrs(hist_data->attrs);
4267	destroy_hist_fields(hist_data);
4268	tracing_map_destroy(hist_data->map);
4269
4270	destroy_actions(hist_data);
4271	destroy_field_vars(hist_data);
4272	destroy_field_var_hists(hist_data);
4273
4274	kfree(hist_data);
4275}
4276
4277static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4278{
4279	struct tracing_map *map = hist_data->map;
4280	struct ftrace_event_field *field;
4281	struct hist_field *hist_field;
4282	int i, idx = 0;
4283
4284	for_each_hist_field(i, hist_data) {
4285		hist_field = hist_data->fields[i];
4286		if (hist_field->flags & HIST_FIELD_FL_KEY) {
4287			tracing_map_cmp_fn_t cmp_fn;
4288
4289			field = hist_field->field;
4290
4291			if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4292				cmp_fn = tracing_map_cmp_none;
4293			else if (!field)
4294				cmp_fn = tracing_map_cmp_num(hist_field->size,
4295							     hist_field->is_signed);
4296			else if (is_string_field(field))
4297				cmp_fn = tracing_map_cmp_string;
4298			else
4299				cmp_fn = tracing_map_cmp_num(field->size,
4300							     field->is_signed);
4301			idx = tracing_map_add_key_field(map,
4302							hist_field->offset,
4303							cmp_fn);
4304		} else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4305			idx = tracing_map_add_sum_field(map);
4306
4307		if (idx < 0)
4308			return idx;
4309
4310		if (hist_field->flags & HIST_FIELD_FL_VAR) {
4311			idx = tracing_map_add_var(map);
4312			if (idx < 0)
4313				return idx;
4314			hist_field->var.idx = idx;
4315			hist_field->var.hist_data = hist_data;
4316		}
4317	}
4318
4319	return 0;
4320}
4321
4322static struct hist_trigger_data *
4323create_hist_data(unsigned int map_bits,
4324		 struct hist_trigger_attrs *attrs,
4325		 struct trace_event_file *file,
4326		 bool remove)
4327{
4328	const struct tracing_map_ops *map_ops = NULL;
4329	struct hist_trigger_data *hist_data;
4330	int ret = 0;
4331
4332	hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4333	if (!hist_data)
4334		return ERR_PTR(-ENOMEM);
4335
4336	hist_data->attrs = attrs;
4337	hist_data->remove = remove;
4338	hist_data->event_file = file;
4339
4340	ret = parse_actions(hist_data);
4341	if (ret)
4342		goto free;
4343
4344	ret = create_hist_fields(hist_data, file);
4345	if (ret)
4346		goto free;
4347
4348	ret = create_sort_keys(hist_data);
4349	if (ret)
4350		goto free;
4351
4352	map_ops = &hist_trigger_elt_data_ops;
4353
4354	hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4355					    map_ops, hist_data);
4356	if (IS_ERR(hist_data->map)) {
4357		ret = PTR_ERR(hist_data->map);
4358		hist_data->map = NULL;
4359		goto free;
4360	}
4361
4362	ret = create_tracing_map_fields(hist_data);
4363	if (ret)
4364		goto free;
4365 out:
4366	return hist_data;
4367 free:
4368	hist_data->attrs = NULL;
4369
4370	destroy_hist_data(hist_data);
4371
4372	hist_data = ERR_PTR(ret);
4373
4374	goto out;
4375}
4376
4377static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4378				    struct tracing_map_elt *elt, void *rec,
4379				    struct ring_buffer_event *rbe,
4380				    u64 *var_ref_vals)
4381{
4382	struct hist_elt_data *elt_data;
4383	struct hist_field *hist_field;
4384	unsigned int i, var_idx;
4385	u64 hist_val;
4386
4387	elt_data = elt->private_data;
4388	elt_data->var_ref_vals = var_ref_vals;
4389
4390	for_each_hist_val_field(i, hist_data) {
4391		hist_field = hist_data->fields[i];
4392		hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4393		if (hist_field->flags & HIST_FIELD_FL_VAR) {
4394			var_idx = hist_field->var.idx;
4395			tracing_map_set_var(elt, var_idx, hist_val);
4396			continue;
4397		}
4398		tracing_map_update_sum(elt, i, hist_val);
4399	}
4400
4401	for_each_hist_key_field(i, hist_data) {
4402		hist_field = hist_data->fields[i];
4403		if (hist_field->flags & HIST_FIELD_FL_VAR) {
4404			hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4405			var_idx = hist_field->var.idx;
4406			tracing_map_set_var(elt, var_idx, hist_val);
4407		}
4408	}
4409
4410	update_field_vars(hist_data, elt, rbe, rec);
4411}
4412
4413static inline void add_to_key(char *compound_key, void *key,
4414			      struct hist_field *key_field, void *rec)
4415{
4416	size_t size = key_field->size;
4417
4418	if (key_field->flags & HIST_FIELD_FL_STRING) {
4419		struct ftrace_event_field *field;
4420
4421		field = key_field->field;
4422		if (field->filter_type == FILTER_DYN_STRING)
4423			size = *(u32 *)(rec + field->offset) >> 16;
4424		else if (field->filter_type == FILTER_PTR_STRING)
4425			size = strlen(key);
4426		else if (field->filter_type == FILTER_STATIC_STRING)
4427			size = field->size;
4428
4429		/* ensure NULL-termination */
4430		if (size > key_field->size - 1)
4431			size = key_field->size - 1;
4432
4433		strncpy(compound_key + key_field->offset, (char *)key, size);
4434	} else
4435		memcpy(compound_key + key_field->offset, key, size);
4436}
4437
4438static void
4439hist_trigger_actions(struct hist_trigger_data *hist_data,
4440		     struct tracing_map_elt *elt, void *rec,
4441		     struct ring_buffer_event *rbe, void *key,
4442		     u64 *var_ref_vals)
4443{
4444	struct action_data *data;
4445	unsigned int i;
4446
4447	for (i = 0; i < hist_data->n_actions; i++) {
4448		data = hist_data->actions[i];
4449		data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals);
4450	}
4451}
4452
4453static void event_hist_trigger(struct event_trigger_data *data, void *rec,
4454			       struct ring_buffer_event *rbe)
4455{
4456	struct hist_trigger_data *hist_data = data->private_data;
4457	bool use_compound_key = (hist_data->n_keys > 1);
4458	unsigned long entries[HIST_STACKTRACE_DEPTH];
4459	u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4460	char compound_key[HIST_KEY_SIZE_MAX];
4461	struct tracing_map_elt *elt = NULL;
4462	struct hist_field *key_field;
4463	u64 field_contents;
4464	void *key = NULL;
4465	unsigned int i;
4466
4467	memset(compound_key, 0, hist_data->key_size);
4468
4469	for_each_hist_key_field(i, hist_data) {
4470		key_field = hist_data->fields[i];
4471
4472		if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4473			memset(entries, 0, HIST_STACKTRACE_SIZE);
4474			stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
4475					 HIST_STACKTRACE_SKIP);
4476			key = entries;
4477		} else {
4478			field_contents = key_field->fn(key_field, elt, rbe, rec);
4479			if (key_field->flags & HIST_FIELD_FL_STRING) {
4480				key = (void *)(unsigned long)field_contents;
4481				use_compound_key = true;
4482			} else
4483				key = (void *)&field_contents;
4484		}
4485
4486		if (use_compound_key)
4487			add_to_key(compound_key, key, key_field, rec);
4488	}
4489
4490	if (use_compound_key)
4491		key = compound_key;
4492
4493	if (hist_data->n_var_refs &&
4494	    !resolve_var_refs(hist_data, key, var_ref_vals, false))
4495		return;
4496
4497	elt = tracing_map_insert(hist_data->map, key);
4498	if (!elt)
4499		return;
4500
4501	hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
4502
4503	if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4504		hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals);
4505}
4506
4507static void hist_trigger_stacktrace_print(struct seq_file *m,
4508					  unsigned long *stacktrace_entries,
4509					  unsigned int max_entries)
4510{
4511	char str[KSYM_SYMBOL_LEN];
4512	unsigned int spaces = 8;
4513	unsigned int i;
4514
4515	for (i = 0; i < max_entries; i++) {
4516		if (!stacktrace_entries[i])
4517			return;
4518
4519		seq_printf(m, "%*c", 1 + spaces, ' ');
4520		sprint_symbol(str, stacktrace_entries[i]);
4521		seq_printf(m, "%s\n", str);
4522	}
4523}
4524
4525static void hist_trigger_print_key(struct seq_file *m,
4526				   struct hist_trigger_data *hist_data,
4527				   void *key,
4528				   struct tracing_map_elt *elt)
4529{
4530	struct hist_field *key_field;
4531	char str[KSYM_SYMBOL_LEN];
4532	bool multiline = false;
4533	const char *field_name;
4534	unsigned int i;
4535	u64 uval;
4536
4537	seq_puts(m, "{ ");
4538
4539	for_each_hist_key_field(i, hist_data) {
4540		key_field = hist_data->fields[i];
4541
4542		if (i > hist_data->n_vals)
4543			seq_puts(m, ", ");
4544
4545		field_name = hist_field_name(key_field, 0);
4546
4547		if (key_field->flags & HIST_FIELD_FL_HEX) {
4548			uval = *(u64 *)(key + key_field->offset);
4549			seq_printf(m, "%s: %llx", field_name, uval);
4550		} else if (key_field->flags & HIST_FIELD_FL_SYM) {
4551			uval = *(u64 *)(key + key_field->offset);
4552			sprint_symbol_no_offset(str, uval);
4553			seq_printf(m, "%s: [%llx] %-45s", field_name,
4554				   uval, str);
4555		} else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4556			uval = *(u64 *)(key + key_field->offset);
4557			sprint_symbol(str, uval);
4558			seq_printf(m, "%s: [%llx] %-55s", field_name,
4559				   uval, str);
4560		} else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
4561			struct hist_elt_data *elt_data = elt->private_data;
4562			char *comm;
4563
4564			if (WARN_ON_ONCE(!elt_data))
4565				return;
4566
4567			comm = elt_data->comm;
4568
4569			uval = *(u64 *)(key + key_field->offset);
4570			seq_printf(m, "%s: %-16s[%10llu]", field_name,
4571				   comm, uval);
4572		} else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4573			const char *syscall_name;
4574
4575			uval = *(u64 *)(key + key_field->offset);
4576			syscall_name = get_syscall_name(uval);
4577			if (!syscall_name)
4578				syscall_name = "unknown_syscall";
4579
4580			seq_printf(m, "%s: %-30s[%3llu]", field_name,
4581				   syscall_name, uval);
4582		} else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4583			seq_puts(m, "stacktrace:\n");
4584			hist_trigger_stacktrace_print(m,
4585						      key + key_field->offset,
4586						      HIST_STACKTRACE_DEPTH);
4587			multiline = true;
4588		} else if (key_field->flags & HIST_FIELD_FL_LOG2) {
4589			seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4590				   *(u64 *)(key + key_field->offset));
4591		} else if (key_field->flags & HIST_FIELD_FL_STRING) {
4592			seq_printf(m, "%s: %-50s", field_name,
4593				   (char *)(key + key_field->offset));
4594		} else {
4595			uval = *(u64 *)(key + key_field->offset);
4596			seq_printf(m, "%s: %10llu", field_name, uval);
4597		}
4598	}
4599
4600	if (!multiline)
4601		seq_puts(m, " ");
4602
4603	seq_puts(m, "}");
4604}
4605
4606static void hist_trigger_entry_print(struct seq_file *m,
4607				     struct hist_trigger_data *hist_data,
4608				     void *key,
4609				     struct tracing_map_elt *elt)
4610{
4611	const char *field_name;
4612	unsigned int i;
4613
4614	hist_trigger_print_key(m, hist_data, key, elt);
4615
4616	seq_printf(m, " hitcount: %10llu",
4617		   tracing_map_read_sum(elt, HITCOUNT_IDX));
4618
4619	for (i = 1; i < hist_data->n_vals; i++) {
4620		field_name = hist_field_name(hist_data->fields[i], 0);
4621
4622		if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4623		    hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
4624			continue;
4625
4626		if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
4627			seq_printf(m, "  %s: %10llx", field_name,
4628				   tracing_map_read_sum(elt, i));
4629		} else {
4630			seq_printf(m, "  %s: %10llu", field_name,
4631				   tracing_map_read_sum(elt, i));
4632		}
4633	}
4634
4635	print_actions(m, hist_data, elt);
4636
4637	seq_puts(m, "\n");
4638}
4639
4640static int print_entries(struct seq_file *m,
4641			 struct hist_trigger_data *hist_data)
4642{
4643	struct tracing_map_sort_entry **sort_entries = NULL;
4644	struct tracing_map *map = hist_data->map;
4645	int i, n_entries;
4646
4647	n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4648					     hist_data->n_sort_keys,
4649					     &sort_entries);
4650	if (n_entries < 0)
4651		return n_entries;
4652
4653	for (i = 0; i < n_entries; i++)
4654		hist_trigger_entry_print(m, hist_data,
4655					 sort_entries[i]->key,
4656					 sort_entries[i]->elt);
4657
4658	tracing_map_destroy_sort_entries(sort_entries, n_entries);
4659
4660	return n_entries;
4661}
4662
4663static void hist_trigger_show(struct seq_file *m,
4664			      struct event_trigger_data *data, int n)
4665{
4666	struct hist_trigger_data *hist_data;
4667	int n_entries;
4668
4669	if (n > 0)
4670		seq_puts(m, "\n\n");
4671
4672	seq_puts(m, "# event histogram\n#\n# trigger info: ");
4673	data->ops->print(m, data->ops, data);
4674	seq_puts(m, "#\n\n");
4675
4676	hist_data = data->private_data;
4677	n_entries = print_entries(m, hist_data);
4678	if (n_entries < 0)
4679		n_entries = 0;
4680
4681	track_data_snapshot_print(m, hist_data);
4682
4683	seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
4684		   (u64)atomic64_read(&hist_data->map->hits),
4685		   n_entries, (u64)atomic64_read(&hist_data->map->drops));
4686}
4687
4688static int hist_show(struct seq_file *m, void *v)
4689{
4690	struct event_trigger_data *data;
4691	struct trace_event_file *event_file;
4692	int n = 0, ret = 0;
4693
4694	mutex_lock(&event_mutex);
4695
4696	event_file = event_file_data(m->private);
4697	if (unlikely(!event_file)) {
4698		ret = -ENODEV;
4699		goto out_unlock;
4700	}
4701
4702	list_for_each_entry(data, &event_file->triggers, list) {
4703		if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4704			hist_trigger_show(m, data, n++);
4705	}
4706
4707 out_unlock:
4708	mutex_unlock(&event_mutex);
4709
4710	return ret;
4711}
4712
4713static int event_hist_open(struct inode *inode, struct file *file)
4714{
4715	int ret;
4716
4717	ret = security_locked_down(LOCKDOWN_TRACEFS);
4718	if (ret)
4719		return ret;
4720
4721	return single_open(file, hist_show, file);
4722}
4723
4724const struct file_operations event_hist_fops = {
4725	.open = event_hist_open,
4726	.read = seq_read,
4727	.llseek = seq_lseek,
4728	.release = single_release,
4729};
4730
4731#ifdef CONFIG_HIST_TRIGGERS_DEBUG
4732static void hist_field_debug_show_flags(struct seq_file *m,
4733					unsigned long flags)
4734{
4735	seq_puts(m, "      flags:\n");
4736
4737	if (flags & HIST_FIELD_FL_KEY)
4738		seq_puts(m, "        HIST_FIELD_FL_KEY\n");
4739	else if (flags & HIST_FIELD_FL_HITCOUNT)
4740		seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
4741	else if (flags & HIST_FIELD_FL_VAR)
4742		seq_puts(m, "        HIST_FIELD_FL_VAR\n");
4743	else if (flags & HIST_FIELD_FL_VAR_REF)
4744		seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
4745	else
4746		seq_puts(m, "        VAL: normal u64 value\n");
4747
4748	if (flags & HIST_FIELD_FL_ALIAS)
4749		seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
4750}
4751
4752static int hist_field_debug_show(struct seq_file *m,
4753				 struct hist_field *field, unsigned long flags)
4754{
4755	if ((field->flags & flags) != flags) {
4756		seq_printf(m, "ERROR: bad flags - %lx\n", flags);
4757		return -EINVAL;
4758	}
4759
4760	hist_field_debug_show_flags(m, field->flags);
4761	if (field->field)
4762		seq_printf(m, "      ftrace_event_field name: %s\n",
4763			   field->field->name);
4764
4765	if (field->flags & HIST_FIELD_FL_VAR) {
4766		seq_printf(m, "      var.name: %s\n", field->var.name);
4767		seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4768			   field->var.idx);
4769	}
4770
4771	if (field->flags & HIST_FIELD_FL_ALIAS)
4772		seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4773			   field->var_ref_idx);
4774
4775	if (field->flags & HIST_FIELD_FL_VAR_REF) {
4776		seq_printf(m, "      name: %s\n", field->name);
4777		seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4778			   field->var.idx);
4779		seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
4780		seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4781			   field->var_ref_idx);
4782		if (field->system)
4783			seq_printf(m, "      system: %s\n", field->system);
4784		if (field->event_name)
4785			seq_printf(m, "      event_name: %s\n", field->event_name);
4786	}
4787
4788	seq_printf(m, "      type: %s\n", field->type);
4789	seq_printf(m, "      size: %u\n", field->size);
4790	seq_printf(m, "      is_signed: %u\n", field->is_signed);
4791
4792	return 0;
4793}
4794
4795static int field_var_debug_show(struct seq_file *m,
4796				struct field_var *field_var, unsigned int i,
4797				bool save_vars)
4798{
4799	const char *vars_name = save_vars ? "save_vars" : "field_vars";
4800	struct hist_field *field;
4801	int ret = 0;
4802
4803	seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
4804
4805	field = field_var->var;
4806
4807	seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
4808
4809	hist_field_debug_show_flags(m, field->flags);
4810	seq_printf(m, "      var.name: %s\n", field->var.name);
4811	seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4812		   field->var.idx);
4813
4814	field = field_var->val;
4815
4816	seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
4817	if (field->field)
4818		seq_printf(m, "      ftrace_event_field name: %s\n",
4819			   field->field->name);
4820	else {
4821		ret = -EINVAL;
4822		goto out;
4823	}
4824
4825	seq_printf(m, "      type: %s\n", field->type);
4826	seq_printf(m, "      size: %u\n", field->size);
4827	seq_printf(m, "      is_signed: %u\n", field->is_signed);
4828out:
4829	return ret;
4830}
4831
4832static int hist_action_debug_show(struct seq_file *m,
4833				  struct action_data *data, int i)
4834{
4835	int ret = 0;
4836
4837	if (data->handler == HANDLER_ONMAX ||
4838	    data->handler == HANDLER_ONCHANGE) {
4839		seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
4840		ret = hist_field_debug_show(m, data->track_data.var_ref,
4841					    HIST_FIELD_FL_VAR_REF);
4842		if (ret)
4843			goto out;
4844
4845		seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
4846		ret = hist_field_debug_show(m, data->track_data.track_var,
4847					    HIST_FIELD_FL_VAR);
4848		if (ret)
4849			goto out;
4850	}
4851
4852	if (data->handler == HANDLER_ONMATCH) {
4853		seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
4854			   i, data->match_data.event_system);
4855		seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
4856			   i, data->match_data.event);
4857	}
4858out:
4859	return ret;
4860}
4861
4862static int hist_actions_debug_show(struct seq_file *m,
4863				   struct hist_trigger_data *hist_data)
4864{
4865	int i, ret = 0;
4866
4867	if (hist_data->n_actions)
4868		seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
4869
4870	for (i = 0; i < hist_data->n_actions; i++) {
4871		struct action_data *action = hist_data->actions[i];
4872
4873		ret = hist_action_debug_show(m, action, i);
4874		if (ret)
4875			goto out;
4876	}
4877
4878	if (hist_data->n_save_vars)
4879		seq_puts(m, "\n  save action variables (save() params):\n");
4880
4881	for (i = 0; i < hist_data->n_save_vars; i++) {
4882		ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
4883		if (ret)
4884			goto out;
4885	}
4886out:
4887	return ret;
4888}
4889
4890static void hist_trigger_debug_show(struct seq_file *m,
4891				    struct event_trigger_data *data, int n)
4892{
4893	struct hist_trigger_data *hist_data;
4894	int i, ret;
4895
4896	if (n > 0)
4897		seq_puts(m, "\n\n");
4898
4899	seq_puts(m, "# event histogram\n#\n# trigger info: ");
4900	data->ops->print(m, data->ops, data);
4901	seq_puts(m, "#\n\n");
4902
4903	hist_data = data->private_data;
4904
4905	seq_printf(m, "hist_data: %p\n\n", hist_data);
4906	seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
4907	seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
4908	seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
4909
4910	seq_puts(m, "\n  val fields:\n\n");
4911
4912	seq_puts(m, "    hist_data->fields[0]:\n");
4913	ret = hist_field_debug_show(m, hist_data->fields[0],
4914				    HIST_FIELD_FL_HITCOUNT);
4915	if (ret)
4916		return;
4917
4918	for (i = 1; i < hist_data->n_vals; i++) {
4919		seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
4920		ret = hist_field_debug_show(m, hist_data->fields[i], 0);
4921		if (ret)
4922			return;
4923	}
4924
4925	seq_puts(m, "\n  key fields:\n");
4926
4927	for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
4928		seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
4929		ret = hist_field_debug_show(m, hist_data->fields[i],
4930					    HIST_FIELD_FL_KEY);
4931		if (ret)
4932			return;
4933	}
4934
4935	if (hist_data->n_var_refs)
4936		seq_puts(m, "\n  variable reference fields:\n");
4937
4938	for (i = 0; i < hist_data->n_var_refs; i++) {
4939		seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
4940		ret = hist_field_debug_show(m, hist_data->var_refs[i],
4941					    HIST_FIELD_FL_VAR_REF);
4942		if (ret)
4943			return;
4944	}
4945
4946	if (hist_data->n_field_vars)
4947		seq_puts(m, "\n  field variables:\n");
4948
4949	for (i = 0; i < hist_data->n_field_vars; i++) {
4950		ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
4951		if (ret)
4952			return;
4953	}
4954
4955	ret = hist_actions_debug_show(m, hist_data);
4956	if (ret)
4957		return;
4958}
4959
4960static int hist_debug_show(struct seq_file *m, void *v)
4961{
4962	struct event_trigger_data *data;
4963	struct trace_event_file *event_file;
4964	int n = 0, ret = 0;
4965
4966	mutex_lock(&event_mutex);
4967
4968	event_file = event_file_data(m->private);
4969	if (unlikely(!event_file)) {
4970		ret = -ENODEV;
4971		goto out_unlock;
4972	}
4973
4974	list_for_each_entry(data, &event_file->triggers, list) {
4975		if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4976			hist_trigger_debug_show(m, data, n++);
4977	}
4978
4979 out_unlock:
4980	mutex_unlock(&event_mutex);
4981
4982	return ret;
4983}
4984
4985static int event_hist_debug_open(struct inode *inode, struct file *file)
4986{
4987	int ret;
4988
4989	ret = security_locked_down(LOCKDOWN_TRACEFS);
4990	if (ret)
4991		return ret;
4992
4993	return single_open(file, hist_debug_show, file);
4994}
4995
4996const struct file_operations event_hist_debug_fops = {
4997	.open = event_hist_debug_open,
4998	.read = seq_read,
4999	.llseek = seq_lseek,
5000	.release = single_release,
5001};
5002#endif
5003
5004static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5005{
5006	const char *field_name = hist_field_name(hist_field, 0);
5007
5008	if (hist_field->var.name)
5009		seq_printf(m, "%s=", hist_field->var.name);
5010
5011	if (hist_field->flags & HIST_FIELD_FL_CPU)
5012		seq_puts(m, "cpu");
5013	else if (field_name) {
5014		if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5015		    hist_field->flags & HIST_FIELD_FL_ALIAS)
5016			seq_putc(m, '$');
5017		seq_printf(m, "%s", field_name);
5018	} else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5019		seq_puts(m, "common_timestamp");
5020
5021	if (hist_field->flags) {
5022		if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5023		    !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5024			const char *flags = get_hist_field_flags(hist_field);
5025
5026			if (flags)
5027				seq_printf(m, ".%s", flags);
5028		}
5029	}
5030}
5031
5032static int event_hist_trigger_print(struct seq_file *m,
5033				    struct event_trigger_ops *ops,
5034				    struct event_trigger_data *data)
5035{
5036	struct hist_trigger_data *hist_data = data->private_data;
5037	struct hist_field *field;
5038	bool have_var = false;
5039	unsigned int i;
5040
5041	seq_puts(m, "hist:");
5042
5043	if (data->name)
5044		seq_printf(m, "%s:", data->name);
5045
5046	seq_puts(m, "keys=");
5047
5048	for_each_hist_key_field(i, hist_data) {
5049		field = hist_data->fields[i];
5050
5051		if (i > hist_data->n_vals)
5052			seq_puts(m, ",");
5053
5054		if (field->flags & HIST_FIELD_FL_STACKTRACE)
5055			seq_puts(m, "stacktrace");
5056		else
5057			hist_field_print(m, field);
5058	}
5059
5060	seq_puts(m, ":vals=");
5061
5062	for_each_hist_val_field(i, hist_data) {
5063		field = hist_data->fields[i];
5064		if (field->flags & HIST_FIELD_FL_VAR) {
5065			have_var = true;
5066			continue;
5067		}
5068
5069		if (i == HITCOUNT_IDX)
5070			seq_puts(m, "hitcount");
5071		else {
5072			seq_puts(m, ",");
5073			hist_field_print(m, field);
5074		}
5075	}
5076
5077	if (have_var) {
5078		unsigned int n = 0;
5079
5080		seq_puts(m, ":");
5081
5082		for_each_hist_val_field(i, hist_data) {
5083			field = hist_data->fields[i];
5084
5085			if (field->flags & HIST_FIELD_FL_VAR) {
5086				if (n++)
5087					seq_puts(m, ",");
5088				hist_field_print(m, field);
5089			}
5090		}
5091	}
5092
5093	seq_puts(m, ":sort=");
5094
5095	for (i = 0; i < hist_data->n_sort_keys; i++) {
5096		struct tracing_map_sort_key *sort_key;
5097		unsigned int idx, first_key_idx;
5098
5099		/* skip VAR vals */
5100		first_key_idx = hist_data->n_vals - hist_data->n_vars;
5101
5102		sort_key = &hist_data->sort_keys[i];
5103		idx = sort_key->field_idx;
5104
5105		if (WARN_ON(idx >= HIST_FIELDS_MAX))
5106			return -EINVAL;
5107
5108		if (i > 0)
5109			seq_puts(m, ",");
5110
5111		if (idx == HITCOUNT_IDX)
5112			seq_puts(m, "hitcount");
5113		else {
5114			if (idx >= first_key_idx)
5115				idx += hist_data->n_vars;
5116			hist_field_print(m, hist_data->fields[idx]);
5117		}
5118
5119		if (sort_key->descending)
5120			seq_puts(m, ".descending");
5121	}
5122	seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5123	if (hist_data->enable_timestamps)
5124		seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5125
5126	print_actions_spec(m, hist_data);
5127
5128	if (data->filter_str)
5129		seq_printf(m, " if %s", data->filter_str);
5130
5131	if (data->paused)
5132		seq_puts(m, " [paused]");
5133	else
5134		seq_puts(m, " [active]");
5135
5136	seq_putc(m, '\n');
5137
5138	return 0;
5139}
5140
5141static int event_hist_trigger_init(struct event_trigger_ops *ops,
5142				   struct event_trigger_data *data)
5143{
5144	struct hist_trigger_data *hist_data = data->private_data;
5145
5146	if (!data->ref && hist_data->attrs->name)
5147		save_named_trigger(hist_data->attrs->name, data);
5148
5149	data->ref++;
5150
5151	return 0;
5152}
5153
5154static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5155{
5156	struct trace_event_file *file;
5157	unsigned int i;
5158	char *cmd;
5159	int ret;
5160
5161	for (i = 0; i < hist_data->n_field_var_hists; i++) {
5162		file = hist_data->field_var_hists[i]->hist_data->event_file;
5163		cmd = hist_data->field_var_hists[i]->cmd;
5164		ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5165					      "!hist", "hist", cmd);
5166	}
5167}
5168
5169static void event_hist_trigger_free(struct event_trigger_ops *ops,
5170				    struct event_trigger_data *data)
5171{
5172	struct hist_trigger_data *hist_data = data->private_data;
5173
5174	if (WARN_ON_ONCE(data->ref <= 0))
5175		return;
5176
5177	data->ref--;
5178	if (!data->ref) {
5179		if (data->name)
5180			del_named_trigger(data);
5181
5182		trigger_data_free(data);
5183
5184		remove_hist_vars(hist_data);
5185
5186		unregister_field_var_hists(hist_data);
5187
5188		destroy_hist_data(hist_data);
5189	}
5190}
5191
5192static struct event_trigger_ops event_hist_trigger_ops = {
5193	.func			= event_hist_trigger,
5194	.print			= event_hist_trigger_print,
5195	.init			= event_hist_trigger_init,
5196	.free			= event_hist_trigger_free,
5197};
5198
5199static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5200					 struct event_trigger_data *data)
5201{
5202	data->ref++;
5203
5204	save_named_trigger(data->named_data->name, data);
5205
5206	event_hist_trigger_init(ops, data->named_data);
5207
5208	return 0;
5209}
5210
5211static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5212					  struct event_trigger_data *data)
5213{
5214	if (WARN_ON_ONCE(data->ref <= 0))
5215		return;
5216
5217	event_hist_trigger_free(ops, data->named_data);
5218
5219	data->ref--;
5220	if (!data->ref) {
5221		del_named_trigger(data);
5222		trigger_data_free(data);
5223	}
5224}
5225
5226static struct event_trigger_ops event_hist_trigger_named_ops = {
5227	.func			= event_hist_trigger,
5228	.print			= event_hist_trigger_print,
5229	.init			= event_hist_trigger_named_init,
5230	.free			= event_hist_trigger_named_free,
5231};
5232
5233static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5234							    char *param)
5235{
5236	return &event_hist_trigger_ops;
5237}
5238
5239static void hist_clear(struct event_trigger_data *data)
5240{
5241	struct hist_trigger_data *hist_data = data->private_data;
5242
5243	if (data->name)
5244		pause_named_trigger(data);
5245
5246	tracepoint_synchronize_unregister();
5247
5248	tracing_map_clear(hist_data->map);
5249
5250	if (data->name)
5251		unpause_named_trigger(data);
5252}
5253
5254static bool compatible_field(struct ftrace_event_field *field,
5255			     struct ftrace_event_field *test_field)
5256{
5257	if (field == test_field)
5258		return true;
5259	if (field == NULL || test_field == NULL)
5260		return false;
5261	if (strcmp(field->name, test_field->name) != 0)
5262		return false;
5263	if (strcmp(field->type, test_field->type) != 0)
5264		return false;
5265	if (field->size != test_field->size)
5266		return false;
5267	if (field->is_signed != test_field->is_signed)
5268		return false;
5269
5270	return true;
5271}
5272
5273static bool hist_trigger_match(struct event_trigger_data *data,
5274			       struct event_trigger_data *data_test,
5275			       struct event_trigger_data *named_data,
5276			       bool ignore_filter)
5277{
5278	struct tracing_map_sort_key *sort_key, *sort_key_test;
5279	struct hist_trigger_data *hist_data, *hist_data_test;
5280	struct hist_field *key_field, *key_field_test;
5281	unsigned int i;
5282
5283	if (named_data && (named_data != data_test) &&
5284	    (named_data != data_test->named_data))
5285		return false;
5286
5287	if (!named_data && is_named_trigger(data_test))
5288		return false;
5289
5290	hist_data = data->private_data;
5291	hist_data_test = data_test->private_data;
5292
5293	if (hist_data->n_vals != hist_data_test->n_vals ||
5294	    hist_data->n_fields != hist_data_test->n_fields ||
5295	    hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5296		return false;
5297
5298	if (!ignore_filter) {
5299		if ((data->filter_str && !data_test->filter_str) ||
5300		   (!data->filter_str && data_test->filter_str))
5301			return false;
5302	}
5303
5304	for_each_hist_field(i, hist_data) {
5305		key_field = hist_data->fields[i];
5306		key_field_test = hist_data_test->fields[i];
5307
5308		if (key_field->flags != key_field_test->flags)
5309			return false;
5310		if (!compatible_field(key_field->field, key_field_test->field))
5311			return false;
5312		if (key_field->offset != key_field_test->offset)
5313			return false;
5314		if (key_field->size != key_field_test->size)
5315			return false;
5316		if (key_field->is_signed != key_field_test->is_signed)
5317			return false;
5318		if (!!key_field->var.name != !!key_field_test->var.name)
5319			return false;
5320		if (key_field->var.name &&
5321		    strcmp(key_field->var.name, key_field_test->var.name) != 0)
5322			return false;
5323	}
5324
5325	for (i = 0; i < hist_data->n_sort_keys; i++) {
5326		sort_key = &hist_data->sort_keys[i];
5327		sort_key_test = &hist_data_test->sort_keys[i];
5328
5329		if (sort_key->field_idx != sort_key_test->field_idx ||
5330		    sort_key->descending != sort_key_test->descending)
5331			return false;
5332	}
5333
5334	if (!ignore_filter && data->filter_str &&
5335	    (strcmp(data->filter_str, data_test->filter_str) != 0))
5336		return false;
5337
5338	if (!actions_match(hist_data, hist_data_test))
5339		return false;
5340
5341	return true;
5342}
5343
5344static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5345				 struct event_trigger_data *data,
5346				 struct trace_event_file *file)
5347{
5348	struct hist_trigger_data *hist_data = data->private_data;
5349	struct event_trigger_data *test, *named_data = NULL;
5350	struct trace_array *tr = file->tr;
5351	int ret = 0;
5352
5353	if (hist_data->attrs->name) {
5354		named_data = find_named_trigger(hist_data->attrs->name);
5355		if (named_data) {
5356			if (!hist_trigger_match(data, named_data, named_data,
5357						true)) {
5358				hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5359				ret = -EINVAL;
5360				goto out;
5361			}
5362		}
5363	}
5364
5365	if (hist_data->attrs->name && !named_data)
5366		goto new;
5367
5368	lockdep_assert_held(&event_mutex);
5369
5370	list_for_each_entry(test, &file->triggers, list) {
5371		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5372			if (!hist_trigger_match(data, test, named_data, false))
5373				continue;
5374			if (hist_data->attrs->pause)
5375				test->paused = true;
5376			else if (hist_data->attrs->cont)
5377				test->paused = false;
5378			else if (hist_data->attrs->clear)
5379				hist_clear(test);
5380			else {
5381				hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5382				ret = -EEXIST;
5383			}
5384			goto out;
5385		}
5386	}
5387 new:
5388	if (hist_data->attrs->cont || hist_data->attrs->clear) {
5389		hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5390		ret = -ENOENT;
5391		goto out;
5392	}
5393
5394	if (hist_data->attrs->pause)
5395		data->paused = true;
5396
5397	if (named_data) {
5398		data->private_data = named_data->private_data;
5399		set_named_trigger_data(data, named_data);
5400		data->ops = &event_hist_trigger_named_ops;
5401	}
5402
5403	if (data->ops->init) {
5404		ret = data->ops->init(data->ops, data);
5405		if (ret < 0)
5406			goto out;
5407	}
5408
5409	if (hist_data->enable_timestamps) {
5410		char *clock = hist_data->attrs->clock;
5411
5412		ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5413		if (ret) {
5414			hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5415			goto out;
5416		}
5417
5418		tracing_set_time_stamp_abs(file->tr, true);
5419	}
5420
5421	if (named_data)
5422		destroy_hist_data(hist_data);
5423
5424	ret++;
5425 out:
5426	return ret;
5427}
5428
5429static int hist_trigger_enable(struct event_trigger_data *data,
5430			       struct trace_event_file *file)
5431{
5432	int ret = 0;
5433
5434	list_add_tail_rcu(&data->list, &file->triggers);
5435
5436	update_cond_flag(file);
5437
5438	if (trace_event_trigger_enable_disable(file, 1) < 0) {
5439		list_del_rcu(&data->list);
5440		update_cond_flag(file);
5441		ret--;
5442	}
5443
5444	return ret;
5445}
5446
5447static bool have_hist_trigger_match(struct event_trigger_data *data,
5448				    struct trace_event_file *file)
5449{
5450	struct hist_trigger_data *hist_data = data->private_data;
5451	struct event_trigger_data *test, *named_data = NULL;
5452	bool match = false;
5453
5454	lockdep_assert_held(&event_mutex);
5455
5456	if (hist_data->attrs->name)
5457		named_data = find_named_trigger(hist_data->attrs->name);
5458
5459	list_for_each_entry(test, &file->triggers, list) {
5460		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5461			if (hist_trigger_match(data, test, named_data, false)) {
5462				match = true;
5463				break;
5464			}
5465		}
5466	}
5467
5468	return match;
5469}
5470
5471static bool hist_trigger_check_refs(struct event_trigger_data *data,
5472				    struct trace_event_file *file)
5473{
5474	struct hist_trigger_data *hist_data = data->private_data;
5475	struct event_trigger_data *test, *named_data = NULL;
5476
5477	lockdep_assert_held(&event_mutex);
5478
5479	if (hist_data->attrs->name)
5480		named_data = find_named_trigger(hist_data->attrs->name);
5481
5482	list_for_each_entry(test, &file->triggers, list) {
5483		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5484			if (!hist_trigger_match(data, test, named_data, false))
5485				continue;
5486			hist_data = test->private_data;
5487			if (check_var_refs(hist_data))
5488				return true;
5489			break;
5490		}
5491	}
5492
5493	return false;
5494}
5495
5496static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5497				    struct event_trigger_data *data,
5498				    struct trace_event_file *file)
5499{
5500	struct hist_trigger_data *hist_data = data->private_data;
5501	struct event_trigger_data *test, *named_data = NULL;
5502	bool unregistered = false;
5503
5504	lockdep_assert_held(&event_mutex);
5505
5506	if (hist_data->attrs->name)
5507		named_data = find_named_trigger(hist_data->attrs->name);
5508
5509	list_for_each_entry(test, &file->triggers, list) {
5510		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5511			if (!hist_trigger_match(data, test, named_data, false))
5512				continue;
5513			unregistered = true;
5514			list_del_rcu(&test->list);
5515			trace_event_trigger_enable_disable(file, 0);
5516			update_cond_flag(file);
5517			break;
5518		}
5519	}
5520
5521	if (unregistered && test->ops->free)
5522		test->ops->free(test->ops, test);
5523
5524	if (hist_data->enable_timestamps) {
5525		if (!hist_data->remove || unregistered)
5526			tracing_set_time_stamp_abs(file->tr, false);
5527	}
5528}
5529
5530static bool hist_file_check_refs(struct trace_event_file *file)
5531{
5532	struct hist_trigger_data *hist_data;
5533	struct event_trigger_data *test;
5534
5535	lockdep_assert_held(&event_mutex);
5536
5537	list_for_each_entry(test, &file->triggers, list) {
5538		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5539			hist_data = test->private_data;
5540			if (check_var_refs(hist_data))
5541				return true;
5542		}
5543	}
5544
5545	return false;
5546}
5547
5548static void hist_unreg_all(struct trace_event_file *file)
5549{
5550	struct event_trigger_data *test, *n;
5551	struct hist_trigger_data *hist_data;
5552	struct synth_event *se;
5553	const char *se_name;
5554
5555	lockdep_assert_held(&event_mutex);
5556
5557	if (hist_file_check_refs(file))
5558		return;
5559
5560	list_for_each_entry_safe(test, n, &file->triggers, list) {
5561		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5562			hist_data = test->private_data;
5563			list_del_rcu(&test->list);
5564			trace_event_trigger_enable_disable(file, 0);
5565
5566			se_name = trace_event_name(file->event_call);
5567			se = find_synth_event(se_name);
5568			if (se)
5569				se->ref--;
5570
5571			update_cond_flag(file);
5572			if (hist_data->enable_timestamps)
5573				tracing_set_time_stamp_abs(file->tr, false);
5574			if (test->ops->free)
5575				test->ops->free(test->ops, test);
5576		}
5577	}
5578}
5579
5580static int event_hist_trigger_func(struct event_command *cmd_ops,
5581				   struct trace_event_file *file,
5582				   char *glob, char *cmd, char *param)
5583{
5584	unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5585	struct event_trigger_data *trigger_data;
5586	struct hist_trigger_attrs *attrs;
5587	struct event_trigger_ops *trigger_ops;
5588	struct hist_trigger_data *hist_data;
5589	struct synth_event *se;
5590	const char *se_name;
5591	bool remove = false;
5592	char *trigger, *p;
5593	int ret = 0;
5594
5595	lockdep_assert_held(&event_mutex);
5596
5597	if (glob && strlen(glob)) {
5598		hist_err_clear();
5599		last_cmd_set(file, param);
5600	}
5601
5602	if (!param)
5603		return -EINVAL;
5604
5605	if (glob[0] == '!')
5606		remove = true;
5607
5608	/*
5609	 * separate the trigger from the filter (k:v [if filter])
5610	 * allowing for whitespace in the trigger
5611	 */
5612	p = trigger = param;
5613	do {
5614		p = strstr(p, "if");
5615		if (!p)
5616			break;
5617		if (p == param)
5618			return -EINVAL;
5619		if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5620			p++;
5621			continue;
5622		}
5623		if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
5624			return -EINVAL;
5625		if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
5626			p++;
5627			continue;
5628		}
5629		break;
5630	} while (p);
5631
5632	if (!p)
5633		param = NULL;
5634	else {
5635		*(p - 1) = '\0';
5636		param = strstrip(p);
5637		trigger = strstrip(trigger);
5638	}
5639
5640	attrs = parse_hist_trigger_attrs(file->tr, trigger);
5641	if (IS_ERR(attrs))
5642		return PTR_ERR(attrs);
5643
5644	if (attrs->map_bits)
5645		hist_trigger_bits = attrs->map_bits;
5646
5647	hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
5648	if (IS_ERR(hist_data)) {
5649		destroy_hist_trigger_attrs(attrs);
5650		return PTR_ERR(hist_data);
5651	}
5652
5653	trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5654
5655	trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
5656	if (!trigger_data) {
5657		ret = -ENOMEM;
5658		goto out_free;
5659	}
5660
5661	trigger_data->count = -1;
5662	trigger_data->ops = trigger_ops;
5663	trigger_data->cmd_ops = cmd_ops;
5664
5665	INIT_LIST_HEAD(&trigger_data->list);
5666	RCU_INIT_POINTER(trigger_data->filter, NULL);
5667
5668	trigger_data->private_data = hist_data;
5669
5670	/* if param is non-empty, it's supposed to be a filter */
5671	if (param && cmd_ops->set_filter) {
5672		ret = cmd_ops->set_filter(param, trigger_data, file);
5673		if (ret < 0)
5674			goto out_free;
5675	}
5676
5677	if (remove) {
5678		if (!have_hist_trigger_match(trigger_data, file))
5679			goto out_free;
5680
5681		if (hist_trigger_check_refs(trigger_data, file)) {
5682			ret = -EBUSY;
5683			goto out_free;
5684		}
5685
5686		cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5687		se_name = trace_event_name(file->event_call);
5688		se = find_synth_event(se_name);
5689		if (se)
5690			se->ref--;
5691		ret = 0;
5692		goto out_free;
5693	}
5694
5695	ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5696	/*
5697	 * The above returns on success the # of triggers registered,
5698	 * but if it didn't register any it returns zero.  Consider no
5699	 * triggers registered a failure too.
5700	 */
5701	if (!ret) {
5702		if (!(attrs->pause || attrs->cont || attrs->clear))
5703			ret = -ENOENT;
5704		goto out_free;
5705	} else if (ret < 0)
5706		goto out_free;
5707
5708	if (get_named_trigger_data(trigger_data))
5709		goto enable;
5710
5711	if (has_hist_vars(hist_data))
5712		save_hist_vars(hist_data);
5713
5714	ret = create_actions(hist_data);
5715	if (ret)
5716		goto out_unreg;
5717
5718	ret = tracing_map_init(hist_data->map);
5719	if (ret)
5720		goto out_unreg;
5721enable:
5722	ret = hist_trigger_enable(trigger_data, file);
5723	if (ret)
5724		goto out_unreg;
5725
5726	se_name = trace_event_name(file->event_call);
5727	se = find_synth_event(se_name);
5728	if (se)
5729		se->ref++;
5730	/* Just return zero, not the number of registered triggers */
5731	ret = 0;
5732 out:
5733	if (ret == 0)
5734		hist_err_clear();
5735
5736	return ret;
5737 out_unreg:
5738	cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5739 out_free:
5740	if (cmd_ops->set_filter)
5741		cmd_ops->set_filter(NULL, trigger_data, NULL);
5742
5743	remove_hist_vars(hist_data);
5744
5745	kfree(trigger_data);
5746
5747	destroy_hist_data(hist_data);
5748	goto out;
5749}
5750
5751static struct event_command trigger_hist_cmd = {
5752	.name			= "hist",
5753	.trigger_type		= ETT_EVENT_HIST,
5754	.flags			= EVENT_CMD_FL_NEEDS_REC,
5755	.func			= event_hist_trigger_func,
5756	.reg			= hist_register_trigger,
5757	.unreg			= hist_unregister_trigger,
5758	.unreg_all		= hist_unreg_all,
5759	.get_trigger_ops	= event_hist_get_trigger_ops,
5760	.set_filter		= set_trigger_filter,
5761};
5762
5763__init int register_trigger_hist_cmd(void)
5764{
5765	int ret;
5766
5767	ret = register_event_command(&trigger_hist_cmd);
5768	WARN_ON(ret < 0);
5769
5770	return ret;
5771}
5772
5773static void
5774hist_enable_trigger(struct event_trigger_data *data, void *rec,
5775		    struct ring_buffer_event *event)
5776{
5777	struct enable_trigger_data *enable_data = data->private_data;
5778	struct event_trigger_data *test;
5779
5780	list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
5781				lockdep_is_held(&event_mutex)) {
5782		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5783			if (enable_data->enable)
5784				test->paused = false;
5785			else
5786				test->paused = true;
5787		}
5788	}
5789}
5790
5791static void
5792hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
5793			  struct ring_buffer_event *event)
5794{
5795	if (!data->count)
5796		return;
5797
5798	if (data->count != -1)
5799		(data->count)--;
5800
5801	hist_enable_trigger(data, rec, event);
5802}
5803
5804static struct event_trigger_ops hist_enable_trigger_ops = {
5805	.func			= hist_enable_trigger,
5806	.print			= event_enable_trigger_print,
5807	.init			= event_trigger_init,
5808	.free			= event_enable_trigger_free,
5809};
5810
5811static struct event_trigger_ops hist_enable_count_trigger_ops = {
5812	.func			= hist_enable_count_trigger,
5813	.print			= event_enable_trigger_print,
5814	.init			= event_trigger_init,
5815	.free			= event_enable_trigger_free,
5816};
5817
5818static struct event_trigger_ops hist_disable_trigger_ops = {
5819	.func			= hist_enable_trigger,
5820	.print			= event_enable_trigger_print,
5821	.init			= event_trigger_init,
5822	.free			= event_enable_trigger_free,
5823};
5824
5825static struct event_trigger_ops hist_disable_count_trigger_ops = {
5826	.func			= hist_enable_count_trigger,
5827	.print			= event_enable_trigger_print,
5828	.init			= event_trigger_init,
5829	.free			= event_enable_trigger_free,
5830};
5831
5832static struct event_trigger_ops *
5833hist_enable_get_trigger_ops(char *cmd, char *param)
5834{
5835	struct event_trigger_ops *ops;
5836	bool enable;
5837
5838	enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5839
5840	if (enable)
5841		ops = param ? &hist_enable_count_trigger_ops :
5842			&hist_enable_trigger_ops;
5843	else
5844		ops = param ? &hist_disable_count_trigger_ops :
5845			&hist_disable_trigger_ops;
5846
5847	return ops;
5848}
5849
5850static void hist_enable_unreg_all(struct trace_event_file *file)
5851{
5852	struct event_trigger_data *test, *n;
5853
5854	list_for_each_entry_safe(test, n, &file->triggers, list) {
5855		if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5856			list_del_rcu(&test->list);
5857			update_cond_flag(file);
5858			trace_event_trigger_enable_disable(file, 0);
5859			if (test->ops->free)
5860				test->ops->free(test->ops, test);
5861		}
5862	}
5863}
5864
5865static struct event_command trigger_hist_enable_cmd = {
5866	.name			= ENABLE_HIST_STR,
5867	.trigger_type		= ETT_HIST_ENABLE,
5868	.func			= event_enable_trigger_func,
5869	.reg			= event_enable_register_trigger,
5870	.unreg			= event_enable_unregister_trigger,
5871	.unreg_all		= hist_enable_unreg_all,
5872	.get_trigger_ops	= hist_enable_get_trigger_ops,
5873	.set_filter		= set_trigger_filter,
5874};
5875
5876static struct event_command trigger_hist_disable_cmd = {
5877	.name			= DISABLE_HIST_STR,
5878	.trigger_type		= ETT_HIST_ENABLE,
5879	.func			= event_enable_trigger_func,
5880	.reg			= event_enable_register_trigger,
5881	.unreg			= event_enable_unregister_trigger,
5882	.unreg_all		= hist_enable_unreg_all,
5883	.get_trigger_ops	= hist_enable_get_trigger_ops,
5884	.set_filter		= set_trigger_filter,
5885};
5886
5887static __init void unregister_trigger_hist_enable_disable_cmds(void)
5888{
5889	unregister_event_command(&trigger_hist_enable_cmd);
5890	unregister_event_command(&trigger_hist_disable_cmd);
5891}
5892
5893__init int register_trigger_hist_enable_disable_cmds(void)
5894{
5895	int ret;
5896
5897	ret = register_event_command(&trigger_hist_enable_cmd);
5898	if (WARN_ON(ret < 0))
5899		return ret;
5900	ret = register_event_command(&trigger_hist_disable_cmd);
5901	if (WARN_ON(ret < 0))
5902		unregister_trigger_hist_enable_disable_cmds();
5903
5904	return ret;
5905}