1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * trace_events_hist - trace event hist triggers
   4 *
   5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6 */
   7
   8#include <linux/module.h>
   9#include <linux/kallsyms.h>
  10#include <linux/security.h>
  11#include <linux/mutex.h>
  12#include <linux/slab.h>
  13#include <linux/stacktrace.h>
  14#include <linux/rculist.h>
  15#include <linux/tracefs.h>
  16
  17/* for gfp flag names */
  18#include <linux/trace_events.h>
  19#include <trace/events/mmflags.h>
  20
  21#include "tracing_map.h"
  22#include "trace_synth.h"
  23
  24#define ERRORS								\
  25	C(NONE,			"No error"),				\
  26	C(DUPLICATE_VAR,	"Variable already defined"),		\
  27	C(VAR_NOT_UNIQUE,	"Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28	C(TOO_MANY_VARS,	"Too many variables defined"),		\
  29	C(MALFORMED_ASSIGNMENT,	"Malformed assignment"),		\
  30	C(NAMED_MISMATCH,	"Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31	C(TRIGGER_EEXIST,	"Hist trigger already exists"),		\
  32	C(TRIGGER_ENOENT_CLEAR,	"Can't clear or continue a nonexistent hist trigger"), \
  33	C(SET_CLOCK_FAIL,	"Couldn't set trace_clock"),		\
  34	C(BAD_FIELD_MODIFIER,	"Invalid field modifier"),		\
  35	C(TOO_MANY_SUBEXPR,	"Too many subexpressions (3 max)"),	\
  36	C(TIMESTAMP_MISMATCH,	"Timestamp units in expression don't match"), \
  37	C(TOO_MANY_FIELD_VARS,	"Too many field variables defined"),	\
  38	C(EVENT_FILE_NOT_FOUND,	"Event file not found"),		\
  39	C(HIST_NOT_FOUND,	"Matching event histogram not found"),	\
  40	C(HIST_CREATE_FAIL,	"Couldn't create histogram for field"),	\
  41	C(SYNTH_VAR_NOT_FOUND,	"Couldn't find synthetic variable"),	\
  42	C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),	\
  43	C(SYNTH_TYPE_MISMATCH,	"Param type doesn't match synthetic event field type"), \
  44	C(SYNTH_COUNT_MISMATCH,	"Param count doesn't match synthetic event field count"), \
  45	C(FIELD_VAR_PARSE_FAIL,	"Couldn't parse field variable"),	\
  46	C(VAR_CREATE_FIND_FAIL,	"Couldn't create or find variable"),	\
  47	C(ONX_NOT_VAR,		"For onmax(x) or onchange(x), x must be a variable"), \
  48	C(ONX_VAR_NOT_FOUND,	"Couldn't find onmax or onchange variable"), \
  49	C(ONX_VAR_CREATE_FAIL,	"Couldn't create onmax or onchange variable"), \
  50	C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),	\
  51	C(TOO_MANY_PARAMS,	"Too many action params"),		\
  52	C(PARAM_NOT_FOUND,	"Couldn't find param"),			\
  53	C(INVALID_PARAM,	"Invalid action param"),		\
  54	C(ACTION_NOT_FOUND,	"No action found"),			\
  55	C(NO_SAVE_PARAMS,	"No params found for save()"),		\
  56	C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57	C(ACTION_MISMATCH,	"Handler doesn't support action"),	\
  58	C(NO_CLOSING_PAREN,	"No closing paren found"),		\
  59	C(SUBSYS_NOT_FOUND,	"Missing subsystem"),			\
  60	C(INVALID_SUBSYS_EVENT,	"Invalid subsystem or event name"),	\
  61	C(INVALID_REF_KEY,	"Using variable references in keys not supported"), \
  62	C(VAR_NOT_FOUND,	"Couldn't find variable"),		\
  63	C(FIELD_NOT_FOUND,	"Couldn't find field"),			\
  64	C(EMPTY_ASSIGNMENT,	"Empty assignment"),			\
  65	C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),		\
  66	C(EMPTY_SORT_FIELD,	"Empty sort field"),			\
  67	C(TOO_MANY_SORT_FIELDS,	"Too many sort fields (Max = 2)"),	\
  68	C(INVALID_SORT_FIELD,	"Sort field must be a key or a val"),	\
  69	C(INVALID_STR_OPERAND,	"String type can not be an operand in expression"),
  70
  71#undef C
  72#define C(a, b)		HIST_ERR_##a
  73
  74enum { ERRORS };
  75
  76#undef C
  77#define C(a, b)		b
  78
  79static const char *err_text[] = { ERRORS };
  80
  81struct hist_field;
  82
  83typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  84				struct tracing_map_elt *elt,
  85				struct trace_buffer *buffer,
  86				struct ring_buffer_event *rbe,
  87				void *event);
  88
  89#define HIST_FIELD_OPERANDS_MAX	2
  90#define HIST_FIELDS_MAX		(TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  91#define HIST_ACTIONS_MAX	8
  92
  93enum field_op_id {
  94	FIELD_OP_NONE,
  95	FIELD_OP_PLUS,
  96	FIELD_OP_MINUS,
  97	FIELD_OP_UNARY_MINUS,
  98};
  99
 100/*
 101 * A hist_var (histogram variable) contains variable information for
 102 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 103 * flag set.  A hist_var has a variable name e.g. ts0, and is
 104 * associated with a given histogram trigger, as specified by
 105 * hist_data.  The hist_var idx is the unique index assigned to the
 106 * variable by the hist trigger's tracing_map.  The idx is what is
 107 * used to set a variable's value and, by a variable reference, to
 108 * retrieve it.
 109 */
 110struct hist_var {
 111	char				*name;
 112	struct hist_trigger_data	*hist_data;
 113	unsigned int			idx;
 114};
 115
 116struct hist_field {
 117	struct ftrace_event_field	*field;
 118	unsigned long			flags;
 119	hist_field_fn_t			fn;
 120	unsigned int			ref;
 121	unsigned int			size;
 122	unsigned int			offset;
 123	unsigned int                    is_signed;
 124	const char			*type;
 125	struct hist_field		*operands[HIST_FIELD_OPERANDS_MAX];
 126	struct hist_trigger_data	*hist_data;
 127
 128	/*
 129	 * Variable fields contain variable-specific info in var.
 130	 */
 131	struct hist_var			var;
 132	enum field_op_id		operator;
 133	char				*system;
 134	char				*event_name;
 135
 136	/*
 137	 * The name field is used for EXPR and VAR_REF fields.  VAR
 138	 * fields contain the variable name in var.name.
 139	 */
 140	char				*name;
 141
 142	/*
 143	 * When a histogram trigger is hit, if it has any references
 144	 * to variables, the values of those variables are collected
 145	 * into a var_ref_vals array by resolve_var_refs().  The
 146	 * current value of each variable is read from the tracing_map
 147	 * using the hist field's hist_var.idx and entered into the
 148	 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 149	 */
 150	unsigned int			var_ref_idx;
 151	bool                            read_once;
 152
 153	unsigned int			var_str_idx;
 154};
 155
 156static u64 hist_field_none(struct hist_field *field,
 157			   struct tracing_map_elt *elt,
 158			   struct trace_buffer *buffer,
 159			   struct ring_buffer_event *rbe,
 160			   void *event)
 161{
 162	return 0;
 163}
 164
 165static u64 hist_field_counter(struct hist_field *field,
 166			      struct tracing_map_elt *elt,
 167			      struct trace_buffer *buffer,
 168			      struct ring_buffer_event *rbe,
 169			      void *event)
 170{
 171	return 1;
 172}
 173
 174static u64 hist_field_string(struct hist_field *hist_field,
 175			     struct tracing_map_elt *elt,
 176			     struct trace_buffer *buffer,
 177			     struct ring_buffer_event *rbe,
 178			     void *event)
 179{
 180	char *addr = (char *)(event + hist_field->field->offset);
 181
 182	return (u64)(unsigned long)addr;
 183}
 184
 185static u64 hist_field_dynstring(struct hist_field *hist_field,
 186				struct tracing_map_elt *elt,
 187				struct trace_buffer *buffer,
 188				struct ring_buffer_event *rbe,
 189				void *event)
 190{
 191	u32 str_item = *(u32 *)(event + hist_field->field->offset);
 192	int str_loc = str_item & 0xffff;
 193	char *addr = (char *)(event + str_loc);
 194
 195	return (u64)(unsigned long)addr;
 196}
 197
 198static u64 hist_field_pstring(struct hist_field *hist_field,
 199			      struct tracing_map_elt *elt,
 200			      struct trace_buffer *buffer,
 201			      struct ring_buffer_event *rbe,
 202			      void *event)
 203{
 204	char **addr = (char **)(event + hist_field->field->offset);
 205
 206	return (u64)(unsigned long)*addr;
 207}
 208
 209static u64 hist_field_log2(struct hist_field *hist_field,
 210			   struct tracing_map_elt *elt,
 211			   struct trace_buffer *buffer,
 212			   struct ring_buffer_event *rbe,
 213			   void *event)
 214{
 215	struct hist_field *operand = hist_field->operands[0];
 216
 217	u64 val = operand->fn(operand, elt, buffer, rbe, event);
 218
 219	return (u64) ilog2(roundup_pow_of_two(val));
 220}
 221
 222static u64 hist_field_plus(struct hist_field *hist_field,
 223			   struct tracing_map_elt *elt,
 224			   struct trace_buffer *buffer,
 225			   struct ring_buffer_event *rbe,
 226			   void *event)
 227{
 228	struct hist_field *operand1 = hist_field->operands[0];
 229	struct hist_field *operand2 = hist_field->operands[1];
 230
 231	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 232	u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 233
 234	return val1 + val2;
 235}
 236
 237static u64 hist_field_minus(struct hist_field *hist_field,
 238			    struct tracing_map_elt *elt,
 239			    struct trace_buffer *buffer,
 240			    struct ring_buffer_event *rbe,
 241			    void *event)
 242{
 243	struct hist_field *operand1 = hist_field->operands[0];
 244	struct hist_field *operand2 = hist_field->operands[1];
 245
 246	u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 247	u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 248
 249	return val1 - val2;
 250}
 251
 252static u64 hist_field_unary_minus(struct hist_field *hist_field,
 253				  struct tracing_map_elt *elt,
 254				  struct trace_buffer *buffer,
 255				  struct ring_buffer_event *rbe,
 256				  void *event)
 257{
 258	struct hist_field *operand = hist_field->operands[0];
 259
 260	s64 sval = (s64)operand->fn(operand, elt, buffer, rbe, event);
 261	u64 val = (u64)-sval;
 262
 263	return val;
 264}
 265
 266#define DEFINE_HIST_FIELD_FN(type)					\
 267	static u64 hist_field_##type(struct hist_field *hist_field,	\
 268				     struct tracing_map_elt *elt,	\
 269				     struct trace_buffer *buffer,	\
 270				     struct ring_buffer_event *rbe,	\
 271				     void *event)			\
 272{									\
 273	type *addr = (type *)(event + hist_field->field->offset);	\
 274									\
 275	return (u64)(unsigned long)*addr;				\
 276}
 277
 278DEFINE_HIST_FIELD_FN(s64);
 279DEFINE_HIST_FIELD_FN(u64);
 280DEFINE_HIST_FIELD_FN(s32);
 281DEFINE_HIST_FIELD_FN(u32);
 282DEFINE_HIST_FIELD_FN(s16);
 283DEFINE_HIST_FIELD_FN(u16);
 284DEFINE_HIST_FIELD_FN(s8);
 285DEFINE_HIST_FIELD_FN(u8);
 286
 287#define for_each_hist_field(i, hist_data)	\
 288	for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 289
 290#define for_each_hist_val_field(i, hist_data)	\
 291	for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 292
 293#define for_each_hist_key_field(i, hist_data)	\
 294	for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 295
 296#define HIST_STACKTRACE_DEPTH	16
 297#define HIST_STACKTRACE_SIZE	(HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 298#define HIST_STACKTRACE_SKIP	5
 299
 300#define HITCOUNT_IDX		0
 301#define HIST_KEY_SIZE_MAX	(MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 302
 303enum hist_field_flags {
 304	HIST_FIELD_FL_HITCOUNT		= 1 << 0,
 305	HIST_FIELD_FL_KEY		= 1 << 1,
 306	HIST_FIELD_FL_STRING		= 1 << 2,
 307	HIST_FIELD_FL_HEX		= 1 << 3,
 308	HIST_FIELD_FL_SYM		= 1 << 4,
 309	HIST_FIELD_FL_SYM_OFFSET	= 1 << 5,
 310	HIST_FIELD_FL_EXECNAME		= 1 << 6,
 311	HIST_FIELD_FL_SYSCALL		= 1 << 7,
 312	HIST_FIELD_FL_STACKTRACE	= 1 << 8,
 313	HIST_FIELD_FL_LOG2		= 1 << 9,
 314	HIST_FIELD_FL_TIMESTAMP		= 1 << 10,
 315	HIST_FIELD_FL_TIMESTAMP_USECS	= 1 << 11,
 316	HIST_FIELD_FL_VAR		= 1 << 12,
 317	HIST_FIELD_FL_EXPR		= 1 << 13,
 318	HIST_FIELD_FL_VAR_REF		= 1 << 14,
 319	HIST_FIELD_FL_CPU		= 1 << 15,
 320	HIST_FIELD_FL_ALIAS		= 1 << 16,
 321};
 322
 323struct var_defs {
 324	unsigned int	n_vars;
 325	char		*name[TRACING_MAP_VARS_MAX];
 326	char		*expr[TRACING_MAP_VARS_MAX];
 327};
 328
 329struct hist_trigger_attrs {
 330	char		*keys_str;
 331	char		*vals_str;
 332	char		*sort_key_str;
 333	char		*name;
 334	char		*clock;
 335	bool		pause;
 336	bool		cont;
 337	bool		clear;
 338	bool		ts_in_usecs;
 339	unsigned int	map_bits;
 340
 341	char		*assignment_str[TRACING_MAP_VARS_MAX];
 342	unsigned int	n_assignments;
 343
 344	char		*action_str[HIST_ACTIONS_MAX];
 345	unsigned int	n_actions;
 346
 347	struct var_defs	var_defs;
 348};
 349
 350struct field_var {
 351	struct hist_field	*var;
 352	struct hist_field	*val;
 353};
 354
 355struct field_var_hist {
 356	struct hist_trigger_data	*hist_data;
 357	char				*cmd;
 358};
 359
 360struct hist_trigger_data {
 361	struct hist_field               *fields[HIST_FIELDS_MAX];
 362	unsigned int			n_vals;
 363	unsigned int			n_keys;
 364	unsigned int			n_fields;
 365	unsigned int			n_vars;
 366	unsigned int			n_var_str;
 367	unsigned int			key_size;
 368	struct tracing_map_sort_key	sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 369	unsigned int			n_sort_keys;
 370	struct trace_event_file		*event_file;
 371	struct hist_trigger_attrs	*attrs;
 372	struct tracing_map		*map;
 373	bool				enable_timestamps;
 374	bool				remove;
 375	struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 376	unsigned int			n_var_refs;
 377
 378	struct action_data		*actions[HIST_ACTIONS_MAX];
 379	unsigned int			n_actions;
 380
 381	struct field_var		*field_vars[SYNTH_FIELDS_MAX];
 382	unsigned int			n_field_vars;
 383	unsigned int			n_field_var_str;
 384	struct field_var_hist		*field_var_hists[SYNTH_FIELDS_MAX];
 385	unsigned int			n_field_var_hists;
 386
 387	struct field_var		*save_vars[SYNTH_FIELDS_MAX];
 388	unsigned int			n_save_vars;
 389	unsigned int			n_save_var_str;
 390};
 391
 392struct action_data;
 393
 394typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 395			     struct tracing_map_elt *elt,
 396			     struct trace_buffer *buffer, void *rec,
 397			     struct ring_buffer_event *rbe, void *key,
 398			     struct action_data *data, u64 *var_ref_vals);
 399
 400typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 401
 402enum handler_id {
 403	HANDLER_ONMATCH = 1,
 404	HANDLER_ONMAX,
 405	HANDLER_ONCHANGE,
 406};
 407
 408enum action_id {
 409	ACTION_SAVE = 1,
 410	ACTION_TRACE,
 411	ACTION_SNAPSHOT,
 412};
 413
 414struct action_data {
 415	enum handler_id		handler;
 416	enum action_id		action;
 417	char			*action_name;
 418	action_fn_t		fn;
 419
 420	unsigned int		n_params;
 421	char			*params[SYNTH_FIELDS_MAX];
 422
 423	/*
 424	 * When a histogram trigger is hit, the values of any
 425	 * references to variables, including variables being passed
 426	 * as parameters to synthetic events, are collected into a
 427	 * var_ref_vals array.  This var_ref_idx array is an array of
 428	 * indices into the var_ref_vals array, one for each synthetic
 429	 * event param, and is passed to the synthetic event
 430	 * invocation.
 431	 */
 432	unsigned int		var_ref_idx[TRACING_MAP_VARS_MAX];
 433	struct synth_event	*synth_event;
 434	bool			use_trace_keyword;
 435	char			*synth_event_name;
 436
 437	union {
 438		struct {
 439			char			*event;
 440			char			*event_system;
 441		} match_data;
 442
 443		struct {
 444			/*
 445			 * var_str contains the $-unstripped variable
 446			 * name referenced by var_ref, and used when
 447			 * printing the action.  Because var_ref
 448			 * creation is deferred to create_actions(),
 449			 * we need a per-action way to save it until
 450			 * then, thus var_str.
 451			 */
 452			char			*var_str;
 453
 454			/*
 455			 * var_ref refers to the variable being
 456			 * tracked e.g onmax($var).
 457			 */
 458			struct hist_field	*var_ref;
 459
 460			/*
 461			 * track_var contains the 'invisible' tracking
 462			 * variable created to keep the current
 463			 * e.g. max value.
 464			 */
 465			struct hist_field	*track_var;
 466
 467			check_track_val_fn_t	check_val;
 468			action_fn_t		save_data;
 469		} track_data;
 470	};
 471};
 472
 473struct track_data {
 474	u64				track_val;
 475	bool				updated;
 476
 477	unsigned int			key_len;
 478	void				*key;
 479	struct tracing_map_elt		elt;
 480
 481	struct action_data		*action_data;
 482	struct hist_trigger_data	*hist_data;
 483};
 484
 485struct hist_elt_data {
 486	char *comm;
 487	u64 *var_ref_vals;
 488	char *field_var_str[SYNTH_FIELDS_MAX];
 489};
 490
 491struct snapshot_context {
 492	struct tracing_map_elt	*elt;
 493	void			*key;
 494};
 495
 496static void track_data_free(struct track_data *track_data)
 497{
 498	struct hist_elt_data *elt_data;
 499
 500	if (!track_data)
 501		return;
 502
 503	kfree(track_data->key);
 504
 505	elt_data = track_data->elt.private_data;
 506	if (elt_data) {
 507		kfree(elt_data->comm);
 508		kfree(elt_data);
 509	}
 510
 511	kfree(track_data);
 512}
 513
 514static struct track_data *track_data_alloc(unsigned int key_len,
 515					   struct action_data *action_data,
 516					   struct hist_trigger_data *hist_data)
 517{
 518	struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 519	struct hist_elt_data *elt_data;
 520
 521	if (!data)
 522		return ERR_PTR(-ENOMEM);
 523
 524	data->key = kzalloc(key_len, GFP_KERNEL);
 525	if (!data->key) {
 526		track_data_free(data);
 527		return ERR_PTR(-ENOMEM);
 528	}
 529
 530	data->key_len = key_len;
 531	data->action_data = action_data;
 532	data->hist_data = hist_data;
 533
 534	elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 535	if (!elt_data) {
 536		track_data_free(data);
 537		return ERR_PTR(-ENOMEM);
 538	}
 539
 540	data->elt.private_data = elt_data;
 541
 542	elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 543	if (!elt_data->comm) {
 544		track_data_free(data);
 545		return ERR_PTR(-ENOMEM);
 546	}
 547
 548	return data;
 549}
 550
 551static char last_cmd[MAX_FILTER_STR_VAL];
 552static char last_cmd_loc[MAX_FILTER_STR_VAL];
 553
 554static int errpos(char *str)
 555{
 556	return err_pos(last_cmd, str);
 557}
 558
 559static void last_cmd_set(struct trace_event_file *file, char *str)
 560{
 561	const char *system = NULL, *name = NULL;
 562	struct trace_event_call *call;
 563
 564	if (!str)
 565		return;
 566
 567	strcpy(last_cmd, "hist:");
 568	strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
 569
 570	if (file) {
 571		call = file->event_call;
 572		system = call->class->system;
 573		if (system) {
 574			name = trace_event_name(call);
 575			if (!name)
 576				system = NULL;
 577		}
 578	}
 579
 580	if (system)
 581		snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 582}
 583
 584static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 585{
 586	tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 587			err_type, err_pos);
 588}
 589
 590static void hist_err_clear(void)
 591{
 592	last_cmd[0] = '\0';
 593	last_cmd_loc[0] = '\0';
 594}
 595
 596typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 597				    unsigned int *var_ref_idx);
 598
 599static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 600			       unsigned int *var_ref_idx)
 601{
 602	struct tracepoint *tp = event->tp;
 603
 604	if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 605		struct tracepoint_func *probe_func_ptr;
 606		synth_probe_func_t probe_func;
 607		void *__data;
 608
 609		if (!(cpu_online(raw_smp_processor_id())))
 610			return;
 611
 612		probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 613		if (probe_func_ptr) {
 614			do {
 615				probe_func = probe_func_ptr->func;
 616				__data = probe_func_ptr->data;
 617				probe_func(__data, var_ref_vals, var_ref_idx);
 618			} while ((++probe_func_ptr)->func);
 619		}
 620	}
 621}
 622
 623static void action_trace(struct hist_trigger_data *hist_data,
 624			 struct tracing_map_elt *elt,
 625			 struct trace_buffer *buffer, void *rec,
 626			 struct ring_buffer_event *rbe, void *key,
 627			 struct action_data *data, u64 *var_ref_vals)
 628{
 629	struct synth_event *event = data->synth_event;
 630
 631	trace_synth(event, var_ref_vals, data->var_ref_idx);
 632}
 633
 634struct hist_var_data {
 635	struct list_head list;
 636	struct hist_trigger_data *hist_data;
 637};
 638
 639static u64 hist_field_timestamp(struct hist_field *hist_field,
 640				struct tracing_map_elt *elt,
 641				struct trace_buffer *buffer,
 642				struct ring_buffer_event *rbe,
 643				void *event)
 644{
 645	struct hist_trigger_data *hist_data = hist_field->hist_data;
 646	struct trace_array *tr = hist_data->event_file->tr;
 647
 648	u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 649
 650	if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 651		ts = ns2usecs(ts);
 652
 653	return ts;
 654}
 655
 656static u64 hist_field_cpu(struct hist_field *hist_field,
 657			  struct tracing_map_elt *elt,
 658			  struct trace_buffer *buffer,
 659			  struct ring_buffer_event *rbe,
 660			  void *event)
 661{
 662	int cpu = smp_processor_id();
 663
 664	return cpu;
 665}
 666
 667/**
 668 * check_field_for_var_ref - Check if a VAR_REF field references a variable
 669 * @hist_field: The VAR_REF field to check
 670 * @var_data: The hist trigger that owns the variable
 671 * @var_idx: The trigger variable identifier
 672 *
 673 * Check the given VAR_REF field to see whether or not it references
 674 * the given variable associated with the given trigger.
 675 *
 676 * Return: The VAR_REF field if it does reference the variable, NULL if not
 677 */
 678static struct hist_field *
 679check_field_for_var_ref(struct hist_field *hist_field,
 680			struct hist_trigger_data *var_data,
 681			unsigned int var_idx)
 682{
 683	WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 684
 685	if (hist_field && hist_field->var.idx == var_idx &&
 686	    hist_field->var.hist_data == var_data)
 687		return hist_field;
 688
 689	return NULL;
 690}
 691
 692/**
 693 * find_var_ref - Check if a trigger has a reference to a trigger variable
 694 * @hist_data: The hist trigger that might have a reference to the variable
 695 * @var_data: The hist trigger that owns the variable
 696 * @var_idx: The trigger variable identifier
 697 *
 698 * Check the list of var_refs[] on the first hist trigger to see
 699 * whether any of them are references to the variable on the second
 700 * trigger.
 701 *
 702 * Return: The VAR_REF field referencing the variable if so, NULL if not
 703 */
 704static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 705				       struct hist_trigger_data *var_data,
 706				       unsigned int var_idx)
 707{
 708	struct hist_field *hist_field;
 709	unsigned int i;
 710
 711	for (i = 0; i < hist_data->n_var_refs; i++) {
 712		hist_field = hist_data->var_refs[i];
 713		if (check_field_for_var_ref(hist_field, var_data, var_idx))
 714			return hist_field;
 715	}
 716
 717	return NULL;
 718}
 719
 720/**
 721 * find_any_var_ref - Check if there is a reference to a given trigger variable
 722 * @hist_data: The hist trigger
 723 * @var_idx: The trigger variable identifier
 724 *
 725 * Check to see whether the given variable is currently referenced by
 726 * any other trigger.
 727 *
 728 * The trigger the variable is defined on is explicitly excluded - the
 729 * assumption being that a self-reference doesn't prevent a trigger
 730 * from being removed.
 731 *
 732 * Return: The VAR_REF field referencing the variable if so, NULL if not
 733 */
 734static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 735					   unsigned int var_idx)
 736{
 737	struct trace_array *tr = hist_data->event_file->tr;
 738	struct hist_field *found = NULL;
 739	struct hist_var_data *var_data;
 740
 741	list_for_each_entry(var_data, &tr->hist_vars, list) {
 742		if (var_data->hist_data == hist_data)
 743			continue;
 744		found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 745		if (found)
 746			break;
 747	}
 748
 749	return found;
 750}
 751
 752/**
 753 * check_var_refs - Check if there is a reference to any of trigger's variables
 754 * @hist_data: The hist trigger
 755 *
 756 * A trigger can define one or more variables.  If any one of them is
 757 * currently referenced by any other trigger, this function will
 758 * determine that.
 759
 760 * Typically used to determine whether or not a trigger can be removed
 761 * - if there are any references to a trigger's variables, it cannot.
 762 *
 763 * Return: True if there is a reference to any of trigger's variables
 764 */
 765static bool check_var_refs(struct hist_trigger_data *hist_data)
 766{
 767	struct hist_field *field;
 768	bool found = false;
 769	int i;
 770
 771	for_each_hist_field(i, hist_data) {
 772		field = hist_data->fields[i];
 773		if (field && field->flags & HIST_FIELD_FL_VAR) {
 774			if (find_any_var_ref(hist_data, field->var.idx)) {
 775				found = true;
 776				break;
 777			}
 778		}
 779	}
 780
 781	return found;
 782}
 783
 784static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
 785{
 786	struct trace_array *tr = hist_data->event_file->tr;
 787	struct hist_var_data *var_data, *found = NULL;
 788
 789	list_for_each_entry(var_data, &tr->hist_vars, list) {
 790		if (var_data->hist_data == hist_data) {
 791			found = var_data;
 792			break;
 793		}
 794	}
 795
 796	return found;
 797}
 798
 799static bool field_has_hist_vars(struct hist_field *hist_field,
 800				unsigned int level)
 801{
 802	int i;
 803
 804	if (level > 3)
 805		return false;
 806
 807	if (!hist_field)
 808		return false;
 809
 810	if (hist_field->flags & HIST_FIELD_FL_VAR ||
 811	    hist_field->flags & HIST_FIELD_FL_VAR_REF)
 812		return true;
 813
 814	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
 815		struct hist_field *operand;
 816
 817		operand = hist_field->operands[i];
 818		if (field_has_hist_vars(operand, level + 1))
 819			return true;
 820	}
 821
 822	return false;
 823}
 824
 825static bool has_hist_vars(struct hist_trigger_data *hist_data)
 826{
 827	struct hist_field *hist_field;
 828	int i;
 829
 830	for_each_hist_field(i, hist_data) {
 831		hist_field = hist_data->fields[i];
 832		if (field_has_hist_vars(hist_field, 0))
 833			return true;
 834	}
 835
 836	return false;
 837}
 838
 839static int save_hist_vars(struct hist_trigger_data *hist_data)
 840{
 841	struct trace_array *tr = hist_data->event_file->tr;
 842	struct hist_var_data *var_data;
 843
 844	var_data = find_hist_vars(hist_data);
 845	if (var_data)
 846		return 0;
 847
 848	if (tracing_check_open_get_tr(tr))
 849		return -ENODEV;
 850
 851	var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
 852	if (!var_data) {
 853		trace_array_put(tr);
 854		return -ENOMEM;
 855	}
 856
 857	var_data->hist_data = hist_data;
 858	list_add(&var_data->list, &tr->hist_vars);
 859
 860	return 0;
 861}
 862
 863static void remove_hist_vars(struct hist_trigger_data *hist_data)
 864{
 865	struct trace_array *tr = hist_data->event_file->tr;
 866	struct hist_var_data *var_data;
 867
 868	var_data = find_hist_vars(hist_data);
 869	if (!var_data)
 870		return;
 871
 872	if (WARN_ON(check_var_refs(hist_data)))
 873		return;
 874
 875	list_del(&var_data->list);
 876
 877	kfree(var_data);
 878
 879	trace_array_put(tr);
 880}
 881
 882static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
 883					 const char *var_name)
 884{
 885	struct hist_field *hist_field, *found = NULL;
 886	int i;
 887
 888	for_each_hist_field(i, hist_data) {
 889		hist_field = hist_data->fields[i];
 890		if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
 891		    strcmp(hist_field->var.name, var_name) == 0) {
 892			found = hist_field;
 893			break;
 894		}
 895	}
 896
 897	return found;
 898}
 899
 900static struct hist_field *find_var(struct hist_trigger_data *hist_data,
 901				   struct trace_event_file *file,
 902				   const char *var_name)
 903{
 904	struct hist_trigger_data *test_data;
 905	struct event_trigger_data *test;
 906	struct hist_field *hist_field;
 907
 908	lockdep_assert_held(&event_mutex);
 909
 910	hist_field = find_var_field(hist_data, var_name);
 911	if (hist_field)
 912		return hist_field;
 913
 914	list_for_each_entry(test, &file->triggers, list) {
 915		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 916			test_data = test->private_data;
 917			hist_field = find_var_field(test_data, var_name);
 918			if (hist_field)
 919				return hist_field;
 920		}
 921	}
 922
 923	return NULL;
 924}
 925
 926static struct trace_event_file *find_var_file(struct trace_array *tr,
 927					      char *system,
 928					      char *event_name,
 929					      char *var_name)
 930{
 931	struct hist_trigger_data *var_hist_data;
 932	struct hist_var_data *var_data;
 933	struct trace_event_file *file, *found = NULL;
 934
 935	if (system)
 936		return find_event_file(tr, system, event_name);
 937
 938	list_for_each_entry(var_data, &tr->hist_vars, list) {
 939		var_hist_data = var_data->hist_data;
 940		file = var_hist_data->event_file;
 941		if (file == found)
 942			continue;
 943
 944		if (find_var_field(var_hist_data, var_name)) {
 945			if (found) {
 946				hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
 947				return NULL;
 948			}
 949
 950			found = file;
 951		}
 952	}
 953
 954	return found;
 955}
 956
 957static struct hist_field *find_file_var(struct trace_event_file *file,
 958					const char *var_name)
 959{
 960	struct hist_trigger_data *test_data;
 961	struct event_trigger_data *test;
 962	struct hist_field *hist_field;
 963
 964	lockdep_assert_held(&event_mutex);
 965
 966	list_for_each_entry(test, &file->triggers, list) {
 967		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 968			test_data = test->private_data;
 969			hist_field = find_var_field(test_data, var_name);
 970			if (hist_field)
 971				return hist_field;
 972		}
 973	}
 974
 975	return NULL;
 976}
 977
 978static struct hist_field *
 979find_match_var(struct hist_trigger_data *hist_data, char *var_name)
 980{
 981	struct trace_array *tr = hist_data->event_file->tr;
 982	struct hist_field *hist_field, *found = NULL;
 983	struct trace_event_file *file;
 984	unsigned int i;
 985
 986	for (i = 0; i < hist_data->n_actions; i++) {
 987		struct action_data *data = hist_data->actions[i];
 988
 989		if (data->handler == HANDLER_ONMATCH) {
 990			char *system = data->match_data.event_system;
 991			char *event_name = data->match_data.event;
 992
 993			file = find_var_file(tr, system, event_name, var_name);
 994			if (!file)
 995				continue;
 996			hist_field = find_file_var(file, var_name);
 997			if (hist_field) {
 998				if (found) {
 999					hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1000						 errpos(var_name));
1001					return ERR_PTR(-EINVAL);
1002				}
1003
1004				found = hist_field;
1005			}
1006		}
1007	}
1008	return found;
1009}
1010
1011static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1012					 char *system,
1013					 char *event_name,
1014					 char *var_name)
1015{
1016	struct trace_array *tr = hist_data->event_file->tr;
1017	struct hist_field *hist_field = NULL;
1018	struct trace_event_file *file;
1019
1020	if (!system || !event_name) {
1021		hist_field = find_match_var(hist_data, var_name);
1022		if (IS_ERR(hist_field))
1023			return NULL;
1024		if (hist_field)
1025			return hist_field;
1026	}
1027
1028	file = find_var_file(tr, system, event_name, var_name);
1029	if (!file)
1030		return NULL;
1031
1032	hist_field = find_file_var(file, var_name);
1033
1034	return hist_field;
1035}
1036
1037static u64 hist_field_var_ref(struct hist_field *hist_field,
1038			      struct tracing_map_elt *elt,
1039			      struct trace_buffer *buffer,
1040			      struct ring_buffer_event *rbe,
1041			      void *event)
1042{
1043	struct hist_elt_data *elt_data;
1044	u64 var_val = 0;
1045
1046	if (WARN_ON_ONCE(!elt))
1047		return var_val;
1048
1049	elt_data = elt->private_data;
1050	var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1051
1052	return var_val;
1053}
1054
1055static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1056			     u64 *var_ref_vals, bool self)
1057{
1058	struct hist_trigger_data *var_data;
1059	struct tracing_map_elt *var_elt;
1060	struct hist_field *hist_field;
1061	unsigned int i, var_idx;
1062	bool resolved = true;
1063	u64 var_val = 0;
1064
1065	for (i = 0; i < hist_data->n_var_refs; i++) {
1066		hist_field = hist_data->var_refs[i];
1067		var_idx = hist_field->var.idx;
1068		var_data = hist_field->var.hist_data;
1069
1070		if (var_data == NULL) {
1071			resolved = false;
1072			break;
1073		}
1074
1075		if ((self && var_data != hist_data) ||
1076		    (!self && var_data == hist_data))
1077			continue;
1078
1079		var_elt = tracing_map_lookup(var_data->map, key);
1080		if (!var_elt) {
1081			resolved = false;
1082			break;
1083		}
1084
1085		if (!tracing_map_var_set(var_elt, var_idx)) {
1086			resolved = false;
1087			break;
1088		}
1089
1090		if (self || !hist_field->read_once)
1091			var_val = tracing_map_read_var(var_elt, var_idx);
1092		else
1093			var_val = tracing_map_read_var_once(var_elt, var_idx);
1094
1095		var_ref_vals[i] = var_val;
1096	}
1097
1098	return resolved;
1099}
1100
1101static const char *hist_field_name(struct hist_field *field,
1102				   unsigned int level)
1103{
1104	const char *field_name = "";
1105
1106	if (level > 1)
1107		return field_name;
1108
1109	if (field->field)
1110		field_name = field->field->name;
1111	else if (field->flags & HIST_FIELD_FL_LOG2 ||
1112		 field->flags & HIST_FIELD_FL_ALIAS)
1113		field_name = hist_field_name(field->operands[0], ++level);
1114	else if (field->flags & HIST_FIELD_FL_CPU)
1115		field_name = "common_cpu";
1116	else if (field->flags & HIST_FIELD_FL_EXPR ||
1117		 field->flags & HIST_FIELD_FL_VAR_REF) {
1118		if (field->system) {
1119			static char full_name[MAX_FILTER_STR_VAL];
1120
1121			strcat(full_name, field->system);
1122			strcat(full_name, ".");
1123			strcat(full_name, field->event_name);
1124			strcat(full_name, ".");
1125			strcat(full_name, field->name);
1126			field_name = full_name;
1127		} else
1128			field_name = field->name;
1129	} else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1130		field_name = "common_timestamp";
1131
1132	if (field_name == NULL)
1133		field_name = "";
1134
1135	return field_name;
1136}
1137
1138static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1139{
1140	hist_field_fn_t fn = NULL;
1141
1142	switch (field_size) {
1143	case 8:
1144		if (field_is_signed)
1145			fn = hist_field_s64;
1146		else
1147			fn = hist_field_u64;
1148		break;
1149	case 4:
1150		if (field_is_signed)
1151			fn = hist_field_s32;
1152		else
1153			fn = hist_field_u32;
1154		break;
1155	case 2:
1156		if (field_is_signed)
1157			fn = hist_field_s16;
1158		else
1159			fn = hist_field_u16;
1160		break;
1161	case 1:
1162		if (field_is_signed)
1163			fn = hist_field_s8;
1164		else
1165			fn = hist_field_u8;
1166		break;
1167	}
1168
1169	return fn;
1170}
1171
1172static int parse_map_size(char *str)
1173{
1174	unsigned long size, map_bits;
1175	int ret;
1176
1177	ret = kstrtoul(str, 0, &size);
1178	if (ret)
1179		goto out;
1180
1181	map_bits = ilog2(roundup_pow_of_two(size));
1182	if (map_bits < TRACING_MAP_BITS_MIN ||
1183	    map_bits > TRACING_MAP_BITS_MAX)
1184		ret = -EINVAL;
1185	else
1186		ret = map_bits;
1187 out:
1188	return ret;
1189}
1190
1191static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1192{
1193	unsigned int i;
1194
1195	if (!attrs)
1196		return;
1197
1198	for (i = 0; i < attrs->n_assignments; i++)
1199		kfree(attrs->assignment_str[i]);
1200
1201	for (i = 0; i < attrs->n_actions; i++)
1202		kfree(attrs->action_str[i]);
1203
1204	kfree(attrs->name);
1205	kfree(attrs->sort_key_str);
1206	kfree(attrs->keys_str);
1207	kfree(attrs->vals_str);
1208	kfree(attrs->clock);
1209	kfree(attrs);
1210}
1211
1212static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1213{
1214	int ret = -EINVAL;
1215
1216	if (attrs->n_actions >= HIST_ACTIONS_MAX)
1217		return ret;
1218
1219	if ((str_has_prefix(str, "onmatch(")) ||
1220	    (str_has_prefix(str, "onmax(")) ||
1221	    (str_has_prefix(str, "onchange("))) {
1222		attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1223		if (!attrs->action_str[attrs->n_actions]) {
1224			ret = -ENOMEM;
1225			return ret;
1226		}
1227		attrs->n_actions++;
1228		ret = 0;
1229	}
1230	return ret;
1231}
1232
1233static int parse_assignment(struct trace_array *tr,
1234			    char *str, struct hist_trigger_attrs *attrs)
1235{
1236	int len, ret = 0;
1237
1238	if ((len = str_has_prefix(str, "key=")) ||
1239	    (len = str_has_prefix(str, "keys="))) {
1240		attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1241		if (!attrs->keys_str) {
1242			ret = -ENOMEM;
1243			goto out;
1244		}
1245	} else if ((len = str_has_prefix(str, "val=")) ||
1246		   (len = str_has_prefix(str, "vals=")) ||
1247		   (len = str_has_prefix(str, "values="))) {
1248		attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1249		if (!attrs->vals_str) {
1250			ret = -ENOMEM;
1251			goto out;
1252		}
1253	} else if ((len = str_has_prefix(str, "sort="))) {
1254		attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1255		if (!attrs->sort_key_str) {
1256			ret = -ENOMEM;
1257			goto out;
1258		}
1259	} else if (str_has_prefix(str, "name=")) {
1260		attrs->name = kstrdup(str, GFP_KERNEL);
1261		if (!attrs->name) {
1262			ret = -ENOMEM;
1263			goto out;
1264		}
1265	} else if ((len = str_has_prefix(str, "clock="))) {
1266		str += len;
1267
1268		str = strstrip(str);
1269		attrs->clock = kstrdup(str, GFP_KERNEL);
1270		if (!attrs->clock) {
1271			ret = -ENOMEM;
1272			goto out;
1273		}
1274	} else if ((len = str_has_prefix(str, "size="))) {
1275		int map_bits = parse_map_size(str + len);
1276
1277		if (map_bits < 0) {
1278			ret = map_bits;
1279			goto out;
1280		}
1281		attrs->map_bits = map_bits;
1282	} else {
1283		char *assignment;
1284
1285		if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1286			hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1287			ret = -EINVAL;
1288			goto out;
1289		}
1290
1291		assignment = kstrdup(str, GFP_KERNEL);
1292		if (!assignment) {
1293			ret = -ENOMEM;
1294			goto out;
1295		}
1296
1297		attrs->assignment_str[attrs->n_assignments++] = assignment;
1298	}
1299 out:
1300	return ret;
1301}
1302
1303static struct hist_trigger_attrs *
1304parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1305{
1306	struct hist_trigger_attrs *attrs;
1307	int ret = 0;
1308
1309	attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1310	if (!attrs)
1311		return ERR_PTR(-ENOMEM);
1312
1313	while (trigger_str) {
1314		char *str = strsep(&trigger_str, ":");
1315		char *rhs;
1316
1317		rhs = strchr(str, '=');
1318		if (rhs) {
1319			if (!strlen(++rhs)) {
1320				ret = -EINVAL;
1321				hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1322				goto free;
1323			}
1324			ret = parse_assignment(tr, str, attrs);
1325			if (ret)
1326				goto free;
1327		} else if (strcmp(str, "pause") == 0)
1328			attrs->pause = true;
1329		else if ((strcmp(str, "cont") == 0) ||
1330			 (strcmp(str, "continue") == 0))
1331			attrs->cont = true;
1332		else if (strcmp(str, "clear") == 0)
1333			attrs->clear = true;
1334		else {
1335			ret = parse_action(str, attrs);
1336			if (ret)
1337				goto free;
1338		}
1339	}
1340
1341	if (!attrs->keys_str) {
1342		ret = -EINVAL;
1343		goto free;
1344	}
1345
1346	if (!attrs->clock) {
1347		attrs->clock = kstrdup("global", GFP_KERNEL);
1348		if (!attrs->clock) {
1349			ret = -ENOMEM;
1350			goto free;
1351		}
1352	}
1353
1354	return attrs;
1355 free:
1356	destroy_hist_trigger_attrs(attrs);
1357
1358	return ERR_PTR(ret);
1359}
1360
1361static inline void save_comm(char *comm, struct task_struct *task)
1362{
1363	if (!task->pid) {
1364		strcpy(comm, "<idle>");
1365		return;
1366	}
1367
1368	if (WARN_ON_ONCE(task->pid < 0)) {
1369		strcpy(comm, "<XXX>");
1370		return;
1371	}
1372
1373	strncpy(comm, task->comm, TASK_COMM_LEN);
1374}
1375
1376static void hist_elt_data_free(struct hist_elt_data *elt_data)
1377{
1378	unsigned int i;
1379
1380	for (i = 0; i < SYNTH_FIELDS_MAX; i++)
1381		kfree(elt_data->field_var_str[i]);
1382
1383	kfree(elt_data->comm);
1384	kfree(elt_data);
1385}
1386
1387static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1388{
1389	struct hist_elt_data *elt_data = elt->private_data;
1390
1391	hist_elt_data_free(elt_data);
1392}
1393
1394static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1395{
1396	struct hist_trigger_data *hist_data = elt->map->private_data;
1397	unsigned int size = TASK_COMM_LEN;
1398	struct hist_elt_data *elt_data;
1399	struct hist_field *key_field;
1400	unsigned int i, n_str;
1401
1402	elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1403	if (!elt_data)
1404		return -ENOMEM;
1405
1406	for_each_hist_key_field(i, hist_data) {
1407		key_field = hist_data->fields[i];
1408
1409		if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
1410			elt_data->comm = kzalloc(size, GFP_KERNEL);
1411			if (!elt_data->comm) {
1412				kfree(elt_data);
1413				return -ENOMEM;
1414			}
1415			break;
1416		}
1417	}
1418
1419	n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1420		hist_data->n_var_str;
1421	if (n_str > SYNTH_FIELDS_MAX) {
1422		hist_elt_data_free(elt_data);
1423		return -EINVAL;
1424	}
1425
1426	BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1427
1428	size = STR_VAR_LEN_MAX;
1429
1430	for (i = 0; i < n_str; i++) {
1431		elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1432		if (!elt_data->field_var_str[i]) {
1433			hist_elt_data_free(elt_data);
1434			return -ENOMEM;
1435		}
1436	}
1437
1438	elt->private_data = elt_data;
1439
1440	return 0;
1441}
1442
1443static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1444{
1445	struct hist_elt_data *elt_data = elt->private_data;
1446
1447	if (elt_data->comm)
1448		save_comm(elt_data->comm, current);
1449}
1450
1451static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1452	.elt_alloc	= hist_trigger_elt_data_alloc,
1453	.elt_free	= hist_trigger_elt_data_free,
1454	.elt_init	= hist_trigger_elt_data_init,
1455};
1456
1457static const char *get_hist_field_flags(struct hist_field *hist_field)
1458{
1459	const char *flags_str = NULL;
1460
1461	if (hist_field->flags & HIST_FIELD_FL_HEX)
1462		flags_str = "hex";
1463	else if (hist_field->flags & HIST_FIELD_FL_SYM)
1464		flags_str = "sym";
1465	else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1466		flags_str = "sym-offset";
1467	else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1468		flags_str = "execname";
1469	else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1470		flags_str = "syscall";
1471	else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1472		flags_str = "log2";
1473	else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1474		flags_str = "usecs";
1475
1476	return flags_str;
1477}
1478
1479static void expr_field_str(struct hist_field *field, char *expr)
1480{
1481	if (field->flags & HIST_FIELD_FL_VAR_REF)
1482		strcat(expr, "$");
1483
1484	strcat(expr, hist_field_name(field, 0));
1485
1486	if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1487		const char *flags_str = get_hist_field_flags(field);
1488
1489		if (flags_str) {
1490			strcat(expr, ".");
1491			strcat(expr, flags_str);
1492		}
1493	}
1494}
1495
1496static char *expr_str(struct hist_field *field, unsigned int level)
1497{
1498	char *expr;
1499
1500	if (level > 1)
1501		return NULL;
1502
1503	expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1504	if (!expr)
1505		return NULL;
1506
1507	if (!field->operands[0]) {
1508		expr_field_str(field, expr);
1509		return expr;
1510	}
1511
1512	if (field->operator == FIELD_OP_UNARY_MINUS) {
1513		char *subexpr;
1514
1515		strcat(expr, "-(");
1516		subexpr = expr_str(field->operands[0], ++level);
1517		if (!subexpr) {
1518			kfree(expr);
1519			return NULL;
1520		}
1521		strcat(expr, subexpr);
1522		strcat(expr, ")");
1523
1524		kfree(subexpr);
1525
1526		return expr;
1527	}
1528
1529	expr_field_str(field->operands[0], expr);
1530
1531	switch (field->operator) {
1532	case FIELD_OP_MINUS:
1533		strcat(expr, "-");
1534		break;
1535	case FIELD_OP_PLUS:
1536		strcat(expr, "+");
1537		break;
1538	default:
1539		kfree(expr);
1540		return NULL;
1541	}
1542
1543	expr_field_str(field->operands[1], expr);
1544
1545	return expr;
1546}
1547
1548static int contains_operator(char *str)
1549{
1550	enum field_op_id field_op = FIELD_OP_NONE;
1551	char *op;
1552
1553	op = strpbrk(str, "+-");
1554	if (!op)
1555		return FIELD_OP_NONE;
1556
1557	switch (*op) {
1558	case '-':
1559		/*
1560		 * Unfortunately, the modifier ".sym-offset"
1561		 * can confuse things.
1562		 */
1563		if (op - str >= 4 && !strncmp(op - 4, ".sym-offset", 11))
1564			return FIELD_OP_NONE;
1565
1566		if (*str == '-')
1567			field_op = FIELD_OP_UNARY_MINUS;
1568		else
1569			field_op = FIELD_OP_MINUS;
1570		break;
1571	case '+':
1572		field_op = FIELD_OP_PLUS;
1573		break;
1574	default:
1575		break;
1576	}
1577
1578	return field_op;
1579}
1580
1581static void get_hist_field(struct hist_field *hist_field)
1582{
1583	hist_field->ref++;
1584}
1585
1586static void __destroy_hist_field(struct hist_field *hist_field)
1587{
1588	if (--hist_field->ref > 1)
1589		return;
1590
1591	kfree(hist_field->var.name);
1592	kfree(hist_field->name);
1593	kfree(hist_field->type);
1594
1595	kfree(hist_field->system);
1596	kfree(hist_field->event_name);
1597
1598	kfree(hist_field);
1599}
1600
1601static void destroy_hist_field(struct hist_field *hist_field,
1602			       unsigned int level)
1603{
1604	unsigned int i;
1605
1606	if (level > 3)
1607		return;
1608
1609	if (!hist_field)
1610		return;
1611
1612	if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1613		return; /* var refs will be destroyed separately */
1614
1615	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1616		destroy_hist_field(hist_field->operands[i], level + 1);
1617
1618	__destroy_hist_field(hist_field);
1619}
1620
1621static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1622					    struct ftrace_event_field *field,
1623					    unsigned long flags,
1624					    char *var_name)
1625{
1626	struct hist_field *hist_field;
1627
1628	if (field && is_function_field(field))
1629		return NULL;
1630
1631	hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1632	if (!hist_field)
1633		return NULL;
1634
1635	hist_field->ref = 1;
1636
1637	hist_field->hist_data = hist_data;
1638
1639	if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1640		goto out; /* caller will populate */
1641
1642	if (flags & HIST_FIELD_FL_VAR_REF) {
1643		hist_field->fn = hist_field_var_ref;
1644		goto out;
1645	}
1646
1647	if (flags & HIST_FIELD_FL_HITCOUNT) {
1648		hist_field->fn = hist_field_counter;
1649		hist_field->size = sizeof(u64);
1650		hist_field->type = kstrdup("u64", GFP_KERNEL);
1651		if (!hist_field->type)
1652			goto free;
1653		goto out;
1654	}
1655
1656	if (flags & HIST_FIELD_FL_STACKTRACE) {
1657		hist_field->fn = hist_field_none;
1658		goto out;
1659	}
1660
1661	if (flags & HIST_FIELD_FL_LOG2) {
1662		unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
1663		hist_field->fn = hist_field_log2;
1664		hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1665		hist_field->size = hist_field->operands[0]->size;
1666		hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
1667		if (!hist_field->type)
1668			goto free;
1669		goto out;
1670	}
1671
1672	if (flags & HIST_FIELD_FL_TIMESTAMP) {
1673		hist_field->fn = hist_field_timestamp;
1674		hist_field->size = sizeof(u64);
1675		hist_field->type = kstrdup("u64", GFP_KERNEL);
1676		if (!hist_field->type)
1677			goto free;
1678		goto out;
1679	}
1680
1681	if (flags & HIST_FIELD_FL_CPU) {
1682		hist_field->fn = hist_field_cpu;
1683		hist_field->size = sizeof(int);
1684		hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
1685		if (!hist_field->type)
1686			goto free;
1687		goto out;
1688	}
1689
1690	if (WARN_ON_ONCE(!field))
1691		goto out;
1692
1693	/* Pointers to strings are just pointers and dangerous to dereference */
1694	if (is_string_field(field) &&
1695	    (field->filter_type != FILTER_PTR_STRING)) {
1696		flags |= HIST_FIELD_FL_STRING;
1697
1698		hist_field->size = MAX_FILTER_STR_VAL;
1699		hist_field->type = kstrdup(field->type, GFP_KERNEL);
1700		if (!hist_field->type)
1701			goto free;
1702
1703		if (field->filter_type == FILTER_STATIC_STRING)
1704			hist_field->fn = hist_field_string;
1705		else if (field->filter_type == FILTER_DYN_STRING)
1706			hist_field->fn = hist_field_dynstring;
1707		else
1708			hist_field->fn = hist_field_pstring;
1709	} else {
1710		hist_field->size = field->size;
1711		hist_field->is_signed = field->is_signed;
1712		hist_field->type = kstrdup(field->type, GFP_KERNEL);
1713		if (!hist_field->type)
1714			goto free;
1715
1716		hist_field->fn = select_value_fn(field->size,
1717						 field->is_signed);
1718		if (!hist_field->fn) {
1719			destroy_hist_field(hist_field, 0);
1720			return NULL;
1721		}
1722	}
1723 out:
1724	hist_field->field = field;
1725	hist_field->flags = flags;
1726
1727	if (var_name) {
1728		hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1729		if (!hist_field->var.name)
1730			goto free;
1731	}
1732
1733	return hist_field;
1734 free:
1735	destroy_hist_field(hist_field, 0);
1736	return NULL;
1737}
1738
1739static void destroy_hist_fields(struct hist_trigger_data *hist_data)
1740{
1741	unsigned int i;
1742
1743	for (i = 0; i < HIST_FIELDS_MAX; i++) {
1744		if (hist_data->fields[i]) {
1745			destroy_hist_field(hist_data->fields[i], 0);
1746			hist_data->fields[i] = NULL;
1747		}
1748	}
1749
1750	for (i = 0; i < hist_data->n_var_refs; i++) {
1751		WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
1752		__destroy_hist_field(hist_data->var_refs[i]);
1753		hist_data->var_refs[i] = NULL;
1754	}
1755}
1756
1757static int init_var_ref(struct hist_field *ref_field,
1758			struct hist_field *var_field,
1759			char *system, char *event_name)
1760{
1761	int err = 0;
1762
1763	ref_field->var.idx = var_field->var.idx;
1764	ref_field->var.hist_data = var_field->hist_data;
1765	ref_field->size = var_field->size;
1766	ref_field->is_signed = var_field->is_signed;
1767	ref_field->flags |= var_field->flags &
1768		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
1769
1770	if (system) {
1771		ref_field->system = kstrdup(system, GFP_KERNEL);
1772		if (!ref_field->system)
1773			return -ENOMEM;
1774	}
1775
1776	if (event_name) {
1777		ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
1778		if (!ref_field->event_name) {
1779			err = -ENOMEM;
1780			goto free;
1781		}
1782	}
1783
1784	if (var_field->var.name) {
1785		ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
1786		if (!ref_field->name) {
1787			err = -ENOMEM;
1788			goto free;
1789		}
1790	} else if (var_field->name) {
1791		ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
1792		if (!ref_field->name) {
1793			err = -ENOMEM;
1794			goto free;
1795		}
1796	}
1797
1798	ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
1799	if (!ref_field->type) {
1800		err = -ENOMEM;
1801		goto free;
1802	}
1803 out:
1804	return err;
1805 free:
1806	kfree(ref_field->system);
1807	kfree(ref_field->event_name);
1808	kfree(ref_field->name);
1809
1810	goto out;
1811}
1812
1813static int find_var_ref_idx(struct hist_trigger_data *hist_data,
1814			    struct hist_field *var_field)
1815{
1816	struct hist_field *ref_field;
1817	int i;
1818
1819	for (i = 0; i < hist_data->n_var_refs; i++) {
1820		ref_field = hist_data->var_refs[i];
1821		if (ref_field->var.idx == var_field->var.idx &&
1822		    ref_field->var.hist_data == var_field->hist_data)
1823			return i;
1824	}
1825
1826	return -ENOENT;
1827}
1828
1829/**
1830 * create_var_ref - Create a variable reference and attach it to trigger
1831 * @hist_data: The trigger that will be referencing the variable
1832 * @var_field: The VAR field to create a reference to
1833 * @system: The optional system string
1834 * @event_name: The optional event_name string
1835 *
1836 * Given a variable hist_field, create a VAR_REF hist_field that
1837 * represents a reference to it.
1838 *
1839 * This function also adds the reference to the trigger that
1840 * now references the variable.
1841 *
1842 * Return: The VAR_REF field if successful, NULL if not
1843 */
1844static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
1845					 struct hist_field *var_field,
1846					 char *system, char *event_name)
1847{
1848	unsigned long flags = HIST_FIELD_FL_VAR_REF;
1849	struct hist_field *ref_field;
1850	int i;
1851
1852	/* Check if the variable already exists */
1853	for (i = 0; i < hist_data->n_var_refs; i++) {
1854		ref_field = hist_data->var_refs[i];
1855		if (ref_field->var.idx == var_field->var.idx &&
1856		    ref_field->var.hist_data == var_field->hist_data) {
1857			get_hist_field(ref_field);
1858			return ref_field;
1859		}
1860	}
1861
1862	ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
1863	if (ref_field) {
1864		if (init_var_ref(ref_field, var_field, system, event_name)) {
1865			destroy_hist_field(ref_field, 0);
1866			return NULL;
1867		}
1868
1869		hist_data->var_refs[hist_data->n_var_refs] = ref_field;
1870		ref_field->var_ref_idx = hist_data->n_var_refs++;
1871	}
1872
1873	return ref_field;
1874}
1875
1876static bool is_var_ref(char *var_name)
1877{
1878	if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
1879		return false;
1880
1881	return true;
1882}
1883
1884static char *field_name_from_var(struct hist_trigger_data *hist_data,
1885				 char *var_name)
1886{
1887	char *name, *field;
1888	unsigned int i;
1889
1890	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
1891		name = hist_data->attrs->var_defs.name[i];
1892
1893		if (strcmp(var_name, name) == 0) {
1894			field = hist_data->attrs->var_defs.expr[i];
1895			if (contains_operator(field) || is_var_ref(field))
1896				continue;
1897			return field;
1898		}
1899	}
1900
1901	return NULL;
1902}
1903
1904static char *local_field_var_ref(struct hist_trigger_data *hist_data,
1905				 char *system, char *event_name,
1906				 char *var_name)
1907{
1908	struct trace_event_call *call;
1909
1910	if (system && event_name) {
1911		call = hist_data->event_file->event_call;
1912
1913		if (strcmp(system, call->class->system) != 0)
1914			return NULL;
1915
1916		if (strcmp(event_name, trace_event_name(call)) != 0)
1917			return NULL;
1918	}
1919
1920	if (!!system != !!event_name)
1921		return NULL;
1922
1923	if (!is_var_ref(var_name))
1924		return NULL;
1925
1926	var_name++;
1927
1928	return field_name_from_var(hist_data, var_name);
1929}
1930
1931static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
1932					char *system, char *event_name,
1933					char *var_name)
1934{
1935	struct hist_field *var_field = NULL, *ref_field = NULL;
1936	struct trace_array *tr = hist_data->event_file->tr;
1937
1938	if (!is_var_ref(var_name))
1939		return NULL;
1940
1941	var_name++;
1942
1943	var_field = find_event_var(hist_data, system, event_name, var_name);
1944	if (var_field)
1945		ref_field = create_var_ref(hist_data, var_field,
1946					   system, event_name);
1947
1948	if (!ref_field)
1949		hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
1950
1951	return ref_field;
1952}
1953
1954static struct ftrace_event_field *
1955parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
1956	    char *field_str, unsigned long *flags)
1957{
1958	struct ftrace_event_field *field = NULL;
1959	char *field_name, *modifier, *str;
1960	struct trace_array *tr = file->tr;
1961
1962	modifier = str = kstrdup(field_str, GFP_KERNEL);
1963	if (!modifier)
1964		return ERR_PTR(-ENOMEM);
1965
1966	field_name = strsep(&modifier, ".");
1967	if (modifier) {
1968		if (strcmp(modifier, "hex") == 0)
1969			*flags |= HIST_FIELD_FL_HEX;
1970		else if (strcmp(modifier, "sym") == 0)
1971			*flags |= HIST_FIELD_FL_SYM;
1972		else if (strcmp(modifier, "sym-offset") == 0)
1973			*flags |= HIST_FIELD_FL_SYM_OFFSET;
1974		else if ((strcmp(modifier, "execname") == 0) &&
1975			 (strcmp(field_name, "common_pid") == 0))
1976			*flags |= HIST_FIELD_FL_EXECNAME;
1977		else if (strcmp(modifier, "syscall") == 0)
1978			*flags |= HIST_FIELD_FL_SYSCALL;
1979		else if (strcmp(modifier, "log2") == 0)
1980			*flags |= HIST_FIELD_FL_LOG2;
1981		else if (strcmp(modifier, "usecs") == 0)
1982			*flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
1983		else {
1984			hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
1985			field = ERR_PTR(-EINVAL);
1986			goto out;
1987		}
1988	}
1989
1990	if (strcmp(field_name, "common_timestamp") == 0) {
1991		*flags |= HIST_FIELD_FL_TIMESTAMP;
1992		hist_data->enable_timestamps = true;
1993		if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1994			hist_data->attrs->ts_in_usecs = true;
1995	} else if (strcmp(field_name, "common_cpu") == 0)
1996		*flags |= HIST_FIELD_FL_CPU;
1997	else {
1998		field = trace_find_event_field(file->event_call, field_name);
1999		if (!field || !field->size) {
2000			/*
2001			 * For backward compatibility, if field_name
2002			 * was "cpu", then we treat this the same as
2003			 * common_cpu.
2004			 */
2005			if (strcmp(field_name, "cpu") == 0) {
2006				*flags |= HIST_FIELD_FL_CPU;
2007			} else {
2008				hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2009					 errpos(field_name));
2010				field = ERR_PTR(-EINVAL);
2011				goto out;
2012			}
2013		}
2014	}
2015 out:
2016	kfree(str);
2017
2018	return field;
2019}
2020
2021static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2022				       struct hist_field *var_ref,
2023				       char *var_name)
2024{
2025	struct hist_field *alias = NULL;
2026	unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2027
2028	alias = create_hist_field(hist_data, NULL, flags, var_name);
2029	if (!alias)
2030		return NULL;
2031
2032	alias->fn = var_ref->fn;
2033	alias->operands[0] = var_ref;
2034
2035	if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2036		destroy_hist_field(alias, 0);
2037		return NULL;
2038	}
2039
2040	alias->var_ref_idx = var_ref->var_ref_idx;
2041
2042	return alias;
2043}
2044
2045static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2046				     struct trace_event_file *file, char *str,
2047				     unsigned long *flags, char *var_name)
2048{
2049	char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2050	struct ftrace_event_field *field = NULL;
2051	struct hist_field *hist_field = NULL;
2052	int ret = 0;
2053
2054	s = strchr(str, '.');
2055	if (s) {
2056		s = strchr(++s, '.');
2057		if (s) {
2058			ref_system = strsep(&str, ".");
2059			if (!str) {
2060				ret = -EINVAL;
2061				goto out;
2062			}
2063			ref_event = strsep(&str, ".");
2064			if (!str) {
2065				ret = -EINVAL;
2066				goto out;
2067			}
2068			ref_var = str;
2069		}
2070	}
2071
2072	s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2073	if (!s) {
2074		hist_field = parse_var_ref(hist_data, ref_system,
2075					   ref_event, ref_var);
2076		if (hist_field) {
2077			if (var_name) {
2078				hist_field = create_alias(hist_data, hist_field, var_name);
2079				if (!hist_field) {
2080					ret = -ENOMEM;
2081					goto out;
2082				}
2083			}
2084			return hist_field;
2085		}
2086	} else
2087		str = s;
2088
2089	field = parse_field(hist_data, file, str, flags);
2090	if (IS_ERR(field)) {
2091		ret = PTR_ERR(field);
2092		goto out;
2093	}
2094
2095	hist_field = create_hist_field(hist_data, field, *flags, var_name);
2096	if (!hist_field) {
2097		ret = -ENOMEM;
2098		goto out;
2099	}
2100
2101	return hist_field;
2102 out:
2103	return ERR_PTR(ret);
2104}
2105
2106static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2107				     struct trace_event_file *file,
2108				     char *str, unsigned long flags,
2109				     char *var_name, unsigned int level);
2110
2111static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2112				      struct trace_event_file *file,
2113				      char *str, unsigned long flags,
2114				      char *var_name, unsigned int level)
2115{
2116	struct hist_field *operand1, *expr = NULL;
2117	unsigned long operand_flags;
2118	int ret = 0;
2119	char *s;
2120
2121	/* we support only -(xxx) i.e. explicit parens required */
2122
2123	if (level > 3) {
2124		hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2125		ret = -EINVAL;
2126		goto free;
2127	}
2128
2129	str++; /* skip leading '-' */
2130
2131	s = strchr(str, '(');
2132	if (s)
2133		str++;
2134	else {
2135		ret = -EINVAL;
2136		goto free;
2137	}
2138
2139	s = strrchr(str, ')');
2140	if (s)
2141		*s = '\0';
2142	else {
2143		ret = -EINVAL; /* no closing ')' */
2144		goto free;
2145	}
2146
2147	flags |= HIST_FIELD_FL_EXPR;
2148	expr = create_hist_field(hist_data, NULL, flags, var_name);
2149	if (!expr) {
2150		ret = -ENOMEM;
2151		goto free;
2152	}
2153
2154	operand_flags = 0;
2155	operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2156	if (IS_ERR(operand1)) {
2157		ret = PTR_ERR(operand1);
2158		goto free;
2159	}
2160	if (operand1->flags & HIST_FIELD_FL_STRING) {
2161		/* String type can not be the operand of unary operator. */
2162		hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2163		destroy_hist_field(operand1, 0);
2164		ret = -EINVAL;
2165		goto free;
2166	}
2167
2168	expr->flags |= operand1->flags &
2169		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2170	expr->fn = hist_field_unary_minus;
2171	expr->operands[0] = operand1;
2172	expr->operator = FIELD_OP_UNARY_MINUS;
2173	expr->name = expr_str(expr, 0);
2174	expr->type = kstrdup(operand1->type, GFP_KERNEL);
2175	if (!expr->type) {
2176		ret = -ENOMEM;
2177		goto free;
2178	}
2179
2180	return expr;
2181 free:
2182	destroy_hist_field(expr, 0);
2183	return ERR_PTR(ret);
2184}
2185
2186static int check_expr_operands(struct trace_array *tr,
2187			       struct hist_field *operand1,
2188			       struct hist_field *operand2)
2189{
2190	unsigned long operand1_flags = operand1->flags;
2191	unsigned long operand2_flags = operand2->flags;
2192
2193	if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2194	    (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2195		struct hist_field *var;
2196
2197		var = find_var_field(operand1->var.hist_data, operand1->name);
2198		if (!var)
2199			return -EINVAL;
2200		operand1_flags = var->flags;
2201	}
2202
2203	if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2204	    (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2205		struct hist_field *var;
2206
2207		var = find_var_field(operand2->var.hist_data, operand2->name);
2208		if (!var)
2209			return -EINVAL;
2210		operand2_flags = var->flags;
2211	}
2212
2213	if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2214	    (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2215		hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2216		return -EINVAL;
2217	}
2218
2219	return 0;
2220}
2221
2222static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2223				     struct trace_event_file *file,
2224				     char *str, unsigned long flags,
2225				     char *var_name, unsigned int level)
2226{
2227	struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2228	unsigned long operand_flags;
2229	int field_op, ret = -EINVAL;
2230	char *sep, *operand1_str;
2231
2232	if (level > 3) {
2233		hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2234		return ERR_PTR(-EINVAL);
2235	}
2236
2237	field_op = contains_operator(str);
2238
2239	if (field_op == FIELD_OP_NONE)
2240		return parse_atom(hist_data, file, str, &flags, var_name);
2241
2242	if (field_op == FIELD_OP_UNARY_MINUS)
2243		return parse_unary(hist_data, file, str, flags, var_name, ++level);
2244
2245	switch (field_op) {
2246	case FIELD_OP_MINUS:
2247		sep = "-";
2248		break;
2249	case FIELD_OP_PLUS:
2250		sep = "+";
2251		break;
2252	default:
2253		goto free;
2254	}
2255
2256	operand1_str = strsep(&str, sep);
2257	if (!operand1_str || !str)
2258		goto free;
2259
2260	operand_flags = 0;
2261	operand1 = parse_atom(hist_data, file, operand1_str,
2262			      &operand_flags, NULL);
2263	if (IS_ERR(operand1)) {
2264		ret = PTR_ERR(operand1);
2265		operand1 = NULL;
2266		goto free;
2267	}
2268	if (operand1->flags & HIST_FIELD_FL_STRING) {
2269		hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2270		ret = -EINVAL;
2271		goto free;
2272	}
2273
2274	/* rest of string could be another expression e.g. b+c in a+b+c */
2275	operand_flags = 0;
2276	operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2277	if (IS_ERR(operand2)) {
2278		ret = PTR_ERR(operand2);
2279		operand2 = NULL;
2280		goto free;
2281	}
2282	if (operand2->flags & HIST_FIELD_FL_STRING) {
2283		hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2284		ret = -EINVAL;
2285		goto free;
2286	}
2287
2288	ret = check_expr_operands(file->tr, operand1, operand2);
2289	if (ret)
2290		goto free;
2291
2292	flags |= HIST_FIELD_FL_EXPR;
2293
2294	flags |= operand1->flags &
2295		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2296
2297	expr = create_hist_field(hist_data, NULL, flags, var_name);
2298	if (!expr) {
2299		ret = -ENOMEM;
2300		goto free;
2301	}
2302
2303	operand1->read_once = true;
2304	operand2->read_once = true;
2305
2306	expr->operands[0] = operand1;
2307	expr->operands[1] = operand2;
2308
2309	/* The operand sizes should be the same, so just pick one */
2310	expr->size = operand1->size;
2311
2312	expr->operator = field_op;
2313	expr->name = expr_str(expr, 0);
2314	expr->type = kstrdup(operand1->type, GFP_KERNEL);
2315	if (!expr->type) {
2316		ret = -ENOMEM;
2317		goto free;
2318	}
2319
2320	switch (field_op) {
2321	case FIELD_OP_MINUS:
2322		expr->fn = hist_field_minus;
2323		break;
2324	case FIELD_OP_PLUS:
2325		expr->fn = hist_field_plus;
2326		break;
2327	default:
2328		ret = -EINVAL;
2329		goto free;
2330	}
2331
2332	return expr;
2333 free:
2334	destroy_hist_field(operand1, 0);
2335	destroy_hist_field(operand2, 0);
2336	destroy_hist_field(expr, 0);
2337
2338	return ERR_PTR(ret);
2339}
2340
2341static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2342				 struct trace_event_file *file)
2343{
2344	struct event_trigger_data *test;
2345
2346	lockdep_assert_held(&event_mutex);
2347
2348	list_for_each_entry(test, &file->triggers, list) {
2349		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2350			if (test->private_data == hist_data)
2351				return test->filter_str;
2352		}
2353	}
2354
2355	return NULL;
2356}
2357
2358static struct event_command trigger_hist_cmd;
2359static int event_hist_trigger_func(struct event_command *cmd_ops,
2360				   struct trace_event_file *file,
2361				   char *glob, char *cmd, char *param);
2362
2363static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2364			    struct hist_trigger_data *hist_data,
2365			    unsigned int n_keys)
2366{
2367	struct hist_field *target_hist_field, *hist_field;
2368	unsigned int n, i, j;
2369
2370	if (hist_data->n_fields - hist_data->n_vals != n_keys)
2371		return false;
2372
2373	i = hist_data->n_vals;
2374	j = target_hist_data->n_vals;
2375
2376	for (n = 0; n < n_keys; n++) {
2377		hist_field = hist_data->fields[i + n];
2378		target_hist_field = target_hist_data->fields[j + n];
2379
2380		if (strcmp(hist_field->type, target_hist_field->type) != 0)
2381			return false;
2382		if (hist_field->size != target_hist_field->size)
2383			return false;
2384		if (hist_field->is_signed != target_hist_field->is_signed)
2385			return false;
2386	}
2387
2388	return true;
2389}
2390
2391static struct hist_trigger_data *
2392find_compatible_hist(struct hist_trigger_data *target_hist_data,
2393		     struct trace_event_file *file)
2394{
2395	struct hist_trigger_data *hist_data;
2396	struct event_trigger_data *test;
2397	unsigned int n_keys;
2398
2399	lockdep_assert_held(&event_mutex);
2400
2401	n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2402
2403	list_for_each_entry(test, &file->triggers, list) {
2404		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2405			hist_data = test->private_data;
2406
2407			if (compatible_keys(target_hist_data, hist_data, n_keys))
2408				return hist_data;
2409		}
2410	}
2411
2412	return NULL;
2413}
2414
2415static struct trace_event_file *event_file(struct trace_array *tr,
2416					   char *system, char *event_name)
2417{
2418	struct trace_event_file *file;
2419
2420	file = __find_event_file(tr, system, event_name);
2421	if (!file)
2422		return ERR_PTR(-EINVAL);
2423
2424	return file;
2425}
2426
2427static struct hist_field *
2428find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2429			 char *system, char *event_name, char *field_name)
2430{
2431	struct hist_field *event_var;
2432	char *synthetic_name;
2433
2434	synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2435	if (!synthetic_name)
2436		return ERR_PTR(-ENOMEM);
2437
2438	strcpy(synthetic_name, "synthetic_");
2439	strcat(synthetic_name, field_name);
2440
2441	event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2442
2443	kfree(synthetic_name);
2444
2445	return event_var;
2446}
2447
2448/**
2449 * create_field_var_hist - Automatically create a histogram and var for a field
2450 * @target_hist_data: The target hist trigger
2451 * @subsys_name: Optional subsystem name
2452 * @event_name: Optional event name
2453 * @field_name: The name of the field (and the resulting variable)
2454 *
2455 * Hist trigger actions fetch data from variables, not directly from
2456 * events.  However, for convenience, users are allowed to directly
2457 * specify an event field in an action, which will be automatically
2458 * converted into a variable on their behalf.
2459
2460 * If a user specifies a field on an event that isn't the event the
2461 * histogram currently being defined (the target event histogram), the
2462 * only way that can be accomplished is if a new hist trigger is
2463 * created and the field variable defined on that.
2464 *
2465 * This function creates a new histogram compatible with the target
2466 * event (meaning a histogram with the same key as the target
2467 * histogram), and creates a variable for the specified field, but
2468 * with 'synthetic_' prepended to the variable name in order to avoid
2469 * collision with normal field variables.
2470 *
2471 * Return: The variable created for the field.
2472 */
2473static struct hist_field *
2474create_field_var_hist(struct hist_trigger_data *target_hist_data,
2475		      char *subsys_name, char *event_name, char *field_name)
2476{
2477	struct trace_array *tr = target_hist_data->event_file->tr;
2478	struct hist_trigger_data *hist_data;
2479	unsigned int i, n, first = true;
2480	struct field_var_hist *var_hist;
2481	struct trace_event_file *file;
2482	struct hist_field *key_field;
2483	struct hist_field *event_var;
2484	char *saved_filter;
2485	char *cmd;
2486	int ret;
2487
2488	if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2489		hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2490		return ERR_PTR(-EINVAL);
2491	}
2492
2493	file = event_file(tr, subsys_name, event_name);
2494
2495	if (IS_ERR(file)) {
2496		hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2497		ret = PTR_ERR(file);
2498		return ERR_PTR(ret);
2499	}
2500
2501	/*
2502	 * Look for a histogram compatible with target.  We'll use the
2503	 * found histogram specification to create a new matching
2504	 * histogram with our variable on it.  target_hist_data is not
2505	 * yet a registered histogram so we can't use that.
2506	 */
2507	hist_data = find_compatible_hist(target_hist_data, file);
2508	if (!hist_data) {
2509		hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2510		return ERR_PTR(-EINVAL);
2511	}
2512
2513	/* See if a synthetic field variable has already been created */
2514	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2515					     event_name, field_name);
2516	if (!IS_ERR_OR_NULL(event_var))
2517		return event_var;
2518
2519	var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2520	if (!var_hist)
2521		return ERR_PTR(-ENOMEM);
2522
2523	cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2524	if (!cmd) {
2525		kfree(var_hist);
2526		return ERR_PTR(-ENOMEM);
2527	}
2528
2529	/* Use the same keys as the compatible histogram */
2530	strcat(cmd, "keys=");
2531
2532	for_each_hist_key_field(i, hist_data) {
2533		key_field = hist_data->fields[i];
2534		if (!first)
2535			strcat(cmd, ",");
2536		strcat(cmd, key_field->field->name);
2537		first = false;
2538	}
2539
2540	/* Create the synthetic field variable specification */
2541	strcat(cmd, ":synthetic_");
2542	strcat(cmd, field_name);
2543	strcat(cmd, "=");
2544	strcat(cmd, field_name);
2545
2546	/* Use the same filter as the compatible histogram */
2547	saved_filter = find_trigger_filter(hist_data, file);
2548	if (saved_filter) {
2549		strcat(cmd, " if ");
2550		strcat(cmd, saved_filter);
2551	}
2552
2553	var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2554	if (!var_hist->cmd) {
2555		kfree(cmd);
2556		kfree(var_hist);
2557		return ERR_PTR(-ENOMEM);
2558	}
2559
2560	/* Save the compatible histogram information */
2561	var_hist->hist_data = hist_data;
2562
2563	/* Create the new histogram with our variable */
2564	ret = event_hist_trigger_func(&trigger_hist_cmd, file,
2565				      "", "hist", cmd);
2566	if (ret) {
2567		kfree(cmd);
2568		kfree(var_hist->cmd);
2569		kfree(var_hist);
2570		hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2571		return ERR_PTR(ret);
2572	}
2573
2574	kfree(cmd);
2575
2576	/* If we can't find the variable, something went wrong */
2577	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2578					     event_name, field_name);
2579	if (IS_ERR_OR_NULL(event_var)) {
2580		kfree(var_hist->cmd);
2581		kfree(var_hist);
2582		hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
2583		return ERR_PTR(-EINVAL);
2584	}
2585
2586	n = target_hist_data->n_field_var_hists;
2587	target_hist_data->field_var_hists[n] = var_hist;
2588	target_hist_data->n_field_var_hists++;
2589
2590	return event_var;
2591}
2592
2593static struct hist_field *
2594find_target_event_var(struct hist_trigger_data *hist_data,
2595		      char *subsys_name, char *event_name, char *var_name)
2596{
2597	struct trace_event_file *file = hist_data->event_file;
2598	struct hist_field *hist_field = NULL;
2599
2600	if (subsys_name) {
2601		struct trace_event_call *call;
2602
2603		if (!event_name)
2604			return NULL;
2605
2606		call = file->event_call;
2607
2608		if (strcmp(subsys_name, call->class->system) != 0)
2609			return NULL;
2610
2611		if (strcmp(event_name, trace_event_name(call)) != 0)
2612			return NULL;
2613	}
2614
2615	hist_field = find_var_field(hist_data, var_name);
2616
2617	return hist_field;
2618}
2619
2620static inline void __update_field_vars(struct tracing_map_elt *elt,
2621				       struct trace_buffer *buffer,
2622				       struct ring_buffer_event *rbe,
2623				       void *rec,
2624				       struct field_var **field_vars,
2625				       unsigned int n_field_vars,
2626				       unsigned int field_var_str_start)
2627{
2628	struct hist_elt_data *elt_data = elt->private_data;
2629	unsigned int i, j, var_idx;
2630	u64 var_val;
2631
2632	for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
2633		struct field_var *field_var = field_vars[i];
2634		struct hist_field *var = field_var->var;
2635		struct hist_field *val = field_var->val;
2636
2637		var_val = val->fn(val, elt, buffer, rbe, rec);
2638		var_idx = var->var.idx;
2639
2640		if (val->flags & HIST_FIELD_FL_STRING) {
2641			char *str = elt_data->field_var_str[j++];
2642			char *val_str = (char *)(uintptr_t)var_val;
2643
2644			strscpy(str, val_str, STR_VAR_LEN_MAX);
2645			var_val = (u64)(uintptr_t)str;
2646		}
2647		tracing_map_set_var(elt, var_idx, var_val);
2648	}
2649}
2650
2651static void update_field_vars(struct hist_trigger_data *hist_data,
2652			      struct tracing_map_elt *elt,
2653			      struct trace_buffer *buffer,
2654			      struct ring_buffer_event *rbe,
2655			      void *rec)
2656{
2657	__update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
2658			    hist_data->n_field_vars, 0);
2659}
2660
2661static void save_track_data_vars(struct hist_trigger_data *hist_data,
2662				 struct tracing_map_elt *elt,
2663				 struct trace_buffer *buffer,  void *rec,
2664				 struct ring_buffer_event *rbe, void *key,
2665				 struct action_data *data, u64 *var_ref_vals)
2666{
2667	__update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
2668			    hist_data->n_save_vars, hist_data->n_field_var_str);
2669}
2670
2671static struct hist_field *create_var(struct hist_trigger_data *hist_data,
2672				     struct trace_event_file *file,
2673				     char *name, int size, const char *type)
2674{
2675	struct hist_field *var;
2676	int idx;
2677
2678	if (find_var(hist_data, file, name) && !hist_data->remove) {
2679		var = ERR_PTR(-EINVAL);
2680		goto out;
2681	}
2682
2683	var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2684	if (!var) {
2685		var = ERR_PTR(-ENOMEM);
2686		goto out;
2687	}
2688
2689	idx = tracing_map_add_var(hist_data->map);
2690	if (idx < 0) {
2691		kfree(var);
2692		var = ERR_PTR(-EINVAL);
2693		goto out;
2694	}
2695
2696	var->ref = 1;
2697	var->flags = HIST_FIELD_FL_VAR;
2698	var->var.idx = idx;
2699	var->var.hist_data = var->hist_data = hist_data;
2700	var->size = size;
2701	var->var.name = kstrdup(name, GFP_KERNEL);
2702	var->type = kstrdup(type, GFP_KERNEL);
2703	if (!var->var.name || !var->type) {
2704		kfree(var->var.name);
2705		kfree(var->type);
2706		kfree(var);
2707		var = ERR_PTR(-ENOMEM);
2708	}
2709 out:
2710	return var;
2711}
2712
2713static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
2714					  struct trace_event_file *file,
2715					  char *field_name)
2716{
2717	struct hist_field *val = NULL, *var = NULL;
2718	unsigned long flags = HIST_FIELD_FL_VAR;
2719	struct trace_array *tr = file->tr;
2720	struct field_var *field_var;
2721	int ret = 0;
2722
2723	if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
2724		hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2725		ret = -EINVAL;
2726		goto err;
2727	}
2728
2729	val = parse_atom(hist_data, file, field_name, &flags, NULL);
2730	if (IS_ERR(val)) {
2731		hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
2732		ret = PTR_ERR(val);
2733		goto err;
2734	}
2735
2736	var = create_var(hist_data, file, field_name, val->size, val->type);
2737	if (IS_ERR(var)) {
2738		hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
2739		kfree(val);
2740		ret = PTR_ERR(var);
2741		goto err;
2742	}
2743
2744	field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
2745	if (!field_var) {
2746		kfree(val);
2747		kfree(var);
2748		ret =  -ENOMEM;
2749		goto err;
2750	}
2751
2752	field_var->var = var;
2753	field_var->val = val;
2754 out:
2755	return field_var;
2756 err:
2757	field_var = ERR_PTR(ret);
2758	goto out;
2759}
2760
2761/**
2762 * create_target_field_var - Automatically create a variable for a field
2763 * @target_hist_data: The target hist trigger
2764 * @subsys_name: Optional subsystem name
2765 * @event_name: Optional event name
2766 * @var_name: The name of the field (and the resulting variable)
2767 *
2768 * Hist trigger actions fetch data from variables, not directly from
2769 * events.  However, for convenience, users are allowed to directly
2770 * specify an event field in an action, which will be automatically
2771 * converted into a variable on their behalf.
2772
2773 * This function creates a field variable with the name var_name on
2774 * the hist trigger currently being defined on the target event.  If
2775 * subsys_name and event_name are specified, this function simply
2776 * verifies that they do in fact match the target event subsystem and
2777 * event name.
2778 *
2779 * Return: The variable created for the field.
2780 */
2781static struct field_var *
2782create_target_field_var(struct hist_trigger_data *target_hist_data,
2783			char *subsys_name, char *event_name, char *var_name)
2784{
2785	struct trace_event_file *file = target_hist_data->event_file;
2786
2787	if (subsys_name) {
2788		struct trace_event_call *call;
2789
2790		if (!event_name)
2791			return NULL;
2792
2793		call = file->event_call;
2794
2795		if (strcmp(subsys_name, call->class->system) != 0)
2796			return NULL;
2797
2798		if (strcmp(event_name, trace_event_name(call)) != 0)
2799			return NULL;
2800	}
2801
2802	return create_field_var(target_hist_data, file, var_name);
2803}
2804
2805static bool check_track_val_max(u64 track_val, u64 var_val)
2806{
2807	if (var_val <= track_val)
2808		return false;
2809
2810	return true;
2811}
2812
2813static bool check_track_val_changed(u64 track_val, u64 var_val)
2814{
2815	if (var_val == track_val)
2816		return false;
2817
2818	return true;
2819}
2820
2821static u64 get_track_val(struct hist_trigger_data *hist_data,
2822			 struct tracing_map_elt *elt,
2823			 struct action_data *data)
2824{
2825	unsigned int track_var_idx = data->track_data.track_var->var.idx;
2826	u64 track_val;
2827
2828	track_val = tracing_map_read_var(elt, track_var_idx);
2829
2830	return track_val;
2831}
2832
2833static void save_track_val(struct hist_trigger_data *hist_data,
2834			   struct tracing_map_elt *elt,
2835			   struct action_data *data, u64 var_val)
2836{
2837	unsigned int track_var_idx = data->track_data.track_var->var.idx;
2838
2839	tracing_map_set_var(elt, track_var_idx, var_val);
2840}
2841
2842static void save_track_data(struct hist_trigger_data *hist_data,
2843			    struct tracing_map_elt *elt,
2844			    struct trace_buffer *buffer, void *rec,
2845			    struct ring_buffer_event *rbe, void *key,
2846			    struct action_data *data, u64 *var_ref_vals)
2847{
2848	if (data->track_data.save_data)
2849		data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
2850					   key, data, var_ref_vals);
2851}
2852
2853static bool check_track_val(struct tracing_map_elt *elt,
2854			    struct action_data *data,
2855			    u64 var_val)
2856{
2857	struct hist_trigger_data *hist_data;
2858	u64 track_val;
2859
2860	hist_data = data->track_data.track_var->hist_data;
2861	track_val = get_track_val(hist_data, elt, data);
2862
2863	return data->track_data.check_val(track_val, var_val);
2864}
2865
2866#ifdef CONFIG_TRACER_SNAPSHOT
2867static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2868{
2869	/* called with tr->max_lock held */
2870	struct track_data *track_data = tr->cond_snapshot->cond_data;
2871	struct hist_elt_data *elt_data, *track_elt_data;
2872	struct snapshot_context *context = cond_data;
2873	struct action_data *action;
2874	u64 track_val;
2875
2876	if (!track_data)
2877		return false;
2878
2879	action = track_data->action_data;
2880
2881	track_val = get_track_val(track_data->hist_data, context->elt,
2882				  track_data->action_data);
2883
2884	if (!action->track_data.check_val(track_data->track_val, track_val))
2885		return false;
2886
2887	track_data->track_val = track_val;
2888	memcpy(track_data->key, context->key, track_data->key_len);
2889
2890	elt_data = context->elt->private_data;
2891	track_elt_data = track_data->elt.private_data;
2892	if (elt_data->comm)
2893		strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
2894
2895	track_data->updated = true;
2896
2897	return true;
2898}
2899
2900static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2901				     struct tracing_map_elt *elt,
2902				     struct trace_buffer *buffer, void *rec,
2903				     struct ring_buffer_event *rbe, void *key,
2904				     struct action_data *data,
2905				     u64 *var_ref_vals)
2906{
2907	struct trace_event_file *file = hist_data->event_file;
2908	struct snapshot_context context;
2909
2910	context.elt = elt;
2911	context.key = key;
2912
2913	tracing_snapshot_cond(file->tr, &context);
2914}
2915
2916static void hist_trigger_print_key(struct seq_file *m,
2917				   struct hist_trigger_data *hist_data,
2918				   void *key,
2919				   struct tracing_map_elt *elt);
2920
2921static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
2922{
2923	unsigned int i;
2924
2925	if (!hist_data->n_actions)
2926		return NULL;
2927
2928	for (i = 0; i < hist_data->n_actions; i++) {
2929		struct action_data *data = hist_data->actions[i];
2930
2931		if (data->action == ACTION_SNAPSHOT)
2932			return data;
2933	}
2934
2935	return NULL;
2936}
2937
2938static void track_data_snapshot_print(struct seq_file *m,
2939				      struct hist_trigger_data *hist_data)
2940{
2941	struct trace_event_file *file = hist_data->event_file;
2942	struct track_data *track_data;
2943	struct action_data *action;
2944
2945	track_data = tracing_cond_snapshot_data(file->tr);
2946	if (!track_data)
2947		return;
2948
2949	if (!track_data->updated)
2950		return;
2951
2952	action = snapshot_action(hist_data);
2953	if (!action)
2954		return;
2955
2956	seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
2957	seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
2958		   action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
2959		   action->track_data.var_str, track_data->track_val);
2960
2961	seq_puts(m, "\ttriggered by event with key: ");
2962	hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
2963	seq_putc(m, '\n');
2964}
2965#else
2966static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2967{
2968	return false;
2969}
2970static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2971				     struct tracing_map_elt *elt,
2972				     struct trace_buffer *buffer, void *rec,
2973				     struct ring_buffer_event *rbe, void *key,
2974				     struct action_data *data,
2975				     u64 *var_ref_vals) {}
2976static void track_data_snapshot_print(struct seq_file *m,
2977				      struct hist_trigger_data *hist_data) {}
2978#endif /* CONFIG_TRACER_SNAPSHOT */
2979
2980static void track_data_print(struct seq_file *m,
2981			     struct hist_trigger_data *hist_data,
2982			     struct tracing_map_elt *elt,
2983			     struct action_data *data)
2984{
2985	u64 track_val = get_track_val(hist_data, elt, data);
2986	unsigned int i, save_var_idx;
2987
2988	if (data->handler == HANDLER_ONMAX)
2989		seq_printf(m, "\n\tmax: %10llu", track_val);
2990	else if (data->handler == HANDLER_ONCHANGE)
2991		seq_printf(m, "\n\tchanged: %10llu", track_val);
2992
2993	if (data->action == ACTION_SNAPSHOT)
2994		return;
2995
2996	for (i = 0; i < hist_data->n_save_vars; i++) {
2997		struct hist_field *save_val = hist_data->save_vars[i]->val;
2998		struct hist_field *save_var = hist_data->save_vars[i]->var;
2999		u64 val;
3000
3001		save_var_idx = save_var->var.idx;
3002
3003		val = tracing_map_read_var(elt, save_var_idx);
3004
3005		if (save_val->flags & HIST_FIELD_FL_STRING) {
3006			seq_printf(m, "  %s: %-32s", save_var->var.name,
3007				   (char *)(uintptr_t)(val));
3008		} else
3009			seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3010	}
3011}
3012
3013static void ontrack_action(struct hist_trigger_data *hist_data,
3014			   struct tracing_map_elt *elt,
3015			   struct trace_buffer *buffer, void *rec,
3016			   struct ring_buffer_event *rbe, void *key,
3017			   struct action_data *data, u64 *var_ref_vals)
3018{
3019	u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3020
3021	if (check_track_val(elt, data, var_val)) {
3022		save_track_val(hist_data, elt, data, var_val);
3023		save_track_data(hist_data, elt, buffer, rec, rbe,
3024				key, data, var_ref_vals);
3025	}
3026}
3027
3028static void action_data_destroy(struct action_data *data)
3029{
3030	unsigned int i;
3031
3032	lockdep_assert_held(&event_mutex);
3033
3034	kfree(data->action_name);
3035
3036	for (i = 0; i < data->n_params; i++)
3037		kfree(data->params[i]);
3038
3039	if (data->synth_event)
3040		data->synth_event->ref--;
3041
3042	kfree(data->synth_event_name);
3043
3044	kfree(data);
3045}
3046
3047static void track_data_destroy(struct hist_trigger_data *hist_data,
3048			       struct action_data *data)
3049{
3050	struct trace_event_file *file = hist_data->event_file;
3051
3052	destroy_hist_field(data->track_data.track_var, 0);
3053
3054	if (data->action == ACTION_SNAPSHOT) {
3055		struct track_data *track_data;
3056
3057		track_data = tracing_cond_snapshot_data(file->tr);
3058		if (track_data && track_data->hist_data == hist_data) {
3059			tracing_snapshot_cond_disable(file->tr);
3060			track_data_free(track_data);
3061		}
3062	}
3063
3064	kfree(data->track_data.var_str);
3065
3066	action_data_destroy(data);
3067}
3068
3069static int action_create(struct hist_trigger_data *hist_data,
3070			 struct action_data *data);
3071
3072static int track_data_create(struct hist_trigger_data *hist_data,
3073			     struct action_data *data)
3074{
3075	struct hist_field *var_field, *ref_field, *track_var = NULL;
3076	struct trace_event_file *file = hist_data->event_file;
3077	struct trace_array *tr = file->tr;
3078	char *track_data_var_str;
3079	int ret = 0;
3080
3081	track_data_var_str = data->track_data.var_str;
3082	if (track_data_var_str[0] != '$') {
3083		hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3084		return -EINVAL;
3085	}
3086	track_data_var_str++;
3087
3088	var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3089	if (!var_field) {
3090		hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3091		return -EINVAL;
3092	}
3093
3094	ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3095	if (!ref_field)
3096		return -ENOMEM;
3097
3098	data->track_data.var_ref = ref_field;
3099
3100	if (data->handler == HANDLER_ONMAX)
3101		track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3102	if (IS_ERR(track_var)) {
3103		hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3104		ret = PTR_ERR(track_var);
3105		goto out;
3106	}
3107
3108	if (data->handler == HANDLER_ONCHANGE)
3109		track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3110	if (IS_ERR(track_var)) {
3111		hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3112		ret = PTR_ERR(track_var);
3113		goto out;
3114	}
3115	data->track_data.track_var = track_var;
3116
3117	ret = action_create(hist_data, data);
3118 out:
3119	return ret;
3120}
3121
3122static int parse_action_params(struct trace_array *tr, char *params,
3123			       struct action_data *data)
3124{
3125	char *param, *saved_param;
3126	bool first_param = true;
3127	int ret = 0;
3128
3129	while (params) {
3130		if (data->n_params >= SYNTH_FIELDS_MAX) {
3131			hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3132			goto out;
3133		}
3134
3135		param = strsep(&params, ",");
3136		if (!param) {
3137			hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3138			ret = -EINVAL;
3139			goto out;
3140		}
3141
3142		param = strstrip(param);
3143		if (strlen(param) < 2) {
3144			hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3145			ret = -EINVAL;
3146			goto out;
3147		}
3148
3149		saved_param = kstrdup(param, GFP_KERNEL);
3150		if (!saved_param) {
3151			ret = -ENOMEM;
3152			goto out;
3153		}
3154
3155		if (first_param && data->use_trace_keyword) {
3156			data->synth_event_name = saved_param;
3157			first_param = false;
3158			continue;
3159		}
3160		first_param = false;
3161
3162		data->params[data->n_params++] = saved_param;
3163	}
3164 out:
3165	return ret;
3166}
3167
3168static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3169			enum handler_id handler)
3170{
3171	char *action_name;
3172	int ret = 0;
3173
3174	strsep(&str, ".");
3175	if (!str) {
3176		hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3177		ret = -EINVAL;
3178		goto out;
3179	}
3180
3181	action_name = strsep(&str, "(");
3182	if (!action_name || !str) {
3183		hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3184		ret = -EINVAL;
3185		goto out;
3186	}
3187
3188	if (str_has_prefix(action_name, "save")) {
3189		char *params = strsep(&str, ")");
3190
3191		if (!params) {
3192			hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3193			ret = -EINVAL;
3194			goto out;
3195		}
3196
3197		ret = parse_action_params(tr, params, data);
3198		if (ret)
3199			goto out;
3200
3201		if (handler == HANDLER_ONMAX)
3202			data->track_data.check_val = check_track_val_max;
3203		else if (handler == HANDLER_ONCHANGE)
3204			data->track_data.check_val = check_track_val_changed;
3205		else {
3206			hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3207			ret = -EINVAL;
3208			goto out;
3209		}
3210
3211		data->track_data.save_data = save_track_data_vars;
3212		data->fn = ontrack_action;
3213		data->action = ACTION_SAVE;
3214	} else if (str_has_prefix(action_name, "snapshot")) {
3215		char *params = strsep(&str, ")");
3216
3217		if (!str) {
3218			hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3219			ret = -EINVAL;
3220			goto out;
3221		}
3222
3223		if (handler == HANDLER_ONMAX)
3224			data->track_data.check_val = check_track_val_max;
3225		else if (handler == HANDLER_ONCHANGE)
3226			data->track_data.check_val = check_track_val_changed;
3227		else {
3228			hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3229			ret = -EINVAL;
3230			goto out;
3231		}
3232
3233		data->track_data.save_data = save_track_data_snapshot;
3234		data->fn = ontrack_action;
3235		data->action = ACTION_SNAPSHOT;
3236	} else {
3237		char *params = strsep(&str, ")");
3238
3239		if (str_has_prefix(action_name, "trace"))
3240			data->use_trace_keyword = true;
3241
3242		if (params) {
3243			ret = parse_action_params(tr, params, data);
3244			if (ret)
3245				goto out;
3246		}
3247
3248		if (handler == HANDLER_ONMAX)
3249			data->track_data.check_val = check_track_val_max;
3250		else if (handler == HANDLER_ONCHANGE)
3251			data->track_data.check_val = check_track_val_changed;
3252
3253		if (handler != HANDLER_ONMATCH) {
3254			data->track_data.save_data = action_trace;
3255			data->fn = ontrack_action;
3256		} else
3257			data->fn = action_trace;
3258
3259		data->action = ACTION_TRACE;
3260	}
3261
3262	data->action_name = kstrdup(action_name, GFP_KERNEL);
3263	if (!data->action_name) {
3264		ret = -ENOMEM;
3265		goto out;
3266	}
3267
3268	data->handler = handler;
3269 out:
3270	return ret;
3271}
3272
3273static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3274					    char *str, enum handler_id handler)
3275{
3276	struct action_data *data;
3277	int ret = -EINVAL;
3278	char *var_str;
3279
3280	data = kzalloc(sizeof(*data), GFP_KERNEL);
3281	if (!data)
3282		return ERR_PTR(-ENOMEM);
3283
3284	var_str = strsep(&str, ")");
3285	if (!var_str || !str) {
3286		ret = -EINVAL;
3287		goto free;
3288	}
3289
3290	data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3291	if (!data->track_data.var_str) {
3292		ret = -ENOMEM;
3293		goto free;
3294	}
3295
3296	ret = action_parse(hist_data->event_file->tr, str, data, handler);
3297	if (ret)
3298		goto free;
3299 out:
3300	return data;
3301 free:
3302	track_data_destroy(hist_data, data);
3303	data = ERR_PTR(ret);
3304	goto out;
3305}
3306
3307static void onmatch_destroy(struct action_data *data)
3308{
3309	kfree(data->match_data.event);
3310	kfree(data->match_data.event_system);
3311
3312	action_data_destroy(data);
3313}
3314
3315static void destroy_field_var(struct field_var *field_var)
3316{
3317	if (!field_var)
3318		return;
3319
3320	destroy_hist_field(field_var->var, 0);
3321	destroy_hist_field(field_var->val, 0);
3322
3323	kfree(field_var);
3324}
3325
3326static void destroy_field_vars(struct hist_trigger_data *hist_data)
3327{
3328	unsigned int i;
3329
3330	for (i = 0; i < hist_data->n_field_vars; i++)
3331		destroy_field_var(hist_data->field_vars[i]);
3332
3333	for (i = 0; i < hist_data->n_save_vars; i++)
3334		destroy_field_var(hist_data->save_vars[i]);
3335}
3336
3337static void save_field_var(struct hist_trigger_data *hist_data,
3338			   struct field_var *field_var)
3339{
3340	hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3341
3342	if (field_var->val->flags & HIST_FIELD_FL_STRING)
3343		hist_data->n_field_var_str++;
3344}
3345
3346
3347static int check_synth_field(struct synth_event *event,
3348			     struct hist_field *hist_field,
3349			     unsigned int field_pos)
3350{
3351	struct synth_field *field;
3352
3353	if (field_pos >= event->n_fields)
3354		return -EINVAL;
3355
3356	field = event->fields[field_pos];
3357
3358	/*
3359	 * A dynamic string synth field can accept static or
3360	 * dynamic. A static string synth field can only accept a
3361	 * same-sized static string, which is checked for later.
3362	 */
3363	if (strstr(hist_field->type, "char[") && field->is_string
3364	    && field->is_dynamic)
3365		return 0;
3366
3367	if (strcmp(field->type, hist_field->type) != 0) {
3368		if (field->size != hist_field->size ||
3369		    field->is_signed != hist_field->is_signed)
3370			return -EINVAL;
3371	}
3372
3373	return 0;
3374}
3375
3376static struct hist_field *
3377trace_action_find_var(struct hist_trigger_data *hist_data,
3378		      struct action_data *data,
3379		      char *system, char *event, char *var)
3380{
3381	struct trace_array *tr = hist_data->event_file->tr;
3382	struct hist_field *hist_field;
3383
3384	var++; /* skip '$' */
3385
3386	hist_field = find_target_event_var(hist_data, system, event, var);
3387	if (!hist_field) {
3388		if (!system && data->handler == HANDLER_ONMATCH) {
3389			system = data->match_data.event_system;
3390			event = data->match_data.event;
3391		}
3392
3393		hist_field = find_event_var(hist_data, system, event, var);
3394	}
3395
3396	if (!hist_field)
3397		hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3398
3399	return hist_field;
3400}
3401
3402static struct hist_field *
3403trace_action_create_field_var(struct hist_trigger_data *hist_data,
3404			      struct action_data *data, char *system,
3405			      char *event, char *var)
3406{
3407	struct hist_field *hist_field = NULL;
3408	struct field_var *field_var;
3409
3410	/*
3411	 * First try to create a field var on the target event (the
3412	 * currently being defined).  This will create a variable for
3413	 * unqualified fields on the target event, or if qualified,
3414	 * target fields that have qualified names matching the target.
3415	 */
3416	field_var = create_target_field_var(hist_data, system, event, var);
3417
3418	if (field_var && !IS_ERR(field_var)) {
3419		save_field_var(hist_data, field_var);
3420		hist_field = field_var->var;
3421	} else {
3422		field_var = NULL;
3423		/*
3424		 * If no explicit system.event is specified, default to
3425		 * looking for fields on the onmatch(system.event.xxx)
3426		 * event.
3427		 */
3428		if (!system && data->handler == HANDLER_ONMATCH) {
3429			system = data->match_data.event_system;
3430			event = data->match_data.event;
3431		}
3432
3433		if (!event)
3434			goto free;
3435		/*
3436		 * At this point, we're looking at a field on another
3437		 * event.  Because we can't modify a hist trigger on
3438		 * another event to add a variable for a field, we need
3439		 * to create a new trigger on that event and create the
3440		 * variable at the same time.
3441		 */
3442		hist_field = create_field_var_hist(hist_data, system, event, var);
3443		if (IS_ERR(hist_field))
3444			goto free;
3445	}
3446 out:
3447	return hist_field;
3448 free:
3449	destroy_field_var(field_var);
3450	hist_field = NULL;
3451	goto out;
3452}
3453
3454static int trace_action_create(struct hist_trigger_data *hist_data,
3455			       struct action_data *data)
3456{
3457	struct trace_array *tr = hist_data->event_file->tr;
3458	char *event_name, *param, *system = NULL;
3459	struct hist_field *hist_field, *var_ref;
3460	unsigned int i;
3461	unsigned int field_pos = 0;
3462	struct synth_event *event;
3463	char *synth_event_name;
3464	int var_ref_idx, ret = 0;
3465
3466	lockdep_assert_held(&event_mutex);
3467
3468	if (data->use_trace_keyword)
3469		synth_event_name = data->synth_event_name;
3470	else
3471		synth_event_name = data->action_name;
3472
3473	event = find_synth_event(synth_event_name);
3474	if (!event) {
3475		hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3476		return -EINVAL;
3477	}
3478
3479	event->ref++;
3480
3481	for (i = 0; i < data->n_params; i++) {
3482		char *p;
3483
3484		p = param = kstrdup(data->params[i], GFP_KERNEL);
3485		if (!param) {
3486			ret = -ENOMEM;
3487			goto err;
3488		}
3489
3490		system = strsep(&param, ".");
3491		if (!param) {
3492			param = (char *)system;
3493			system = event_name = NULL;
3494		} else {
3495			event_name = strsep(&param, ".");
3496			if (!param) {
3497				kfree(p);
3498				ret = -EINVAL;
3499				goto err;
3500			}
3501		}
3502
3503		if (param[0] == '$')
3504			hist_field = trace_action_find_var(hist_data, data,
3505							   system, event_name,
3506							   param);
3507		else
3508			hist_field = trace_action_create_field_var(hist_data,
3509								   data,
3510								   system,
3511								   event_name,
3512								   param);
3513
3514		if (!hist_field) {
3515			kfree(p);
3516			ret = -EINVAL;
3517			goto err;
3518		}
3519
3520		if (check_synth_field(event, hist_field, field_pos) == 0) {
3521			var_ref = create_var_ref(hist_data, hist_field,
3522						 system, event_name);
3523			if (!var_ref) {
3524				kfree(p);
3525				ret = -ENOMEM;
3526				goto err;
3527			}
3528
3529			var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3530			if (WARN_ON(var_ref_idx < 0)) {
3531				ret = var_ref_idx;
3532				goto err;
3533			}
3534
3535			data->var_ref_idx[i] = var_ref_idx;
3536
3537			field_pos++;
3538			kfree(p);
3539			continue;
3540		}
3541
3542		hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3543		kfree(p);
3544		ret = -EINVAL;
3545		goto err;
3546	}
3547
3548	if (field_pos != event->n_fields) {
3549		hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3550		ret = -EINVAL;
3551		goto err;
3552	}
3553
3554	data->synth_event = event;
3555 out:
3556	return ret;
3557 err:
3558	event->ref--;
3559
3560	goto out;
3561}
3562
3563static int action_create(struct hist_trigger_data *hist_data,
3564			 struct action_data *data)
3565{
3566	struct trace_event_file *file = hist_data->event_file;
3567	struct trace_array *tr = file->tr;
3568	struct track_data *track_data;
3569	struct field_var *field_var;
3570	unsigned int i;
3571	char *param;
3572	int ret = 0;
3573
3574	if (data->action == ACTION_TRACE)
3575		return trace_action_create(hist_data, data);
3576
3577	if (data->action == ACTION_SNAPSHOT) {
3578		track_data = track_data_alloc(hist_data->key_size, data, hist_data);
3579		if (IS_ERR(track_data)) {
3580			ret = PTR_ERR(track_data);
3581			goto out;
3582		}
3583
3584		ret = tracing_snapshot_cond_enable(file->tr, track_data,
3585						   cond_snapshot_update);
3586		if (ret)
3587			track_data_free(track_data);
3588
3589		goto out;
3590	}
3591
3592	if (data->action == ACTION_SAVE) {
3593		if (hist_data->n_save_vars) {
3594			ret = -EEXIST;
3595			hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
3596			goto out;
3597		}
3598
3599		for (i = 0; i < data->n_params; i++) {
3600			param = kstrdup(data->params[i], GFP_KERNEL);
3601			if (!param) {
3602				ret = -ENOMEM;
3603				goto out;
3604			}
3605
3606			field_var = create_target_field_var(hist_data, NULL, NULL, param);
3607			if (IS_ERR(field_var)) {
3608				hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
3609					 errpos(param));
3610				ret = PTR_ERR(field_var);
3611				kfree(param);
3612				goto out;
3613			}
3614
3615			hist_data->save_vars[hist_data->n_save_vars++] = field_var;
3616			if (field_var->val->flags & HIST_FIELD_FL_STRING)
3617				hist_data->n_save_var_str++;
3618			kfree(param);
3619		}
3620	}
3621 out:
3622	return ret;
3623}
3624
3625static int onmatch_create(struct hist_trigger_data *hist_data,
3626			  struct action_data *data)
3627{
3628	return action_create(hist_data, data);
3629}
3630
3631static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3632{
3633	char *match_event, *match_event_system;
3634	struct action_data *data;
3635	int ret = -EINVAL;
3636
3637	data = kzalloc(sizeof(*data), GFP_KERNEL);
3638	if (!data)
3639		return ERR_PTR(-ENOMEM);
3640
3641	match_event = strsep(&str, ")");
3642	if (!match_event || !str) {
3643		hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
3644		goto free;
3645	}
3646
3647	match_event_system = strsep(&match_event, ".");
3648	if (!match_event) {
3649		hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
3650		goto free;
3651	}
3652
3653	if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3654		hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
3655		goto free;
3656	}
3657
3658	data->match_data.event = kstrdup(match_event, GFP_KERNEL);
3659	if (!data->match_data.event) {
3660		ret = -ENOMEM;
3661		goto free;
3662	}
3663
3664	data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
3665	if (!data->match_data.event_system) {
3666		ret = -ENOMEM;
3667		goto free;
3668	}
3669
3670	ret = action_parse(tr, str, data, HANDLER_ONMATCH);
3671	if (ret)
3672		goto free;
3673 out:
3674	return data;
3675 free:
3676	onmatch_destroy(data);
3677	data = ERR_PTR(ret);
3678	goto out;
3679}
3680
3681static int create_hitcount_val(struct hist_trigger_data *hist_data)
3682{
3683	hist_data->fields[HITCOUNT_IDX] =
3684		create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
3685	if (!hist_data->fields[HITCOUNT_IDX])
3686		return -ENOMEM;
3687
3688	hist_data->n_vals++;
3689	hist_data->n_fields++;
3690
3691	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3692		return -EINVAL;
3693
3694	return 0;
3695}
3696
3697static int __create_val_field(struct hist_trigger_data *hist_data,
3698			      unsigned int val_idx,
3699			      struct trace_event_file *file,
3700			      char *var_name, char *field_str,
3701			      unsigned long flags)
3702{
3703	struct hist_field *hist_field;
3704	int ret = 0;
3705
3706	hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3707	if (IS_ERR(hist_field)) {
3708		ret = PTR_ERR(hist_field);
3709		goto out;
3710	}
3711
3712	hist_data->fields[val_idx] = hist_field;
3713
3714	++hist_data->n_vals;
3715	++hist_data->n_fields;
3716
3717	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3718		ret = -EINVAL;
3719 out:
3720	return ret;
3721}
3722
3723static int create_val_field(struct hist_trigger_data *hist_data,
3724			    unsigned int val_idx,
3725			    struct trace_event_file *file,
3726			    char *field_str)
3727{
3728	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3729		return -EINVAL;
3730
3731	return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3732}
3733
3734static int create_var_field(struct hist_trigger_data *hist_data,
3735			    unsigned int val_idx,
3736			    struct trace_event_file *file,
3737			    char *var_name, char *expr_str)
3738{
3739	struct trace_array *tr = hist_data->event_file->tr;
3740	unsigned long flags = 0;
3741	int ret;
3742
3743	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3744		return -EINVAL;
3745
3746	if (find_var(hist_data, file, var_name) && !hist_data->remove) {
3747		hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
3748		return -EINVAL;
3749	}
3750
3751	flags |= HIST_FIELD_FL_VAR;
3752	hist_data->n_vars++;
3753	if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3754		return -EINVAL;
3755
3756	ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3757
3758	if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_STRING)
3759		hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
3760
3761	return ret;
3762}
3763
3764static int create_val_fields(struct hist_trigger_data *hist_data,
3765			     struct trace_event_file *file)
3766{
3767	char *fields_str, *field_str;
3768	unsigned int i, j = 1;
3769	int ret;
3770
3771	ret = create_hitcount_val(hist_data);
3772	if (ret)
3773		goto out;
3774
3775	fields_str = hist_data->attrs->vals_str;
3776	if (!fields_str)
3777		goto out;
3778
3779	for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3780		     j < TRACING_MAP_VALS_MAX; i++) {
3781		field_str = strsep(&fields_str, ",");
3782		if (!field_str)
3783			break;
3784
3785		if (strcmp(field_str, "hitcount") == 0)
3786			continue;
3787
3788		ret = create_val_field(hist_data, j++, file, field_str);
3789		if (ret)
3790			goto out;
3791	}
3792
3793	if (fields_str && (strcmp(fields_str, "hitcount") != 0))
3794		ret = -EINVAL;
3795 out:
3796	return ret;
3797}
3798
3799static int create_key_field(struct hist_trigger_data *hist_data,
3800			    unsigned int key_idx,
3801			    unsigned int key_offset,
3802			    struct trace_event_file *file,
3803			    char *field_str)
3804{
3805	struct trace_array *tr = hist_data->event_file->tr;
3806	struct hist_field *hist_field = NULL;
3807	unsigned long flags = 0;
3808	unsigned int key_size;
3809	int ret = 0;
3810
3811	if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
3812		return -EINVAL;
3813
3814	flags |= HIST_FIELD_FL_KEY;
3815
3816	if (strcmp(field_str, "stacktrace") == 0) {
3817		flags |= HIST_FIELD_FL_STACKTRACE;
3818		key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
3819		hist_field = create_hist_field(hist_data, NULL, flags, NULL);
3820	} else {
3821		hist_field = parse_expr(hist_data, file, field_str, flags,
3822					NULL, 0);
3823		if (IS_ERR(hist_field)) {
3824			ret = PTR_ERR(hist_field);
3825			goto out;
3826		}
3827
3828		if (field_has_hist_vars(hist_field, 0))	{
3829			hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
3830			destroy_hist_field(hist_field, 0);
3831			ret = -EINVAL;
3832			goto out;
3833		}
3834
3835		key_size = hist_field->size;
3836	}
3837
3838	hist_data->fields[key_idx] = hist_field;
3839
3840	key_size = ALIGN(key_size, sizeof(u64));
3841	hist_data->fields[key_idx]->size = key_size;
3842	hist_data->fields[key_idx]->offset = key_offset;
3843
3844	hist_data->key_size += key_size;
3845
3846	if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
3847		ret = -EINVAL;
3848		goto out;
3849	}
3850
3851	hist_data->n_keys++;
3852	hist_data->n_fields++;
3853
3854	if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
3855		return -EINVAL;
3856
3857	ret = key_size;
3858 out:
3859	return ret;
3860}
3861
3862static int create_key_fields(struct hist_trigger_data *hist_data,
3863			     struct trace_event_file *file)
3864{
3865	unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
3866	char *fields_str, *field_str;
3867	int ret = -EINVAL;
3868
3869	fields_str = hist_data->attrs->keys_str;
3870	if (!fields_str)
3871		goto out;
3872
3873	for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
3874		field_str = strsep(&fields_str, ",");
3875		if (!field_str)
3876			break;
3877		ret = create_key_field(hist_data, i, key_offset,
3878				       file, field_str);
3879		if (ret < 0)
3880			goto out;
3881		key_offset += ret;
3882	}
3883	if (fields_str) {
3884		ret = -EINVAL;
3885		goto out;
3886	}
3887	ret = 0;
3888 out:
3889	return ret;
3890}
3891
3892static int create_var_fields(struct hist_trigger_data *hist_data,
3893			     struct trace_event_file *file)
3894{
3895	unsigned int i, j = hist_data->n_vals;
3896	int ret = 0;
3897
3898	unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
3899
3900	for (i = 0; i < n_vars; i++) {
3901		char *var_name = hist_data->attrs->var_defs.name[i];
3902		char *expr = hist_data->attrs->var_defs.expr[i];
3903
3904		ret = create_var_field(hist_data, j++, file, var_name, expr);
3905		if (ret)
3906			goto out;
3907	}
3908 out:
3909	return ret;
3910}
3911
3912static void free_var_defs(struct hist_trigger_data *hist_data)
3913{
3914	unsigned int i;
3915
3916	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
3917		kfree(hist_data->attrs->var_defs.name[i]);
3918		kfree(hist_data->attrs->var_defs.expr[i]);
3919	}
3920
3921	hist_data->attrs->var_defs.n_vars = 0;
3922}
3923
3924static int parse_var_defs(struct hist_trigger_data *hist_data)
3925{
3926	struct trace_array *tr = hist_data->event_file->tr;
3927	char *s, *str, *var_name, *field_str;
3928	unsigned int i, j, n_vars = 0;
3929	int ret = 0;
3930
3931	for (i = 0; i < hist_data->attrs->n_assignments; i++) {
3932		str = hist_data->attrs->assignment_str[i];
3933		for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
3934			field_str = strsep(&str, ",");
3935			if (!field_str)
3936				break;
3937
3938			var_name = strsep(&field_str, "=");
3939			if (!var_name || !field_str) {
3940				hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
3941					 errpos(var_name));
3942				ret = -EINVAL;
3943				goto free;
3944			}
3945
3946			if (n_vars == TRACING_MAP_VARS_MAX) {
3947				hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
3948				ret = -EINVAL;
3949				goto free;
3950			}
3951
3952			s = kstrdup(var_name, GFP_KERNEL);
3953			if (!s) {
3954				ret = -ENOMEM;
3955				goto free;
3956			}
3957			hist_data->attrs->var_defs.name[n_vars] = s;
3958
3959			s = kstrdup(field_str, GFP_KERNEL);
3960			if (!s) {
3961				ret = -ENOMEM;
3962				goto free;
3963			}
3964			hist_data->attrs->var_defs.expr[n_vars++] = s;
3965
3966			hist_data->attrs->var_defs.n_vars = n_vars;
3967		}
3968	}
3969
3970	return ret;
3971 free:
3972	free_var_defs(hist_data);
3973
3974	return ret;
3975}
3976
3977static int create_hist_fields(struct hist_trigger_data *hist_data,
3978			      struct trace_event_file *file)
3979{
3980	int ret;
3981
3982	ret = parse_var_defs(hist_data);
3983	if (ret)
3984		goto out;
3985
3986	ret = create_val_fields(hist_data, file);
3987	if (ret)
3988		goto out;
3989
3990	ret = create_var_fields(hist_data, file);
3991	if (ret)
3992		goto out;
3993
3994	ret = create_key_fields(hist_data, file);
3995	if (ret)
3996		goto out;
3997 out:
3998	free_var_defs(hist_data);
3999
4000	return ret;
4001}
4002
4003static int is_descending(struct trace_array *tr, const char *str)
4004{
4005	if (!str)
4006		return 0;
4007
4008	if (strcmp(str, "descending") == 0)
4009		return 1;
4010
4011	if (strcmp(str, "ascending") == 0)
4012		return 0;
4013
4014	hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4015
4016	return -EINVAL;
4017}
4018
4019static int create_sort_keys(struct hist_trigger_data *hist_data)
4020{
4021	struct trace_array *tr = hist_data->event_file->tr;
4022	char *fields_str = hist_data->attrs->sort_key_str;
4023	struct tracing_map_sort_key *sort_key;
4024	int descending, ret = 0;
4025	unsigned int i, j, k;
4026
4027	hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4028
4029	if (!fields_str)
4030		goto out;
4031
4032	for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4033		struct hist_field *hist_field;
4034		char *field_str, *field_name;
4035		const char *test_name;
4036
4037		sort_key = &hist_data->sort_keys[i];
4038
4039		field_str = strsep(&fields_str, ",");
4040		if (!field_str)
4041			break;
4042
4043		if (!*field_str) {
4044			ret = -EINVAL;
4045			hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4046			break;
4047		}
4048
4049		if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4050			hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4051			ret = -EINVAL;
4052			break;
4053		}
4054
4055		field_name = strsep(&field_str, ".");
4056		if (!field_name || !*field_name) {
4057			ret = -EINVAL;
4058			hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4059			break;
4060		}
4061
4062		if (strcmp(field_name, "hitcount") == 0) {
4063			descending = is_descending(tr, field_str);
4064			if (descending < 0) {
4065				ret = descending;
4066				break;
4067			}
4068			sort_key->descending = descending;
4069			continue;
4070		}
4071
4072		for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4073			unsigned int idx;
4074
4075			hist_field = hist_data->fields[j];
4076			if (hist_field->flags & HIST_FIELD_FL_VAR)
4077				continue;
4078
4079			idx = k++;
4080
4081			test_name = hist_field_name(hist_field, 0);
4082
4083			if (strcmp(field_name, test_name) == 0) {
4084				sort_key->field_idx = idx;
4085				descending = is_descending(tr, field_str);
4086				if (descending < 0) {
4087					ret = descending;
4088					goto out;
4089				}
4090				sort_key->descending = descending;
4091				break;
4092			}
4093		}
4094		if (j == hist_data->n_fields) {
4095			ret = -EINVAL;
4096			hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4097			break;
4098		}
4099	}
4100
4101	hist_data->n_sort_keys = i;
4102 out:
4103	return ret;
4104}
4105
4106static void destroy_actions(struct hist_trigger_data *hist_data)
4107{
4108	unsigned int i;
4109
4110	for (i = 0; i < hist_data->n_actions; i++) {
4111		struct action_data *data = hist_data->actions[i];
4112
4113		if (data->handler == HANDLER_ONMATCH)
4114			onmatch_destroy(data);
4115		else if (data->handler == HANDLER_ONMAX ||
4116			 data->handler == HANDLER_ONCHANGE)
4117			track_data_destroy(hist_data, data);
4118		else
4119			kfree(data);
4120	}
4121}
4122
4123static int parse_actions(struct hist_trigger_data *hist_data)
4124{
4125	struct trace_array *tr = hist_data->event_file->tr;
4126	struct action_data *data;
4127	unsigned int i;
4128	int ret = 0;
4129	char *str;
4130	int len;
4131
4132	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4133		str = hist_data->attrs->action_str[i];
4134
4135		if ((len = str_has_prefix(str, "onmatch("))) {
4136			char *action_str = str + len;
4137
4138			data = onmatch_parse(tr, action_str);
4139			if (IS_ERR(data)) {
4140				ret = PTR_ERR(data);
4141				break;
4142			}
4143		} else if ((len = str_has_prefix(str, "onmax("))) {
4144			char *action_str = str + len;
4145
4146			data = track_data_parse(hist_data, action_str,
4147						HANDLER_ONMAX);
4148			if (IS_ERR(data)) {
4149				ret = PTR_ERR(data);
4150				break;
4151			}
4152		} else if ((len = str_has_prefix(str, "onchange("))) {
4153			char *action_str = str + len;
4154
4155			data = track_data_parse(hist_data, action_str,
4156						HANDLER_ONCHANGE);
4157			if (IS_ERR(data)) {
4158				ret = PTR_ERR(data);
4159				break;
4160			}
4161		} else {
4162			ret = -EINVAL;
4163			break;
4164		}
4165
4166		hist_data->actions[hist_data->n_actions++] = data;
4167	}
4168
4169	return ret;
4170}
4171
4172static int create_actions(struct hist_trigger_data *hist_data)
4173{
4174	struct action_data *data;
4175	unsigned int i;
4176	int ret = 0;
4177
4178	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4179		data = hist_data->actions[i];
4180
4181		if (data->handler == HANDLER_ONMATCH) {
4182			ret = onmatch_create(hist_data, data);
4183			if (ret)
4184				break;
4185		} else if (data->handler == HANDLER_ONMAX ||
4186			   data->handler == HANDLER_ONCHANGE) {
4187			ret = track_data_create(hist_data, data);
4188			if (ret)
4189				break;
4190		} else {
4191			ret = -EINVAL;
4192			break;
4193		}
4194	}
4195
4196	return ret;
4197}
4198
4199static void print_actions(struct seq_file *m,
4200			  struct hist_trigger_data *hist_data,
4201			  struct tracing_map_elt *elt)
4202{
4203	unsigned int i;
4204
4205	for (i = 0; i < hist_data->n_actions; i++) {
4206		struct action_data *data = hist_data->actions[i];
4207
4208		if (data->action == ACTION_SNAPSHOT)
4209			continue;
4210
4211		if (data->handler == HANDLER_ONMAX ||
4212		    data->handler == HANDLER_ONCHANGE)
4213			track_data_print(m, hist_data, elt, data);
4214	}
4215}
4216
4217static void print_action_spec(struct seq_file *m,
4218			      struct hist_trigger_data *hist_data,
4219			      struct action_data *data)
4220{
4221	unsigned int i;
4222
4223	if (data->action == ACTION_SAVE) {
4224		for (i = 0; i < hist_data->n_save_vars; i++) {
4225			seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4226			if (i < hist_data->n_save_vars - 1)
4227				seq_puts(m, ",");
4228		}
4229	} else if (data->action == ACTION_TRACE) {
4230		if (data->use_trace_keyword)
4231			seq_printf(m, "%s", data->synth_event_name);
4232		for (i = 0; i < data->n_params; i++) {
4233			if (i || data->use_trace_keyword)
4234				seq_puts(m, ",");
4235			seq_printf(m, "%s", data->params[i]);
4236		}
4237	}
4238}
4239
4240static void print_track_data_spec(struct seq_file *m,
4241				  struct hist_trigger_data *hist_data,
4242				  struct action_data *data)
4243{
4244	if (data->handler == HANDLER_ONMAX)
4245		seq_puts(m, ":onmax(");
4246	else if (data->handler == HANDLER_ONCHANGE)
4247		seq_puts(m, ":onchange(");
4248	seq_printf(m, "%s", data->track_data.var_str);
4249	seq_printf(m, ").%s(", data->action_name);
4250
4251	print_action_spec(m, hist_data, data);
4252
4253	seq_puts(m, ")");
4254}
4255
4256static void print_onmatch_spec(struct seq_file *m,
4257			       struct hist_trigger_data *hist_data,
4258			       struct action_data *data)
4259{
4260	seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4261		   data->match_data.event);
4262
4263	seq_printf(m, "%s(", data->action_name);
4264
4265	print_action_spec(m, hist_data, data);
4266
4267	seq_puts(m, ")");
4268}
4269
4270static bool actions_match(struct hist_trigger_data *hist_data,
4271			  struct hist_trigger_data *hist_data_test)
4272{
4273	unsigned int i, j;
4274
4275	if (hist_data->n_actions != hist_data_test->n_actions)
4276		return false;
4277
4278	for (i = 0; i < hist_data->n_actions; i++) {
4279		struct action_data *data = hist_data->actions[i];
4280		struct action_data *data_test = hist_data_test->actions[i];
4281		char *action_name, *action_name_test;
4282
4283		if (data->handler != data_test->handler)
4284			return false;
4285		if (data->action != data_test->action)
4286			return false;
4287
4288		if (data->n_params != data_test->n_params)
4289			return false;
4290
4291		for (j = 0; j < data->n_params; j++) {
4292			if (strcmp(data->params[j], data_test->params[j]) != 0)
4293				return false;
4294		}
4295
4296		if (data->use_trace_keyword)
4297			action_name = data->synth_event_name;
4298		else
4299			action_name = data->action_name;
4300
4301		if (data_test->use_trace_keyword)
4302			action_name_test = data_test->synth_event_name;
4303		else
4304			action_name_test = data_test->action_name;
4305
4306		if (strcmp(action_name, action_name_test) != 0)
4307			return false;
4308
4309		if (data->handler == HANDLER_ONMATCH) {
4310			if (strcmp(data->match_data.event_system,
4311				   data_test->match_data.event_system) != 0)
4312				return false;
4313			if (strcmp(data->match_data.event,
4314				   data_test->match_data.event) != 0)
4315				return false;
4316		} else if (data->handler == HANDLER_ONMAX ||
4317			   data->handler == HANDLER_ONCHANGE) {
4318			if (strcmp(data->track_data.var_str,
4319				   data_test->track_data.var_str) != 0)
4320				return false;
4321		}
4322	}
4323
4324	return true;
4325}
4326
4327
4328static void print_actions_spec(struct seq_file *m,
4329			       struct hist_trigger_data *hist_data)
4330{
4331	unsigned int i;
4332
4333	for (i = 0; i < hist_data->n_actions; i++) {
4334		struct action_data *data = hist_data->actions[i];
4335
4336		if (data->handler == HANDLER_ONMATCH)
4337			print_onmatch_spec(m, hist_data, data);
4338		else if (data->handler == HANDLER_ONMAX ||
4339			 data->handler == HANDLER_ONCHANGE)
4340			print_track_data_spec(m, hist_data, data);
4341	}
4342}
4343
4344static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4345{
4346	unsigned int i;
4347
4348	for (i = 0; i < hist_data->n_field_var_hists; i++) {
4349		kfree(hist_data->field_var_hists[i]->cmd);
4350		kfree(hist_data->field_var_hists[i]);
4351	}
4352}
4353
4354static void destroy_hist_data(struct hist_trigger_data *hist_data)
4355{
4356	if (!hist_data)
4357		return;
4358
4359	destroy_hist_trigger_attrs(hist_data->attrs);
4360	destroy_hist_fields(hist_data);
4361	tracing_map_destroy(hist_data->map);
4362
4363	destroy_actions(hist_data);
4364	destroy_field_vars(hist_data);
4365	destroy_field_var_hists(hist_data);
4366
4367	kfree(hist_data);
4368}
4369
4370static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4371{
4372	struct tracing_map *map = hist_data->map;
4373	struct ftrace_event_field *field;
4374	struct hist_field *hist_field;
4375	int i, idx = 0;
4376
4377	for_each_hist_field(i, hist_data) {
4378		hist_field = hist_data->fields[i];
4379		if (hist_field->flags & HIST_FIELD_FL_KEY) {
4380			tracing_map_cmp_fn_t cmp_fn;
4381
4382			field = hist_field->field;
4383
4384			if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4385				cmp_fn = tracing_map_cmp_none;
4386			else if (!field)
4387				cmp_fn = tracing_map_cmp_num(hist_field->size,
4388							     hist_field->is_signed);
4389			else if (is_string_field(field))
4390				cmp_fn = tracing_map_cmp_string;
4391			else
4392				cmp_fn = tracing_map_cmp_num(field->size,
4393							     field->is_signed);
4394			idx = tracing_map_add_key_field(map,
4395							hist_field->offset,
4396							cmp_fn);
4397		} else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4398			idx = tracing_map_add_sum_field(map);
4399
4400		if (idx < 0)
4401			return idx;
4402
4403		if (hist_field->flags & HIST_FIELD_FL_VAR) {
4404			idx = tracing_map_add_var(map);
4405			if (idx < 0)
4406				return idx;
4407			hist_field->var.idx = idx;
4408			hist_field->var.hist_data = hist_data;
4409		}
4410	}
4411
4412	return 0;
4413}
4414
4415static struct hist_trigger_data *
4416create_hist_data(unsigned int map_bits,
4417		 struct hist_trigger_attrs *attrs,
4418		 struct trace_event_file *file,
4419		 bool remove)
4420{
4421	const struct tracing_map_ops *map_ops = NULL;
4422	struct hist_trigger_data *hist_data;
4423	int ret = 0;
4424
4425	hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4426	if (!hist_data)
4427		return ERR_PTR(-ENOMEM);
4428
4429	hist_data->attrs = attrs;
4430	hist_data->remove = remove;
4431	hist_data->event_file = file;
4432
4433	ret = parse_actions(hist_data);
4434	if (ret)
4435		goto free;
4436
4437	ret = create_hist_fields(hist_data, file);
4438	if (ret)
4439		goto free;
4440
4441	ret = create_sort_keys(hist_data);
4442	if (ret)
4443		goto free;
4444
4445	map_ops = &hist_trigger_elt_data_ops;
4446
4447	hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4448					    map_ops, hist_data);
4449	if (IS_ERR(hist_data->map)) {
4450		ret = PTR_ERR(hist_data->map);
4451		hist_data->map = NULL;
4452		goto free;
4453	}
4454
4455	ret = create_tracing_map_fields(hist_data);
4456	if (ret)
4457		goto free;
4458 out:
4459	return hist_data;
4460 free:
4461	hist_data->attrs = NULL;
4462
4463	destroy_hist_data(hist_data);
4464
4465	hist_data = ERR_PTR(ret);
4466
4467	goto out;
4468}
4469
4470static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4471				    struct tracing_map_elt *elt,
4472				    struct trace_buffer *buffer, void *rec,
4473				    struct ring_buffer_event *rbe,
4474				    u64 *var_ref_vals)
4475{
4476	struct hist_elt_data *elt_data;
4477	struct hist_field *hist_field;
4478	unsigned int i, var_idx;
4479	u64 hist_val;
4480
4481	elt_data = elt->private_data;
4482	elt_data->var_ref_vals = var_ref_vals;
4483
4484	for_each_hist_val_field(i, hist_data) {
4485		hist_field = hist_data->fields[i];
4486		hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4487		if (hist_field->flags & HIST_FIELD_FL_VAR) {
4488			var_idx = hist_field->var.idx;
4489
4490			if (hist_field->flags & HIST_FIELD_FL_STRING) {
4491				unsigned int str_start, var_str_idx, idx;
4492				char *str, *val_str;
4493
4494				str_start = hist_data->n_field_var_str +
4495					hist_data->n_save_var_str;
4496				var_str_idx = hist_field->var_str_idx;
4497				idx = str_start + var_str_idx;
4498
4499				str = elt_data->field_var_str[idx];
4500				val_str = (char *)(uintptr_t)hist_val;
4501				strscpy(str, val_str, STR_VAR_LEN_MAX);
4502
4503				hist_val = (u64)(uintptr_t)str;
4504			}
4505			tracing_map_set_var(elt, var_idx, hist_val);
4506			continue;
4507		}
4508		tracing_map_update_sum(elt, i, hist_val);
4509	}
4510
4511	for_each_hist_key_field(i, hist_data) {
4512		hist_field = hist_data->fields[i];
4513		if (hist_field->flags & HIST_FIELD_FL_VAR) {
4514			hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4515			var_idx = hist_field->var.idx;
4516			tracing_map_set_var(elt, var_idx, hist_val);
4517		}
4518	}
4519
4520	update_field_vars(hist_data, elt, buffer, rbe, rec);
4521}
4522
4523static inline void add_to_key(char *compound_key, void *key,
4524			      struct hist_field *key_field, void *rec)
4525{
4526	size_t size = key_field->size;
4527
4528	if (key_field->flags & HIST_FIELD_FL_STRING) {
4529		struct ftrace_event_field *field;
4530
4531		field = key_field->field;
4532		if (field->filter_type == FILTER_DYN_STRING)
4533			size = *(u32 *)(rec + field->offset) >> 16;
4534		else if (field->filter_type == FILTER_STATIC_STRING)
4535			size = field->size;
4536
4537		/* ensure NULL-termination */
4538		if (size > key_field->size - 1)
4539			size = key_field->size - 1;
4540
4541		strncpy(compound_key + key_field->offset, (char *)key, size);
4542	} else
4543		memcpy(compound_key + key_field->offset, key, size);
4544}
4545
4546static void
4547hist_trigger_actions(struct hist_trigger_data *hist_data,
4548		     struct tracing_map_elt *elt,
4549		     struct trace_buffer *buffer, void *rec,
4550		     struct ring_buffer_event *rbe, void *key,
4551		     u64 *var_ref_vals)
4552{
4553	struct action_data *data;
4554	unsigned int i;
4555
4556	for (i = 0; i < hist_data->n_actions; i++) {
4557		data = hist_data->actions[i];
4558		data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
4559	}
4560}
4561
4562static void event_hist_trigger(struct event_trigger_data *data,
4563			       struct trace_buffer *buffer, void *rec,
4564			       struct ring_buffer_event *rbe)
4565{
4566	struct hist_trigger_data *hist_data = data->private_data;
4567	bool use_compound_key = (hist_data->n_keys > 1);
4568	unsigned long entries[HIST_STACKTRACE_DEPTH];
4569	u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4570	char compound_key[HIST_KEY_SIZE_MAX];
4571	struct tracing_map_elt *elt = NULL;
4572	struct hist_field *key_field;
4573	u64 field_contents;
4574	void *key = NULL;
4575	unsigned int i;
4576
4577	memset(compound_key, 0, hist_data->key_size);
4578
4579	for_each_hist_key_field(i, hist_data) {
4580		key_field = hist_data->fields[i];
4581
4582		if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4583			memset(entries, 0, HIST_STACKTRACE_SIZE);
4584			stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
4585					 HIST_STACKTRACE_SKIP);
4586			key = entries;
4587		} else {
4588			field_contents = key_field->fn(key_field, elt, buffer, rbe, rec);
4589			if (key_field->flags & HIST_FIELD_FL_STRING) {
4590				key = (void *)(unsigned long)field_contents;
4591				use_compound_key = true;
4592			} else
4593				key = (void *)&field_contents;
4594		}
4595
4596		if (use_compound_key)
4597			add_to_key(compound_key, key, key_field, rec);
4598	}
4599
4600	if (use_compound_key)
4601		key = compound_key;
4602
4603	if (hist_data->n_var_refs &&
4604	    !resolve_var_refs(hist_data, key, var_ref_vals, false))
4605		return;
4606
4607	elt = tracing_map_insert(hist_data->map, key);
4608	if (!elt)
4609		return;
4610
4611	hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
4612
4613	if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4614		hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
4615}
4616
4617static void hist_trigger_stacktrace_print(struct seq_file *m,
4618					  unsigned long *stacktrace_entries,
4619					  unsigned int max_entries)
4620{
4621	char str[KSYM_SYMBOL_LEN];
4622	unsigned int spaces = 8;
4623	unsigned int i;
4624
4625	for (i = 0; i < max_entries; i++) {
4626		if (!stacktrace_entries[i])
4627			return;
4628
4629		seq_printf(m, "%*c", 1 + spaces, ' ');
4630		sprint_symbol(str, stacktrace_entries[i]);
4631		seq_printf(m, "%s\n", str);
4632	}
4633}
4634
4635static void hist_trigger_print_key(struct seq_file *m,
4636				   struct hist_trigger_data *hist_data,
4637				   void *key,
4638				   struct tracing_map_elt *elt)
4639{
4640	struct hist_field *key_field;
4641	char str[KSYM_SYMBOL_LEN];
4642	bool multiline = false;
4643	const char *field_name;
4644	unsigned int i;
4645	u64 uval;
4646
4647	seq_puts(m, "{ ");
4648
4649	for_each_hist_key_field(i, hist_data) {
4650		key_field = hist_data->fields[i];
4651
4652		if (i > hist_data->n_vals)
4653			seq_puts(m, ", ");
4654
4655		field_name = hist_field_name(key_field, 0);
4656
4657		if (key_field->flags & HIST_FIELD_FL_HEX) {
4658			uval = *(u64 *)(key + key_field->offset);
4659			seq_printf(m, "%s: %llx", field_name, uval);
4660		} else if (key_field->flags & HIST_FIELD_FL_SYM) {
4661			uval = *(u64 *)(key + key_field->offset);
4662			sprint_symbol_no_offset(str, uval);
4663			seq_printf(m, "%s: [%llx] %-45s", field_name,
4664				   uval, str);
4665		} else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4666			uval = *(u64 *)(key + key_field->offset);
4667			sprint_symbol(str, uval);
4668			seq_printf(m, "%s: [%llx] %-55s", field_name,
4669				   uval, str);
4670		} else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
4671			struct hist_elt_data *elt_data = elt->private_data;
4672			char *comm;
4673
4674			if (WARN_ON_ONCE(!elt_data))
4675				return;
4676
4677			comm = elt_data->comm;
4678
4679			uval = *(u64 *)(key + key_field->offset);
4680			seq_printf(m, "%s: %-16s[%10llu]", field_name,
4681				   comm, uval);
4682		} else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4683			const char *syscall_name;
4684
4685			uval = *(u64 *)(key + key_field->offset);
4686			syscall_name = get_syscall_name(uval);
4687			if (!syscall_name)
4688				syscall_name = "unknown_syscall";
4689
4690			seq_printf(m, "%s: %-30s[%3llu]", field_name,
4691				   syscall_name, uval);
4692		} else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4693			seq_puts(m, "stacktrace:\n");
4694			hist_trigger_stacktrace_print(m,
4695						      key + key_field->offset,
4696						      HIST_STACKTRACE_DEPTH);
4697			multiline = true;
4698		} else if (key_field->flags & HIST_FIELD_FL_LOG2) {
4699			seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4700				   *(u64 *)(key + key_field->offset));
4701		} else if (key_field->flags & HIST_FIELD_FL_STRING) {
4702			seq_printf(m, "%s: %-50s", field_name,
4703				   (char *)(key + key_field->offset));
4704		} else {
4705			uval = *(u64 *)(key + key_field->offset);
4706			seq_printf(m, "%s: %10llu", field_name, uval);
4707		}
4708	}
4709
4710	if (!multiline)
4711		seq_puts(m, " ");
4712
4713	seq_puts(m, "}");
4714}
4715
4716static void hist_trigger_entry_print(struct seq_file *m,
4717				     struct hist_trigger_data *hist_data,
4718				     void *key,
4719				     struct tracing_map_elt *elt)
4720{
4721	const char *field_name;
4722	unsigned int i;
4723
4724	hist_trigger_print_key(m, hist_data, key, elt);
4725
4726	seq_printf(m, " hitcount: %10llu",
4727		   tracing_map_read_sum(elt, HITCOUNT_IDX));
4728
4729	for (i = 1; i < hist_data->n_vals; i++) {
4730		field_name = hist_field_name(hist_data->fields[i], 0);
4731
4732		if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4733		    hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
4734			continue;
4735
4736		if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
4737			seq_printf(m, "  %s: %10llx", field_name,
4738				   tracing_map_read_sum(elt, i));
4739		} else {
4740			seq_printf(m, "  %s: %10llu", field_name,
4741				   tracing_map_read_sum(elt, i));
4742		}
4743	}
4744
4745	print_actions(m, hist_data, elt);
4746
4747	seq_puts(m, "\n");
4748}
4749
4750static int print_entries(struct seq_file *m,
4751			 struct hist_trigger_data *hist_data)
4752{
4753	struct tracing_map_sort_entry **sort_entries = NULL;
4754	struct tracing_map *map = hist_data->map;
4755	int i, n_entries;
4756
4757	n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4758					     hist_data->n_sort_keys,
4759					     &sort_entries);
4760	if (n_entries < 0)
4761		return n_entries;
4762
4763	for (i = 0; i < n_entries; i++)
4764		hist_trigger_entry_print(m, hist_data,
4765					 sort_entries[i]->key,
4766					 sort_entries[i]->elt);
4767
4768	tracing_map_destroy_sort_entries(sort_entries, n_entries);
4769
4770	return n_entries;
4771}
4772
4773static void hist_trigger_show(struct seq_file *m,
4774			      struct event_trigger_data *data, int n)
4775{
4776	struct hist_trigger_data *hist_data;
4777	int n_entries;
4778
4779	if (n > 0)
4780		seq_puts(m, "\n\n");
4781
4782	seq_puts(m, "# event histogram\n#\n# trigger info: ");
4783	data->ops->print(m, data->ops, data);
4784	seq_puts(m, "#\n\n");
4785
4786	hist_data = data->private_data;
4787	n_entries = print_entries(m, hist_data);
4788	if (n_entries < 0)
4789		n_entries = 0;
4790
4791	track_data_snapshot_print(m, hist_data);
4792
4793	seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
4794		   (u64)atomic64_read(&hist_data->map->hits),
4795		   n_entries, (u64)atomic64_read(&hist_data->map->drops));
4796}
4797
4798static int hist_show(struct seq_file *m, void *v)
4799{
4800	struct event_trigger_data *data;
4801	struct trace_event_file *event_file;
4802	int n = 0, ret = 0;
4803
4804	mutex_lock(&event_mutex);
4805
4806	event_file = event_file_data(m->private);
4807	if (unlikely(!event_file)) {
4808		ret = -ENODEV;
4809		goto out_unlock;
4810	}
4811
4812	list_for_each_entry(data, &event_file->triggers, list) {
4813		if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4814			hist_trigger_show(m, data, n++);
4815	}
4816
4817 out_unlock:
4818	mutex_unlock(&event_mutex);
4819
4820	return ret;
4821}
4822
4823static int event_hist_open(struct inode *inode, struct file *file)
4824{
4825	int ret;
4826
4827	ret = security_locked_down(LOCKDOWN_TRACEFS);
4828	if (ret)
4829		return ret;
4830
4831	return single_open(file, hist_show, file);
4832}
4833
4834const struct file_operations event_hist_fops = {
4835	.open = event_hist_open,
4836	.read = seq_read,
4837	.llseek = seq_lseek,
4838	.release = single_release,
4839};
4840
4841#ifdef CONFIG_HIST_TRIGGERS_DEBUG
4842static void hist_field_debug_show_flags(struct seq_file *m,
4843					unsigned long flags)
4844{
4845	seq_puts(m, "      flags:\n");
4846
4847	if (flags & HIST_FIELD_FL_KEY)
4848		seq_puts(m, "        HIST_FIELD_FL_KEY\n");
4849	else if (flags & HIST_FIELD_FL_HITCOUNT)
4850		seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
4851	else if (flags & HIST_FIELD_FL_VAR)
4852		seq_puts(m, "        HIST_FIELD_FL_VAR\n");
4853	else if (flags & HIST_FIELD_FL_VAR_REF)
4854		seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
4855	else
4856		seq_puts(m, "        VAL: normal u64 value\n");
4857
4858	if (flags & HIST_FIELD_FL_ALIAS)
4859		seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
4860}
4861
4862static int hist_field_debug_show(struct seq_file *m,
4863				 struct hist_field *field, unsigned long flags)
4864{
4865	if ((field->flags & flags) != flags) {
4866		seq_printf(m, "ERROR: bad flags - %lx\n", flags);
4867		return -EINVAL;
4868	}
4869
4870	hist_field_debug_show_flags(m, field->flags);
4871	if (field->field)
4872		seq_printf(m, "      ftrace_event_field name: %s\n",
4873			   field->field->name);
4874
4875	if (field->flags & HIST_FIELD_FL_VAR) {
4876		seq_printf(m, "      var.name: %s\n", field->var.name);
4877		seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4878			   field->var.idx);
4879	}
4880
4881	if (field->flags & HIST_FIELD_FL_ALIAS)
4882		seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4883			   field->var_ref_idx);
4884
4885	if (field->flags & HIST_FIELD_FL_VAR_REF) {
4886		seq_printf(m, "      name: %s\n", field->name);
4887		seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4888			   field->var.idx);
4889		seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
4890		seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4891			   field->var_ref_idx);
4892		if (field->system)
4893			seq_printf(m, "      system: %s\n", field->system);
4894		if (field->event_name)
4895			seq_printf(m, "      event_name: %s\n", field->event_name);
4896	}
4897
4898	seq_printf(m, "      type: %s\n", field->type);
4899	seq_printf(m, "      size: %u\n", field->size);
4900	seq_printf(m, "      is_signed: %u\n", field->is_signed);
4901
4902	return 0;
4903}
4904
4905static int field_var_debug_show(struct seq_file *m,
4906				struct field_var *field_var, unsigned int i,
4907				bool save_vars)
4908{
4909	const char *vars_name = save_vars ? "save_vars" : "field_vars";
4910	struct hist_field *field;
4911	int ret = 0;
4912
4913	seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
4914
4915	field = field_var->var;
4916
4917	seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
4918
4919	hist_field_debug_show_flags(m, field->flags);
4920	seq_printf(m, "      var.name: %s\n", field->var.name);
4921	seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4922		   field->var.idx);
4923
4924	field = field_var->val;
4925
4926	seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
4927	if (field->field)
4928		seq_printf(m, "      ftrace_event_field name: %s\n",
4929			   field->field->name);
4930	else {
4931		ret = -EINVAL;
4932		goto out;
4933	}
4934
4935	seq_printf(m, "      type: %s\n", field->type);
4936	seq_printf(m, "      size: %u\n", field->size);
4937	seq_printf(m, "      is_signed: %u\n", field->is_signed);
4938out:
4939	return ret;
4940}
4941
4942static int hist_action_debug_show(struct seq_file *m,
4943				  struct action_data *data, int i)
4944{
4945	int ret = 0;
4946
4947	if (data->handler == HANDLER_ONMAX ||
4948	    data->handler == HANDLER_ONCHANGE) {
4949		seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
4950		ret = hist_field_debug_show(m, data->track_data.var_ref,
4951					    HIST_FIELD_FL_VAR_REF);
4952		if (ret)
4953			goto out;
4954
4955		seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
4956		ret = hist_field_debug_show(m, data->track_data.track_var,
4957					    HIST_FIELD_FL_VAR);
4958		if (ret)
4959			goto out;
4960	}
4961
4962	if (data->handler == HANDLER_ONMATCH) {
4963		seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
4964			   i, data->match_data.event_system);
4965		seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
4966			   i, data->match_data.event);
4967	}
4968out:
4969	return ret;
4970}
4971
4972static int hist_actions_debug_show(struct seq_file *m,
4973				   struct hist_trigger_data *hist_data)
4974{
4975	int i, ret = 0;
4976
4977	if (hist_data->n_actions)
4978		seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
4979
4980	for (i = 0; i < hist_data->n_actions; i++) {
4981		struct action_data *action = hist_data->actions[i];
4982
4983		ret = hist_action_debug_show(m, action, i);
4984		if (ret)
4985			goto out;
4986	}
4987
4988	if (hist_data->n_save_vars)
4989		seq_puts(m, "\n  save action variables (save() params):\n");
4990
4991	for (i = 0; i < hist_data->n_save_vars; i++) {
4992		ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
4993		if (ret)
4994			goto out;
4995	}
4996out:
4997	return ret;
4998}
4999
5000static void hist_trigger_debug_show(struct seq_file *m,
5001				    struct event_trigger_data *data, int n)
5002{
5003	struct hist_trigger_data *hist_data;
5004	int i, ret;
5005
5006	if (n > 0)
5007		seq_puts(m, "\n\n");
5008
5009	seq_puts(m, "# event histogram\n#\n# trigger info: ");
5010	data->ops->print(m, data->ops, data);
5011	seq_puts(m, "#\n\n");
5012
5013	hist_data = data->private_data;
5014
5015	seq_printf(m, "hist_data: %p\n\n", hist_data);
5016	seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
5017	seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
5018	seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
5019
5020	seq_puts(m, "\n  val fields:\n\n");
5021
5022	seq_puts(m, "    hist_data->fields[0]:\n");
5023	ret = hist_field_debug_show(m, hist_data->fields[0],
5024				    HIST_FIELD_FL_HITCOUNT);
5025	if (ret)
5026		return;
5027
5028	for (i = 1; i < hist_data->n_vals; i++) {
5029		seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5030		ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5031		if (ret)
5032			return;
5033	}
5034
5035	seq_puts(m, "\n  key fields:\n");
5036
5037	for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5038		seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5039		ret = hist_field_debug_show(m, hist_data->fields[i],
5040					    HIST_FIELD_FL_KEY);
5041		if (ret)
5042			return;
5043	}
5044
5045	if (hist_data->n_var_refs)
5046		seq_puts(m, "\n  variable reference fields:\n");
5047
5048	for (i = 0; i < hist_data->n_var_refs; i++) {
5049		seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5050		ret = hist_field_debug_show(m, hist_data->var_refs[i],
5051					    HIST_FIELD_FL_VAR_REF);
5052		if (ret)
5053			return;
5054	}
5055
5056	if (hist_data->n_field_vars)
5057		seq_puts(m, "\n  field variables:\n");
5058
5059	for (i = 0; i < hist_data->n_field_vars; i++) {
5060		ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5061		if (ret)
5062			return;
5063	}
5064
5065	ret = hist_actions_debug_show(m, hist_data);
5066	if (ret)
5067		return;
5068}
5069
5070static int hist_debug_show(struct seq_file *m, void *v)
5071{
5072	struct event_trigger_data *data;
5073	struct trace_event_file *event_file;
5074	int n = 0, ret = 0;
5075
5076	mutex_lock(&event_mutex);
5077
5078	event_file = event_file_data(m->private);
5079	if (unlikely(!event_file)) {
5080		ret = -ENODEV;
5081		goto out_unlock;
5082	}
5083
5084	list_for_each_entry(data, &event_file->triggers, list) {
5085		if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5086			hist_trigger_debug_show(m, data, n++);
5087	}
5088
5089 out_unlock:
5090	mutex_unlock(&event_mutex);
5091
5092	return ret;
5093}
5094
5095static int event_hist_debug_open(struct inode *inode, struct file *file)
5096{
5097	int ret;
5098
5099	ret = security_locked_down(LOCKDOWN_TRACEFS);
5100	if (ret)
5101		return ret;
5102
5103	return single_open(file, hist_debug_show, file);
5104}
5105
5106const struct file_operations event_hist_debug_fops = {
5107	.open = event_hist_debug_open,
5108	.read = seq_read,
5109	.llseek = seq_lseek,
5110	.release = single_release,
5111};
5112#endif
5113
5114static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5115{
5116	const char *field_name = hist_field_name(hist_field, 0);
5117
5118	if (hist_field->var.name)
5119		seq_printf(m, "%s=", hist_field->var.name);
5120
5121	if (hist_field->flags & HIST_FIELD_FL_CPU)
5122		seq_puts(m, "common_cpu");
5123	else if (field_name) {
5124		if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5125		    hist_field->flags & HIST_FIELD_FL_ALIAS)
5126			seq_putc(m, '$');
5127		seq_printf(m, "%s", field_name);
5128	} else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5129		seq_puts(m, "common_timestamp");
5130
5131	if (hist_field->flags) {
5132		if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5133		    !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5134			const char *flags = get_hist_field_flags(hist_field);
5135
5136			if (flags)
5137				seq_printf(m, ".%s", flags);
5138		}
5139	}
5140}
5141
5142static int event_hist_trigger_print(struct seq_file *m,
5143				    struct event_trigger_ops *ops,
5144				    struct event_trigger_data *data)
5145{
5146	struct hist_trigger_data *hist_data = data->private_data;
5147	struct hist_field *field;
5148	bool have_var = false;
5149	unsigned int i;
5150
5151	seq_puts(m, "hist:");
5152
5153	if (data->name)
5154		seq_printf(m, "%s:", data->name);
5155
5156	seq_puts(m, "keys=");
5157
5158	for_each_hist_key_field(i, hist_data) {
5159		field = hist_data->fields[i];
5160
5161		if (i > hist_data->n_vals)
5162			seq_puts(m, ",");
5163
5164		if (field->flags & HIST_FIELD_FL_STACKTRACE)
5165			seq_puts(m, "stacktrace");
5166		else
5167			hist_field_print(m, field);
5168	}
5169
5170	seq_puts(m, ":vals=");
5171
5172	for_each_hist_val_field(i, hist_data) {
5173		field = hist_data->fields[i];
5174		if (field->flags & HIST_FIELD_FL_VAR) {
5175			have_var = true;
5176			continue;
5177		}
5178
5179		if (i == HITCOUNT_IDX)
5180			seq_puts(m, "hitcount");
5181		else {
5182			seq_puts(m, ",");
5183			hist_field_print(m, field);
5184		}
5185	}
5186
5187	if (have_var) {
5188		unsigned int n = 0;
5189
5190		seq_puts(m, ":");
5191
5192		for_each_hist_val_field(i, hist_data) {
5193			field = hist_data->fields[i];
5194
5195			if (field->flags & HIST_FIELD_FL_VAR) {
5196				if (n++)
5197					seq_puts(m, ",");
5198				hist_field_print(m, field);
5199			}
5200		}
5201	}
5202
5203	seq_puts(m, ":sort=");
5204
5205	for (i = 0; i < hist_data->n_sort_keys; i++) {
5206		struct tracing_map_sort_key *sort_key;
5207		unsigned int idx, first_key_idx;
5208
5209		/* skip VAR vals */
5210		first_key_idx = hist_data->n_vals - hist_data->n_vars;
5211
5212		sort_key = &hist_data->sort_keys[i];
5213		idx = sort_key->field_idx;
5214
5215		if (WARN_ON(idx >= HIST_FIELDS_MAX))
5216			return -EINVAL;
5217
5218		if (i > 0)
5219			seq_puts(m, ",");
5220
5221		if (idx == HITCOUNT_IDX)
5222			seq_puts(m, "hitcount");
5223		else {
5224			if (idx >= first_key_idx)
5225				idx += hist_data->n_vars;
5226			hist_field_print(m, hist_data->fields[idx]);
5227		}
5228
5229		if (sort_key->descending)
5230			seq_puts(m, ".descending");
5231	}
5232	seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5233	if (hist_data->enable_timestamps)
5234		seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5235
5236	print_actions_spec(m, hist_data);
5237
5238	if (data->filter_str)
5239		seq_printf(m, " if %s", data->filter_str);
5240
5241	if (data->paused)
5242		seq_puts(m, " [paused]");
5243	else
5244		seq_puts(m, " [active]");
5245
5246	seq_putc(m, '\n');
5247
5248	return 0;
5249}
5250
5251static int event_hist_trigger_init(struct event_trigger_ops *ops,
5252				   struct event_trigger_data *data)
5253{
5254	struct hist_trigger_data *hist_data = data->private_data;
5255
5256	if (!data->ref && hist_data->attrs->name)
5257		save_named_trigger(hist_data->attrs->name, data);
5258
5259	data->ref++;
5260
5261	return 0;
5262}
5263
5264static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5265{
5266	struct trace_event_file *file;
5267	unsigned int i;
5268	char *cmd;
5269	int ret;
5270
5271	for (i = 0; i < hist_data->n_field_var_hists; i++) {
5272		file = hist_data->field_var_hists[i]->hist_data->event_file;
5273		cmd = hist_data->field_var_hists[i]->cmd;
5274		ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5275					      "!hist", "hist", cmd);
5276		WARN_ON_ONCE(ret < 0);
5277	}
5278}
5279
5280static void event_hist_trigger_free(struct event_trigger_ops *ops,
5281				    struct event_trigger_data *data)
5282{
5283	struct hist_trigger_data *hist_data = data->private_data;
5284
5285	if (WARN_ON_ONCE(data->ref <= 0))
5286		return;
5287
5288	data->ref--;
5289	if (!data->ref) {
5290		if (data->name)
5291			del_named_trigger(data);
5292
5293		trigger_data_free(data);
5294
5295		remove_hist_vars(hist_data);
5296
5297		unregister_field_var_hists(hist_data);
5298
5299		destroy_hist_data(hist_data);
5300	}
5301}
5302
5303static struct event_trigger_ops event_hist_trigger_ops = {
5304	.func			= event_hist_trigger,
5305	.print			= event_hist_trigger_print,
5306	.init			= event_hist_trigger_init,
5307	.free			= event_hist_trigger_free,
5308};
5309
5310static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5311					 struct event_trigger_data *data)
5312{
5313	data->ref++;
5314
5315	save_named_trigger(data->named_data->name, data);
5316
5317	event_hist_trigger_init(ops, data->named_data);
5318
5319	return 0;
5320}
5321
5322static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5323					  struct event_trigger_data *data)
5324{
5325	if (WARN_ON_ONCE(data->ref <= 0))
5326		return;
5327
5328	event_hist_trigger_free(ops, data->named_data);
5329
5330	data->ref--;
5331	if (!data->ref) {
5332		del_named_trigger(data);
5333		trigger_data_free(data);
5334	}
5335}
5336
5337static struct event_trigger_ops event_hist_trigger_named_ops = {
5338	.func			= event_hist_trigger,
5339	.print			= event_hist_trigger_print,
5340	.init			= event_hist_trigger_named_init,
5341	.free			= event_hist_trigger_named_free,
5342};
5343
5344static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5345							    char *param)
5346{
5347	return &event_hist_trigger_ops;
5348}
5349
5350static void hist_clear(struct event_trigger_data *data)
5351{
5352	struct hist_trigger_data *hist_data = data->private_data;
5353
5354	if (data->name)
5355		pause_named_trigger(data);
5356
5357	tracepoint_synchronize_unregister();
5358
5359	tracing_map_clear(hist_data->map);
5360
5361	if (data->name)
5362		unpause_named_trigger(data);
5363}
5364
5365static bool compatible_field(struct ftrace_event_field *field,
5366			     struct ftrace_event_field *test_field)
5367{
5368	if (field == test_field)
5369		return true;
5370	if (field == NULL || test_field == NULL)
5371		return false;
5372	if (strcmp(field->name, test_field->name) != 0)
5373		return false;
5374	if (strcmp(field->type, test_field->type) != 0)
5375		return false;
5376	if (field->size != test_field->size)
5377		return false;
5378	if (field->is_signed != test_field->is_signed)
5379		return false;
5380
5381	return true;
5382}
5383
5384static bool hist_trigger_match(struct event_trigger_data *data,
5385			       struct event_trigger_data *data_test,
5386			       struct event_trigger_data *named_data,
5387			       bool ignore_filter)
5388{
5389	struct tracing_map_sort_key *sort_key, *sort_key_test;
5390	struct hist_trigger_data *hist_data, *hist_data_test;
5391	struct hist_field *key_field, *key_field_test;
5392	unsigned int i;
5393
5394	if (named_data && (named_data != data_test) &&
5395	    (named_data != data_test->named_data))
5396		return false;
5397
5398	if (!named_data && is_named_trigger(data_test))
5399		return false;
5400
5401	hist_data = data->private_data;
5402	hist_data_test = data_test->private_data;
5403
5404	if (hist_data->n_vals != hist_data_test->n_vals ||
5405	    hist_data->n_fields != hist_data_test->n_fields ||
5406	    hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5407		return false;
5408
5409	if (!ignore_filter) {
5410		if ((data->filter_str && !data_test->filter_str) ||
5411		   (!data->filter_str && data_test->filter_str))
5412			return false;
5413	}
5414
5415	for_each_hist_field(i, hist_data) {
5416		key_field = hist_data->fields[i];
5417		key_field_test = hist_data_test->fields[i];
5418
5419		if (key_field->flags != key_field_test->flags)
5420			return false;
5421		if (!compatible_field(key_field->field, key_field_test->field))
5422			return false;
5423		if (key_field->offset != key_field_test->offset)
5424			return false;
5425		if (key_field->size != key_field_test->size)
5426			return false;
5427		if (key_field->is_signed != key_field_test->is_signed)
5428			return false;
5429		if (!!key_field->var.name != !!key_field_test->var.name)
5430			return false;
5431		if (key_field->var.name &&
5432		    strcmp(key_field->var.name, key_field_test->var.name) != 0)
5433			return false;
5434	}
5435
5436	for (i = 0; i < hist_data->n_sort_keys; i++) {
5437		sort_key = &hist_data->sort_keys[i];
5438		sort_key_test = &hist_data_test->sort_keys[i];
5439
5440		if (sort_key->field_idx != sort_key_test->field_idx ||
5441		    sort_key->descending != sort_key_test->descending)
5442			return false;
5443	}
5444
5445	if (!ignore_filter && data->filter_str &&
5446	    (strcmp(data->filter_str, data_test->filter_str) != 0))
5447		return false;
5448
5449	if (!actions_match(hist_data, hist_data_test))
5450		return false;
5451
5452	return true;
5453}
5454
5455static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5456				 struct event_trigger_data *data,
5457				 struct trace_event_file *file)
5458{
5459	struct hist_trigger_data *hist_data = data->private_data;
5460	struct event_trigger_data *test, *named_data = NULL;
5461	struct trace_array *tr = file->tr;
5462	int ret = 0;
5463
5464	if (hist_data->attrs->name) {
5465		named_data = find_named_trigger(hist_data->attrs->name);
5466		if (named_data) {
5467			if (!hist_trigger_match(data, named_data, named_data,
5468						true)) {
5469				hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5470				ret = -EINVAL;
5471				goto out;
5472			}
5473		}
5474	}
5475
5476	if (hist_data->attrs->name && !named_data)
5477		goto new;
5478
5479	lockdep_assert_held(&event_mutex);
5480
5481	list_for_each_entry(test, &file->triggers, list) {
5482		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5483			if (!hist_trigger_match(data, test, named_data, false))
5484				continue;
5485			if (hist_data->attrs->pause)
5486				test->paused = true;
5487			else if (hist_data->attrs->cont)
5488				test->paused = false;
5489			else if (hist_data->attrs->clear)
5490				hist_clear(test);
5491			else {
5492				hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5493				ret = -EEXIST;
5494			}
5495			goto out;
5496		}
5497	}
5498 new:
5499	if (hist_data->attrs->cont || hist_data->attrs->clear) {
5500		hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5501		ret = -ENOENT;
5502		goto out;
5503	}
5504
5505	if (hist_data->attrs->pause)
5506		data->paused = true;
5507
5508	if (named_data) {
5509		data->private_data = named_data->private_data;
5510		set_named_trigger_data(data, named_data);
5511		data->ops = &event_hist_trigger_named_ops;
5512	}
5513
5514	if (data->ops->init) {
5515		ret = data->ops->init(data->ops, data);
5516		if (ret < 0)
5517			goto out;
5518	}
5519
5520	if (hist_data->enable_timestamps) {
5521		char *clock = hist_data->attrs->clock;
5522
5523		ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5524		if (ret) {
5525			hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5526			goto out;
5527		}
5528
5529		tracing_set_filter_buffering(file->tr, true);
5530	}
5531
5532	if (named_data)
5533		destroy_hist_data(hist_data);
5534
5535	ret++;
5536 out:
5537	return ret;
5538}
5539
5540static int hist_trigger_enable(struct event_trigger_data *data,
5541			       struct trace_event_file *file)
5542{
5543	int ret = 0;
5544
5545	list_add_tail_rcu(&data->list, &file->triggers);
5546
5547	update_cond_flag(file);
5548
5549	if (trace_event_trigger_enable_disable(file, 1) < 0) {
5550		list_del_rcu(&data->list);
5551		update_cond_flag(file);
5552		ret--;
5553	}
5554
5555	return ret;
5556}
5557
5558static bool have_hist_trigger_match(struct event_trigger_data *data,
5559				    struct trace_event_file *file)
5560{
5561	struct hist_trigger_data *hist_data = data->private_data;
5562	struct event_trigger_data *test, *named_data = NULL;
5563	bool match = false;
5564
5565	lockdep_assert_held(&event_mutex);
5566
5567	if (hist_data->attrs->name)
5568		named_data = find_named_trigger(hist_data->attrs->name);
5569
5570	list_for_each_entry(test, &file->triggers, list) {
5571		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5572			if (hist_trigger_match(data, test, named_data, false)) {
5573				match = true;
5574				break;
5575			}
5576		}
5577	}
5578
5579	return match;
5580}
5581
5582static bool hist_trigger_check_refs(struct event_trigger_data *data,
5583				    struct trace_event_file *file)
5584{
5585	struct hist_trigger_data *hist_data = data->private_data;
5586	struct event_trigger_data *test, *named_data = NULL;
5587
5588	lockdep_assert_held(&event_mutex);
5589
5590	if (hist_data->attrs->name)
5591		named_data = find_named_trigger(hist_data->attrs->name);
5592
5593	list_for_each_entry(test, &file->triggers, list) {
5594		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5595			if (!hist_trigger_match(data, test, named_data, false))
5596				continue;
5597			hist_data = test->private_data;
5598			if (check_var_refs(hist_data))
5599				return true;
5600			break;
5601		}
5602	}
5603
5604	return false;
5605}
5606
5607static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5608				    struct event_trigger_data *data,
5609				    struct trace_event_file *file)
5610{
5611	struct hist_trigger_data *hist_data = data->private_data;
5612	struct event_trigger_data *test, *named_data = NULL;
5613	bool unregistered = false;
5614
5615	lockdep_assert_held(&event_mutex);
5616
5617	if (hist_data->attrs->name)
5618		named_data = find_named_trigger(hist_data->attrs->name);
5619
5620	list_for_each_entry(test, &file->triggers, list) {
5621		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5622			if (!hist_trigger_match(data, test, named_data, false))
5623				continue;
5624			unregistered = true;
5625			list_del_rcu(&test->list);
5626			trace_event_trigger_enable_disable(file, 0);
5627			update_cond_flag(file);
5628			break;
5629		}
5630	}
5631
5632	if (unregistered && test->ops->free)
5633		test->ops->free(test->ops, test);
5634
5635	if (hist_data->enable_timestamps) {
5636		if (!hist_data->remove || unregistered)
5637			tracing_set_filter_buffering(file->tr, false);
5638	}
5639}
5640
5641static bool hist_file_check_refs(struct trace_event_file *file)
5642{
5643	struct hist_trigger_data *hist_data;
5644	struct event_trigger_data *test;
5645
5646	lockdep_assert_held(&event_mutex);
5647
5648	list_for_each_entry(test, &file->triggers, list) {
5649		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5650			hist_data = test->private_data;
5651			if (check_var_refs(hist_data))
5652				return true;
5653		}
5654	}
5655
5656	return false;
5657}
5658
5659static void hist_unreg_all(struct trace_event_file *file)
5660{
5661	struct event_trigger_data *test, *n;
5662	struct hist_trigger_data *hist_data;
5663	struct synth_event *se;
5664	const char *se_name;
5665
5666	lockdep_assert_held(&event_mutex);
5667
5668	if (hist_file_check_refs(file))
5669		return;
5670
5671	list_for_each_entry_safe(test, n, &file->triggers, list) {
5672		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5673			hist_data = test->private_data;
5674			list_del_rcu(&test->list);
5675			trace_event_trigger_enable_disable(file, 0);
5676
5677			se_name = trace_event_name(file->event_call);
5678			se = find_synth_event(se_name);
5679			if (se)
5680				se->ref--;
5681
5682			update_cond_flag(file);
5683			if (hist_data->enable_timestamps)
5684				tracing_set_filter_buffering(file->tr, false);
5685			if (test->ops->free)
5686				test->ops->free(test->ops, test);
5687		}
5688	}
5689}
5690
5691static int event_hist_trigger_func(struct event_command *cmd_ops,
5692				   struct trace_event_file *file,
5693				   char *glob, char *cmd, char *param)
5694{
5695	unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5696	struct event_trigger_data *trigger_data;
5697	struct hist_trigger_attrs *attrs;
5698	struct event_trigger_ops *trigger_ops;
5699	struct hist_trigger_data *hist_data;
5700	struct synth_event *se;
5701	const char *se_name;
5702	bool remove = false;
5703	char *trigger, *p;
5704	int ret = 0;
5705
5706	lockdep_assert_held(&event_mutex);
5707
5708	if (glob && strlen(glob)) {
5709		hist_err_clear();
5710		last_cmd_set(file, param);
5711	}
5712
5713	if (!param)
5714		return -EINVAL;
5715
5716	if (glob[0] == '!')
5717		remove = true;
5718
5719	/*
5720	 * separate the trigger from the filter (k:v [if filter])
5721	 * allowing for whitespace in the trigger
5722	 */
5723	p = trigger = param;
5724	do {
5725		p = strstr(p, "if");
5726		if (!p)
5727			break;
5728		if (p == param)
5729			return -EINVAL;
5730		if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5731			p++;
5732			continue;
5733		}
5734		if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
5735			return -EINVAL;
5736		if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
5737			p++;
5738			continue;
5739		}
5740		break;
5741	} while (p);
5742
5743	if (!p)
5744		param = NULL;
5745	else {
5746		*(p - 1) = '\0';
5747		param = strstrip(p);
5748		trigger = strstrip(trigger);
5749	}
5750
5751	attrs = parse_hist_trigger_attrs(file->tr, trigger);
5752	if (IS_ERR(attrs))
5753		return PTR_ERR(attrs);
5754
5755	if (attrs->map_bits)
5756		hist_trigger_bits = attrs->map_bits;
5757
5758	hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
5759	if (IS_ERR(hist_data)) {
5760		destroy_hist_trigger_attrs(attrs);
5761		return PTR_ERR(hist_data);
5762	}
5763
5764	trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5765
5766	trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
5767	if (!trigger_data) {
5768		ret = -ENOMEM;
5769		goto out_free;
5770	}
5771
5772	trigger_data->count = -1;
5773	trigger_data->ops = trigger_ops;
5774	trigger_data->cmd_ops = cmd_ops;
5775
5776	INIT_LIST_HEAD(&trigger_data->list);
5777	RCU_INIT_POINTER(trigger_data->filter, NULL);
5778
5779	trigger_data->private_data = hist_data;
5780
5781	/* if param is non-empty, it's supposed to be a filter */
5782	if (param && cmd_ops->set_filter) {
5783		ret = cmd_ops->set_filter(param, trigger_data, file);
5784		if (ret < 0)
5785			goto out_free;
5786	}
5787
5788	if (remove) {
5789		if (!have_hist_trigger_match(trigger_data, file))
5790			goto out_free;
5791
5792		if (hist_trigger_check_refs(trigger_data, file)) {
5793			ret = -EBUSY;
5794			goto out_free;
5795		}
5796
5797		cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5798		se_name = trace_event_name(file->event_call);
5799		se = find_synth_event(se_name);
5800		if (se)
5801			se->ref--;
5802		ret = 0;
5803		goto out_free;
5804	}
5805
5806	ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5807	/*
5808	 * The above returns on success the # of triggers registered,
5809	 * but if it didn't register any it returns zero.  Consider no
5810	 * triggers registered a failure too.
5811	 */
5812	if (!ret) {
5813		if (!(attrs->pause || attrs->cont || attrs->clear))
5814			ret = -ENOENT;
5815		goto out_free;
5816	} else if (ret < 0)
5817		goto out_free;
5818
5819	if (get_named_trigger_data(trigger_data))
5820		goto enable;
5821
5822	if (has_hist_vars(hist_data))
5823		save_hist_vars(hist_data);
5824
5825	ret = create_actions(hist_data);
5826	if (ret)
5827		goto out_unreg;
5828
5829	ret = tracing_map_init(hist_data->map);
5830	if (ret)
5831		goto out_unreg;
5832enable:
5833	ret = hist_trigger_enable(trigger_data, file);
5834	if (ret)
5835		goto out_unreg;
5836
5837	se_name = trace_event_name(file->event_call);
5838	se = find_synth_event(se_name);
5839	if (se)
5840		se->ref++;
5841	/* Just return zero, not the number of registered triggers */
5842	ret = 0;
5843 out:
5844	if (ret == 0)
5845		hist_err_clear();
5846
5847	return ret;
5848 out_unreg:
5849	cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5850 out_free:
5851	if (cmd_ops->set_filter)
5852		cmd_ops->set_filter(NULL, trigger_data, NULL);
5853
5854	remove_hist_vars(hist_data);
5855
5856	kfree(trigger_data);
5857
5858	destroy_hist_data(hist_data);
5859	goto out;
5860}
5861
5862static struct event_command trigger_hist_cmd = {
5863	.name			= "hist",
5864	.trigger_type		= ETT_EVENT_HIST,
5865	.flags			= EVENT_CMD_FL_NEEDS_REC,
5866	.func			= event_hist_trigger_func,
5867	.reg			= hist_register_trigger,
5868	.unreg			= hist_unregister_trigger,
5869	.unreg_all		= hist_unreg_all,
5870	.get_trigger_ops	= event_hist_get_trigger_ops,
5871	.set_filter		= set_trigger_filter,
5872};
5873
5874__init int register_trigger_hist_cmd(void)
5875{
5876	int ret;
5877
5878	ret = register_event_command(&trigger_hist_cmd);
5879	WARN_ON(ret < 0);
5880
5881	return ret;
5882}
5883
5884static void
5885hist_enable_trigger(struct event_trigger_data *data,
5886		    struct trace_buffer *buffer,  void *rec,
5887		    struct ring_buffer_event *event)
5888{
5889	struct enable_trigger_data *enable_data = data->private_data;
5890	struct event_trigger_data *test;
5891
5892	list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
5893				lockdep_is_held(&event_mutex)) {
5894		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5895			if (enable_data->enable)
5896				test->paused = false;
5897			else
5898				test->paused = true;
5899		}
5900	}
5901}
5902
5903static void
5904hist_enable_count_trigger(struct event_trigger_data *data,
5905			  struct trace_buffer *buffer,  void *rec,
5906			  struct ring_buffer_event *event)
5907{
5908	if (!data->count)
5909		return;
5910
5911	if (data->count != -1)
5912		(data->count)--;
5913
5914	hist_enable_trigger(data, buffer, rec, event);
5915}
5916
5917static struct event_trigger_ops hist_enable_trigger_ops = {
5918	.func			= hist_enable_trigger,
5919	.print			= event_enable_trigger_print,
5920	.init			= event_trigger_init,
5921	.free			= event_enable_trigger_free,
5922};
5923
5924static struct event_trigger_ops hist_enable_count_trigger_ops = {
5925	.func			= hist_enable_count_trigger,
5926	.print			= event_enable_trigger_print,
5927	.init			= event_trigger_init,
5928	.free			= event_enable_trigger_free,
5929};
5930
5931static struct event_trigger_ops hist_disable_trigger_ops = {
5932	.func			= hist_enable_trigger,
5933	.print			= event_enable_trigger_print,
5934	.init			= event_trigger_init,
5935	.free			= event_enable_trigger_free,
5936};
5937
5938static struct event_trigger_ops hist_disable_count_trigger_ops = {
5939	.func			= hist_enable_count_trigger,
5940	.print			= event_enable_trigger_print,
5941	.init			= event_trigger_init,
5942	.free			= event_enable_trigger_free,
5943};
5944
5945static struct event_trigger_ops *
5946hist_enable_get_trigger_ops(char *cmd, char *param)
5947{
5948	struct event_trigger_ops *ops;
5949	bool enable;
5950
5951	enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5952
5953	if (enable)
5954		ops = param ? &hist_enable_count_trigger_ops :
5955			&hist_enable_trigger_ops;
5956	else
5957		ops = param ? &hist_disable_count_trigger_ops :
5958			&hist_disable_trigger_ops;
5959
5960	return ops;
5961}
5962
5963static void hist_enable_unreg_all(struct trace_event_file *file)
5964{
5965	struct event_trigger_data *test, *n;
5966
5967	list_for_each_entry_safe(test, n, &file->triggers, list) {
5968		if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5969			list_del_rcu(&test->list);
5970			update_cond_flag(file);
5971			trace_event_trigger_enable_disable(file, 0);
5972			if (test->ops->free)
5973				test->ops->free(test->ops, test);
5974		}
5975	}
5976}
5977
5978static struct event_command trigger_hist_enable_cmd = {
5979	.name			= ENABLE_HIST_STR,
5980	.trigger_type		= ETT_HIST_ENABLE,
5981	.func			= event_enable_trigger_func,
5982	.reg			= event_enable_register_trigger,
5983	.unreg			= event_enable_unregister_trigger,
5984	.unreg_all		= hist_enable_unreg_all,
5985	.get_trigger_ops	= hist_enable_get_trigger_ops,
5986	.set_filter		= set_trigger_filter,
5987};
5988
5989static struct event_command trigger_hist_disable_cmd = {
5990	.name			= DISABLE_HIST_STR,
5991	.trigger_type		= ETT_HIST_ENABLE,
5992	.func			= event_enable_trigger_func,
5993	.reg			= event_enable_register_trigger,
5994	.unreg			= event_enable_unregister_trigger,
5995	.unreg_all		= hist_enable_unreg_all,
5996	.get_trigger_ops	= hist_enable_get_trigger_ops,
5997	.set_filter		= set_trigger_filter,
5998};
5999
6000static __init void unregister_trigger_hist_enable_disable_cmds(void)
6001{
6002	unregister_event_command(&trigger_hist_enable_cmd);
6003	unregister_event_command(&trigger_hist_disable_cmd);
6004}
6005
6006__init int register_trigger_hist_enable_disable_cmds(void)
6007{
6008	int ret;
6009
6010	ret = register_event_command(&trigger_hist_enable_cmd);
6011	if (WARN_ON(ret < 0))
6012		return ret;
6013	ret = register_event_command(&trigger_hist_disable_cmd);
6014	if (WARN_ON(ret < 0))
6015		unregister_trigger_hist_enable_disable_cmds();
6016
6017	return ret;
6018}