1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * trace_events_hist - trace event hist triggers
   4 *
   5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6 */
   7
   8#include <linux/module.h>
   9#include <linux/kallsyms.h>
  10#include <linux/security.h>
  11#include <linux/mutex.h>
  12#include <linux/slab.h>
  13#include <linux/stacktrace.h>
  14#include <linux/rculist.h>
  15#include <linux/tracefs.h>
  16
  17/* for gfp flag names */
  18#include <linux/trace_events.h>
  19#include <trace/events/mmflags.h>
  20
  21#include "tracing_map.h"
  22#include "trace_synth.h"
 
 
 
 
 
 
  23
  24#define ERRORS								\
  25	C(NONE,			"No error"),				\
  26	C(DUPLICATE_VAR,	"Variable already defined"),		\
  27	C(VAR_NOT_UNIQUE,	"Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28	C(TOO_MANY_VARS,	"Too many variables defined"),		\
  29	C(MALFORMED_ASSIGNMENT,	"Malformed assignment"),		\
  30	C(NAMED_MISMATCH,	"Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31	C(TRIGGER_EEXIST,	"Hist trigger already exists"),		\
  32	C(TRIGGER_ENOENT_CLEAR,	"Can't clear or continue a nonexistent hist trigger"), \
  33	C(SET_CLOCK_FAIL,	"Couldn't set trace_clock"),		\
  34	C(BAD_FIELD_MODIFIER,	"Invalid field modifier"),		\
  35	C(TOO_MANY_SUBEXPR,	"Too many subexpressions (3 max)"),	\
  36	C(TIMESTAMP_MISMATCH,	"Timestamp units in expression don't match"), \
  37	C(TOO_MANY_FIELD_VARS,	"Too many field variables defined"),	\
  38	C(EVENT_FILE_NOT_FOUND,	"Event file not found"),		\
  39	C(HIST_NOT_FOUND,	"Matching event histogram not found"),	\
  40	C(HIST_CREATE_FAIL,	"Couldn't create histogram for field"),	\
  41	C(SYNTH_VAR_NOT_FOUND,	"Couldn't find synthetic variable"),	\
  42	C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),	\
  43	C(SYNTH_TYPE_MISMATCH,	"Param type doesn't match synthetic event field type"), \
  44	C(SYNTH_COUNT_MISMATCH,	"Param count doesn't match synthetic event field count"), \
  45	C(FIELD_VAR_PARSE_FAIL,	"Couldn't parse field variable"),	\
  46	C(VAR_CREATE_FIND_FAIL,	"Couldn't create or find variable"),	\
  47	C(ONX_NOT_VAR,		"For onmax(x) or onchange(x), x must be a variable"), \
  48	C(ONX_VAR_NOT_FOUND,	"Couldn't find onmax or onchange variable"), \
  49	C(ONX_VAR_CREATE_FAIL,	"Couldn't create onmax or onchange variable"), \
  50	C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),	\
  51	C(TOO_MANY_PARAMS,	"Too many action params"),		\
  52	C(PARAM_NOT_FOUND,	"Couldn't find param"),			\
  53	C(INVALID_PARAM,	"Invalid action param"),		\
  54	C(ACTION_NOT_FOUND,	"No action found"),			\
  55	C(NO_SAVE_PARAMS,	"No params found for save()"),		\
  56	C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57	C(ACTION_MISMATCH,	"Handler doesn't support action"),	\
  58	C(NO_CLOSING_PAREN,	"No closing paren found"),		\
  59	C(SUBSYS_NOT_FOUND,	"Missing subsystem"),			\
  60	C(INVALID_SUBSYS_EVENT,	"Invalid subsystem or event name"),	\
  61	C(INVALID_REF_KEY,	"Using variable references in keys not supported"), \
  62	C(VAR_NOT_FOUND,	"Couldn't find variable"),		\
  63	C(FIELD_NOT_FOUND,	"Couldn't find field"),			\
  64	C(EMPTY_ASSIGNMENT,	"Empty assignment"),			\
  65	C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),		\
  66	C(EMPTY_SORT_FIELD,	"Empty sort field"),			\
  67	C(TOO_MANY_SORT_FIELDS,	"Too many sort fields (Max = 2)"),	\
  68	C(INVALID_SORT_FIELD,	"Sort field must be a key or a val"),	\
  69	C(INVALID_STR_OPERAND,	"String type can not be an operand in expression"), \
  70	C(EXPECT_NUMBER,	"Expecting numeric literal"),		\
  71	C(UNARY_MINUS_SUBEXPR,	"Unary minus not supported in sub-expressions"), \
  72	C(DIVISION_BY_ZERO,	"Division by zero"),			\
  73	C(NEED_NOHC_VAL,	"Non-hitcount value is required for 'nohitcount'"),
  74
  75#undef C
  76#define C(a, b)		HIST_ERR_##a
  77
  78enum { ERRORS };
  79
  80#undef C
  81#define C(a, b)		b
  82
  83static const char *err_text[] = { ERRORS };
  84
  85struct hist_field;
  86
  87typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  88				struct tracing_map_elt *elt,
  89				struct trace_buffer *buffer,
  90				struct ring_buffer_event *rbe,
  91				void *event);
  92
  93#define HIST_FIELD_OPERANDS_MAX	2
  94#define HIST_FIELDS_MAX		(TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  95#define HIST_ACTIONS_MAX	8
  96#define HIST_CONST_DIGITS_MAX	21
  97#define HIST_DIV_SHIFT		20  /* For optimizing division by constants */
  98
  99enum field_op_id {
 100	FIELD_OP_NONE,
 101	FIELD_OP_PLUS,
 102	FIELD_OP_MINUS,
 103	FIELD_OP_UNARY_MINUS,
 104	FIELD_OP_DIV,
 105	FIELD_OP_MULT,
 106};
 107
 108enum hist_field_fn {
 109	HIST_FIELD_FN_NOP,
 110	HIST_FIELD_FN_VAR_REF,
 111	HIST_FIELD_FN_COUNTER,
 112	HIST_FIELD_FN_CONST,
 113	HIST_FIELD_FN_LOG2,
 114	HIST_FIELD_FN_BUCKET,
 115	HIST_FIELD_FN_TIMESTAMP,
 116	HIST_FIELD_FN_CPU,
 117	HIST_FIELD_FN_STRING,
 118	HIST_FIELD_FN_DYNSTRING,
 119	HIST_FIELD_FN_RELDYNSTRING,
 120	HIST_FIELD_FN_PSTRING,
 121	HIST_FIELD_FN_S64,
 122	HIST_FIELD_FN_U64,
 123	HIST_FIELD_FN_S32,
 124	HIST_FIELD_FN_U32,
 125	HIST_FIELD_FN_S16,
 126	HIST_FIELD_FN_U16,
 127	HIST_FIELD_FN_S8,
 128	HIST_FIELD_FN_U8,
 129	HIST_FIELD_FN_UMINUS,
 130	HIST_FIELD_FN_MINUS,
 131	HIST_FIELD_FN_PLUS,
 132	HIST_FIELD_FN_DIV,
 133	HIST_FIELD_FN_MULT,
 134	HIST_FIELD_FN_DIV_POWER2,
 135	HIST_FIELD_FN_DIV_NOT_POWER2,
 136	HIST_FIELD_FN_DIV_MULT_SHIFT,
 137	HIST_FIELD_FN_EXECNAME,
 138	HIST_FIELD_FN_STACK,
 139};
 140
 141/*
 142 * A hist_var (histogram variable) contains variable information for
 143 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 144 * flag set.  A hist_var has a variable name e.g. ts0, and is
 145 * associated with a given histogram trigger, as specified by
 146 * hist_data.  The hist_var idx is the unique index assigned to the
 147 * variable by the hist trigger's tracing_map.  The idx is what is
 148 * used to set a variable's value and, by a variable reference, to
 149 * retrieve it.
 150 */
 151struct hist_var {
 152	char				*name;
 153	struct hist_trigger_data	*hist_data;
 154	unsigned int			idx;
 155};
 156
 157struct hist_field {
 158	struct ftrace_event_field	*field;
 159	unsigned long			flags;
 160	unsigned long			buckets;
 161	const char			*type;
 162	struct hist_field		*operands[HIST_FIELD_OPERANDS_MAX];
 163	struct hist_trigger_data	*hist_data;
 164	enum hist_field_fn		fn_num;
 165	unsigned int			ref;
 166	unsigned int			size;
 167	unsigned int			offset;
 168	unsigned int                    is_signed;
 
 
 
 169
 170	/*
 171	 * Variable fields contain variable-specific info in var.
 172	 */
 173	struct hist_var			var;
 174	enum field_op_id		operator;
 175	char				*system;
 176	char				*event_name;
 177
 178	/*
 179	 * The name field is used for EXPR and VAR_REF fields.  VAR
 180	 * fields contain the variable name in var.name.
 181	 */
 182	char				*name;
 183
 184	/*
 185	 * When a histogram trigger is hit, if it has any references
 186	 * to variables, the values of those variables are collected
 187	 * into a var_ref_vals array by resolve_var_refs().  The
 188	 * current value of each variable is read from the tracing_map
 189	 * using the hist field's hist_var.idx and entered into the
 190	 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 191	 */
 192	unsigned int			var_ref_idx;
 193	bool                            read_once;
 194
 195	unsigned int			var_str_idx;
 196
 197	/* Numeric literals are represented as u64 */
 198	u64				constant;
 199	/* Used to optimize division by constants */
 200	u64				div_multiplier;
 201};
 202
 203static u64 hist_fn_call(struct hist_field *hist_field,
 204			struct tracing_map_elt *elt,
 205			struct trace_buffer *buffer,
 206			struct ring_buffer_event *rbe,
 207			void *event);
 208
 209static u64 hist_field_const(struct hist_field *field,
 210			   struct tracing_map_elt *elt,
 211			   struct trace_buffer *buffer,
 212			   struct ring_buffer_event *rbe,
 213			   void *event)
 214{
 215	return field->constant;
 216}
 217
 218static u64 hist_field_counter(struct hist_field *field,
 219			      struct tracing_map_elt *elt,
 220			      struct trace_buffer *buffer,
 221			      struct ring_buffer_event *rbe,
 222			      void *event)
 223{
 224	return 1;
 225}
 226
 227static u64 hist_field_string(struct hist_field *hist_field,
 228			     struct tracing_map_elt *elt,
 229			     struct trace_buffer *buffer,
 230			     struct ring_buffer_event *rbe,
 231			     void *event)
 232{
 233	char *addr = (char *)(event + hist_field->field->offset);
 234
 235	return (u64)(unsigned long)addr;
 236}
 237
 238static u64 hist_field_dynstring(struct hist_field *hist_field,
 239				struct tracing_map_elt *elt,
 240				struct trace_buffer *buffer,
 241				struct ring_buffer_event *rbe,
 242				void *event)
 243{
 244	u32 str_item = *(u32 *)(event + hist_field->field->offset);
 245	int str_loc = str_item & 0xffff;
 246	char *addr = (char *)(event + str_loc);
 247
 248	return (u64)(unsigned long)addr;
 249}
 250
 251static u64 hist_field_reldynstring(struct hist_field *hist_field,
 252				   struct tracing_map_elt *elt,
 253				   struct trace_buffer *buffer,
 254				   struct ring_buffer_event *rbe,
 255				   void *event)
 256{
 257	u32 *item = event + hist_field->field->offset;
 258	u32 str_item = *item;
 259	int str_loc = str_item & 0xffff;
 260	char *addr = (char *)&item[1] + str_loc;
 261
 262	return (u64)(unsigned long)addr;
 263}
 264
 265static u64 hist_field_pstring(struct hist_field *hist_field,
 266			      struct tracing_map_elt *elt,
 267			      struct trace_buffer *buffer,
 268			      struct ring_buffer_event *rbe,
 269			      void *event)
 270{
 271	char **addr = (char **)(event + hist_field->field->offset);
 272
 273	return (u64)(unsigned long)*addr;
 274}
 275
 276static u64 hist_field_log2(struct hist_field *hist_field,
 277			   struct tracing_map_elt *elt,
 278			   struct trace_buffer *buffer,
 279			   struct ring_buffer_event *rbe,
 280			   void *event)
 281{
 282	struct hist_field *operand = hist_field->operands[0];
 283
 284	u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 285
 286	return (u64) ilog2(roundup_pow_of_two(val));
 287}
 288
 289static u64 hist_field_bucket(struct hist_field *hist_field,
 290			     struct tracing_map_elt *elt,
 291			     struct trace_buffer *buffer,
 292			     struct ring_buffer_event *rbe,
 293			     void *event)
 294{
 295	struct hist_field *operand = hist_field->operands[0];
 296	unsigned long buckets = hist_field->buckets;
 297
 298	u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 299
 300	if (WARN_ON_ONCE(!buckets))
 301		return val;
 302
 303	if (val >= LONG_MAX)
 304		val = div64_ul(val, buckets);
 305	else
 306		val = (u64)((unsigned long)val / buckets);
 307	return val * buckets;
 308}
 309
 310static u64 hist_field_plus(struct hist_field *hist_field,
 311			   struct tracing_map_elt *elt,
 312			   struct trace_buffer *buffer,
 313			   struct ring_buffer_event *rbe,
 314			   void *event)
 315{
 316	struct hist_field *operand1 = hist_field->operands[0];
 317	struct hist_field *operand2 = hist_field->operands[1];
 318
 319	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 320	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 321
 322	return val1 + val2;
 323}
 324
 325static u64 hist_field_minus(struct hist_field *hist_field,
 326			    struct tracing_map_elt *elt,
 327			    struct trace_buffer *buffer,
 328			    struct ring_buffer_event *rbe,
 329			    void *event)
 330{
 331	struct hist_field *operand1 = hist_field->operands[0];
 332	struct hist_field *operand2 = hist_field->operands[1];
 333
 334	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 335	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 336
 337	return val1 - val2;
 338}
 339
 340static u64 hist_field_div(struct hist_field *hist_field,
 341			   struct tracing_map_elt *elt,
 342			   struct trace_buffer *buffer,
 343			   struct ring_buffer_event *rbe,
 344			   void *event)
 345{
 346	struct hist_field *operand1 = hist_field->operands[0];
 347	struct hist_field *operand2 = hist_field->operands[1];
 348
 349	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 350	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 351
 352	/* Return -1 for the undefined case */
 353	if (!val2)
 354		return -1;
 355
 356	/* Use shift if the divisor is a power of 2 */
 357	if (!(val2 & (val2 - 1)))
 358		return val1 >> __ffs64(val2);
 359
 360	return div64_u64(val1, val2);
 361}
 362
 363static u64 div_by_power_of_two(struct hist_field *hist_field,
 364				struct tracing_map_elt *elt,
 365				struct trace_buffer *buffer,
 366				struct ring_buffer_event *rbe,
 367				void *event)
 368{
 369	struct hist_field *operand1 = hist_field->operands[0];
 370	struct hist_field *operand2 = hist_field->operands[1];
 371
 372	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 373
 374	return val1 >> __ffs64(operand2->constant);
 375}
 376
 377static u64 div_by_not_power_of_two(struct hist_field *hist_field,
 378				struct tracing_map_elt *elt,
 379				struct trace_buffer *buffer,
 380				struct ring_buffer_event *rbe,
 381				void *event)
 382{
 383	struct hist_field *operand1 = hist_field->operands[0];
 384	struct hist_field *operand2 = hist_field->operands[1];
 385
 386	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 387
 388	return div64_u64(val1, operand2->constant);
 389}
 390
 391static u64 div_by_mult_and_shift(struct hist_field *hist_field,
 392				struct tracing_map_elt *elt,
 393				struct trace_buffer *buffer,
 394				struct ring_buffer_event *rbe,
 395				void *event)
 396{
 397	struct hist_field *operand1 = hist_field->operands[0];
 398	struct hist_field *operand2 = hist_field->operands[1];
 399
 400	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 401
 402	/*
 403	 * If the divisor is a constant, do a multiplication and shift instead.
 404	 *
 405	 * Choose Z = some power of 2. If Y <= Z, then:
 406	 *     X / Y = (X * (Z / Y)) / Z
 407	 *
 408	 * (Z / Y) is a constant (mult) which is calculated at parse time, so:
 409	 *     X / Y = (X * mult) / Z
 410	 *
 411	 * The division by Z can be replaced by a shift since Z is a power of 2:
 412	 *     X / Y = (X * mult) >> HIST_DIV_SHIFT
 413	 *
 414	 * As long, as X < Z the results will not be off by more than 1.
 415	 */
 416	if (val1 < (1 << HIST_DIV_SHIFT)) {
 417		u64 mult = operand2->div_multiplier;
 418
 419		return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT;
 420	}
 421
 422	return div64_u64(val1, operand2->constant);
 423}
 424
 425static u64 hist_field_mult(struct hist_field *hist_field,
 426			   struct tracing_map_elt *elt,
 427			   struct trace_buffer *buffer,
 428			   struct ring_buffer_event *rbe,
 429			   void *event)
 430{
 431	struct hist_field *operand1 = hist_field->operands[0];
 432	struct hist_field *operand2 = hist_field->operands[1];
 433
 434	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 435	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 436
 437	return val1 * val2;
 438}
 439
 440static u64 hist_field_unary_minus(struct hist_field *hist_field,
 441				  struct tracing_map_elt *elt,
 442				  struct trace_buffer *buffer,
 443				  struct ring_buffer_event *rbe,
 444				  void *event)
 445{
 446	struct hist_field *operand = hist_field->operands[0];
 447
 448	s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event);
 449	u64 val = (u64)-sval;
 450
 451	return val;
 452}
 453
 454#define DEFINE_HIST_FIELD_FN(type)					\
 455	static u64 hist_field_##type(struct hist_field *hist_field,	\
 456				     struct tracing_map_elt *elt,	\
 457				     struct trace_buffer *buffer,	\
 458				     struct ring_buffer_event *rbe,	\
 459				     void *event)			\
 460{									\
 461	type *addr = (type *)(event + hist_field->field->offset);	\
 462									\
 463	return (u64)(unsigned long)*addr;				\
 464}
 465
 466DEFINE_HIST_FIELD_FN(s64);
 467DEFINE_HIST_FIELD_FN(u64);
 468DEFINE_HIST_FIELD_FN(s32);
 469DEFINE_HIST_FIELD_FN(u32);
 470DEFINE_HIST_FIELD_FN(s16);
 471DEFINE_HIST_FIELD_FN(u16);
 472DEFINE_HIST_FIELD_FN(s8);
 473DEFINE_HIST_FIELD_FN(u8);
 474
 475#define for_each_hist_field(i, hist_data)	\
 476	for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 477
 478#define for_each_hist_val_field(i, hist_data)	\
 479	for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 480
 481#define for_each_hist_key_field(i, hist_data)	\
 482	for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 483
 
 
 
 
 484#define HITCOUNT_IDX		0
 485#define HIST_KEY_SIZE_MAX	(MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 486
 487enum hist_field_flags {
 488	HIST_FIELD_FL_HITCOUNT		= 1 << 0,
 489	HIST_FIELD_FL_KEY		= 1 << 1,
 490	HIST_FIELD_FL_STRING		= 1 << 2,
 491	HIST_FIELD_FL_HEX		= 1 << 3,
 492	HIST_FIELD_FL_SYM		= 1 << 4,
 493	HIST_FIELD_FL_SYM_OFFSET	= 1 << 5,
 494	HIST_FIELD_FL_EXECNAME		= 1 << 6,
 495	HIST_FIELD_FL_SYSCALL		= 1 << 7,
 496	HIST_FIELD_FL_STACKTRACE	= 1 << 8,
 497	HIST_FIELD_FL_LOG2		= 1 << 9,
 498	HIST_FIELD_FL_TIMESTAMP		= 1 << 10,
 499	HIST_FIELD_FL_TIMESTAMP_USECS	= 1 << 11,
 500	HIST_FIELD_FL_VAR		= 1 << 12,
 501	HIST_FIELD_FL_EXPR		= 1 << 13,
 502	HIST_FIELD_FL_VAR_REF		= 1 << 14,
 503	HIST_FIELD_FL_CPU		= 1 << 15,
 504	HIST_FIELD_FL_ALIAS		= 1 << 16,
 505	HIST_FIELD_FL_BUCKET		= 1 << 17,
 506	HIST_FIELD_FL_CONST		= 1 << 18,
 507	HIST_FIELD_FL_PERCENT		= 1 << 19,
 508	HIST_FIELD_FL_GRAPH		= 1 << 20,
 509};
 510
 511struct var_defs {
 512	unsigned int	n_vars;
 513	char		*name[TRACING_MAP_VARS_MAX];
 514	char		*expr[TRACING_MAP_VARS_MAX];
 515};
 516
 517struct hist_trigger_attrs {
 518	char		*keys_str;
 519	char		*vals_str;
 520	char		*sort_key_str;
 521	char		*name;
 522	char		*clock;
 523	bool		pause;
 524	bool		cont;
 525	bool		clear;
 526	bool		ts_in_usecs;
 527	bool		no_hitcount;
 528	unsigned int	map_bits;
 529
 530	char		*assignment_str[TRACING_MAP_VARS_MAX];
 531	unsigned int	n_assignments;
 532
 533	char		*action_str[HIST_ACTIONS_MAX];
 534	unsigned int	n_actions;
 535
 536	struct var_defs	var_defs;
 537};
 538
 539struct field_var {
 540	struct hist_field	*var;
 541	struct hist_field	*val;
 542};
 543
 544struct field_var_hist {
 545	struct hist_trigger_data	*hist_data;
 546	char				*cmd;
 547};
 548
 549struct hist_trigger_data {
 550	struct hist_field               *fields[HIST_FIELDS_MAX];
 551	unsigned int			n_vals;
 552	unsigned int			n_keys;
 553	unsigned int			n_fields;
 554	unsigned int			n_vars;
 555	unsigned int			n_var_str;
 556	unsigned int			key_size;
 557	struct tracing_map_sort_key	sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 558	unsigned int			n_sort_keys;
 559	struct trace_event_file		*event_file;
 560	struct hist_trigger_attrs	*attrs;
 561	struct tracing_map		*map;
 562	bool				enable_timestamps;
 563	bool				remove;
 564	struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 565	unsigned int			n_var_refs;
 566
 567	struct action_data		*actions[HIST_ACTIONS_MAX];
 568	unsigned int			n_actions;
 569
 570	struct field_var		*field_vars[SYNTH_FIELDS_MAX];
 571	unsigned int			n_field_vars;
 572	unsigned int			n_field_var_str;
 573	struct field_var_hist		*field_var_hists[SYNTH_FIELDS_MAX];
 574	unsigned int			n_field_var_hists;
 575
 576	struct field_var		*save_vars[SYNTH_FIELDS_MAX];
 577	unsigned int			n_save_vars;
 578	unsigned int			n_save_var_str;
 579};
 580
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 581struct action_data;
 582
 583typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 584			     struct tracing_map_elt *elt,
 585			     struct trace_buffer *buffer, void *rec,
 586			     struct ring_buffer_event *rbe, void *key,
 587			     struct action_data *data, u64 *var_ref_vals);
 588
 589typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 590
 591enum handler_id {
 592	HANDLER_ONMATCH = 1,
 593	HANDLER_ONMAX,
 594	HANDLER_ONCHANGE,
 595};
 596
 597enum action_id {
 598	ACTION_SAVE = 1,
 599	ACTION_TRACE,
 600	ACTION_SNAPSHOT,
 601};
 602
 603struct action_data {
 604	enum handler_id		handler;
 605	enum action_id		action;
 606	char			*action_name;
 607	action_fn_t		fn;
 608
 609	unsigned int		n_params;
 610	char			*params[SYNTH_FIELDS_MAX];
 611
 612	/*
 613	 * When a histogram trigger is hit, the values of any
 614	 * references to variables, including variables being passed
 615	 * as parameters to synthetic events, are collected into a
 616	 * var_ref_vals array.  This var_ref_idx array is an array of
 617	 * indices into the var_ref_vals array, one for each synthetic
 618	 * event param, and is passed to the synthetic event
 619	 * invocation.
 620	 */
 621	unsigned int		var_ref_idx[SYNTH_FIELDS_MAX];
 622	struct synth_event	*synth_event;
 623	bool			use_trace_keyword;
 624	char			*synth_event_name;
 625
 626	union {
 627		struct {
 628			char			*event;
 629			char			*event_system;
 630		} match_data;
 631
 632		struct {
 633			/*
 634			 * var_str contains the $-unstripped variable
 635			 * name referenced by var_ref, and used when
 636			 * printing the action.  Because var_ref
 637			 * creation is deferred to create_actions(),
 638			 * we need a per-action way to save it until
 639			 * then, thus var_str.
 640			 */
 641			char			*var_str;
 642
 643			/*
 644			 * var_ref refers to the variable being
 645			 * tracked e.g onmax($var).
 646			 */
 647			struct hist_field	*var_ref;
 648
 649			/*
 650			 * track_var contains the 'invisible' tracking
 651			 * variable created to keep the current
 652			 * e.g. max value.
 653			 */
 654			struct hist_field	*track_var;
 655
 656			check_track_val_fn_t	check_val;
 657			action_fn_t		save_data;
 658		} track_data;
 659	};
 660};
 661
 662struct track_data {
 663	u64				track_val;
 664	bool				updated;
 665
 666	unsigned int			key_len;
 667	void				*key;
 668	struct tracing_map_elt		elt;
 669
 670	struct action_data		*action_data;
 671	struct hist_trigger_data	*hist_data;
 672};
 673
 674struct hist_elt_data {
 675	char *comm;
 676	u64 *var_ref_vals;
 677	char **field_var_str;
 678	int n_field_var_str;
 679};
 680
 681struct snapshot_context {
 682	struct tracing_map_elt	*elt;
 683	void			*key;
 684};
 685
 686/*
 687 * Returns the specific division function to use if the divisor
 688 * is constant. This avoids extra branches when the trigger is hit.
 689 */
 690static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor)
 691{
 692	u64 div = divisor->constant;
 693
 694	if (!(div & (div - 1)))
 695		return HIST_FIELD_FN_DIV_POWER2;
 696
 697	/* If the divisor is too large, do a regular division */
 698	if (div > (1 << HIST_DIV_SHIFT))
 699		return HIST_FIELD_FN_DIV_NOT_POWER2;
 700
 701	divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
 702	return HIST_FIELD_FN_DIV_MULT_SHIFT;
 703}
 704
 705static void track_data_free(struct track_data *track_data)
 706{
 707	struct hist_elt_data *elt_data;
 708
 709	if (!track_data)
 710		return;
 711
 712	kfree(track_data->key);
 713
 714	elt_data = track_data->elt.private_data;
 715	if (elt_data) {
 716		kfree(elt_data->comm);
 717		kfree(elt_data);
 718	}
 719
 720	kfree(track_data);
 721}
 722
 723static struct track_data *track_data_alloc(unsigned int key_len,
 724					   struct action_data *action_data,
 725					   struct hist_trigger_data *hist_data)
 726{
 727	struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 728	struct hist_elt_data *elt_data;
 729
 730	if (!data)
 731		return ERR_PTR(-ENOMEM);
 732
 733	data->key = kzalloc(key_len, GFP_KERNEL);
 734	if (!data->key) {
 735		track_data_free(data);
 736		return ERR_PTR(-ENOMEM);
 737	}
 738
 739	data->key_len = key_len;
 740	data->action_data = action_data;
 741	data->hist_data = hist_data;
 742
 743	elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 744	if (!elt_data) {
 745		track_data_free(data);
 746		return ERR_PTR(-ENOMEM);
 747	}
 748
 749	data->elt.private_data = elt_data;
 750
 751	elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 752	if (!elt_data->comm) {
 753		track_data_free(data);
 754		return ERR_PTR(-ENOMEM);
 755	}
 756
 757	return data;
 758}
 759
 760#define HIST_PREFIX "hist:"
 761
 762static char *last_cmd;
 763static char last_cmd_loc[MAX_FILTER_STR_VAL];
 764
 765static int errpos(char *str)
 766{
 767	if (!str || !last_cmd)
 768		return 0;
 769
 770	return err_pos(last_cmd, str);
 771}
 772
 773static void last_cmd_set(struct trace_event_file *file, char *str)
 774{
 775	const char *system = NULL, *name = NULL;
 776	struct trace_event_call *call;
 777
 778	if (!str)
 779		return;
 780
 781	kfree(last_cmd);
 782
 783	last_cmd = kasprintf(GFP_KERNEL, HIST_PREFIX "%s", str);
 784	if (!last_cmd)
 785		return;
 786
 787	if (file) {
 788		call = file->event_call;
 
 789		system = call->class->system;
 790		if (system) {
 791			name = trace_event_name(call);
 792			if (!name)
 793				system = NULL;
 794		}
 795	}
 796
 797	if (system)
 798		snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, HIST_PREFIX "%s:%s", system, name);
 799}
 800
 801static void hist_err(struct trace_array *tr, u8 err_type, u16 err_pos)
 802{
 803	if (!last_cmd)
 804		return;
 805
 806	tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 807			err_type, err_pos);
 808}
 809
 810static void hist_err_clear(void)
 811{
 812	if (last_cmd)
 813		last_cmd[0] = '\0';
 814	last_cmd_loc[0] = '\0';
 815}
 816
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 817typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 818				    unsigned int *var_ref_idx);
 819
 820static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 821			       unsigned int *var_ref_idx)
 822{
 823	struct tracepoint *tp = event->tp;
 824
 825	if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 826		struct tracepoint_func *probe_func_ptr;
 827		synth_probe_func_t probe_func;
 828		void *__data;
 829
 830		if (!(cpu_online(raw_smp_processor_id())))
 831			return;
 832
 833		probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 834		if (probe_func_ptr) {
 835			do {
 836				probe_func = probe_func_ptr->func;
 837				__data = probe_func_ptr->data;
 838				probe_func(__data, var_ref_vals, var_ref_idx);
 839			} while ((++probe_func_ptr)->func);
 840		}
 841	}
 842}
 843
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 844static void action_trace(struct hist_trigger_data *hist_data,
 845			 struct tracing_map_elt *elt,
 846			 struct trace_buffer *buffer, void *rec,
 847			 struct ring_buffer_event *rbe, void *key,
 848			 struct action_data *data, u64 *var_ref_vals)
 849{
 850	struct synth_event *event = data->synth_event;
 851
 852	trace_synth(event, var_ref_vals, data->var_ref_idx);
 853}
 854
 855struct hist_var_data {
 856	struct list_head list;
 857	struct hist_trigger_data *hist_data;
 858};
 859
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 860static u64 hist_field_timestamp(struct hist_field *hist_field,
 861				struct tracing_map_elt *elt,
 862				struct trace_buffer *buffer,
 863				struct ring_buffer_event *rbe,
 864				void *event)
 865{
 866	struct hist_trigger_data *hist_data = hist_field->hist_data;
 867	struct trace_array *tr = hist_data->event_file->tr;
 868
 869	u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 870
 871	if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 872		ts = ns2usecs(ts);
 873
 874	return ts;
 875}
 876
 877static u64 hist_field_cpu(struct hist_field *hist_field,
 878			  struct tracing_map_elt *elt,
 879			  struct trace_buffer *buffer,
 880			  struct ring_buffer_event *rbe,
 881			  void *event)
 882{
 883	int cpu = smp_processor_id();
 884
 885	return cpu;
 886}
 887
 888/**
 889 * check_field_for_var_ref - Check if a VAR_REF field references a variable
 890 * @hist_field: The VAR_REF field to check
 891 * @var_data: The hist trigger that owns the variable
 892 * @var_idx: The trigger variable identifier
 893 *
 894 * Check the given VAR_REF field to see whether or not it references
 895 * the given variable associated with the given trigger.
 896 *
 897 * Return: The VAR_REF field if it does reference the variable, NULL if not
 898 */
 899static struct hist_field *
 900check_field_for_var_ref(struct hist_field *hist_field,
 901			struct hist_trigger_data *var_data,
 902			unsigned int var_idx)
 903{
 904	WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 905
 906	if (hist_field && hist_field->var.idx == var_idx &&
 907	    hist_field->var.hist_data == var_data)
 908		return hist_field;
 909
 910	return NULL;
 911}
 912
 913/**
 914 * find_var_ref - Check if a trigger has a reference to a trigger variable
 915 * @hist_data: The hist trigger that might have a reference to the variable
 916 * @var_data: The hist trigger that owns the variable
 917 * @var_idx: The trigger variable identifier
 918 *
 919 * Check the list of var_refs[] on the first hist trigger to see
 920 * whether any of them are references to the variable on the second
 921 * trigger.
 922 *
 923 * Return: The VAR_REF field referencing the variable if so, NULL if not
 924 */
 925static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 926				       struct hist_trigger_data *var_data,
 927				       unsigned int var_idx)
 928{
 929	struct hist_field *hist_field;
 930	unsigned int i;
 931
 932	for (i = 0; i < hist_data->n_var_refs; i++) {
 933		hist_field = hist_data->var_refs[i];
 934		if (check_field_for_var_ref(hist_field, var_data, var_idx))
 935			return hist_field;
 936	}
 937
 938	return NULL;
 939}
 940
 941/**
 942 * find_any_var_ref - Check if there is a reference to a given trigger variable
 943 * @hist_data: The hist trigger
 944 * @var_idx: The trigger variable identifier
 945 *
 946 * Check to see whether the given variable is currently referenced by
 947 * any other trigger.
 948 *
 949 * The trigger the variable is defined on is explicitly excluded - the
 950 * assumption being that a self-reference doesn't prevent a trigger
 951 * from being removed.
 952 *
 953 * Return: The VAR_REF field referencing the variable if so, NULL if not
 954 */
 955static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 956					   unsigned int var_idx)
 957{
 958	struct trace_array *tr = hist_data->event_file->tr;
 959	struct hist_field *found = NULL;
 960	struct hist_var_data *var_data;
 961
 962	list_for_each_entry(var_data, &tr->hist_vars, list) {
 963		if (var_data->hist_data == hist_data)
 964			continue;
 965		found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 966		if (found)
 967			break;
 968	}
 969
 970	return found;
 971}
 972
 973/**
 974 * check_var_refs - Check if there is a reference to any of trigger's variables
 975 * @hist_data: The hist trigger
 976 *
 977 * A trigger can define one or more variables.  If any one of them is
 978 * currently referenced by any other trigger, this function will
 979 * determine that.
 980 *
 981 * Typically used to determine whether or not a trigger can be removed
 982 * - if there are any references to a trigger's variables, it cannot.
 983 *
 984 * Return: True if there is a reference to any of trigger's variables
 985 */
 986static bool check_var_refs(struct hist_trigger_data *hist_data)
 987{
 988	struct hist_field *field;
 989	bool found = false;
 990	int i;
 991
 992	for_each_hist_field(i, hist_data) {
 993		field = hist_data->fields[i];
 994		if (field && field->flags & HIST_FIELD_FL_VAR) {
 995			if (find_any_var_ref(hist_data, field->var.idx)) {
 996				found = true;
 997				break;
 998			}
 999		}
1000	}
1001
1002	return found;
1003}
1004
1005static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1006{
1007	struct trace_array *tr = hist_data->event_file->tr;
1008	struct hist_var_data *var_data, *found = NULL;
1009
1010	list_for_each_entry(var_data, &tr->hist_vars, list) {
1011		if (var_data->hist_data == hist_data) {
1012			found = var_data;
1013			break;
1014		}
1015	}
1016
1017	return found;
1018}
1019
1020static bool field_has_hist_vars(struct hist_field *hist_field,
1021				unsigned int level)
1022{
1023	int i;
1024
1025	if (level > 3)
1026		return false;
1027
1028	if (!hist_field)
1029		return false;
1030
1031	if (hist_field->flags & HIST_FIELD_FL_VAR ||
1032	    hist_field->flags & HIST_FIELD_FL_VAR_REF)
1033		return true;
1034
1035	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1036		struct hist_field *operand;
1037
1038		operand = hist_field->operands[i];
1039		if (field_has_hist_vars(operand, level + 1))
1040			return true;
1041	}
1042
1043	return false;
1044}
1045
1046static bool has_hist_vars(struct hist_trigger_data *hist_data)
1047{
1048	struct hist_field *hist_field;
1049	int i;
1050
1051	for_each_hist_field(i, hist_data) {
1052		hist_field = hist_data->fields[i];
1053		if (field_has_hist_vars(hist_field, 0))
1054			return true;
1055	}
1056
1057	return false;
1058}
1059
1060static int save_hist_vars(struct hist_trigger_data *hist_data)
1061{
1062	struct trace_array *tr = hist_data->event_file->tr;
1063	struct hist_var_data *var_data;
1064
1065	var_data = find_hist_vars(hist_data);
1066	if (var_data)
1067		return 0;
1068
1069	if (tracing_check_open_get_tr(tr))
1070		return -ENODEV;
1071
1072	var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1073	if (!var_data) {
1074		trace_array_put(tr);
1075		return -ENOMEM;
1076	}
1077
1078	var_data->hist_data = hist_data;
1079	list_add(&var_data->list, &tr->hist_vars);
1080
1081	return 0;
1082}
1083
1084static void remove_hist_vars(struct hist_trigger_data *hist_data)
1085{
1086	struct trace_array *tr = hist_data->event_file->tr;
1087	struct hist_var_data *var_data;
1088
1089	var_data = find_hist_vars(hist_data);
1090	if (!var_data)
1091		return;
1092
1093	if (WARN_ON(check_var_refs(hist_data)))
1094		return;
1095
1096	list_del(&var_data->list);
1097
1098	kfree(var_data);
1099
1100	trace_array_put(tr);
1101}
1102
1103static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1104					 const char *var_name)
1105{
1106	struct hist_field *hist_field, *found = NULL;
1107	int i;
1108
1109	for_each_hist_field(i, hist_data) {
1110		hist_field = hist_data->fields[i];
1111		if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1112		    strcmp(hist_field->var.name, var_name) == 0) {
1113			found = hist_field;
1114			break;
1115		}
1116	}
1117
1118	return found;
1119}
1120
1121static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1122				   struct trace_event_file *file,
1123				   const char *var_name)
1124{
1125	struct hist_trigger_data *test_data;
1126	struct event_trigger_data *test;
1127	struct hist_field *hist_field;
1128
1129	lockdep_assert_held(&event_mutex);
1130
1131	hist_field = find_var_field(hist_data, var_name);
1132	if (hist_field)
1133		return hist_field;
1134
1135	list_for_each_entry(test, &file->triggers, list) {
1136		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1137			test_data = test->private_data;
1138			hist_field = find_var_field(test_data, var_name);
1139			if (hist_field)
1140				return hist_field;
1141		}
1142	}
1143
1144	return NULL;
1145}
1146
1147static struct trace_event_file *find_var_file(struct trace_array *tr,
1148					      char *system,
1149					      char *event_name,
1150					      char *var_name)
1151{
1152	struct hist_trigger_data *var_hist_data;
1153	struct hist_var_data *var_data;
1154	struct trace_event_file *file, *found = NULL;
1155
1156	if (system)
1157		return find_event_file(tr, system, event_name);
1158
1159	list_for_each_entry(var_data, &tr->hist_vars, list) {
1160		var_hist_data = var_data->hist_data;
1161		file = var_hist_data->event_file;
1162		if (file == found)
1163			continue;
1164
1165		if (find_var_field(var_hist_data, var_name)) {
1166			if (found) {
1167				hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1168				return NULL;
1169			}
1170
1171			found = file;
1172		}
1173	}
1174
1175	return found;
1176}
1177
1178static struct hist_field *find_file_var(struct trace_event_file *file,
1179					const char *var_name)
1180{
1181	struct hist_trigger_data *test_data;
1182	struct event_trigger_data *test;
1183	struct hist_field *hist_field;
1184
1185	lockdep_assert_held(&event_mutex);
1186
1187	list_for_each_entry(test, &file->triggers, list) {
1188		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1189			test_data = test->private_data;
1190			hist_field = find_var_field(test_data, var_name);
1191			if (hist_field)
1192				return hist_field;
1193		}
1194	}
1195
1196	return NULL;
1197}
1198
1199static struct hist_field *
1200find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1201{
1202	struct trace_array *tr = hist_data->event_file->tr;
1203	struct hist_field *hist_field, *found = NULL;
1204	struct trace_event_file *file;
1205	unsigned int i;
1206
1207	for (i = 0; i < hist_data->n_actions; i++) {
1208		struct action_data *data = hist_data->actions[i];
1209
1210		if (data->handler == HANDLER_ONMATCH) {
1211			char *system = data->match_data.event_system;
1212			char *event_name = data->match_data.event;
1213
1214			file = find_var_file(tr, system, event_name, var_name);
1215			if (!file)
1216				continue;
1217			hist_field = find_file_var(file, var_name);
1218			if (hist_field) {
1219				if (found) {
1220					hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1221						 errpos(var_name));
1222					return ERR_PTR(-EINVAL);
1223				}
1224
1225				found = hist_field;
1226			}
1227		}
1228	}
1229	return found;
1230}
1231
1232static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1233					 char *system,
1234					 char *event_name,
1235					 char *var_name)
1236{
1237	struct trace_array *tr = hist_data->event_file->tr;
1238	struct hist_field *hist_field = NULL;
1239	struct trace_event_file *file;
1240
1241	if (!system || !event_name) {
1242		hist_field = find_match_var(hist_data, var_name);
1243		if (IS_ERR(hist_field))
1244			return NULL;
1245		if (hist_field)
1246			return hist_field;
1247	}
1248
1249	file = find_var_file(tr, system, event_name, var_name);
1250	if (!file)
1251		return NULL;
1252
1253	hist_field = find_file_var(file, var_name);
1254
1255	return hist_field;
1256}
1257
1258static u64 hist_field_var_ref(struct hist_field *hist_field,
1259			      struct tracing_map_elt *elt,
1260			      struct trace_buffer *buffer,
1261			      struct ring_buffer_event *rbe,
1262			      void *event)
1263{
1264	struct hist_elt_data *elt_data;
1265	u64 var_val = 0;
1266
1267	if (WARN_ON_ONCE(!elt))
1268		return var_val;
1269
1270	elt_data = elt->private_data;
1271	var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1272
1273	return var_val;
1274}
1275
1276static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1277			     u64 *var_ref_vals, bool self)
1278{
1279	struct hist_trigger_data *var_data;
1280	struct tracing_map_elt *var_elt;
1281	struct hist_field *hist_field;
1282	unsigned int i, var_idx;
1283	bool resolved = true;
1284	u64 var_val = 0;
1285
1286	for (i = 0; i < hist_data->n_var_refs; i++) {
1287		hist_field = hist_data->var_refs[i];
1288		var_idx = hist_field->var.idx;
1289		var_data = hist_field->var.hist_data;
1290
1291		if (var_data == NULL) {
1292			resolved = false;
1293			break;
1294		}
1295
1296		if ((self && var_data != hist_data) ||
1297		    (!self && var_data == hist_data))
1298			continue;
1299
1300		var_elt = tracing_map_lookup(var_data->map, key);
1301		if (!var_elt) {
1302			resolved = false;
1303			break;
1304		}
1305
1306		if (!tracing_map_var_set(var_elt, var_idx)) {
1307			resolved = false;
1308			break;
1309		}
1310
1311		if (self || !hist_field->read_once)
1312			var_val = tracing_map_read_var(var_elt, var_idx);
1313		else
1314			var_val = tracing_map_read_var_once(var_elt, var_idx);
1315
1316		var_ref_vals[i] = var_val;
1317	}
1318
1319	return resolved;
1320}
1321
1322static const char *hist_field_name(struct hist_field *field,
1323				   unsigned int level)
1324{
1325	const char *field_name = "";
1326
1327	if (WARN_ON_ONCE(!field))
1328		return field_name;
1329
1330	if (level > 1)
1331		return field_name;
1332
1333	if (field->field)
1334		field_name = field->field->name;
1335	else if (field->flags & HIST_FIELD_FL_LOG2 ||
1336		 field->flags & HIST_FIELD_FL_ALIAS ||
1337		 field->flags & HIST_FIELD_FL_BUCKET)
1338		field_name = hist_field_name(field->operands[0], ++level);
1339	else if (field->flags & HIST_FIELD_FL_CPU)
1340		field_name = "common_cpu";
1341	else if (field->flags & HIST_FIELD_FL_EXPR ||
1342		 field->flags & HIST_FIELD_FL_VAR_REF) {
1343		if (field->system) {
1344			static char full_name[MAX_FILTER_STR_VAL];
1345
1346			strcat(full_name, field->system);
1347			strcat(full_name, ".");
1348			strcat(full_name, field->event_name);
1349			strcat(full_name, ".");
1350			strcat(full_name, field->name);
1351			field_name = full_name;
1352		} else
1353			field_name = field->name;
1354	} else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1355		field_name = "common_timestamp";
1356	else if (field->flags & HIST_FIELD_FL_STACKTRACE) {
1357		if (field->field)
1358			field_name = field->field->name;
1359		else
1360			field_name = "common_stacktrace";
1361	} else if (field->flags & HIST_FIELD_FL_HITCOUNT)
1362		field_name = "hitcount";
1363
1364	if (field_name == NULL)
1365		field_name = "";
1366
1367	return field_name;
1368}
1369
1370static enum hist_field_fn select_value_fn(int field_size, int field_is_signed)
1371{
 
 
1372	switch (field_size) {
1373	case 8:
1374		if (field_is_signed)
1375			return HIST_FIELD_FN_S64;
1376		else
1377			return HIST_FIELD_FN_U64;
 
1378	case 4:
1379		if (field_is_signed)
1380			return HIST_FIELD_FN_S32;
1381		else
1382			return HIST_FIELD_FN_U32;
 
1383	case 2:
1384		if (field_is_signed)
1385			return HIST_FIELD_FN_S16;
1386		else
1387			return HIST_FIELD_FN_U16;
 
1388	case 1:
1389		if (field_is_signed)
1390			return HIST_FIELD_FN_S8;
1391		else
1392			return HIST_FIELD_FN_U8;
 
1393	}
1394
1395	return HIST_FIELD_FN_NOP;
1396}
1397
1398static int parse_map_size(char *str)
1399{
1400	unsigned long size, map_bits;
1401	int ret;
1402
 
 
 
 
 
 
1403	ret = kstrtoul(str, 0, &size);
1404	if (ret)
1405		goto out;
1406
1407	map_bits = ilog2(roundup_pow_of_two(size));
1408	if (map_bits < TRACING_MAP_BITS_MIN ||
1409	    map_bits > TRACING_MAP_BITS_MAX)
1410		ret = -EINVAL;
1411	else
1412		ret = map_bits;
1413 out:
1414	return ret;
1415}
1416
1417static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1418{
1419	unsigned int i;
1420
1421	if (!attrs)
1422		return;
1423
1424	for (i = 0; i < attrs->n_assignments; i++)
1425		kfree(attrs->assignment_str[i]);
1426
1427	for (i = 0; i < attrs->n_actions; i++)
1428		kfree(attrs->action_str[i]);
1429
1430	kfree(attrs->name);
1431	kfree(attrs->sort_key_str);
1432	kfree(attrs->keys_str);
1433	kfree(attrs->vals_str);
1434	kfree(attrs->clock);
1435	kfree(attrs);
1436}
1437
1438static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1439{
1440	int ret = -EINVAL;
1441
1442	if (attrs->n_actions >= HIST_ACTIONS_MAX)
1443		return ret;
1444
1445	if ((str_has_prefix(str, "onmatch(")) ||
1446	    (str_has_prefix(str, "onmax(")) ||
1447	    (str_has_prefix(str, "onchange("))) {
1448		attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1449		if (!attrs->action_str[attrs->n_actions]) {
1450			ret = -ENOMEM;
1451			return ret;
1452		}
1453		attrs->n_actions++;
1454		ret = 0;
1455	}
1456	return ret;
1457}
1458
1459static int parse_assignment(struct trace_array *tr,
1460			    char *str, struct hist_trigger_attrs *attrs)
1461{
1462	int len, ret = 0;
1463
1464	if ((len = str_has_prefix(str, "key=")) ||
1465	    (len = str_has_prefix(str, "keys="))) {
1466		attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1467		if (!attrs->keys_str) {
1468			ret = -ENOMEM;
1469			goto out;
1470		}
1471	} else if ((len = str_has_prefix(str, "val=")) ||
1472		   (len = str_has_prefix(str, "vals=")) ||
1473		   (len = str_has_prefix(str, "values="))) {
1474		attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1475		if (!attrs->vals_str) {
1476			ret = -ENOMEM;
1477			goto out;
1478		}
1479	} else if ((len = str_has_prefix(str, "sort="))) {
1480		attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1481		if (!attrs->sort_key_str) {
1482			ret = -ENOMEM;
1483			goto out;
1484		}
1485	} else if (str_has_prefix(str, "name=")) {
1486		attrs->name = kstrdup(str, GFP_KERNEL);
1487		if (!attrs->name) {
1488			ret = -ENOMEM;
1489			goto out;
1490		}
1491	} else if ((len = str_has_prefix(str, "clock="))) {
1492		str += len;
 
 
 
 
1493
1494		str = strstrip(str);
1495		attrs->clock = kstrdup(str, GFP_KERNEL);
1496		if (!attrs->clock) {
1497			ret = -ENOMEM;
1498			goto out;
1499		}
1500	} else if ((len = str_has_prefix(str, "size="))) {
1501		int map_bits = parse_map_size(str + len);
1502
1503		if (map_bits < 0) {
1504			ret = map_bits;
1505			goto out;
1506		}
1507		attrs->map_bits = map_bits;
1508	} else {
1509		char *assignment;
1510
1511		if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1512			hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1513			ret = -EINVAL;
1514			goto out;
1515		}
1516
1517		assignment = kstrdup(str, GFP_KERNEL);
1518		if (!assignment) {
1519			ret = -ENOMEM;
1520			goto out;
1521		}
1522
1523		attrs->assignment_str[attrs->n_assignments++] = assignment;
1524	}
1525 out:
1526	return ret;
1527}
1528
1529static struct hist_trigger_attrs *
1530parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1531{
1532	struct hist_trigger_attrs *attrs;
1533	int ret = 0;
1534
1535	attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1536	if (!attrs)
1537		return ERR_PTR(-ENOMEM);
1538
1539	while (trigger_str) {
1540		char *str = strsep(&trigger_str, ":");
1541		char *rhs;
1542
1543		rhs = strchr(str, '=');
1544		if (rhs) {
1545			if (!strlen(++rhs)) {
1546				ret = -EINVAL;
1547				hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1548				goto free;
1549			}
1550			ret = parse_assignment(tr, str, attrs);
1551			if (ret)
1552				goto free;
1553		} else if (strcmp(str, "nohitcount") == 0 ||
1554			   strcmp(str, "NOHC") == 0)
1555			attrs->no_hitcount = true;
1556		else if (strcmp(str, "pause") == 0)
1557			attrs->pause = true;
1558		else if ((strcmp(str, "cont") == 0) ||
1559			 (strcmp(str, "continue") == 0))
1560			attrs->cont = true;
1561		else if (strcmp(str, "clear") == 0)
1562			attrs->clear = true;
1563		else {
1564			ret = parse_action(str, attrs);
1565			if (ret)
1566				goto free;
1567		}
1568	}
1569
1570	if (!attrs->keys_str) {
1571		ret = -EINVAL;
1572		goto free;
1573	}
1574
1575	if (!attrs->clock) {
1576		attrs->clock = kstrdup("global", GFP_KERNEL);
1577		if (!attrs->clock) {
1578			ret = -ENOMEM;
1579			goto free;
1580		}
1581	}
1582
1583	return attrs;
1584 free:
1585	destroy_hist_trigger_attrs(attrs);
1586
1587	return ERR_PTR(ret);
1588}
1589
1590static inline void save_comm(char *comm, struct task_struct *task)
1591{
1592	if (!task->pid) {
1593		strcpy(comm, "<idle>");
1594		return;
1595	}
1596
1597	if (WARN_ON_ONCE(task->pid < 0)) {
1598		strcpy(comm, "<XXX>");
1599		return;
1600	}
1601
1602	strncpy(comm, task->comm, TASK_COMM_LEN);
1603}
1604
1605static void hist_elt_data_free(struct hist_elt_data *elt_data)
1606{
1607	unsigned int i;
1608
1609	for (i = 0; i < elt_data->n_field_var_str; i++)
1610		kfree(elt_data->field_var_str[i]);
1611
1612	kfree(elt_data->field_var_str);
1613
1614	kfree(elt_data->comm);
1615	kfree(elt_data);
1616}
1617
1618static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1619{
1620	struct hist_elt_data *elt_data = elt->private_data;
1621
1622	hist_elt_data_free(elt_data);
1623}
1624
1625static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1626{
1627	struct hist_trigger_data *hist_data = elt->map->private_data;
1628	unsigned int size = TASK_COMM_LEN;
1629	struct hist_elt_data *elt_data;
1630	struct hist_field *hist_field;
1631	unsigned int i, n_str;
1632
1633	elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1634	if (!elt_data)
1635		return -ENOMEM;
1636
1637	for_each_hist_field(i, hist_data) {
1638		hist_field = hist_data->fields[i];
1639
1640		if (hist_field->flags & HIST_FIELD_FL_EXECNAME) {
1641			elt_data->comm = kzalloc(size, GFP_KERNEL);
1642			if (!elt_data->comm) {
1643				kfree(elt_data);
1644				return -ENOMEM;
1645			}
1646			break;
1647		}
1648	}
1649
1650	n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1651		hist_data->n_var_str;
1652	if (n_str > SYNTH_FIELDS_MAX) {
1653		hist_elt_data_free(elt_data);
1654		return -EINVAL;
1655	}
1656
1657	BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1658
1659	size = STR_VAR_LEN_MAX;
1660
1661	elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL);
1662	if (!elt_data->field_var_str) {
1663		hist_elt_data_free(elt_data);
1664		return -EINVAL;
1665	}
1666	elt_data->n_field_var_str = n_str;
1667
1668	for (i = 0; i < n_str; i++) {
1669		elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1670		if (!elt_data->field_var_str[i]) {
1671			hist_elt_data_free(elt_data);
1672			return -ENOMEM;
1673		}
1674	}
1675
1676	elt->private_data = elt_data;
1677
1678	return 0;
1679}
1680
1681static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1682{
1683	struct hist_elt_data *elt_data = elt->private_data;
1684
1685	if (elt_data->comm)
1686		save_comm(elt_data->comm, current);
1687}
1688
1689static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1690	.elt_alloc	= hist_trigger_elt_data_alloc,
1691	.elt_free	= hist_trigger_elt_data_free,
1692	.elt_init	= hist_trigger_elt_data_init,
1693};
1694
1695static const char *get_hist_field_flags(struct hist_field *hist_field)
1696{
1697	const char *flags_str = NULL;
1698
1699	if (hist_field->flags & HIST_FIELD_FL_HEX)
1700		flags_str = "hex";
1701	else if (hist_field->flags & HIST_FIELD_FL_SYM)
1702		flags_str = "sym";
1703	else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1704		flags_str = "sym-offset";
1705	else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1706		flags_str = "execname";
1707	else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1708		flags_str = "syscall";
1709	else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1710		flags_str = "log2";
1711	else if (hist_field->flags & HIST_FIELD_FL_BUCKET)
1712		flags_str = "buckets";
1713	else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1714		flags_str = "usecs";
1715	else if (hist_field->flags & HIST_FIELD_FL_PERCENT)
1716		flags_str = "percent";
1717	else if (hist_field->flags & HIST_FIELD_FL_GRAPH)
1718		flags_str = "graph";
1719	else if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
1720		flags_str = "stacktrace";
1721
1722	return flags_str;
1723}
1724
1725static void expr_field_str(struct hist_field *field, char *expr)
1726{
1727	if (field->flags & HIST_FIELD_FL_VAR_REF)
1728		strcat(expr, "$");
1729	else if (field->flags & HIST_FIELD_FL_CONST) {
1730		char str[HIST_CONST_DIGITS_MAX];
1731
1732		snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant);
1733		strcat(expr, str);
1734	}
1735
1736	strcat(expr, hist_field_name(field, 0));
1737
1738	if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1739		const char *flags_str = get_hist_field_flags(field);
1740
1741		if (flags_str) {
1742			strcat(expr, ".");
1743			strcat(expr, flags_str);
1744		}
1745	}
1746}
1747
1748static char *expr_str(struct hist_field *field, unsigned int level)
1749{
1750	char *expr;
1751
1752	if (level > 1)
1753		return NULL;
1754
1755	expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1756	if (!expr)
1757		return NULL;
1758
1759	if (!field->operands[0]) {
1760		expr_field_str(field, expr);
1761		return expr;
1762	}
1763
1764	if (field->operator == FIELD_OP_UNARY_MINUS) {
1765		char *subexpr;
1766
1767		strcat(expr, "-(");
1768		subexpr = expr_str(field->operands[0], ++level);
1769		if (!subexpr) {
1770			kfree(expr);
1771			return NULL;
1772		}
1773		strcat(expr, subexpr);
1774		strcat(expr, ")");
1775
1776		kfree(subexpr);
1777
1778		return expr;
1779	}
1780
1781	expr_field_str(field->operands[0], expr);
1782
1783	switch (field->operator) {
1784	case FIELD_OP_MINUS:
1785		strcat(expr, "-");
1786		break;
1787	case FIELD_OP_PLUS:
1788		strcat(expr, "+");
1789		break;
1790	case FIELD_OP_DIV:
1791		strcat(expr, "/");
1792		break;
1793	case FIELD_OP_MULT:
1794		strcat(expr, "*");
1795		break;
1796	default:
1797		kfree(expr);
1798		return NULL;
1799	}
1800
1801	expr_field_str(field->operands[1], expr);
1802
1803	return expr;
1804}
1805
1806/*
1807 * If field_op != FIELD_OP_NONE, *sep points to the root operator
1808 * of the expression tree to be evaluated.
1809 */
1810static int contains_operator(char *str, char **sep)
1811{
1812	enum field_op_id field_op = FIELD_OP_NONE;
1813	char *minus_op, *plus_op, *div_op, *mult_op;
1814
1815
1816	/*
1817	 * Report the last occurrence of the operators first, so that the
1818	 * expression is evaluated left to right. This is important since
1819	 * subtraction and division are not associative.
1820	 *
1821	 *	e.g
1822	 *		64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2
1823	 *		14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2
1824	 */
1825
1826	/*
1827	 * First, find lower precedence addition and subtraction
1828	 * since the expression will be evaluated recursively.
1829	 */
1830	minus_op = strrchr(str, '-');
1831	if (minus_op) {
1832		/*
1833		 * Unary minus is not supported in sub-expressions. If
1834		 * present, it is always the next root operator.
1835		 */
1836		if (minus_op == str) {
1837			field_op = FIELD_OP_UNARY_MINUS;
1838			goto out;
1839		}
1840
1841		field_op = FIELD_OP_MINUS;
1842	}
1843
1844	plus_op = strrchr(str, '+');
1845	if (plus_op || minus_op) {
1846		/*
1847		 * For operators of the same precedence use to rightmost as the
1848		 * root, so that the expression is evaluated left to right.
1849		 */
1850		if (plus_op > minus_op)
1851			field_op = FIELD_OP_PLUS;
1852		goto out;
1853	}
1854
1855	/*
1856	 * Multiplication and division have higher precedence than addition and
1857	 * subtraction.
1858	 */
1859	div_op = strrchr(str, '/');
1860	if (div_op)
1861		field_op = FIELD_OP_DIV;
1862
1863	mult_op = strrchr(str, '*');
1864	/*
1865	 * For operators of the same precedence use to rightmost as the
1866	 * root, so that the expression is evaluated left to right.
1867	 */
1868	if (mult_op > div_op)
1869		field_op = FIELD_OP_MULT;
1870
1871out:
1872	if (sep) {
1873		switch (field_op) {
1874		case FIELD_OP_UNARY_MINUS:
1875		case FIELD_OP_MINUS:
1876			*sep = minus_op;
1877			break;
1878		case FIELD_OP_PLUS:
1879			*sep = plus_op;
1880			break;
1881		case FIELD_OP_DIV:
1882			*sep = div_op;
1883			break;
1884		case FIELD_OP_MULT:
1885			*sep = mult_op;
1886			break;
1887		case FIELD_OP_NONE:
1888		default:
1889			*sep = NULL;
1890			break;
1891		}
1892	}
1893
1894	return field_op;
1895}
1896
1897static void get_hist_field(struct hist_field *hist_field)
1898{
1899	hist_field->ref++;
1900}
1901
1902static void __destroy_hist_field(struct hist_field *hist_field)
1903{
1904	if (--hist_field->ref > 1)
1905		return;
1906
1907	kfree(hist_field->var.name);
1908	kfree(hist_field->name);
1909
1910	/* Can likely be a const */
1911	kfree_const(hist_field->type);
1912
1913	kfree(hist_field->system);
1914	kfree(hist_field->event_name);
1915
1916	kfree(hist_field);
1917}
1918
1919static void destroy_hist_field(struct hist_field *hist_field,
1920			       unsigned int level)
1921{
1922	unsigned int i;
1923
1924	if (level > 3)
1925		return;
1926
1927	if (!hist_field)
1928		return;
1929
1930	if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1931		return; /* var refs will be destroyed separately */
1932
1933	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1934		destroy_hist_field(hist_field->operands[i], level + 1);
1935
1936	__destroy_hist_field(hist_field);
1937}
1938
1939static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1940					    struct ftrace_event_field *field,
1941					    unsigned long flags,
1942					    char *var_name)
1943{
1944	struct hist_field *hist_field;
1945
1946	if (field && is_function_field(field))
1947		return NULL;
1948
1949	hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1950	if (!hist_field)
1951		return NULL;
1952
1953	hist_field->ref = 1;
1954
1955	hist_field->hist_data = hist_data;
1956
1957	if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1958		goto out; /* caller will populate */
1959
1960	if (flags & HIST_FIELD_FL_VAR_REF) {
1961		hist_field->fn_num = HIST_FIELD_FN_VAR_REF;
1962		goto out;
1963	}
1964
1965	if (flags & HIST_FIELD_FL_HITCOUNT) {
1966		hist_field->fn_num = HIST_FIELD_FN_COUNTER;
1967		hist_field->size = sizeof(u64);
1968		hist_field->type = "u64";
1969		goto out;
1970	}
1971
1972	if (flags & HIST_FIELD_FL_CONST) {
1973		hist_field->fn_num = HIST_FIELD_FN_CONST;
1974		hist_field->size = sizeof(u64);
1975		hist_field->type = kstrdup("u64", GFP_KERNEL);
1976		if (!hist_field->type)
1977			goto free;
1978		goto out;
1979	}
1980
1981	if (flags & HIST_FIELD_FL_STACKTRACE) {
1982		if (field)
1983			hist_field->fn_num = HIST_FIELD_FN_STACK;
1984		else
1985			hist_field->fn_num = HIST_FIELD_FN_NOP;
1986		hist_field->size = HIST_STACKTRACE_SIZE;
1987		hist_field->type = kstrdup_const("unsigned long[]", GFP_KERNEL);
1988		if (!hist_field->type)
1989			goto free;
1990		goto out;
1991	}
1992
1993	if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
1994		unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
1995		hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 :
1996			HIST_FIELD_FN_BUCKET;
1997		hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1998		if (!hist_field->operands[0])
1999			goto free;
2000		hist_field->size = hist_field->operands[0]->size;
2001		hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
2002		if (!hist_field->type)
2003			goto free;
2004		goto out;
2005	}
2006
2007	if (flags & HIST_FIELD_FL_TIMESTAMP) {
2008		hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP;
2009		hist_field->size = sizeof(u64);
2010		hist_field->type = "u64";
 
 
2011		goto out;
2012	}
2013
2014	if (flags & HIST_FIELD_FL_CPU) {
2015		hist_field->fn_num = HIST_FIELD_FN_CPU;
2016		hist_field->size = sizeof(int);
2017		hist_field->type = "unsigned int";
 
 
2018		goto out;
2019	}
2020
2021	if (WARN_ON_ONCE(!field))
2022		goto out;
2023
2024	/* Pointers to strings are just pointers and dangerous to dereference */
2025	if (is_string_field(field) &&
2026	    (field->filter_type != FILTER_PTR_STRING)) {
2027		flags |= HIST_FIELD_FL_STRING;
2028
2029		hist_field->size = MAX_FILTER_STR_VAL;
2030		hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2031		if (!hist_field->type)
2032			goto free;
2033
2034		if (field->filter_type == FILTER_STATIC_STRING) {
2035			hist_field->fn_num = HIST_FIELD_FN_STRING;
2036			hist_field->size = field->size;
2037		} else if (field->filter_type == FILTER_DYN_STRING) {
2038			hist_field->fn_num = HIST_FIELD_FN_DYNSTRING;
2039		} else if (field->filter_type == FILTER_RDYN_STRING)
2040			hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING;
2041		else
2042			hist_field->fn_num = HIST_FIELD_FN_PSTRING;
2043	} else {
2044		hist_field->size = field->size;
2045		hist_field->is_signed = field->is_signed;
2046		hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2047		if (!hist_field->type)
2048			goto free;
2049
2050		hist_field->fn_num = select_value_fn(field->size,
2051						     field->is_signed);
2052		if (hist_field->fn_num == HIST_FIELD_FN_NOP) {
2053			destroy_hist_field(hist_field, 0);
2054			return NULL;
2055		}
2056	}
2057 out:
2058	hist_field->field = field;
2059	hist_field->flags = flags;
2060
2061	if (var_name) {
2062		hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2063		if (!hist_field->var.name)
2064			goto free;
2065	}
2066
2067	return hist_field;
2068 free:
2069	destroy_hist_field(hist_field, 0);
2070	return NULL;
2071}
2072
2073static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2074{
2075	unsigned int i;
2076
2077	for (i = 0; i < HIST_FIELDS_MAX; i++) {
2078		if (hist_data->fields[i]) {
2079			destroy_hist_field(hist_data->fields[i], 0);
2080			hist_data->fields[i] = NULL;
2081		}
2082	}
2083
2084	for (i = 0; i < hist_data->n_var_refs; i++) {
2085		WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2086		__destroy_hist_field(hist_data->var_refs[i]);
2087		hist_data->var_refs[i] = NULL;
2088	}
2089}
2090
2091static int init_var_ref(struct hist_field *ref_field,
2092			struct hist_field *var_field,
2093			char *system, char *event_name)
2094{
2095	int err = 0;
2096
2097	ref_field->var.idx = var_field->var.idx;
2098	ref_field->var.hist_data = var_field->hist_data;
2099	ref_field->size = var_field->size;
2100	ref_field->is_signed = var_field->is_signed;
2101	ref_field->flags |= var_field->flags &
2102		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2103
2104	if (system) {
2105		ref_field->system = kstrdup(system, GFP_KERNEL);
2106		if (!ref_field->system)
2107			return -ENOMEM;
2108	}
2109
2110	if (event_name) {
2111		ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2112		if (!ref_field->event_name) {
2113			err = -ENOMEM;
2114			goto free;
2115		}
2116	}
2117
2118	if (var_field->var.name) {
2119		ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2120		if (!ref_field->name) {
2121			err = -ENOMEM;
2122			goto free;
2123		}
2124	} else if (var_field->name) {
2125		ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2126		if (!ref_field->name) {
2127			err = -ENOMEM;
2128			goto free;
2129		}
2130	}
2131
2132	ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL);
2133	if (!ref_field->type) {
2134		err = -ENOMEM;
2135		goto free;
2136	}
2137 out:
2138	return err;
2139 free:
2140	kfree(ref_field->system);
2141	ref_field->system = NULL;
2142	kfree(ref_field->event_name);
2143	ref_field->event_name = NULL;
2144	kfree(ref_field->name);
2145	ref_field->name = NULL;
2146
2147	goto out;
2148}
2149
2150static int find_var_ref_idx(struct hist_trigger_data *hist_data,
2151			    struct hist_field *var_field)
2152{
2153	struct hist_field *ref_field;
2154	int i;
2155
2156	for (i = 0; i < hist_data->n_var_refs; i++) {
2157		ref_field = hist_data->var_refs[i];
2158		if (ref_field->var.idx == var_field->var.idx &&
2159		    ref_field->var.hist_data == var_field->hist_data)
2160			return i;
2161	}
2162
2163	return -ENOENT;
2164}
2165
2166/**
2167 * create_var_ref - Create a variable reference and attach it to trigger
2168 * @hist_data: The trigger that will be referencing the variable
2169 * @var_field: The VAR field to create a reference to
2170 * @system: The optional system string
2171 * @event_name: The optional event_name string
2172 *
2173 * Given a variable hist_field, create a VAR_REF hist_field that
2174 * represents a reference to it.
2175 *
2176 * This function also adds the reference to the trigger that
2177 * now references the variable.
2178 *
2179 * Return: The VAR_REF field if successful, NULL if not
2180 */
2181static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2182					 struct hist_field *var_field,
2183					 char *system, char *event_name)
2184{
2185	unsigned long flags = HIST_FIELD_FL_VAR_REF;
2186	struct hist_field *ref_field;
2187	int i;
2188
2189	/* Check if the variable already exists */
2190	for (i = 0; i < hist_data->n_var_refs; i++) {
2191		ref_field = hist_data->var_refs[i];
2192		if (ref_field->var.idx == var_field->var.idx &&
2193		    ref_field->var.hist_data == var_field->hist_data) {
2194			get_hist_field(ref_field);
2195			return ref_field;
2196		}
2197	}
2198	/* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */
2199	if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX)
2200		return NULL;
2201	ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2202	if (ref_field) {
2203		if (init_var_ref(ref_field, var_field, system, event_name)) {
2204			destroy_hist_field(ref_field, 0);
2205			return NULL;
2206		}
2207
2208		hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2209		ref_field->var_ref_idx = hist_data->n_var_refs++;
2210	}
2211
2212	return ref_field;
2213}
2214
2215static bool is_var_ref(char *var_name)
2216{
2217	if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2218		return false;
2219
2220	return true;
2221}
2222
2223static char *field_name_from_var(struct hist_trigger_data *hist_data,
2224				 char *var_name)
2225{
2226	char *name, *field;
2227	unsigned int i;
2228
2229	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2230		name = hist_data->attrs->var_defs.name[i];
2231
2232		if (strcmp(var_name, name) == 0) {
2233			field = hist_data->attrs->var_defs.expr[i];
2234			if (contains_operator(field, NULL) || is_var_ref(field))
2235				continue;
2236			return field;
2237		}
2238	}
2239
2240	return NULL;
2241}
2242
2243static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2244				 char *system, char *event_name,
2245				 char *var_name)
2246{
2247	struct trace_event_call *call;
2248
2249	if (system && event_name) {
2250		call = hist_data->event_file->event_call;
2251
2252		if (strcmp(system, call->class->system) != 0)
2253			return NULL;
2254
2255		if (strcmp(event_name, trace_event_name(call)) != 0)
2256			return NULL;
2257	}
2258
2259	if (!!system != !!event_name)
2260		return NULL;
2261
2262	if (!is_var_ref(var_name))
2263		return NULL;
2264
2265	var_name++;
2266
2267	return field_name_from_var(hist_data, var_name);
2268}
2269
2270static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2271					char *system, char *event_name,
2272					char *var_name)
2273{
2274	struct hist_field *var_field = NULL, *ref_field = NULL;
2275	struct trace_array *tr = hist_data->event_file->tr;
2276
2277	if (!is_var_ref(var_name))
2278		return NULL;
2279
2280	var_name++;
2281
2282	var_field = find_event_var(hist_data, system, event_name, var_name);
2283	if (var_field)
2284		ref_field = create_var_ref(hist_data, var_field,
2285					   system, event_name);
2286
2287	if (!ref_field)
2288		hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2289
2290	return ref_field;
2291}
2292
2293static struct ftrace_event_field *
2294parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2295	    char *field_str, unsigned long *flags, unsigned long *buckets)
2296{
2297	struct ftrace_event_field *field = NULL;
2298	char *field_name, *modifier, *str;
2299	struct trace_array *tr = file->tr;
2300
2301	modifier = str = kstrdup(field_str, GFP_KERNEL);
2302	if (!modifier)
2303		return ERR_PTR(-ENOMEM);
2304
2305	field_name = strsep(&modifier, ".");
2306	if (modifier) {
2307		if (strcmp(modifier, "hex") == 0)
2308			*flags |= HIST_FIELD_FL_HEX;
2309		else if (strcmp(modifier, "sym") == 0)
2310			*flags |= HIST_FIELD_FL_SYM;
2311		/*
2312		 * 'sym-offset' occurrences in the trigger string are modified
2313		 * to 'symXoffset' to simplify arithmetic expression parsing.
2314		 */
2315		else if (strcmp(modifier, "symXoffset") == 0)
2316			*flags |= HIST_FIELD_FL_SYM_OFFSET;
2317		else if ((strcmp(modifier, "execname") == 0) &&
2318			 (strcmp(field_name, "common_pid") == 0))
2319			*flags |= HIST_FIELD_FL_EXECNAME;
2320		else if (strcmp(modifier, "syscall") == 0)
2321			*flags |= HIST_FIELD_FL_SYSCALL;
2322		else if (strcmp(modifier, "stacktrace") == 0)
2323			*flags |= HIST_FIELD_FL_STACKTRACE;
2324		else if (strcmp(modifier, "log2") == 0)
2325			*flags |= HIST_FIELD_FL_LOG2;
2326		else if (strcmp(modifier, "usecs") == 0)
2327			*flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2328		else if (strncmp(modifier, "bucket", 6) == 0) {
2329			int ret;
2330
2331			modifier += 6;
2332
2333			if (*modifier == 's')
2334				modifier++;
2335			if (*modifier != '=')
2336				goto error;
2337			modifier++;
2338			ret = kstrtoul(modifier, 0, buckets);
2339			if (ret || !(*buckets))
2340				goto error;
2341			*flags |= HIST_FIELD_FL_BUCKET;
2342		} else if (strncmp(modifier, "percent", 7) == 0) {
2343			if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2344				goto error;
2345			*flags |= HIST_FIELD_FL_PERCENT;
2346		} else if (strncmp(modifier, "graph", 5) == 0) {
2347			if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2348				goto error;
2349			*flags |= HIST_FIELD_FL_GRAPH;
2350		} else {
2351 error:
2352			hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2353			field = ERR_PTR(-EINVAL);
2354			goto out;
2355		}
2356	}
2357
2358	if (strcmp(field_name, "common_timestamp") == 0) {
2359		*flags |= HIST_FIELD_FL_TIMESTAMP;
2360		hist_data->enable_timestamps = true;
2361		if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2362			hist_data->attrs->ts_in_usecs = true;
2363	} else if (strcmp(field_name, "common_stacktrace") == 0) {
2364		*flags |= HIST_FIELD_FL_STACKTRACE;
2365	} else if (strcmp(field_name, "common_cpu") == 0)
2366		*flags |= HIST_FIELD_FL_CPU;
2367	else if (strcmp(field_name, "hitcount") == 0)
2368		*flags |= HIST_FIELD_FL_HITCOUNT;
2369	else {
2370		field = trace_find_event_field(file->event_call, field_name);
2371		if (!field || !field->size) {
2372			/*
2373			 * For backward compatibility, if field_name
2374			 * was "cpu" or "stacktrace", then we treat this
2375			 * the same as common_cpu and common_stacktrace
2376			 * respectively. This also works for "CPU", and
2377			 * "STACKTRACE".
2378			 */
2379			if (field && field->filter_type == FILTER_CPU) {
2380				*flags |= HIST_FIELD_FL_CPU;
2381			} else if (field && field->filter_type == FILTER_STACKTRACE) {
2382				*flags |= HIST_FIELD_FL_STACKTRACE;
2383			} else {
2384				hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2385					 errpos(field_name));
2386				field = ERR_PTR(-EINVAL);
2387				goto out;
2388			}
2389		}
2390	}
2391 out:
2392	kfree(str);
2393
2394	return field;
2395}
2396
2397static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2398				       struct hist_field *var_ref,
2399				       char *var_name)
2400{
2401	struct hist_field *alias = NULL;
2402	unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2403
2404	alias = create_hist_field(hist_data, NULL, flags, var_name);
2405	if (!alias)
2406		return NULL;
2407
2408	alias->fn_num = var_ref->fn_num;
2409	alias->operands[0] = var_ref;
2410
2411	if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2412		destroy_hist_field(alias, 0);
2413		return NULL;
2414	}
2415
2416	alias->var_ref_idx = var_ref->var_ref_idx;
2417
2418	return alias;
2419}
2420
2421static struct hist_field *parse_const(struct hist_trigger_data *hist_data,
2422				      char *str, char *var_name,
2423				      unsigned long *flags)
2424{
2425	struct trace_array *tr = hist_data->event_file->tr;
2426	struct hist_field *field = NULL;
2427	u64 constant;
2428
2429	if (kstrtoull(str, 0, &constant)) {
2430		hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str));
2431		return NULL;
2432	}
2433
2434	*flags |= HIST_FIELD_FL_CONST;
2435	field = create_hist_field(hist_data, NULL, *flags, var_name);
2436	if (!field)
2437		return NULL;
2438
2439	field->constant = constant;
2440
2441	return field;
2442}
2443
2444static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2445				     struct trace_event_file *file, char *str,
2446				     unsigned long *flags, char *var_name)
2447{
2448	char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2449	struct ftrace_event_field *field = NULL;
2450	struct hist_field *hist_field = NULL;
2451	unsigned long buckets = 0;
2452	int ret = 0;
2453
2454	if (isdigit(str[0])) {
2455		hist_field = parse_const(hist_data, str, var_name, flags);
2456		if (!hist_field) {
2457			ret = -EINVAL;
2458			goto out;
2459		}
2460		return hist_field;
2461	}
2462
2463	s = strchr(str, '.');
2464	if (s) {
2465		s = strchr(++s, '.');
2466		if (s) {
2467			ref_system = strsep(&str, ".");
2468			if (!str) {
2469				ret = -EINVAL;
2470				goto out;
2471			}
2472			ref_event = strsep(&str, ".");
2473			if (!str) {
2474				ret = -EINVAL;
2475				goto out;
2476			}
2477			ref_var = str;
2478		}
2479	}
2480
2481	s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2482	if (!s) {
2483		hist_field = parse_var_ref(hist_data, ref_system,
2484					   ref_event, ref_var);
2485		if (hist_field) {
2486			if (var_name) {
2487				hist_field = create_alias(hist_data, hist_field, var_name);
2488				if (!hist_field) {
2489					ret = -ENOMEM;
2490					goto out;
2491				}
2492			}
2493			return hist_field;
2494		}
2495	} else
2496		str = s;
2497
2498	field = parse_field(hist_data, file, str, flags, &buckets);
2499	if (IS_ERR(field)) {
2500		ret = PTR_ERR(field);
2501		goto out;
2502	}
2503
2504	hist_field = create_hist_field(hist_data, field, *flags, var_name);
2505	if (!hist_field) {
2506		ret = -ENOMEM;
2507		goto out;
2508	}
2509	hist_field->buckets = buckets;
2510
2511	return hist_field;
2512 out:
2513	return ERR_PTR(ret);
2514}
2515
2516static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2517				     struct trace_event_file *file,
2518				     char *str, unsigned long flags,
2519				     char *var_name, unsigned int *n_subexprs);
2520
2521static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2522				      struct trace_event_file *file,
2523				      char *str, unsigned long flags,
2524				      char *var_name, unsigned int *n_subexprs)
2525{
2526	struct hist_field *operand1, *expr = NULL;
2527	unsigned long operand_flags;
2528	int ret = 0;
2529	char *s;
2530
2531	/* Unary minus operator, increment n_subexprs */
2532	++*n_subexprs;
2533
2534	/* we support only -(xxx) i.e. explicit parens required */
2535
2536	if (*n_subexprs > 3) {
2537		hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2538		ret = -EINVAL;
2539		goto free;
2540	}
2541
2542	str++; /* skip leading '-' */
2543
2544	s = strchr(str, '(');
2545	if (s)
2546		str++;
2547	else {
2548		ret = -EINVAL;
2549		goto free;
2550	}
2551
2552	s = strrchr(str, ')');
2553	if (s) {
2554		 /* unary minus not supported in sub-expressions */
2555		if (*(s+1) != '\0') {
2556			hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR,
2557				 errpos(str));
2558			ret = -EINVAL;
2559			goto free;
2560		}
2561		*s = '\0';
2562	}
2563	else {
2564		ret = -EINVAL; /* no closing ')' */
2565		goto free;
2566	}
2567
2568	flags |= HIST_FIELD_FL_EXPR;
2569	expr = create_hist_field(hist_data, NULL, flags, var_name);
2570	if (!expr) {
2571		ret = -ENOMEM;
2572		goto free;
2573	}
2574
2575	operand_flags = 0;
2576	operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2577	if (IS_ERR(operand1)) {
2578		ret = PTR_ERR(operand1);
2579		goto free;
2580	}
2581	if (operand1->flags & HIST_FIELD_FL_STRING) {
2582		/* String type can not be the operand of unary operator. */
2583		hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2584		destroy_hist_field(operand1, 0);
2585		ret = -EINVAL;
2586		goto free;
2587	}
2588
2589	expr->flags |= operand1->flags &
2590		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2591	expr->fn_num = HIST_FIELD_FN_UMINUS;
2592	expr->operands[0] = operand1;
2593	expr->size = operand1->size;
2594	expr->is_signed = operand1->is_signed;
2595	expr->operator = FIELD_OP_UNARY_MINUS;
2596	expr->name = expr_str(expr, 0);
2597	expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2598	if (!expr->type) {
2599		ret = -ENOMEM;
2600		goto free;
2601	}
2602
2603	return expr;
2604 free:
2605	destroy_hist_field(expr, 0);
2606	return ERR_PTR(ret);
2607}
2608
2609/*
2610 * If the operands are var refs, return pointers the
2611 * variable(s) referenced in var1 and var2, else NULL.
2612 */
2613static int check_expr_operands(struct trace_array *tr,
2614			       struct hist_field *operand1,
2615			       struct hist_field *operand2,
2616			       struct hist_field **var1,
2617			       struct hist_field **var2)
2618{
2619	unsigned long operand1_flags = operand1->flags;
2620	unsigned long operand2_flags = operand2->flags;
2621
2622	if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2623	    (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2624		struct hist_field *var;
2625
2626		var = find_var_field(operand1->var.hist_data, operand1->name);
2627		if (!var)
2628			return -EINVAL;
2629		operand1_flags = var->flags;
2630		*var1 = var;
2631	}
2632
2633	if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2634	    (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2635		struct hist_field *var;
2636
2637		var = find_var_field(operand2->var.hist_data, operand2->name);
2638		if (!var)
2639			return -EINVAL;
2640		operand2_flags = var->flags;
2641		*var2 = var;
2642	}
2643
2644	if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2645	    (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2646		hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2647		return -EINVAL;
2648	}
2649
2650	return 0;
2651}
2652
2653static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2654				     struct trace_event_file *file,
2655				     char *str, unsigned long flags,
2656				     char *var_name, unsigned int *n_subexprs)
2657{
2658	struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2659	struct hist_field *var1 = NULL, *var2 = NULL;
2660	unsigned long operand_flags, operand2_flags;
2661	int field_op, ret = -EINVAL;
2662	char *sep, *operand1_str;
2663	enum hist_field_fn op_fn;
2664	bool combine_consts;
2665
2666	if (*n_subexprs > 3) {
2667		hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2668		return ERR_PTR(-EINVAL);
2669	}
2670
2671	field_op = contains_operator(str, &sep);
2672
2673	if (field_op == FIELD_OP_NONE)
2674		return parse_atom(hist_data, file, str, &flags, var_name);
2675
2676	if (field_op == FIELD_OP_UNARY_MINUS)
2677		return parse_unary(hist_data, file, str, flags, var_name, n_subexprs);
2678
2679	/* Binary operator found, increment n_subexprs */
2680	++*n_subexprs;
2681
2682	/* Split the expression string at the root operator */
2683	if (!sep)
2684		return ERR_PTR(-EINVAL);
2685
2686	*sep = '\0';
2687	operand1_str = str;
2688	str = sep+1;
 
 
 
 
 
 
 
2689
2690	/* Binary operator requires both operands */
2691	if (*operand1_str == '\0' || *str == '\0')
2692		return ERR_PTR(-EINVAL);
2693
2694	operand_flags = 0;
2695
2696	/* LHS of string is an expression e.g. a+b in a+b+c */
2697	operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs);
2698	if (IS_ERR(operand1))
2699		return ERR_CAST(operand1);
2700
2701	if (operand1->flags & HIST_FIELD_FL_STRING) {
2702		hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2703		ret = -EINVAL;
2704		goto free_op1;
2705	}
2706
2707	/* RHS of string is another expression e.g. c in a+b+c */
2708	operand_flags = 0;
2709	operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2710	if (IS_ERR(operand2)) {
2711		ret = PTR_ERR(operand2);
2712		goto free_op1;
2713	}
2714	if (operand2->flags & HIST_FIELD_FL_STRING) {
2715		hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2716		ret = -EINVAL;
2717		goto free_operands;
2718	}
2719
2720	switch (field_op) {
2721	case FIELD_OP_MINUS:
2722		op_fn = HIST_FIELD_FN_MINUS;
2723		break;
2724	case FIELD_OP_PLUS:
2725		op_fn = HIST_FIELD_FN_PLUS;
2726		break;
2727	case FIELD_OP_DIV:
2728		op_fn = HIST_FIELD_FN_DIV;
2729		break;
2730	case FIELD_OP_MULT:
2731		op_fn = HIST_FIELD_FN_MULT;
2732		break;
2733	default:
2734		ret = -EINVAL;
2735		goto free_operands;
2736	}
2737
2738	ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2);
2739	if (ret)
2740		goto free_operands;
2741
2742	operand_flags = var1 ? var1->flags : operand1->flags;
2743	operand2_flags = var2 ? var2->flags : operand2->flags;
2744
2745	/*
2746	 * If both operands are constant, the expression can be
2747	 * collapsed to a single constant.
2748	 */
2749	combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST;
2750
2751	flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR;
2752
2753	flags |= operand1->flags &
2754		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2755
2756	expr = create_hist_field(hist_data, NULL, flags, var_name);
2757	if (!expr) {
2758		ret = -ENOMEM;
2759		goto free_operands;
2760	}
2761
2762	operand1->read_once = true;
2763	operand2->read_once = true;
2764
2765	/* The operands are now owned and free'd by 'expr' */
2766	expr->operands[0] = operand1;
2767	expr->operands[1] = operand2;
2768
2769	if (field_op == FIELD_OP_DIV &&
2770			operand2_flags & HIST_FIELD_FL_CONST) {
2771		u64 divisor = var2 ? var2->constant : operand2->constant;
2772
2773		if (!divisor) {
2774			hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str));
2775			ret = -EDOM;
2776			goto free_expr;
2777		}
2778
2779		/*
2780		 * Copy the divisor here so we don't have to look it up
2781		 * later if this is a var ref
2782		 */
2783		operand2->constant = divisor;
2784		op_fn = hist_field_get_div_fn(operand2);
2785	}
2786
2787	expr->fn_num = op_fn;
2788
2789	if (combine_consts) {
2790		if (var1)
2791			expr->operands[0] = var1;
2792		if (var2)
2793			expr->operands[1] = var2;
2794
2795		expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL);
2796		expr->fn_num = HIST_FIELD_FN_CONST;
2797
2798		expr->operands[0] = NULL;
2799		expr->operands[1] = NULL;
2800
2801		/*
2802		 * var refs won't be destroyed immediately
2803		 * See: destroy_hist_field()
2804		 */
2805		destroy_hist_field(operand2, 0);
2806		destroy_hist_field(operand1, 0);
2807
2808		expr->name = expr_str(expr, 0);
2809	} else {
2810		/* The operand sizes should be the same, so just pick one */
2811		expr->size = operand1->size;
2812		expr->is_signed = operand1->is_signed;
2813
2814		expr->operator = field_op;
2815		expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2816		if (!expr->type) {
2817			ret = -ENOMEM;
2818			goto free_expr;
2819		}
2820
2821		expr->name = expr_str(expr, 0);
2822	}
2823
2824	return expr;
2825
2826free_operands:
2827	destroy_hist_field(operand2, 0);
2828free_op1:
2829	destroy_hist_field(operand1, 0);
2830	return ERR_PTR(ret);
2831
2832free_expr:
2833	destroy_hist_field(expr, 0);
 
2834	return ERR_PTR(ret);
2835}
2836
2837static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2838				 struct trace_event_file *file)
2839{
2840	struct event_trigger_data *test;
2841
2842	lockdep_assert_held(&event_mutex);
2843
2844	list_for_each_entry(test, &file->triggers, list) {
2845		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2846			if (test->private_data == hist_data)
2847				return test->filter_str;
2848		}
2849	}
2850
2851	return NULL;
2852}
2853
2854static struct event_command trigger_hist_cmd;
2855static int event_hist_trigger_parse(struct event_command *cmd_ops,
2856				    struct trace_event_file *file,
2857				    char *glob, char *cmd,
2858				    char *param_and_filter);
2859
2860static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2861			    struct hist_trigger_data *hist_data,
2862			    unsigned int n_keys)
2863{
2864	struct hist_field *target_hist_field, *hist_field;
2865	unsigned int n, i, j;
2866
2867	if (hist_data->n_fields - hist_data->n_vals != n_keys)
2868		return false;
2869
2870	i = hist_data->n_vals;
2871	j = target_hist_data->n_vals;
2872
2873	for (n = 0; n < n_keys; n++) {
2874		hist_field = hist_data->fields[i + n];
2875		target_hist_field = target_hist_data->fields[j + n];
2876
2877		if (strcmp(hist_field->type, target_hist_field->type) != 0)
2878			return false;
2879		if (hist_field->size != target_hist_field->size)
2880			return false;
2881		if (hist_field->is_signed != target_hist_field->is_signed)
2882			return false;
2883	}
2884
2885	return true;
2886}
2887
2888static struct hist_trigger_data *
2889find_compatible_hist(struct hist_trigger_data *target_hist_data,
2890		     struct trace_event_file *file)
2891{
2892	struct hist_trigger_data *hist_data;
2893	struct event_trigger_data *test;
2894	unsigned int n_keys;
2895
2896	lockdep_assert_held(&event_mutex);
2897
2898	n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2899
2900	list_for_each_entry(test, &file->triggers, list) {
2901		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2902			hist_data = test->private_data;
2903
2904			if (compatible_keys(target_hist_data, hist_data, n_keys))
2905				return hist_data;
2906		}
2907	}
2908
2909	return NULL;
2910}
2911
2912static struct trace_event_file *event_file(struct trace_array *tr,
2913					   char *system, char *event_name)
2914{
2915	struct trace_event_file *file;
2916
2917	file = __find_event_file(tr, system, event_name);
2918	if (!file)
2919		return ERR_PTR(-EINVAL);
2920
2921	return file;
2922}
2923
2924static struct hist_field *
2925find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2926			 char *system, char *event_name, char *field_name)
2927{
2928	struct hist_field *event_var;
2929	char *synthetic_name;
2930
2931	synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2932	if (!synthetic_name)
2933		return ERR_PTR(-ENOMEM);
2934
2935	strcpy(synthetic_name, "synthetic_");
2936	strcat(synthetic_name, field_name);
2937
2938	event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2939
2940	kfree(synthetic_name);
2941
2942	return event_var;
2943}
2944
2945/**
2946 * create_field_var_hist - Automatically create a histogram and var for a field
2947 * @target_hist_data: The target hist trigger
2948 * @subsys_name: Optional subsystem name
2949 * @event_name: Optional event name
2950 * @field_name: The name of the field (and the resulting variable)
2951 *
2952 * Hist trigger actions fetch data from variables, not directly from
2953 * events.  However, for convenience, users are allowed to directly
2954 * specify an event field in an action, which will be automatically
2955 * converted into a variable on their behalf.
2956 *
2957 * If a user specifies a field on an event that isn't the event the
2958 * histogram currently being defined (the target event histogram), the
2959 * only way that can be accomplished is if a new hist trigger is
2960 * created and the field variable defined on that.
2961 *
2962 * This function creates a new histogram compatible with the target
2963 * event (meaning a histogram with the same key as the target
2964 * histogram), and creates a variable for the specified field, but
2965 * with 'synthetic_' prepended to the variable name in order to avoid
2966 * collision with normal field variables.
2967 *
2968 * Return: The variable created for the field.
2969 */
2970static struct hist_field *
2971create_field_var_hist(struct hist_trigger_data *target_hist_data,
2972		      char *subsys_name, char *event_name, char *field_name)
2973{
2974	struct trace_array *tr = target_hist_data->event_file->tr;
 
2975	struct hist_trigger_data *hist_data;
2976	unsigned int i, n, first = true;
2977	struct field_var_hist *var_hist;
2978	struct trace_event_file *file;
2979	struct hist_field *key_field;
2980	struct hist_field *event_var;
2981	char *saved_filter;
2982	char *cmd;
2983	int ret;
2984
2985	if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2986		hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2987		return ERR_PTR(-EINVAL);
2988	}
2989
2990	file = event_file(tr, subsys_name, event_name);
2991
2992	if (IS_ERR(file)) {
2993		hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2994		ret = PTR_ERR(file);
2995		return ERR_PTR(ret);
2996	}
2997
2998	/*
2999	 * Look for a histogram compatible with target.  We'll use the
3000	 * found histogram specification to create a new matching
3001	 * histogram with our variable on it.  target_hist_data is not
3002	 * yet a registered histogram so we can't use that.
3003	 */
3004	hist_data = find_compatible_hist(target_hist_data, file);
3005	if (!hist_data) {
3006		hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
3007		return ERR_PTR(-EINVAL);
3008	}
3009
3010	/* See if a synthetic field variable has already been created */
3011	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3012					     event_name, field_name);
3013	if (!IS_ERR_OR_NULL(event_var))
3014		return event_var;
3015
3016	var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3017	if (!var_hist)
3018		return ERR_PTR(-ENOMEM);
3019
3020	cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3021	if (!cmd) {
3022		kfree(var_hist);
3023		return ERR_PTR(-ENOMEM);
3024	}
3025
3026	/* Use the same keys as the compatible histogram */
3027	strcat(cmd, "keys=");
3028
3029	for_each_hist_key_field(i, hist_data) {
3030		key_field = hist_data->fields[i];
3031		if (!first)
3032			strcat(cmd, ",");
3033		strcat(cmd, key_field->field->name);
3034		first = false;
3035	}
3036
3037	/* Create the synthetic field variable specification */
3038	strcat(cmd, ":synthetic_");
3039	strcat(cmd, field_name);
3040	strcat(cmd, "=");
3041	strcat(cmd, field_name);
3042
3043	/* Use the same filter as the compatible histogram */
3044	saved_filter = find_trigger_filter(hist_data, file);
3045	if (saved_filter) {
3046		strcat(cmd, " if ");
3047		strcat(cmd, saved_filter);
3048	}
3049
3050	var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3051	if (!var_hist->cmd) {
3052		kfree(cmd);
3053		kfree(var_hist);
3054		return ERR_PTR(-ENOMEM);
3055	}
3056
3057	/* Save the compatible histogram information */
3058	var_hist->hist_data = hist_data;
3059
3060	/* Create the new histogram with our variable */
3061	ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
3062				       "", "hist", cmd);
3063	if (ret) {
3064		kfree(cmd);
3065		kfree(var_hist->cmd);
3066		kfree(var_hist);
3067		hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
3068		return ERR_PTR(ret);
3069	}
3070
3071	kfree(cmd);
3072
3073	/* If we can't find the variable, something went wrong */
3074	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3075					     event_name, field_name);
3076	if (IS_ERR_OR_NULL(event_var)) {
3077		kfree(var_hist->cmd);
3078		kfree(var_hist);
3079		hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
3080		return ERR_PTR(-EINVAL);
3081	}
3082
3083	n = target_hist_data->n_field_var_hists;
3084	target_hist_data->field_var_hists[n] = var_hist;
3085	target_hist_data->n_field_var_hists++;
3086
3087	return event_var;
3088}
3089
3090static struct hist_field *
3091find_target_event_var(struct hist_trigger_data *hist_data,
3092		      char *subsys_name, char *event_name, char *var_name)
3093{
3094	struct trace_event_file *file = hist_data->event_file;
3095	struct hist_field *hist_field = NULL;
3096
3097	if (subsys_name) {
3098		struct trace_event_call *call;
3099
3100		if (!event_name)
3101			return NULL;
3102
3103		call = file->event_call;
3104
3105		if (strcmp(subsys_name, call->class->system) != 0)
3106			return NULL;
3107
3108		if (strcmp(event_name, trace_event_name(call)) != 0)
3109			return NULL;
3110	}
3111
3112	hist_field = find_var_field(hist_data, var_name);
3113
3114	return hist_field;
3115}
3116
3117static inline void __update_field_vars(struct tracing_map_elt *elt,
3118				       struct trace_buffer *buffer,
3119				       struct ring_buffer_event *rbe,
3120				       void *rec,
3121				       struct field_var **field_vars,
3122				       unsigned int n_field_vars,
3123				       unsigned int field_var_str_start)
3124{
3125	struct hist_elt_data *elt_data = elt->private_data;
3126	unsigned int i, j, var_idx;
3127	u64 var_val;
3128
3129	/* Make sure stacktrace can fit in the string variable length */
3130	BUILD_BUG_ON((HIST_STACKTRACE_DEPTH + 1) * sizeof(long) >= STR_VAR_LEN_MAX);
3131
3132	for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3133		struct field_var *field_var = field_vars[i];
3134		struct hist_field *var = field_var->var;
3135		struct hist_field *val = field_var->val;
3136
3137		var_val = hist_fn_call(val, elt, buffer, rbe, rec);
3138		var_idx = var->var.idx;
3139
3140		if (val->flags & (HIST_FIELD_FL_STRING |
3141				  HIST_FIELD_FL_STACKTRACE)) {
3142			char *str = elt_data->field_var_str[j++];
3143			char *val_str = (char *)(uintptr_t)var_val;
3144			unsigned int size;
3145
3146			if (val->flags & HIST_FIELD_FL_STRING) {
3147				size = min(val->size, STR_VAR_LEN_MAX);
3148				strscpy(str, val_str, size);
3149			} else {
3150				char *stack_start = str + sizeof(unsigned long);
3151				int e;
3152
3153				e = stack_trace_save((void *)stack_start,
3154						     HIST_STACKTRACE_DEPTH,
3155						     HIST_STACKTRACE_SKIP);
3156				if (e < HIST_STACKTRACE_DEPTH - 1)
3157					((unsigned long *)stack_start)[e] = 0;
3158				*((unsigned long *)str) = e;
3159			}
3160			var_val = (u64)(uintptr_t)str;
3161		}
3162		tracing_map_set_var(elt, var_idx, var_val);
3163	}
3164}
3165
3166static void update_field_vars(struct hist_trigger_data *hist_data,
3167			      struct tracing_map_elt *elt,
3168			      struct trace_buffer *buffer,
3169			      struct ring_buffer_event *rbe,
3170			      void *rec)
3171{
3172	__update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
3173			    hist_data->n_field_vars, 0);
3174}
3175
3176static void save_track_data_vars(struct hist_trigger_data *hist_data,
3177				 struct tracing_map_elt *elt,
3178				 struct trace_buffer *buffer,  void *rec,
3179				 struct ring_buffer_event *rbe, void *key,
3180				 struct action_data *data, u64 *var_ref_vals)
3181{
3182	__update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
3183			    hist_data->n_save_vars, hist_data->n_field_var_str);
3184}
3185
3186static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3187				     struct trace_event_file *file,
3188				     char *name, int size, const char *type)
3189{
3190	struct hist_field *var;
3191	int idx;
3192
3193	if (find_var(hist_data, file, name) && !hist_data->remove) {
3194		var = ERR_PTR(-EINVAL);
3195		goto out;
3196	}
3197
3198	var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3199	if (!var) {
3200		var = ERR_PTR(-ENOMEM);
3201		goto out;
3202	}
3203
3204	idx = tracing_map_add_var(hist_data->map);
3205	if (idx < 0) {
3206		kfree(var);
3207		var = ERR_PTR(-EINVAL);
3208		goto out;
3209	}
3210
3211	var->ref = 1;
3212	var->flags = HIST_FIELD_FL_VAR;
3213	var->var.idx = idx;
3214	var->var.hist_data = var->hist_data = hist_data;
3215	var->size = size;
3216	var->var.name = kstrdup(name, GFP_KERNEL);
3217	var->type = kstrdup_const(type, GFP_KERNEL);
3218	if (!var->var.name || !var->type) {
3219		kfree_const(var->type);
3220		kfree(var->var.name);
 
3221		kfree(var);
3222		var = ERR_PTR(-ENOMEM);
3223	}
3224 out:
3225	return var;
3226}
3227
3228static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3229					  struct trace_event_file *file,
3230					  char *field_name)
3231{
3232	struct hist_field *val = NULL, *var = NULL;
3233	unsigned long flags = HIST_FIELD_FL_VAR;
3234	struct trace_array *tr = file->tr;
3235	struct field_var *field_var;
3236	int ret = 0;
3237
3238	if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3239		hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3240		ret = -EINVAL;
3241		goto err;
3242	}
3243
3244	val = parse_atom(hist_data, file, field_name, &flags, NULL);
3245	if (IS_ERR(val)) {
3246		hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3247		ret = PTR_ERR(val);
3248		goto err;
3249	}
3250
3251	var = create_var(hist_data, file, field_name, val->size, val->type);
3252	if (IS_ERR(var)) {
3253		hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3254		kfree(val);
3255		ret = PTR_ERR(var);
3256		goto err;
3257	}
3258
3259	field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3260	if (!field_var) {
3261		kfree(val);
3262		kfree(var);
3263		ret =  -ENOMEM;
3264		goto err;
3265	}
3266
3267	field_var->var = var;
3268	field_var->val = val;
3269 out:
3270	return field_var;
3271 err:
3272	field_var = ERR_PTR(ret);
3273	goto out;
3274}
3275
3276/**
3277 * create_target_field_var - Automatically create a variable for a field
3278 * @target_hist_data: The target hist trigger
3279 * @subsys_name: Optional subsystem name
3280 * @event_name: Optional event name
3281 * @var_name: The name of the field (and the resulting variable)
3282 *
3283 * Hist trigger actions fetch data from variables, not directly from
3284 * events.  However, for convenience, users are allowed to directly
3285 * specify an event field in an action, which will be automatically
3286 * converted into a variable on their behalf.
3287 *
3288 * This function creates a field variable with the name var_name on
3289 * the hist trigger currently being defined on the target event.  If
3290 * subsys_name and event_name are specified, this function simply
3291 * verifies that they do in fact match the target event subsystem and
3292 * event name.
3293 *
3294 * Return: The variable created for the field.
3295 */
3296static struct field_var *
3297create_target_field_var(struct hist_trigger_data *target_hist_data,
3298			char *subsys_name, char *event_name, char *var_name)
3299{
3300	struct trace_event_file *file = target_hist_data->event_file;
3301
3302	if (subsys_name) {
3303		struct trace_event_call *call;
3304
3305		if (!event_name)
3306			return NULL;
3307
3308		call = file->event_call;
3309
3310		if (strcmp(subsys_name, call->class->system) != 0)
3311			return NULL;
3312
3313		if (strcmp(event_name, trace_event_name(call)) != 0)
3314			return NULL;
3315	}
3316
3317	return create_field_var(target_hist_data, file, var_name);
3318}
3319
3320static bool check_track_val_max(u64 track_val, u64 var_val)
3321{
3322	if (var_val <= track_val)
3323		return false;
3324
3325	return true;
3326}
3327
3328static bool check_track_val_changed(u64 track_val, u64 var_val)
3329{
3330	if (var_val == track_val)
3331		return false;
3332
3333	return true;
3334}
3335
3336static u64 get_track_val(struct hist_trigger_data *hist_data,
3337			 struct tracing_map_elt *elt,
3338			 struct action_data *data)
3339{
3340	unsigned int track_var_idx = data->track_data.track_var->var.idx;
3341	u64 track_val;
3342
3343	track_val = tracing_map_read_var(elt, track_var_idx);
3344
3345	return track_val;
3346}
3347
3348static void save_track_val(struct hist_trigger_data *hist_data,
3349			   struct tracing_map_elt *elt,
3350			   struct action_data *data, u64 var_val)
3351{
3352	unsigned int track_var_idx = data->track_data.track_var->var.idx;
3353
3354	tracing_map_set_var(elt, track_var_idx, var_val);
3355}
3356
3357static void save_track_data(struct hist_trigger_data *hist_data,
3358			    struct tracing_map_elt *elt,
3359			    struct trace_buffer *buffer, void *rec,
3360			    struct ring_buffer_event *rbe, void *key,
3361			    struct action_data *data, u64 *var_ref_vals)
3362{
3363	if (data->track_data.save_data)
3364		data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
3365					   key, data, var_ref_vals);
3366}
3367
3368static bool check_track_val(struct tracing_map_elt *elt,
3369			    struct action_data *data,
3370			    u64 var_val)
3371{
3372	struct hist_trigger_data *hist_data;
3373	u64 track_val;
3374
3375	hist_data = data->track_data.track_var->hist_data;
3376	track_val = get_track_val(hist_data, elt, data);
3377
3378	return data->track_data.check_val(track_val, var_val);
3379}
3380
3381#ifdef CONFIG_TRACER_SNAPSHOT
3382static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3383{
3384	/* called with tr->max_lock held */
3385	struct track_data *track_data = tr->cond_snapshot->cond_data;
3386	struct hist_elt_data *elt_data, *track_elt_data;
3387	struct snapshot_context *context = cond_data;
3388	struct action_data *action;
3389	u64 track_val;
3390
3391	if (!track_data)
3392		return false;
3393
3394	action = track_data->action_data;
3395
3396	track_val = get_track_val(track_data->hist_data, context->elt,
3397				  track_data->action_data);
3398
3399	if (!action->track_data.check_val(track_data->track_val, track_val))
3400		return false;
3401
3402	track_data->track_val = track_val;
3403	memcpy(track_data->key, context->key, track_data->key_len);
3404
3405	elt_data = context->elt->private_data;
3406	track_elt_data = track_data->elt.private_data;
3407	if (elt_data->comm)
3408		strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3409
3410	track_data->updated = true;
3411
3412	return true;
3413}
3414
3415static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3416				     struct tracing_map_elt *elt,
3417				     struct trace_buffer *buffer, void *rec,
3418				     struct ring_buffer_event *rbe, void *key,
3419				     struct action_data *data,
3420				     u64 *var_ref_vals)
3421{
3422	struct trace_event_file *file = hist_data->event_file;
3423	struct snapshot_context context;
3424
3425	context.elt = elt;
3426	context.key = key;
3427
3428	tracing_snapshot_cond(file->tr, &context);
3429}
3430
3431static void hist_trigger_print_key(struct seq_file *m,
3432				   struct hist_trigger_data *hist_data,
3433				   void *key,
3434				   struct tracing_map_elt *elt);
3435
3436static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3437{
3438	unsigned int i;
3439
3440	if (!hist_data->n_actions)
3441		return NULL;
3442
3443	for (i = 0; i < hist_data->n_actions; i++) {
3444		struct action_data *data = hist_data->actions[i];
3445
3446		if (data->action == ACTION_SNAPSHOT)
3447			return data;
3448	}
3449
3450	return NULL;
3451}
3452
3453static void track_data_snapshot_print(struct seq_file *m,
3454				      struct hist_trigger_data *hist_data)
3455{
3456	struct trace_event_file *file = hist_data->event_file;
3457	struct track_data *track_data;
3458	struct action_data *action;
3459
3460	track_data = tracing_cond_snapshot_data(file->tr);
3461	if (!track_data)
3462		return;
3463
3464	if (!track_data->updated)
3465		return;
3466
3467	action = snapshot_action(hist_data);
3468	if (!action)
3469		return;
3470
3471	seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3472	seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3473		   action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3474		   action->track_data.var_str, track_data->track_val);
3475
3476	seq_puts(m, "\ttriggered by event with key: ");
3477	hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3478	seq_putc(m, '\n');
3479}
3480#else
3481static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3482{
3483	return false;
3484}
3485static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3486				     struct tracing_map_elt *elt,
3487				     struct trace_buffer *buffer, void *rec,
3488				     struct ring_buffer_event *rbe, void *key,
3489				     struct action_data *data,
3490				     u64 *var_ref_vals) {}
3491static void track_data_snapshot_print(struct seq_file *m,
3492				      struct hist_trigger_data *hist_data) {}
3493#endif /* CONFIG_TRACER_SNAPSHOT */
3494
3495static void track_data_print(struct seq_file *m,
3496			     struct hist_trigger_data *hist_data,
3497			     struct tracing_map_elt *elt,
3498			     struct action_data *data)
3499{
3500	u64 track_val = get_track_val(hist_data, elt, data);
3501	unsigned int i, save_var_idx;
3502
3503	if (data->handler == HANDLER_ONMAX)
3504		seq_printf(m, "\n\tmax: %10llu", track_val);
3505	else if (data->handler == HANDLER_ONCHANGE)
3506		seq_printf(m, "\n\tchanged: %10llu", track_val);
3507
3508	if (data->action == ACTION_SNAPSHOT)
3509		return;
3510
3511	for (i = 0; i < hist_data->n_save_vars; i++) {
3512		struct hist_field *save_val = hist_data->save_vars[i]->val;
3513		struct hist_field *save_var = hist_data->save_vars[i]->var;
3514		u64 val;
3515
3516		save_var_idx = save_var->var.idx;
3517
3518		val = tracing_map_read_var(elt, save_var_idx);
3519
3520		if (save_val->flags & HIST_FIELD_FL_STRING) {
3521			seq_printf(m, "  %s: %-32s", save_var->var.name,
3522				   (char *)(uintptr_t)(val));
3523		} else
3524			seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3525	}
3526}
3527
3528static void ontrack_action(struct hist_trigger_data *hist_data,
3529			   struct tracing_map_elt *elt,
3530			   struct trace_buffer *buffer, void *rec,
3531			   struct ring_buffer_event *rbe, void *key,
3532			   struct action_data *data, u64 *var_ref_vals)
3533{
3534	u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3535
3536	if (check_track_val(elt, data, var_val)) {
3537		save_track_val(hist_data, elt, data, var_val);
3538		save_track_data(hist_data, elt, buffer, rec, rbe,
3539				key, data, var_ref_vals);
3540	}
3541}
3542
3543static void action_data_destroy(struct action_data *data)
3544{
3545	unsigned int i;
3546
3547	lockdep_assert_held(&event_mutex);
3548
3549	kfree(data->action_name);
3550
3551	for (i = 0; i < data->n_params; i++)
3552		kfree(data->params[i]);
3553
3554	if (data->synth_event)
3555		data->synth_event->ref--;
3556
3557	kfree(data->synth_event_name);
3558
3559	kfree(data);
3560}
3561
3562static void track_data_destroy(struct hist_trigger_data *hist_data,
3563			       struct action_data *data)
3564{
3565	struct trace_event_file *file = hist_data->event_file;
3566
3567	destroy_hist_field(data->track_data.track_var, 0);
3568
3569	if (data->action == ACTION_SNAPSHOT) {
3570		struct track_data *track_data;
3571
3572		track_data = tracing_cond_snapshot_data(file->tr);
3573		if (track_data && track_data->hist_data == hist_data) {
3574			tracing_snapshot_cond_disable(file->tr);
3575			track_data_free(track_data);
3576		}
3577	}
3578
3579	kfree(data->track_data.var_str);
3580
3581	action_data_destroy(data);
3582}
3583
3584static int action_create(struct hist_trigger_data *hist_data,
3585			 struct action_data *data);
3586
3587static int track_data_create(struct hist_trigger_data *hist_data,
3588			     struct action_data *data)
3589{
3590	struct hist_field *var_field, *ref_field, *track_var = NULL;
3591	struct trace_event_file *file = hist_data->event_file;
3592	struct trace_array *tr = file->tr;
3593	char *track_data_var_str;
3594	int ret = 0;
3595
3596	track_data_var_str = data->track_data.var_str;
3597	if (track_data_var_str[0] != '$') {
3598		hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3599		return -EINVAL;
3600	}
3601	track_data_var_str++;
3602
3603	var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3604	if (!var_field) {
3605		hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3606		return -EINVAL;
3607	}
3608
3609	ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3610	if (!ref_field)
3611		return -ENOMEM;
3612
3613	data->track_data.var_ref = ref_field;
3614
3615	if (data->handler == HANDLER_ONMAX)
3616		track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3617	if (IS_ERR(track_var)) {
3618		hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3619		ret = PTR_ERR(track_var);
3620		goto out;
3621	}
3622
3623	if (data->handler == HANDLER_ONCHANGE)
3624		track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3625	if (IS_ERR(track_var)) {
3626		hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3627		ret = PTR_ERR(track_var);
3628		goto out;
3629	}
3630	data->track_data.track_var = track_var;
3631
3632	ret = action_create(hist_data, data);
3633 out:
3634	return ret;
3635}
3636
3637static int parse_action_params(struct trace_array *tr, char *params,
3638			       struct action_data *data)
3639{
3640	char *param, *saved_param;
3641	bool first_param = true;
3642	int ret = 0;
3643
3644	while (params) {
3645		if (data->n_params >= SYNTH_FIELDS_MAX) {
3646			hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3647			ret = -EINVAL;
3648			goto out;
3649		}
3650
3651		param = strsep(&params, ",");
3652		if (!param) {
3653			hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3654			ret = -EINVAL;
3655			goto out;
3656		}
3657
3658		param = strstrip(param);
3659		if (strlen(param) < 2) {
3660			hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3661			ret = -EINVAL;
3662			goto out;
3663		}
3664
3665		saved_param = kstrdup(param, GFP_KERNEL);
3666		if (!saved_param) {
3667			ret = -ENOMEM;
3668			goto out;
3669		}
3670
3671		if (first_param && data->use_trace_keyword) {
3672			data->synth_event_name = saved_param;
3673			first_param = false;
3674			continue;
3675		}
3676		first_param = false;
3677
3678		data->params[data->n_params++] = saved_param;
3679	}
3680 out:
3681	return ret;
3682}
3683
3684static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3685			enum handler_id handler)
3686{
3687	char *action_name;
3688	int ret = 0;
3689
3690	strsep(&str, ".");
3691	if (!str) {
3692		hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3693		ret = -EINVAL;
3694		goto out;
3695	}
3696
3697	action_name = strsep(&str, "(");
3698	if (!action_name || !str) {
3699		hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3700		ret = -EINVAL;
3701		goto out;
3702	}
3703
3704	if (str_has_prefix(action_name, "save")) {
3705		char *params = strsep(&str, ")");
3706
3707		if (!params) {
3708			hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3709			ret = -EINVAL;
3710			goto out;
3711		}
3712
3713		ret = parse_action_params(tr, params, data);
3714		if (ret)
3715			goto out;
3716
3717		if (handler == HANDLER_ONMAX)
3718			data->track_data.check_val = check_track_val_max;
3719		else if (handler == HANDLER_ONCHANGE)
3720			data->track_data.check_val = check_track_val_changed;
3721		else {
3722			hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3723			ret = -EINVAL;
3724			goto out;
3725		}
3726
3727		data->track_data.save_data = save_track_data_vars;
3728		data->fn = ontrack_action;
3729		data->action = ACTION_SAVE;
3730	} else if (str_has_prefix(action_name, "snapshot")) {
3731		char *params = strsep(&str, ")");
3732
3733		if (!str) {
3734			hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3735			ret = -EINVAL;
3736			goto out;
3737		}
3738
3739		if (handler == HANDLER_ONMAX)
3740			data->track_data.check_val = check_track_val_max;
3741		else if (handler == HANDLER_ONCHANGE)
3742			data->track_data.check_val = check_track_val_changed;
3743		else {
3744			hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3745			ret = -EINVAL;
3746			goto out;
3747		}
3748
3749		data->track_data.save_data = save_track_data_snapshot;
3750		data->fn = ontrack_action;
3751		data->action = ACTION_SNAPSHOT;
3752	} else {
3753		char *params = strsep(&str, ")");
3754
3755		if (str_has_prefix(action_name, "trace"))
3756			data->use_trace_keyword = true;
3757
3758		if (params) {
3759			ret = parse_action_params(tr, params, data);
3760			if (ret)
3761				goto out;
3762		}
3763
3764		if (handler == HANDLER_ONMAX)
3765			data->track_data.check_val = check_track_val_max;
3766		else if (handler == HANDLER_ONCHANGE)
3767			data->track_data.check_val = check_track_val_changed;
3768
3769		if (handler != HANDLER_ONMATCH) {
3770			data->track_data.save_data = action_trace;
3771			data->fn = ontrack_action;
3772		} else
3773			data->fn = action_trace;
3774
3775		data->action = ACTION_TRACE;
3776	}
3777
3778	data->action_name = kstrdup(action_name, GFP_KERNEL);
3779	if (!data->action_name) {
3780		ret = -ENOMEM;
3781		goto out;
3782	}
3783
3784	data->handler = handler;
3785 out:
3786	return ret;
3787}
3788
3789static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3790					    char *str, enum handler_id handler)
3791{
3792	struct action_data *data;
3793	int ret = -EINVAL;
3794	char *var_str;
3795
3796	data = kzalloc(sizeof(*data), GFP_KERNEL);
3797	if (!data)
3798		return ERR_PTR(-ENOMEM);
3799
3800	var_str = strsep(&str, ")");
3801	if (!var_str || !str) {
3802		ret = -EINVAL;
3803		goto free;
3804	}
3805
3806	data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3807	if (!data->track_data.var_str) {
3808		ret = -ENOMEM;
3809		goto free;
3810	}
3811
3812	ret = action_parse(hist_data->event_file->tr, str, data, handler);
3813	if (ret)
3814		goto free;
3815 out:
3816	return data;
3817 free:
3818	track_data_destroy(hist_data, data);
3819	data = ERR_PTR(ret);
3820	goto out;
3821}
3822
3823static void onmatch_destroy(struct action_data *data)
3824{
3825	kfree(data->match_data.event);
3826	kfree(data->match_data.event_system);
3827
3828	action_data_destroy(data);
3829}
3830
3831static void destroy_field_var(struct field_var *field_var)
3832{
3833	if (!field_var)
3834		return;
3835
3836	destroy_hist_field(field_var->var, 0);
3837	destroy_hist_field(field_var->val, 0);
3838
3839	kfree(field_var);
3840}
3841
3842static void destroy_field_vars(struct hist_trigger_data *hist_data)
3843{
3844	unsigned int i;
3845
3846	for (i = 0; i < hist_data->n_field_vars; i++)
3847		destroy_field_var(hist_data->field_vars[i]);
3848
3849	for (i = 0; i < hist_data->n_save_vars; i++)
3850		destroy_field_var(hist_data->save_vars[i]);
3851}
3852
3853static void save_field_var(struct hist_trigger_data *hist_data,
3854			   struct field_var *field_var)
3855{
3856	hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3857
3858	/* Stack traces are saved in the string storage too */
3859	if (field_var->val->flags & (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
3860		hist_data->n_field_var_str++;
3861}
3862
3863
3864static int check_synth_field(struct synth_event *event,
3865			     struct hist_field *hist_field,
3866			     unsigned int field_pos)
3867{
3868	struct synth_field *field;
3869
3870	if (field_pos >= event->n_fields)
3871		return -EINVAL;
3872
3873	field = event->fields[field_pos];
3874
3875	/*
3876	 * A dynamic string synth field can accept static or
3877	 * dynamic. A static string synth field can only accept a
3878	 * same-sized static string, which is checked for later.
3879	 */
3880	if (strstr(hist_field->type, "char[") && field->is_string
3881	    && field->is_dynamic)
3882		return 0;
3883
3884	if (strstr(hist_field->type, "long[") && field->is_stack)
3885		return 0;
3886
3887	if (strcmp(field->type, hist_field->type) != 0) {
3888		if (field->size != hist_field->size ||
3889		    (!field->is_string && field->is_signed != hist_field->is_signed))
3890			return -EINVAL;
3891	}
3892
3893	return 0;
3894}
3895
3896static struct hist_field *
3897trace_action_find_var(struct hist_trigger_data *hist_data,
3898		      struct action_data *data,
3899		      char *system, char *event, char *var)
3900{
3901	struct trace_array *tr = hist_data->event_file->tr;
3902	struct hist_field *hist_field;
3903
3904	var++; /* skip '$' */
3905
3906	hist_field = find_target_event_var(hist_data, system, event, var);
3907	if (!hist_field) {
3908		if (!system && data->handler == HANDLER_ONMATCH) {
3909			system = data->match_data.event_system;
3910			event = data->match_data.event;
3911		}
3912
3913		hist_field = find_event_var(hist_data, system, event, var);
3914	}
3915
3916	if (!hist_field)
3917		hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3918
3919	return hist_field;
3920}
3921
3922static struct hist_field *
3923trace_action_create_field_var(struct hist_trigger_data *hist_data,
3924			      struct action_data *data, char *system,
3925			      char *event, char *var)
3926{
3927	struct hist_field *hist_field = NULL;
3928	struct field_var *field_var;
3929
3930	/*
3931	 * First try to create a field var on the target event (the
3932	 * currently being defined).  This will create a variable for
3933	 * unqualified fields on the target event, or if qualified,
3934	 * target fields that have qualified names matching the target.
3935	 */
3936	field_var = create_target_field_var(hist_data, system, event, var);
3937
3938	if (field_var && !IS_ERR(field_var)) {
3939		save_field_var(hist_data, field_var);
3940		hist_field = field_var->var;
3941	} else {
3942		field_var = NULL;
3943		/*
3944		 * If no explicit system.event is specified, default to
3945		 * looking for fields on the onmatch(system.event.xxx)
3946		 * event.
3947		 */
3948		if (!system && data->handler == HANDLER_ONMATCH) {
3949			system = data->match_data.event_system;
3950			event = data->match_data.event;
3951		}
3952
3953		if (!event)
3954			goto free;
3955		/*
3956		 * At this point, we're looking at a field on another
3957		 * event.  Because we can't modify a hist trigger on
3958		 * another event to add a variable for a field, we need
3959		 * to create a new trigger on that event and create the
3960		 * variable at the same time.
3961		 */
3962		hist_field = create_field_var_hist(hist_data, system, event, var);
3963		if (IS_ERR(hist_field))
3964			goto free;
3965	}
3966 out:
3967	return hist_field;
3968 free:
3969	destroy_field_var(field_var);
3970	hist_field = NULL;
3971	goto out;
3972}
3973
3974static int trace_action_create(struct hist_trigger_data *hist_data,
3975			       struct action_data *data)
3976{
3977	struct trace_array *tr = hist_data->event_file->tr;
3978	char *event_name, *param, *system = NULL;
3979	struct hist_field *hist_field, *var_ref;
3980	unsigned int i;
3981	unsigned int field_pos = 0;
3982	struct synth_event *event;
3983	char *synth_event_name;
3984	int var_ref_idx, ret = 0;
3985
3986	lockdep_assert_held(&event_mutex);
3987
3988	/* Sanity check to avoid out-of-bound write on 'data->var_ref_idx' */
3989	if (data->n_params > SYNTH_FIELDS_MAX)
3990		return -EINVAL;
3991
3992	if (data->use_trace_keyword)
3993		synth_event_name = data->synth_event_name;
3994	else
3995		synth_event_name = data->action_name;
3996
3997	event = find_synth_event(synth_event_name);
3998	if (!event) {
3999		hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
4000		return -EINVAL;
4001	}
4002
4003	event->ref++;
4004
 
 
4005	for (i = 0; i < data->n_params; i++) {
4006		char *p;
4007
4008		p = param = kstrdup(data->params[i], GFP_KERNEL);
4009		if (!param) {
4010			ret = -ENOMEM;
4011			goto err;
4012		}
4013
4014		system = strsep(&param, ".");
4015		if (!param) {
4016			param = (char *)system;
4017			system = event_name = NULL;
4018		} else {
4019			event_name = strsep(&param, ".");
4020			if (!param) {
4021				kfree(p);
4022				ret = -EINVAL;
4023				goto err;
4024			}
4025		}
4026
4027		if (param[0] == '$')
4028			hist_field = trace_action_find_var(hist_data, data,
4029							   system, event_name,
4030							   param);
4031		else
4032			hist_field = trace_action_create_field_var(hist_data,
4033								   data,
4034								   system,
4035								   event_name,
4036								   param);
4037
4038		if (!hist_field) {
4039			kfree(p);
4040			ret = -EINVAL;
4041			goto err;
4042		}
4043
4044		if (check_synth_field(event, hist_field, field_pos) == 0) {
4045			var_ref = create_var_ref(hist_data, hist_field,
4046						 system, event_name);
4047			if (!var_ref) {
4048				kfree(p);
4049				ret = -ENOMEM;
4050				goto err;
4051			}
4052
4053			var_ref_idx = find_var_ref_idx(hist_data, var_ref);
4054			if (WARN_ON(var_ref_idx < 0)) {
4055				kfree(p);
4056				ret = var_ref_idx;
4057				goto err;
4058			}
4059
4060			data->var_ref_idx[i] = var_ref_idx;
4061
4062			field_pos++;
4063			kfree(p);
4064			continue;
4065		}
4066
4067		hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
4068		kfree(p);
4069		ret = -EINVAL;
4070		goto err;
4071	}
4072
4073	if (field_pos != event->n_fields) {
4074		hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
4075		ret = -EINVAL;
4076		goto err;
4077	}
4078
4079	data->synth_event = event;
 
4080 out:
4081	return ret;
4082 err:
4083	event->ref--;
4084
4085	goto out;
4086}
4087
4088static int action_create(struct hist_trigger_data *hist_data,
4089			 struct action_data *data)
4090{
4091	struct trace_event_file *file = hist_data->event_file;
4092	struct trace_array *tr = file->tr;
4093	struct track_data *track_data;
4094	struct field_var *field_var;
4095	unsigned int i;
4096	char *param;
4097	int ret = 0;
4098
4099	if (data->action == ACTION_TRACE)
4100		return trace_action_create(hist_data, data);
4101
4102	if (data->action == ACTION_SNAPSHOT) {
4103		track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4104		if (IS_ERR(track_data)) {
4105			ret = PTR_ERR(track_data);
4106			goto out;
4107		}
4108
4109		ret = tracing_snapshot_cond_enable(file->tr, track_data,
4110						   cond_snapshot_update);
4111		if (ret)
4112			track_data_free(track_data);
4113
4114		goto out;
4115	}
4116
4117	if (data->action == ACTION_SAVE) {
4118		if (hist_data->n_save_vars) {
4119			ret = -EEXIST;
4120			hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4121			goto out;
4122		}
4123
4124		for (i = 0; i < data->n_params; i++) {
4125			param = kstrdup(data->params[i], GFP_KERNEL);
4126			if (!param) {
4127				ret = -ENOMEM;
4128				goto out;
4129			}
4130
4131			field_var = create_target_field_var(hist_data, NULL, NULL, param);
4132			if (IS_ERR(field_var)) {
4133				hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4134					 errpos(param));
4135				ret = PTR_ERR(field_var);
4136				kfree(param);
4137				goto out;
4138			}
4139
4140			hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4141			if (field_var->val->flags &
4142			    (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
4143				hist_data->n_save_var_str++;
4144			kfree(param);
4145		}
4146	}
4147 out:
4148	return ret;
4149}
4150
4151static int onmatch_create(struct hist_trigger_data *hist_data,
4152			  struct action_data *data)
4153{
4154	return action_create(hist_data, data);
4155}
4156
4157static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4158{
4159	char *match_event, *match_event_system;
4160	struct action_data *data;
4161	int ret = -EINVAL;
4162
4163	data = kzalloc(sizeof(*data), GFP_KERNEL);
4164	if (!data)
4165		return ERR_PTR(-ENOMEM);
4166
4167	match_event = strsep(&str, ")");
4168	if (!match_event || !str) {
4169		hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4170		goto free;
4171	}
4172
4173	match_event_system = strsep(&match_event, ".");
4174	if (!match_event) {
4175		hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4176		goto free;
4177	}
4178
4179	if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4180		hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4181		goto free;
4182	}
4183
4184	data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4185	if (!data->match_data.event) {
4186		ret = -ENOMEM;
4187		goto free;
4188	}
4189
4190	data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4191	if (!data->match_data.event_system) {
4192		ret = -ENOMEM;
4193		goto free;
4194	}
4195
4196	ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4197	if (ret)
4198		goto free;
4199 out:
4200	return data;
4201 free:
4202	onmatch_destroy(data);
4203	data = ERR_PTR(ret);
4204	goto out;
4205}
4206
4207static int create_hitcount_val(struct hist_trigger_data *hist_data)
4208{
4209	hist_data->fields[HITCOUNT_IDX] =
4210		create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4211	if (!hist_data->fields[HITCOUNT_IDX])
4212		return -ENOMEM;
4213
4214	hist_data->n_vals++;
4215	hist_data->n_fields++;
4216
4217	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4218		return -EINVAL;
4219
4220	return 0;
4221}
4222
4223static int __create_val_field(struct hist_trigger_data *hist_data,
4224			      unsigned int val_idx,
4225			      struct trace_event_file *file,
4226			      char *var_name, char *field_str,
4227			      unsigned long flags)
4228{
4229	struct hist_field *hist_field;
4230	int ret = 0, n_subexprs = 0;
4231
4232	hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs);
4233	if (IS_ERR(hist_field)) {
4234		ret = PTR_ERR(hist_field);
4235		goto out;
4236	}
4237
4238	/* values and variables should not have some modifiers */
4239	if (hist_field->flags & HIST_FIELD_FL_VAR) {
4240		/* Variable */
4241		if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT |
4242					 HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2))
4243			goto err;
4244	} else {
4245		/* Value */
4246		if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT |
4247					 HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2 |
4248					 HIST_FIELD_FL_SYM | HIST_FIELD_FL_SYM_OFFSET |
4249					 HIST_FIELD_FL_SYSCALL | HIST_FIELD_FL_STACKTRACE))
4250			goto err;
4251	}
4252
4253	hist_data->fields[val_idx] = hist_field;
4254
4255	++hist_data->n_vals;
4256	++hist_data->n_fields;
4257
4258	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4259		ret = -EINVAL;
4260 out:
4261	return ret;
4262 err:
4263	hist_err(file->tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str));
4264	return -EINVAL;
4265}
4266
4267static int create_val_field(struct hist_trigger_data *hist_data,
4268			    unsigned int val_idx,
4269			    struct trace_event_file *file,
4270			    char *field_str)
4271{
4272	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4273		return -EINVAL;
4274
4275	return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4276}
4277
4278static const char no_comm[] = "(no comm)";
4279
4280static u64 hist_field_execname(struct hist_field *hist_field,
4281			       struct tracing_map_elt *elt,
4282			       struct trace_buffer *buffer,
4283			       struct ring_buffer_event *rbe,
4284			       void *event)
4285{
4286	struct hist_elt_data *elt_data;
4287
4288	if (WARN_ON_ONCE(!elt))
4289		return (u64)(unsigned long)no_comm;
4290
4291	elt_data = elt->private_data;
4292
4293	if (WARN_ON_ONCE(!elt_data->comm))
4294		return (u64)(unsigned long)no_comm;
4295
4296	return (u64)(unsigned long)(elt_data->comm);
4297}
4298
4299static u64 hist_field_stack(struct hist_field *hist_field,
4300			    struct tracing_map_elt *elt,
4301			    struct trace_buffer *buffer,
4302			    struct ring_buffer_event *rbe,
4303			    void *event)
4304{
4305	u32 str_item = *(u32 *)(event + hist_field->field->offset);
4306	int str_loc = str_item & 0xffff;
4307	char *addr = (char *)(event + str_loc);
4308
4309	return (u64)(unsigned long)addr;
4310}
4311
4312static u64 hist_fn_call(struct hist_field *hist_field,
4313			struct tracing_map_elt *elt,
4314			struct trace_buffer *buffer,
4315			struct ring_buffer_event *rbe,
4316			void *event)
4317{
4318	switch (hist_field->fn_num) {
4319	case HIST_FIELD_FN_VAR_REF:
4320		return hist_field_var_ref(hist_field, elt, buffer, rbe, event);
4321	case HIST_FIELD_FN_COUNTER:
4322		return hist_field_counter(hist_field, elt, buffer, rbe, event);
4323	case HIST_FIELD_FN_CONST:
4324		return hist_field_const(hist_field, elt, buffer, rbe, event);
4325	case HIST_FIELD_FN_LOG2:
4326		return hist_field_log2(hist_field, elt, buffer, rbe, event);
4327	case HIST_FIELD_FN_BUCKET:
4328		return hist_field_bucket(hist_field, elt, buffer, rbe, event);
4329	case HIST_FIELD_FN_TIMESTAMP:
4330		return hist_field_timestamp(hist_field, elt, buffer, rbe, event);
4331	case HIST_FIELD_FN_CPU:
4332		return hist_field_cpu(hist_field, elt, buffer, rbe, event);
4333	case HIST_FIELD_FN_STRING:
4334		return hist_field_string(hist_field, elt, buffer, rbe, event);
4335	case HIST_FIELD_FN_DYNSTRING:
4336		return hist_field_dynstring(hist_field, elt, buffer, rbe, event);
4337	case HIST_FIELD_FN_RELDYNSTRING:
4338		return hist_field_reldynstring(hist_field, elt, buffer, rbe, event);
4339	case HIST_FIELD_FN_PSTRING:
4340		return hist_field_pstring(hist_field, elt, buffer, rbe, event);
4341	case HIST_FIELD_FN_S64:
4342		return hist_field_s64(hist_field, elt, buffer, rbe, event);
4343	case HIST_FIELD_FN_U64:
4344		return hist_field_u64(hist_field, elt, buffer, rbe, event);
4345	case HIST_FIELD_FN_S32:
4346		return hist_field_s32(hist_field, elt, buffer, rbe, event);
4347	case HIST_FIELD_FN_U32:
4348		return hist_field_u32(hist_field, elt, buffer, rbe, event);
4349	case HIST_FIELD_FN_S16:
4350		return hist_field_s16(hist_field, elt, buffer, rbe, event);
4351	case HIST_FIELD_FN_U16:
4352		return hist_field_u16(hist_field, elt, buffer, rbe, event);
4353	case HIST_FIELD_FN_S8:
4354		return hist_field_s8(hist_field, elt, buffer, rbe, event);
4355	case HIST_FIELD_FN_U8:
4356		return hist_field_u8(hist_field, elt, buffer, rbe, event);
4357	case HIST_FIELD_FN_UMINUS:
4358		return hist_field_unary_minus(hist_field, elt, buffer, rbe, event);
4359	case HIST_FIELD_FN_MINUS:
4360		return hist_field_minus(hist_field, elt, buffer, rbe, event);
4361	case HIST_FIELD_FN_PLUS:
4362		return hist_field_plus(hist_field, elt, buffer, rbe, event);
4363	case HIST_FIELD_FN_DIV:
4364		return hist_field_div(hist_field, elt, buffer, rbe, event);
4365	case HIST_FIELD_FN_MULT:
4366		return hist_field_mult(hist_field, elt, buffer, rbe, event);
4367	case HIST_FIELD_FN_DIV_POWER2:
4368		return div_by_power_of_two(hist_field, elt, buffer, rbe, event);
4369	case HIST_FIELD_FN_DIV_NOT_POWER2:
4370		return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event);
4371	case HIST_FIELD_FN_DIV_MULT_SHIFT:
4372		return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event);
4373	case HIST_FIELD_FN_EXECNAME:
4374		return hist_field_execname(hist_field, elt, buffer, rbe, event);
4375	case HIST_FIELD_FN_STACK:
4376		return hist_field_stack(hist_field, elt, buffer, rbe, event);
4377	default:
4378		return 0;
4379	}
4380}
4381
4382/* Convert a var that points to common_pid.execname to a string */
4383static void update_var_execname(struct hist_field *hist_field)
4384{
4385	hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR |
4386		HIST_FIELD_FL_EXECNAME;
4387	hist_field->size = MAX_FILTER_STR_VAL;
4388	hist_field->is_signed = 0;
4389
4390	kfree_const(hist_field->type);
4391	hist_field->type = "char[]";
4392
4393	hist_field->fn_num = HIST_FIELD_FN_EXECNAME;
4394}
4395
4396static int create_var_field(struct hist_trigger_data *hist_data,
4397			    unsigned int val_idx,
4398			    struct trace_event_file *file,
4399			    char *var_name, char *expr_str)
4400{
4401	struct trace_array *tr = hist_data->event_file->tr;
4402	unsigned long flags = 0;
4403	int ret;
4404
4405	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4406		return -EINVAL;
4407
4408	if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4409		hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4410		return -EINVAL;
4411	}
4412
4413	flags |= HIST_FIELD_FL_VAR;
4414	hist_data->n_vars++;
4415	if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4416		return -EINVAL;
4417
4418	ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4419
4420	if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME)
4421		update_var_execname(hist_data->fields[val_idx]);
4422
4423	if (!ret && hist_data->fields[val_idx]->flags &
4424	    (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
4425		hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
4426
4427	return ret;
4428}
4429
4430static int create_val_fields(struct hist_trigger_data *hist_data,
4431			     struct trace_event_file *file)
4432{
4433	unsigned int i, j = 1, n_hitcount = 0;
4434	char *fields_str, *field_str;
 
4435	int ret;
4436
4437	ret = create_hitcount_val(hist_data);
4438	if (ret)
4439		goto out;
4440
4441	fields_str = hist_data->attrs->vals_str;
4442	if (!fields_str)
4443		goto out;
4444
 
 
 
 
4445	for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4446		     j < TRACING_MAP_VALS_MAX; i++) {
4447		field_str = strsep(&fields_str, ",");
4448		if (!field_str)
4449			break;
4450
4451		if (strcmp(field_str, "hitcount") == 0) {
4452			if (!n_hitcount++)
4453				continue;
4454		}
4455
4456		ret = create_val_field(hist_data, j++, file, field_str);
4457		if (ret)
4458			goto out;
4459	}
4460
4461	if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4462		ret = -EINVAL;
4463 out:
4464	/* There is only raw hitcount but nohitcount suppresses it. */
4465	if (j == 1 && hist_data->attrs->no_hitcount) {
4466		hist_err(hist_data->event_file->tr, HIST_ERR_NEED_NOHC_VAL, 0);
4467		ret = -ENOENT;
4468	}
4469
4470	return ret;
4471}
4472
4473static int create_key_field(struct hist_trigger_data *hist_data,
4474			    unsigned int key_idx,
4475			    unsigned int key_offset,
4476			    struct trace_event_file *file,
4477			    char *field_str)
4478{
4479	struct trace_array *tr = hist_data->event_file->tr;
4480	struct hist_field *hist_field = NULL;
4481	unsigned long flags = 0;
4482	unsigned int key_size;
4483	int ret = 0, n_subexprs = 0;
4484
4485	if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4486		return -EINVAL;
4487
4488	flags |= HIST_FIELD_FL_KEY;
4489
4490	if (strcmp(field_str, "stacktrace") == 0) {
4491		flags |= HIST_FIELD_FL_STACKTRACE;
4492		key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4493		hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4494	} else {
4495		hist_field = parse_expr(hist_data, file, field_str, flags,
4496					NULL, &n_subexprs);
4497		if (IS_ERR(hist_field)) {
4498			ret = PTR_ERR(hist_field);
4499			goto out;
4500		}
4501
4502		if (field_has_hist_vars(hist_field, 0))	{
4503			hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4504			destroy_hist_field(hist_field, 0);
4505			ret = -EINVAL;
4506			goto out;
4507		}
4508
4509		key_size = hist_field->size;
4510	}
4511
4512	hist_data->fields[key_idx] = hist_field;
4513
4514	key_size = ALIGN(key_size, sizeof(u64));
4515	hist_data->fields[key_idx]->size = key_size;
4516	hist_data->fields[key_idx]->offset = key_offset;
4517
4518	hist_data->key_size += key_size;
4519
4520	if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4521		ret = -EINVAL;
4522		goto out;
4523	}
4524
4525	hist_data->n_keys++;
4526	hist_data->n_fields++;
4527
4528	if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4529		return -EINVAL;
4530
4531	ret = key_size;
4532 out:
4533	return ret;
4534}
4535
4536static int create_key_fields(struct hist_trigger_data *hist_data,
4537			     struct trace_event_file *file)
4538{
4539	unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4540	char *fields_str, *field_str;
4541	int ret = -EINVAL;
4542
4543	fields_str = hist_data->attrs->keys_str;
4544	if (!fields_str)
4545		goto out;
4546
 
 
 
 
4547	for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4548		field_str = strsep(&fields_str, ",");
4549		if (!field_str)
4550			break;
4551		ret = create_key_field(hist_data, i, key_offset,
4552				       file, field_str);
4553		if (ret < 0)
4554			goto out;
4555		key_offset += ret;
4556	}
4557	if (fields_str) {
4558		ret = -EINVAL;
4559		goto out;
4560	}
4561	ret = 0;
4562 out:
4563	return ret;
4564}
4565
4566static int create_var_fields(struct hist_trigger_data *hist_data,
4567			     struct trace_event_file *file)
4568{
4569	unsigned int i, j = hist_data->n_vals;
4570	int ret = 0;
4571
4572	unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4573
4574	for (i = 0; i < n_vars; i++) {
4575		char *var_name = hist_data->attrs->var_defs.name[i];
4576		char *expr = hist_data->attrs->var_defs.expr[i];
4577
4578		ret = create_var_field(hist_data, j++, file, var_name, expr);
4579		if (ret)
4580			goto out;
4581	}
4582 out:
4583	return ret;
4584}
4585
4586static void free_var_defs(struct hist_trigger_data *hist_data)
4587{
4588	unsigned int i;
4589
4590	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4591		kfree(hist_data->attrs->var_defs.name[i]);
4592		kfree(hist_data->attrs->var_defs.expr[i]);
4593	}
4594
4595	hist_data->attrs->var_defs.n_vars = 0;
4596}
4597
4598static int parse_var_defs(struct hist_trigger_data *hist_data)
4599{
4600	struct trace_array *tr = hist_data->event_file->tr;
4601	char *s, *str, *var_name, *field_str;
4602	unsigned int i, j, n_vars = 0;
4603	int ret = 0;
4604
4605	for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4606		str = hist_data->attrs->assignment_str[i];
4607		for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4608			field_str = strsep(&str, ",");
4609			if (!field_str)
4610				break;
4611
4612			var_name = strsep(&field_str, "=");
4613			if (!var_name || !field_str) {
4614				hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4615					 errpos(var_name));
4616				ret = -EINVAL;
4617				goto free;
4618			}
4619
4620			if (n_vars == TRACING_MAP_VARS_MAX) {
4621				hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4622				ret = -EINVAL;
4623				goto free;
4624			}
4625
4626			s = kstrdup(var_name, GFP_KERNEL);
4627			if (!s) {
4628				ret = -ENOMEM;
4629				goto free;
4630			}
4631			hist_data->attrs->var_defs.name[n_vars] = s;
4632
4633			s = kstrdup(field_str, GFP_KERNEL);
4634			if (!s) {
4635				kfree(hist_data->attrs->var_defs.name[n_vars]);
4636				hist_data->attrs->var_defs.name[n_vars] = NULL;
4637				ret = -ENOMEM;
4638				goto free;
4639			}
4640			hist_data->attrs->var_defs.expr[n_vars++] = s;
4641
4642			hist_data->attrs->var_defs.n_vars = n_vars;
4643		}
4644	}
4645
4646	return ret;
4647 free:
4648	free_var_defs(hist_data);
4649
4650	return ret;
4651}
4652
4653static int create_hist_fields(struct hist_trigger_data *hist_data,
4654			      struct trace_event_file *file)
4655{
4656	int ret;
4657
4658	ret = parse_var_defs(hist_data);
4659	if (ret)
4660		return ret;
4661
4662	ret = create_val_fields(hist_data, file);
4663	if (ret)
4664		goto out;
4665
4666	ret = create_var_fields(hist_data, file);
4667	if (ret)
4668		goto out;
4669
4670	ret = create_key_fields(hist_data, file);
4671
 
4672 out:
4673	free_var_defs(hist_data);
4674
4675	return ret;
4676}
4677
4678static int is_descending(struct trace_array *tr, const char *str)
4679{
4680	if (!str)
4681		return 0;
4682
4683	if (strcmp(str, "descending") == 0)
4684		return 1;
4685
4686	if (strcmp(str, "ascending") == 0)
4687		return 0;
4688
4689	hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4690
4691	return -EINVAL;
4692}
4693
4694static int create_sort_keys(struct hist_trigger_data *hist_data)
4695{
4696	struct trace_array *tr = hist_data->event_file->tr;
4697	char *fields_str = hist_data->attrs->sort_key_str;
4698	struct tracing_map_sort_key *sort_key;
4699	int descending, ret = 0;
4700	unsigned int i, j, k;
4701
4702	hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4703
4704	if (!fields_str)
4705		goto out;
4706
 
 
 
 
 
 
4707	for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4708		struct hist_field *hist_field;
4709		char *field_str, *field_name;
4710		const char *test_name;
4711
4712		sort_key = &hist_data->sort_keys[i];
4713
4714		field_str = strsep(&fields_str, ",");
4715		if (!field_str)
4716			break;
4717
4718		if (!*field_str) {
4719			ret = -EINVAL;
4720			hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4721			break;
4722		}
4723
4724		if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4725			hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4726			ret = -EINVAL;
4727			break;
4728		}
4729
4730		field_name = strsep(&field_str, ".");
4731		if (!field_name || !*field_name) {
4732			ret = -EINVAL;
4733			hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4734			break;
4735		}
4736
4737		if (strcmp(field_name, "hitcount") == 0) {
4738			descending = is_descending(tr, field_str);
4739			if (descending < 0) {
4740				ret = descending;
4741				break;
4742			}
4743			sort_key->descending = descending;
4744			continue;
4745		}
4746
4747		for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4748			unsigned int idx;
4749
4750			hist_field = hist_data->fields[j];
4751			if (hist_field->flags & HIST_FIELD_FL_VAR)
4752				continue;
4753
4754			idx = k++;
4755
4756			test_name = hist_field_name(hist_field, 0);
4757
4758			if (strcmp(field_name, test_name) == 0) {
4759				sort_key->field_idx = idx;
4760				descending = is_descending(tr, field_str);
4761				if (descending < 0) {
4762					ret = descending;
4763					goto out;
4764				}
4765				sort_key->descending = descending;
4766				break;
4767			}
4768		}
4769		if (j == hist_data->n_fields) {
4770			ret = -EINVAL;
4771			hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4772			break;
4773		}
4774	}
4775
4776	hist_data->n_sort_keys = i;
4777 out:
4778	return ret;
4779}
4780
4781static void destroy_actions(struct hist_trigger_data *hist_data)
4782{
4783	unsigned int i;
4784
4785	for (i = 0; i < hist_data->n_actions; i++) {
4786		struct action_data *data = hist_data->actions[i];
4787
4788		if (data->handler == HANDLER_ONMATCH)
4789			onmatch_destroy(data);
4790		else if (data->handler == HANDLER_ONMAX ||
4791			 data->handler == HANDLER_ONCHANGE)
4792			track_data_destroy(hist_data, data);
4793		else
4794			kfree(data);
4795	}
4796}
4797
4798static int parse_actions(struct hist_trigger_data *hist_data)
4799{
4800	struct trace_array *tr = hist_data->event_file->tr;
4801	struct action_data *data;
4802	unsigned int i;
4803	int ret = 0;
4804	char *str;
4805	int len;
4806
4807	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4808		enum handler_id hid = 0;
4809		char *action_str;
4810
4811		str = hist_data->attrs->action_str[i];
4812
4813		if ((len = str_has_prefix(str, "onmatch(")))
4814			hid = HANDLER_ONMATCH;
4815		else if ((len = str_has_prefix(str, "onmax(")))
4816			hid = HANDLER_ONMAX;
4817		else if ((len = str_has_prefix(str, "onchange(")))
4818			hid = HANDLER_ONCHANGE;
4819
4820		action_str = str + len;
4821
4822		switch (hid) {
4823		case HANDLER_ONMATCH:
4824			data = onmatch_parse(tr, action_str);
4825			break;
4826		case HANDLER_ONMAX:
4827		case HANDLER_ONCHANGE:
4828			data = track_data_parse(hist_data, action_str, hid);
4829			break;
4830		default:
4831			data = ERR_PTR(-EINVAL);
4832			break;
4833		}
4834
4835		if (IS_ERR(data)) {
4836			ret = PTR_ERR(data);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4837			break;
4838		}
4839
4840		hist_data->actions[hist_data->n_actions++] = data;
4841	}
4842
4843	return ret;
4844}
4845
4846static int create_actions(struct hist_trigger_data *hist_data)
4847{
4848	struct action_data *data;
4849	unsigned int i;
4850	int ret = 0;
4851
4852	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4853		data = hist_data->actions[i];
4854
4855		if (data->handler == HANDLER_ONMATCH) {
4856			ret = onmatch_create(hist_data, data);
4857			if (ret)
4858				break;
4859		} else if (data->handler == HANDLER_ONMAX ||
4860			   data->handler == HANDLER_ONCHANGE) {
4861			ret = track_data_create(hist_data, data);
4862			if (ret)
4863				break;
4864		} else {
4865			ret = -EINVAL;
4866			break;
4867		}
4868	}
4869
4870	return ret;
4871}
4872
4873static void print_actions(struct seq_file *m,
4874			  struct hist_trigger_data *hist_data,
4875			  struct tracing_map_elt *elt)
4876{
4877	unsigned int i;
4878
4879	for (i = 0; i < hist_data->n_actions; i++) {
4880		struct action_data *data = hist_data->actions[i];
4881
4882		if (data->action == ACTION_SNAPSHOT)
4883			continue;
4884
4885		if (data->handler == HANDLER_ONMAX ||
4886		    data->handler == HANDLER_ONCHANGE)
4887			track_data_print(m, hist_data, elt, data);
4888	}
4889}
4890
4891static void print_action_spec(struct seq_file *m,
4892			      struct hist_trigger_data *hist_data,
4893			      struct action_data *data)
4894{
4895	unsigned int i;
4896
4897	if (data->action == ACTION_SAVE) {
4898		for (i = 0; i < hist_data->n_save_vars; i++) {
4899			seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4900			if (i < hist_data->n_save_vars - 1)
4901				seq_puts(m, ",");
4902		}
4903	} else if (data->action == ACTION_TRACE) {
4904		if (data->use_trace_keyword)
4905			seq_printf(m, "%s", data->synth_event_name);
4906		for (i = 0; i < data->n_params; i++) {
4907			if (i || data->use_trace_keyword)
4908				seq_puts(m, ",");
4909			seq_printf(m, "%s", data->params[i]);
4910		}
4911	}
4912}
4913
4914static void print_track_data_spec(struct seq_file *m,
4915				  struct hist_trigger_data *hist_data,
4916				  struct action_data *data)
4917{
4918	if (data->handler == HANDLER_ONMAX)
4919		seq_puts(m, ":onmax(");
4920	else if (data->handler == HANDLER_ONCHANGE)
4921		seq_puts(m, ":onchange(");
4922	seq_printf(m, "%s", data->track_data.var_str);
4923	seq_printf(m, ").%s(", data->action_name);
4924
4925	print_action_spec(m, hist_data, data);
4926
4927	seq_puts(m, ")");
4928}
4929
4930static void print_onmatch_spec(struct seq_file *m,
4931			       struct hist_trigger_data *hist_data,
4932			       struct action_data *data)
4933{
4934	seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4935		   data->match_data.event);
4936
4937	seq_printf(m, "%s(", data->action_name);
4938
4939	print_action_spec(m, hist_data, data);
4940
4941	seq_puts(m, ")");
4942}
4943
4944static bool actions_match(struct hist_trigger_data *hist_data,
4945			  struct hist_trigger_data *hist_data_test)
4946{
4947	unsigned int i, j;
4948
4949	if (hist_data->n_actions != hist_data_test->n_actions)
4950		return false;
4951
4952	for (i = 0; i < hist_data->n_actions; i++) {
4953		struct action_data *data = hist_data->actions[i];
4954		struct action_data *data_test = hist_data_test->actions[i];
4955		char *action_name, *action_name_test;
4956
4957		if (data->handler != data_test->handler)
4958			return false;
4959		if (data->action != data_test->action)
4960			return false;
4961
4962		if (data->n_params != data_test->n_params)
4963			return false;
4964
4965		for (j = 0; j < data->n_params; j++) {
4966			if (strcmp(data->params[j], data_test->params[j]) != 0)
4967				return false;
4968		}
4969
4970		if (data->use_trace_keyword)
4971			action_name = data->synth_event_name;
4972		else
4973			action_name = data->action_name;
4974
4975		if (data_test->use_trace_keyword)
4976			action_name_test = data_test->synth_event_name;
4977		else
4978			action_name_test = data_test->action_name;
4979
4980		if (strcmp(action_name, action_name_test) != 0)
4981			return false;
4982
4983		if (data->handler == HANDLER_ONMATCH) {
4984			if (strcmp(data->match_data.event_system,
4985				   data_test->match_data.event_system) != 0)
4986				return false;
4987			if (strcmp(data->match_data.event,
4988				   data_test->match_data.event) != 0)
4989				return false;
4990		} else if (data->handler == HANDLER_ONMAX ||
4991			   data->handler == HANDLER_ONCHANGE) {
4992			if (strcmp(data->track_data.var_str,
4993				   data_test->track_data.var_str) != 0)
4994				return false;
4995		}
4996	}
4997
4998	return true;
4999}
5000
5001
5002static void print_actions_spec(struct seq_file *m,
5003			       struct hist_trigger_data *hist_data)
5004{
5005	unsigned int i;
5006
5007	for (i = 0; i < hist_data->n_actions; i++) {
5008		struct action_data *data = hist_data->actions[i];
5009
5010		if (data->handler == HANDLER_ONMATCH)
5011			print_onmatch_spec(m, hist_data, data);
5012		else if (data->handler == HANDLER_ONMAX ||
5013			 data->handler == HANDLER_ONCHANGE)
5014			print_track_data_spec(m, hist_data, data);
5015	}
5016}
5017
5018static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
5019{
5020	unsigned int i;
5021
5022	for (i = 0; i < hist_data->n_field_var_hists; i++) {
5023		kfree(hist_data->field_var_hists[i]->cmd);
5024		kfree(hist_data->field_var_hists[i]);
5025	}
5026}
5027
5028static void destroy_hist_data(struct hist_trigger_data *hist_data)
5029{
5030	if (!hist_data)
5031		return;
5032
5033	destroy_hist_trigger_attrs(hist_data->attrs);
5034	destroy_hist_fields(hist_data);
5035	tracing_map_destroy(hist_data->map);
5036
5037	destroy_actions(hist_data);
5038	destroy_field_vars(hist_data);
5039	destroy_field_var_hists(hist_data);
5040
5041	kfree(hist_data);
5042}
5043
5044static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
5045{
5046	struct tracing_map *map = hist_data->map;
5047	struct ftrace_event_field *field;
5048	struct hist_field *hist_field;
5049	int i, idx = 0;
5050
5051	for_each_hist_field(i, hist_data) {
5052		hist_field = hist_data->fields[i];
5053		if (hist_field->flags & HIST_FIELD_FL_KEY) {
5054			tracing_map_cmp_fn_t cmp_fn;
5055
5056			field = hist_field->field;
5057
5058			if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
5059				cmp_fn = tracing_map_cmp_none;
5060			else if (!field || hist_field->flags & HIST_FIELD_FL_CPU)
5061				cmp_fn = tracing_map_cmp_num(hist_field->size,
5062							     hist_field->is_signed);
5063			else if (is_string_field(field))
5064				cmp_fn = tracing_map_cmp_string;
5065			else
5066				cmp_fn = tracing_map_cmp_num(field->size,
5067							     field->is_signed);
5068			idx = tracing_map_add_key_field(map,
5069							hist_field->offset,
5070							cmp_fn);
5071		} else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5072			idx = tracing_map_add_sum_field(map);
5073
5074		if (idx < 0)
5075			return idx;
5076
5077		if (hist_field->flags & HIST_FIELD_FL_VAR) {
5078			idx = tracing_map_add_var(map);
5079			if (idx < 0)
5080				return idx;
5081			hist_field->var.idx = idx;
5082			hist_field->var.hist_data = hist_data;
5083		}
5084	}
5085
5086	return 0;
5087}
5088
5089static struct hist_trigger_data *
5090create_hist_data(unsigned int map_bits,
5091		 struct hist_trigger_attrs *attrs,
5092		 struct trace_event_file *file,
5093		 bool remove)
5094{
5095	const struct tracing_map_ops *map_ops = NULL;
5096	struct hist_trigger_data *hist_data;
5097	int ret = 0;
5098
5099	hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5100	if (!hist_data)
5101		return ERR_PTR(-ENOMEM);
5102
5103	hist_data->attrs = attrs;
5104	hist_data->remove = remove;
5105	hist_data->event_file = file;
5106
5107	ret = parse_actions(hist_data);
5108	if (ret)
5109		goto free;
5110
5111	ret = create_hist_fields(hist_data, file);
5112	if (ret)
5113		goto free;
5114
5115	ret = create_sort_keys(hist_data);
5116	if (ret)
5117		goto free;
5118
5119	map_ops = &hist_trigger_elt_data_ops;
5120
5121	hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5122					    map_ops, hist_data);
5123	if (IS_ERR(hist_data->map)) {
5124		ret = PTR_ERR(hist_data->map);
5125		hist_data->map = NULL;
5126		goto free;
5127	}
5128
5129	ret = create_tracing_map_fields(hist_data);
5130	if (ret)
5131		goto free;
5132 out:
5133	return hist_data;
5134 free:
5135	hist_data->attrs = NULL;
5136
5137	destroy_hist_data(hist_data);
5138
5139	hist_data = ERR_PTR(ret);
5140
5141	goto out;
5142}
5143
5144static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5145				    struct tracing_map_elt *elt,
5146				    struct trace_buffer *buffer, void *rec,
5147				    struct ring_buffer_event *rbe,
5148				    u64 *var_ref_vals)
5149{
5150	struct hist_elt_data *elt_data;
5151	struct hist_field *hist_field;
5152	unsigned int i, var_idx;
5153	u64 hist_val;
5154
5155	elt_data = elt->private_data;
5156	elt_data->var_ref_vals = var_ref_vals;
5157
5158	for_each_hist_val_field(i, hist_data) {
5159		hist_field = hist_data->fields[i];
5160		hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5161		if (hist_field->flags & HIST_FIELD_FL_VAR) {
5162			var_idx = hist_field->var.idx;
5163
5164			if (hist_field->flags &
5165			    (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) {
5166				unsigned int str_start, var_str_idx, idx;
5167				char *str, *val_str;
5168				unsigned int size;
5169
5170				str_start = hist_data->n_field_var_str +
5171					hist_data->n_save_var_str;
5172				var_str_idx = hist_field->var_str_idx;
5173				idx = str_start + var_str_idx;
5174
5175				str = elt_data->field_var_str[idx];
5176				val_str = (char *)(uintptr_t)hist_val;
5177
5178				if (hist_field->flags & HIST_FIELD_FL_STRING) {
5179					size = min(hist_field->size, STR_VAR_LEN_MAX);
5180					strscpy(str, val_str, size);
5181				} else {
5182					char *stack_start = str + sizeof(unsigned long);
5183					int e;
5184
5185					e = stack_trace_save((void *)stack_start,
5186							     HIST_STACKTRACE_DEPTH,
5187							     HIST_STACKTRACE_SKIP);
5188					if (e < HIST_STACKTRACE_DEPTH - 1)
5189						((unsigned long *)stack_start)[e] = 0;
5190					*((unsigned long *)str) = e;
5191				}
5192				hist_val = (u64)(uintptr_t)str;
5193			}
5194			tracing_map_set_var(elt, var_idx, hist_val);
5195			continue;
5196		}
5197		tracing_map_update_sum(elt, i, hist_val);
5198	}
5199
5200	for_each_hist_key_field(i, hist_data) {
5201		hist_field = hist_data->fields[i];
5202		if (hist_field->flags & HIST_FIELD_FL_VAR) {
5203			hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5204			var_idx = hist_field->var.idx;
5205			tracing_map_set_var(elt, var_idx, hist_val);
5206		}
5207	}
5208
5209	update_field_vars(hist_data, elt, buffer, rbe, rec);
5210}
5211
5212static inline void add_to_key(char *compound_key, void *key,
5213			      struct hist_field *key_field, void *rec)
5214{
5215	size_t size = key_field->size;
5216
5217	if (key_field->flags & HIST_FIELD_FL_STRING) {
5218		struct ftrace_event_field *field;
5219
5220		field = key_field->field;
5221		if (field->filter_type == FILTER_DYN_STRING ||
5222		    field->filter_type == FILTER_RDYN_STRING)
5223			size = *(u32 *)(rec + field->offset) >> 16;
 
 
5224		else if (field->filter_type == FILTER_STATIC_STRING)
5225			size = field->size;
5226
5227		/* ensure NULL-termination */
5228		if (size > key_field->size - 1)
5229			size = key_field->size - 1;
5230
5231		strncpy(compound_key + key_field->offset, (char *)key, size);
5232	} else
5233		memcpy(compound_key + key_field->offset, key, size);
5234}
5235
5236static void
5237hist_trigger_actions(struct hist_trigger_data *hist_data,
5238		     struct tracing_map_elt *elt,
5239		     struct trace_buffer *buffer, void *rec,
5240		     struct ring_buffer_event *rbe, void *key,
5241		     u64 *var_ref_vals)
5242{
5243	struct action_data *data;
5244	unsigned int i;
5245
5246	for (i = 0; i < hist_data->n_actions; i++) {
5247		data = hist_data->actions[i];
5248		data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
5249	}
5250}
5251
5252static void event_hist_trigger(struct event_trigger_data *data,
5253			       struct trace_buffer *buffer, void *rec,
5254			       struct ring_buffer_event *rbe)
5255{
5256	struct hist_trigger_data *hist_data = data->private_data;
5257	bool use_compound_key = (hist_data->n_keys > 1);
5258	unsigned long entries[HIST_STACKTRACE_DEPTH];
5259	u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5260	char compound_key[HIST_KEY_SIZE_MAX];
5261	struct tracing_map_elt *elt = NULL;
5262	struct hist_field *key_field;
5263	u64 field_contents;
5264	void *key = NULL;
5265	unsigned int i;
5266
5267	if (unlikely(!rbe))
5268		return;
5269
5270	memset(compound_key, 0, hist_data->key_size);
5271
5272	for_each_hist_key_field(i, hist_data) {
5273		key_field = hist_data->fields[i];
5274
5275		if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5276			memset(entries, 0, HIST_STACKTRACE_SIZE);
5277			if (key_field->field) {
5278				unsigned long *stack, n_entries;
5279
5280				field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
5281				stack = (unsigned long *)(long)field_contents;
5282				n_entries = *stack;
5283				memcpy(entries, ++stack, n_entries * sizeof(unsigned long));
5284			} else {
5285				stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5286						 HIST_STACKTRACE_SKIP);
5287			}
5288			key = entries;
5289		} else {
5290			field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
5291			if (key_field->flags & HIST_FIELD_FL_STRING) {
5292				key = (void *)(unsigned long)field_contents;
5293				use_compound_key = true;
5294			} else
5295				key = (void *)&field_contents;
5296		}
5297
5298		if (use_compound_key)
5299			add_to_key(compound_key, key, key_field, rec);
5300	}
5301
5302	if (use_compound_key)
5303		key = compound_key;
5304
5305	if (hist_data->n_var_refs &&
5306	    !resolve_var_refs(hist_data, key, var_ref_vals, false))
5307		return;
5308
5309	elt = tracing_map_insert(hist_data->map, key);
5310	if (!elt)
5311		return;
5312
5313	hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
5314
5315	if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5316		hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
5317}
5318
5319static void hist_trigger_stacktrace_print(struct seq_file *m,
5320					  unsigned long *stacktrace_entries,
5321					  unsigned int max_entries)
5322{
 
5323	unsigned int spaces = 8;
5324	unsigned int i;
5325
5326	for (i = 0; i < max_entries; i++) {
5327		if (!stacktrace_entries[i])
5328			return;
5329
5330		seq_printf(m, "%*c", 1 + spaces, ' ');
5331		seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]);
 
5332	}
5333}
5334
5335static void hist_trigger_print_key(struct seq_file *m,
5336				   struct hist_trigger_data *hist_data,
5337				   void *key,
5338				   struct tracing_map_elt *elt)
5339{
5340	struct hist_field *key_field;
 
5341	bool multiline = false;
5342	const char *field_name;
5343	unsigned int i;
5344	u64 uval;
5345
5346	seq_puts(m, "{ ");
5347
5348	for_each_hist_key_field(i, hist_data) {
5349		key_field = hist_data->fields[i];
5350
5351		if (i > hist_data->n_vals)
5352			seq_puts(m, ", ");
5353
5354		field_name = hist_field_name(key_field, 0);
5355
5356		if (key_field->flags & HIST_FIELD_FL_HEX) {
5357			uval = *(u64 *)(key + key_field->offset);
5358			seq_printf(m, "%s: %llx", field_name, uval);
5359		} else if (key_field->flags & HIST_FIELD_FL_SYM) {
5360			uval = *(u64 *)(key + key_field->offset);
5361			seq_printf(m, "%s: [%llx] %-45ps", field_name,
5362				   uval, (void *)(uintptr_t)uval);
 
5363		} else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5364			uval = *(u64 *)(key + key_field->offset);
5365			seq_printf(m, "%s: [%llx] %-55pS", field_name,
5366				   uval, (void *)(uintptr_t)uval);
 
5367		} else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5368			struct hist_elt_data *elt_data = elt->private_data;
5369			char *comm;
5370
5371			if (WARN_ON_ONCE(!elt_data))
5372				return;
5373
5374			comm = elt_data->comm;
5375
5376			uval = *(u64 *)(key + key_field->offset);
5377			seq_printf(m, "%s: %-16s[%10llu]", field_name,
5378				   comm, uval);
5379		} else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5380			const char *syscall_name;
5381
5382			uval = *(u64 *)(key + key_field->offset);
5383			syscall_name = get_syscall_name(uval);
5384			if (!syscall_name)
5385				syscall_name = "unknown_syscall";
5386
5387			seq_printf(m, "%s: %-30s[%3llu]", field_name,
5388				   syscall_name, uval);
5389		} else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5390			if (key_field->field)
5391				seq_printf(m, "%s.stacktrace", key_field->field->name);
5392			else
5393				seq_puts(m, "common_stacktrace:\n");
5394			hist_trigger_stacktrace_print(m,
5395						      key + key_field->offset,
5396						      HIST_STACKTRACE_DEPTH);
5397			multiline = true;
5398		} else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5399			seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5400				   *(u64 *)(key + key_field->offset));
5401		} else if (key_field->flags & HIST_FIELD_FL_BUCKET) {
5402			unsigned long buckets = key_field->buckets;
5403			uval = *(u64 *)(key + key_field->offset);
5404			seq_printf(m, "%s: ~ %llu-%llu", field_name,
5405				   uval, uval + buckets -1);
5406		} else if (key_field->flags & HIST_FIELD_FL_STRING) {
5407			seq_printf(m, "%s: %-50s", field_name,
5408				   (char *)(key + key_field->offset));
5409		} else {
5410			uval = *(u64 *)(key + key_field->offset);
5411			seq_printf(m, "%s: %10llu", field_name, uval);
5412		}
5413	}
5414
5415	if (!multiline)
5416		seq_puts(m, " ");
5417
5418	seq_puts(m, "}");
5419}
5420
5421/* Get the 100 times of the percentage of @val in @total */
5422static inline unsigned int __get_percentage(u64 val, u64 total)
5423{
5424	if (!total)
5425		goto div0;
5426
5427	if (val < (U64_MAX / 10000))
5428		return (unsigned int)div64_ul(val * 10000, total);
5429
5430	total = div64_u64(total, 10000);
5431	if (!total)
5432		goto div0;
5433
5434	return (unsigned int)div64_ul(val, total);
5435div0:
5436	return val ? UINT_MAX : 0;
5437}
5438
5439#define BAR_CHAR '#'
5440
5441static inline const char *__fill_bar_str(char *buf, int size, u64 val, u64 max)
5442{
5443	unsigned int len = __get_percentage(val, max);
5444	int i;
5445
5446	if (len == UINT_MAX) {
5447		snprintf(buf, size, "[ERROR]");
5448		return buf;
5449	}
5450
5451	len = len * size / 10000;
5452	for (i = 0; i < len && i < size; i++)
5453		buf[i] = BAR_CHAR;
5454	while (i < size)
5455		buf[i++] = ' ';
5456	buf[size] = '\0';
5457
5458	return buf;
5459}
5460
5461struct hist_val_stat {
5462	u64 max;
5463	u64 total;
5464};
5465
5466static void hist_trigger_print_val(struct seq_file *m, unsigned int idx,
5467				   const char *field_name, unsigned long flags,
5468				   struct hist_val_stat *stats,
5469				   struct tracing_map_elt *elt)
5470{
5471	u64 val = tracing_map_read_sum(elt, idx);
5472	unsigned int pc;
5473	char bar[21];
5474
5475	if (flags & HIST_FIELD_FL_PERCENT) {
5476		pc = __get_percentage(val, stats[idx].total);
5477		if (pc == UINT_MAX)
5478			seq_printf(m, " %s (%%):[ERROR]", field_name);
5479		else
5480			seq_printf(m, " %s (%%): %3u.%02u", field_name,
5481					pc / 100, pc % 100);
5482	} else if (flags & HIST_FIELD_FL_GRAPH) {
5483		seq_printf(m, " %s: %20s", field_name,
5484			   __fill_bar_str(bar, 20, val, stats[idx].max));
5485	} else if (flags & HIST_FIELD_FL_HEX) {
5486		seq_printf(m, " %s: %10llx", field_name, val);
5487	} else {
5488		seq_printf(m, " %s: %10llu", field_name, val);
5489	}
5490}
5491
5492static void hist_trigger_entry_print(struct seq_file *m,
5493				     struct hist_trigger_data *hist_data,
5494				     struct hist_val_stat *stats,
5495				     void *key,
5496				     struct tracing_map_elt *elt)
5497{
5498	const char *field_name;
5499	unsigned int i = HITCOUNT_IDX;
5500	unsigned long flags;
5501
5502	hist_trigger_print_key(m, hist_data, key, elt);
5503
5504	/* At first, show the raw hitcount if !nohitcount */
5505	if (!hist_data->attrs->no_hitcount)
5506		hist_trigger_print_val(m, i, "hitcount", 0, stats, elt);
5507
5508	for (i = 1; i < hist_data->n_vals; i++) {
5509		field_name = hist_field_name(hist_data->fields[i], 0);
5510		flags = hist_data->fields[i]->flags;
5511		if (flags & HIST_FIELD_FL_VAR || flags & HIST_FIELD_FL_EXPR)
 
5512			continue;
5513
5514		seq_puts(m, " ");
5515		hist_trigger_print_val(m, i, field_name, flags, stats, elt);
 
 
 
 
 
5516	}
5517
5518	print_actions(m, hist_data, elt);
5519
5520	seq_puts(m, "\n");
5521}
5522
5523static int print_entries(struct seq_file *m,
5524			 struct hist_trigger_data *hist_data)
5525{
5526	struct tracing_map_sort_entry **sort_entries = NULL;
5527	struct tracing_map *map = hist_data->map;
5528	int i, j, n_entries;
5529	struct hist_val_stat *stats = NULL;
5530	u64 val;
5531
5532	n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5533					     hist_data->n_sort_keys,
5534					     &sort_entries);
5535	if (n_entries < 0)
5536		return n_entries;
5537
5538	/* Calculate the max and the total for each field if needed. */
5539	for (j = 0; j < hist_data->n_vals; j++) {
5540		if (!(hist_data->fields[j]->flags &
5541			(HIST_FIELD_FL_PERCENT | HIST_FIELD_FL_GRAPH)))
5542			continue;
5543		if (!stats) {
5544			stats = kcalloc(hist_data->n_vals, sizeof(*stats),
5545				       GFP_KERNEL);
5546			if (!stats) {
5547				n_entries = -ENOMEM;
5548				goto out;
5549			}
5550		}
5551		for (i = 0; i < n_entries; i++) {
5552			val = tracing_map_read_sum(sort_entries[i]->elt, j);
5553			stats[j].total += val;
5554			if (stats[j].max < val)
5555				stats[j].max = val;
5556		}
5557	}
5558
5559	for (i = 0; i < n_entries; i++)
5560		hist_trigger_entry_print(m, hist_data, stats,
5561					 sort_entries[i]->key,
5562					 sort_entries[i]->elt);
5563
5564	kfree(stats);
5565out:
5566	tracing_map_destroy_sort_entries(sort_entries, n_entries);
5567
5568	return n_entries;
5569}
5570
5571static void hist_trigger_show(struct seq_file *m,
5572			      struct event_trigger_data *data, int n)
5573{
5574	struct hist_trigger_data *hist_data;
5575	int n_entries;
5576
5577	if (n > 0)
5578		seq_puts(m, "\n\n");
5579
5580	seq_puts(m, "# event histogram\n#\n# trigger info: ");
5581	data->ops->print(m, data);
5582	seq_puts(m, "#\n\n");
5583
5584	hist_data = data->private_data;
5585	n_entries = print_entries(m, hist_data);
5586	if (n_entries < 0)
5587		n_entries = 0;
5588
5589	track_data_snapshot_print(m, hist_data);
5590
5591	seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5592		   (u64)atomic64_read(&hist_data->map->hits),
5593		   n_entries, (u64)atomic64_read(&hist_data->map->drops));
5594}
5595
5596static int hist_show(struct seq_file *m, void *v)
5597{
5598	struct event_trigger_data *data;
5599	struct trace_event_file *event_file;
5600	int n = 0, ret = 0;
5601
5602	mutex_lock(&event_mutex);
5603
5604	event_file = event_file_data(m->private);
5605	if (unlikely(!event_file)) {
5606		ret = -ENODEV;
5607		goto out_unlock;
5608	}
5609
5610	list_for_each_entry(data, &event_file->triggers, list) {
5611		if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5612			hist_trigger_show(m, data, n++);
5613	}
5614
5615 out_unlock:
5616	mutex_unlock(&event_mutex);
5617
5618	return ret;
5619}
5620
5621static int event_hist_open(struct inode *inode, struct file *file)
5622{
5623	int ret;
5624
5625	ret = tracing_open_file_tr(inode, file);
5626	if (ret)
5627		return ret;
5628
5629	/* Clear private_data to avoid warning in single_open() */
5630	file->private_data = NULL;
5631	return single_open(file, hist_show, file);
5632}
5633
5634const struct file_operations event_hist_fops = {
5635	.open = event_hist_open,
5636	.read = seq_read,
5637	.llseek = seq_lseek,
5638	.release = tracing_single_release_file_tr,
5639};
5640
5641#ifdef CONFIG_HIST_TRIGGERS_DEBUG
5642static void hist_field_debug_show_flags(struct seq_file *m,
5643					unsigned long flags)
5644{
5645	seq_puts(m, "      flags:\n");
5646
5647	if (flags & HIST_FIELD_FL_KEY)
5648		seq_puts(m, "        HIST_FIELD_FL_KEY\n");
5649	else if (flags & HIST_FIELD_FL_HITCOUNT)
5650		seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
5651	else if (flags & HIST_FIELD_FL_VAR)
5652		seq_puts(m, "        HIST_FIELD_FL_VAR\n");
5653	else if (flags & HIST_FIELD_FL_VAR_REF)
5654		seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
5655	else
5656		seq_puts(m, "        VAL: normal u64 value\n");
5657
5658	if (flags & HIST_FIELD_FL_ALIAS)
5659		seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
5660	else if (flags & HIST_FIELD_FL_CONST)
5661		seq_puts(m, "        HIST_FIELD_FL_CONST\n");
5662}
5663
5664static int hist_field_debug_show(struct seq_file *m,
5665				 struct hist_field *field, unsigned long flags)
5666{
5667	if ((field->flags & flags) != flags) {
5668		seq_printf(m, "ERROR: bad flags - %lx\n", flags);
5669		return -EINVAL;
5670	}
5671
5672	hist_field_debug_show_flags(m, field->flags);
5673	if (field->field)
5674		seq_printf(m, "      ftrace_event_field name: %s\n",
5675			   field->field->name);
5676
5677	if (field->flags & HIST_FIELD_FL_VAR) {
5678		seq_printf(m, "      var.name: %s\n", field->var.name);
5679		seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5680			   field->var.idx);
5681	}
5682
5683	if (field->flags & HIST_FIELD_FL_CONST)
5684		seq_printf(m, "      constant: %llu\n", field->constant);
5685
5686	if (field->flags & HIST_FIELD_FL_ALIAS)
5687		seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5688			   field->var_ref_idx);
5689
5690	if (field->flags & HIST_FIELD_FL_VAR_REF) {
5691		seq_printf(m, "      name: %s\n", field->name);
5692		seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5693			   field->var.idx);
5694		seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
5695		seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5696			   field->var_ref_idx);
5697		if (field->system)
5698			seq_printf(m, "      system: %s\n", field->system);
5699		if (field->event_name)
5700			seq_printf(m, "      event_name: %s\n", field->event_name);
5701	}
5702
5703	seq_printf(m, "      type: %s\n", field->type);
5704	seq_printf(m, "      size: %u\n", field->size);
5705	seq_printf(m, "      is_signed: %u\n", field->is_signed);
5706
5707	return 0;
5708}
5709
5710static int field_var_debug_show(struct seq_file *m,
5711				struct field_var *field_var, unsigned int i,
5712				bool save_vars)
5713{
5714	const char *vars_name = save_vars ? "save_vars" : "field_vars";
5715	struct hist_field *field;
5716	int ret = 0;
5717
5718	seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
5719
5720	field = field_var->var;
5721
5722	seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
5723
5724	hist_field_debug_show_flags(m, field->flags);
5725	seq_printf(m, "      var.name: %s\n", field->var.name);
5726	seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5727		   field->var.idx);
5728
5729	field = field_var->val;
5730
5731	seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
5732	if (field->field)
5733		seq_printf(m, "      ftrace_event_field name: %s\n",
5734			   field->field->name);
5735	else {
5736		ret = -EINVAL;
5737		goto out;
5738	}
5739
5740	seq_printf(m, "      type: %s\n", field->type);
5741	seq_printf(m, "      size: %u\n", field->size);
5742	seq_printf(m, "      is_signed: %u\n", field->is_signed);
5743out:
5744	return ret;
5745}
5746
5747static int hist_action_debug_show(struct seq_file *m,
5748				  struct action_data *data, int i)
5749{
5750	int ret = 0;
5751
5752	if (data->handler == HANDLER_ONMAX ||
5753	    data->handler == HANDLER_ONCHANGE) {
5754		seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
5755		ret = hist_field_debug_show(m, data->track_data.var_ref,
5756					    HIST_FIELD_FL_VAR_REF);
5757		if (ret)
5758			goto out;
5759
5760		seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
5761		ret = hist_field_debug_show(m, data->track_data.track_var,
5762					    HIST_FIELD_FL_VAR);
5763		if (ret)
5764			goto out;
5765	}
5766
5767	if (data->handler == HANDLER_ONMATCH) {
5768		seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
5769			   i, data->match_data.event_system);
5770		seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
5771			   i, data->match_data.event);
5772	}
5773out:
5774	return ret;
5775}
5776
5777static int hist_actions_debug_show(struct seq_file *m,
5778				   struct hist_trigger_data *hist_data)
5779{
5780	int i, ret = 0;
5781
5782	if (hist_data->n_actions)
5783		seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
5784
5785	for (i = 0; i < hist_data->n_actions; i++) {
5786		struct action_data *action = hist_data->actions[i];
5787
5788		ret = hist_action_debug_show(m, action, i);
5789		if (ret)
5790			goto out;
5791	}
5792
5793	if (hist_data->n_save_vars)
5794		seq_puts(m, "\n  save action variables (save() params):\n");
5795
5796	for (i = 0; i < hist_data->n_save_vars; i++) {
5797		ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
5798		if (ret)
5799			goto out;
5800	}
5801out:
5802	return ret;
5803}
5804
5805static void hist_trigger_debug_show(struct seq_file *m,
5806				    struct event_trigger_data *data, int n)
5807{
5808	struct hist_trigger_data *hist_data;
5809	int i, ret;
5810
5811	if (n > 0)
5812		seq_puts(m, "\n\n");
5813
5814	seq_puts(m, "# event histogram\n#\n# trigger info: ");
5815	data->ops->print(m, data);
5816	seq_puts(m, "#\n\n");
5817
5818	hist_data = data->private_data;
5819
5820	seq_printf(m, "hist_data: %p\n\n", hist_data);
5821	seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
5822	seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
5823	seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
5824
5825	seq_puts(m, "\n  val fields:\n\n");
5826
5827	seq_puts(m, "    hist_data->fields[0]:\n");
5828	ret = hist_field_debug_show(m, hist_data->fields[0],
5829				    HIST_FIELD_FL_HITCOUNT);
5830	if (ret)
5831		return;
5832
5833	for (i = 1; i < hist_data->n_vals; i++) {
5834		seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5835		ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5836		if (ret)
5837			return;
5838	}
5839
5840	seq_puts(m, "\n  key fields:\n");
5841
5842	for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5843		seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5844		ret = hist_field_debug_show(m, hist_data->fields[i],
5845					    HIST_FIELD_FL_KEY);
5846		if (ret)
5847			return;
5848	}
5849
5850	if (hist_data->n_var_refs)
5851		seq_puts(m, "\n  variable reference fields:\n");
5852
5853	for (i = 0; i < hist_data->n_var_refs; i++) {
5854		seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5855		ret = hist_field_debug_show(m, hist_data->var_refs[i],
5856					    HIST_FIELD_FL_VAR_REF);
5857		if (ret)
5858			return;
5859	}
5860
5861	if (hist_data->n_field_vars)
5862		seq_puts(m, "\n  field variables:\n");
5863
5864	for (i = 0; i < hist_data->n_field_vars; i++) {
5865		ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5866		if (ret)
5867			return;
5868	}
5869
5870	ret = hist_actions_debug_show(m, hist_data);
5871	if (ret)
5872		return;
5873}
5874
5875static int hist_debug_show(struct seq_file *m, void *v)
5876{
5877	struct event_trigger_data *data;
5878	struct trace_event_file *event_file;
5879	int n = 0, ret = 0;
5880
5881	mutex_lock(&event_mutex);
5882
5883	event_file = event_file_data(m->private);
5884	if (unlikely(!event_file)) {
5885		ret = -ENODEV;
5886		goto out_unlock;
5887	}
5888
5889	list_for_each_entry(data, &event_file->triggers, list) {
5890		if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5891			hist_trigger_debug_show(m, data, n++);
5892	}
5893
5894 out_unlock:
5895	mutex_unlock(&event_mutex);
5896
5897	return ret;
5898}
5899
5900static int event_hist_debug_open(struct inode *inode, struct file *file)
5901{
5902	int ret;
5903
5904	ret = tracing_open_file_tr(inode, file);
5905	if (ret)
5906		return ret;
5907
5908	/* Clear private_data to avoid warning in single_open() */
5909	file->private_data = NULL;
5910	return single_open(file, hist_debug_show, file);
5911}
5912
5913const struct file_operations event_hist_debug_fops = {
5914	.open = event_hist_debug_open,
5915	.read = seq_read,
5916	.llseek = seq_lseek,
5917	.release = tracing_single_release_file_tr,
5918};
5919#endif
5920
5921static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5922{
5923	const char *field_name = hist_field_name(hist_field, 0);
5924
5925	if (hist_field->var.name)
5926		seq_printf(m, "%s=", hist_field->var.name);
5927
5928	if (hist_field->flags & HIST_FIELD_FL_CPU)
5929		seq_puts(m, "common_cpu");
5930	else if (hist_field->flags & HIST_FIELD_FL_CONST)
5931		seq_printf(m, "%llu", hist_field->constant);
5932	else if (field_name) {
5933		if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5934		    hist_field->flags & HIST_FIELD_FL_ALIAS)
5935			seq_putc(m, '$');
5936		seq_printf(m, "%s", field_name);
5937	} else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5938		seq_puts(m, "common_timestamp");
5939
5940	if (hist_field->flags) {
5941		if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5942		    !(hist_field->flags & HIST_FIELD_FL_EXPR) &&
5943		    !(hist_field->flags & HIST_FIELD_FL_STACKTRACE)) {
5944			const char *flags = get_hist_field_flags(hist_field);
5945
5946			if (flags)
5947				seq_printf(m, ".%s", flags);
5948		}
5949	}
5950	if (hist_field->buckets)
5951		seq_printf(m, "=%ld", hist_field->buckets);
5952}
5953
5954static int event_hist_trigger_print(struct seq_file *m,
 
5955				    struct event_trigger_data *data)
5956{
5957	struct hist_trigger_data *hist_data = data->private_data;
5958	struct hist_field *field;
5959	bool have_var = false;
5960	bool show_val = false;
5961	unsigned int i;
5962
5963	seq_puts(m, HIST_PREFIX);
5964
5965	if (data->name)
5966		seq_printf(m, "%s:", data->name);
5967
5968	seq_puts(m, "keys=");
5969
5970	for_each_hist_key_field(i, hist_data) {
5971		field = hist_data->fields[i];
5972
5973		if (i > hist_data->n_vals)
5974			seq_puts(m, ",");
5975
5976		if (field->flags & HIST_FIELD_FL_STACKTRACE) {
5977			if (field->field)
5978				seq_printf(m, "%s.stacktrace", field->field->name);
5979			else
5980				seq_puts(m, "common_stacktrace");
5981		} else
5982			hist_field_print(m, field);
5983	}
5984
5985	seq_puts(m, ":vals=");
5986
5987	for_each_hist_val_field(i, hist_data) {
5988		field = hist_data->fields[i];
5989		if (field->flags & HIST_FIELD_FL_VAR) {
5990			have_var = true;
5991			continue;
5992		}
5993
5994		if (i == HITCOUNT_IDX) {
5995			if (hist_data->attrs->no_hitcount)
5996				continue;
5997			seq_puts(m, "hitcount");
5998		} else {
5999			if (show_val)
6000				seq_puts(m, ",");
6001			hist_field_print(m, field);
6002		}
6003		show_val = true;
6004	}
6005
6006	if (have_var) {
6007		unsigned int n = 0;
6008
6009		seq_puts(m, ":");
6010
6011		for_each_hist_val_field(i, hist_data) {
6012			field = hist_data->fields[i];
6013
6014			if (field->flags & HIST_FIELD_FL_VAR) {
6015				if (n++)
6016					seq_puts(m, ",");
6017				hist_field_print(m, field);
6018			}
6019		}
6020	}
6021
6022	seq_puts(m, ":sort=");
6023
6024	for (i = 0; i < hist_data->n_sort_keys; i++) {
6025		struct tracing_map_sort_key *sort_key;
6026		unsigned int idx, first_key_idx;
6027
6028		/* skip VAR vals */
6029		first_key_idx = hist_data->n_vals - hist_data->n_vars;
6030
6031		sort_key = &hist_data->sort_keys[i];
6032		idx = sort_key->field_idx;
6033
6034		if (WARN_ON(idx >= HIST_FIELDS_MAX))
6035			return -EINVAL;
6036
6037		if (i > 0)
6038			seq_puts(m, ",");
6039
6040		if (idx == HITCOUNT_IDX)
6041			seq_puts(m, "hitcount");
6042		else {
6043			if (idx >= first_key_idx)
6044				idx += hist_data->n_vars;
6045			hist_field_print(m, hist_data->fields[idx]);
6046		}
6047
6048		if (sort_key->descending)
6049			seq_puts(m, ".descending");
6050	}
6051	seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
6052	if (hist_data->enable_timestamps)
6053		seq_printf(m, ":clock=%s", hist_data->attrs->clock);
6054	if (hist_data->attrs->no_hitcount)
6055		seq_puts(m, ":nohitcount");
6056
6057	print_actions_spec(m, hist_data);
6058
6059	if (data->filter_str)
6060		seq_printf(m, " if %s", data->filter_str);
6061
6062	if (data->paused)
6063		seq_puts(m, " [paused]");
6064	else
6065		seq_puts(m, " [active]");
6066
6067	seq_putc(m, '\n');
6068
6069	return 0;
6070}
6071
6072static int event_hist_trigger_init(struct event_trigger_data *data)
 
6073{
6074	struct hist_trigger_data *hist_data = data->private_data;
6075
6076	if (!data->ref && hist_data->attrs->name)
6077		save_named_trigger(hist_data->attrs->name, data);
6078
6079	data->ref++;
6080
6081	return 0;
6082}
6083
6084static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
6085{
6086	struct trace_event_file *file;
6087	unsigned int i;
6088	char *cmd;
6089	int ret;
6090
6091	for (i = 0; i < hist_data->n_field_var_hists; i++) {
6092		file = hist_data->field_var_hists[i]->hist_data->event_file;
6093		cmd = hist_data->field_var_hists[i]->cmd;
6094		ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
6095					       "!hist", "hist", cmd);
6096		WARN_ON_ONCE(ret < 0);
6097	}
6098}
6099
6100static void event_hist_trigger_free(struct event_trigger_data *data)
 
6101{
6102	struct hist_trigger_data *hist_data = data->private_data;
6103
6104	if (WARN_ON_ONCE(data->ref <= 0))
6105		return;
6106
6107	data->ref--;
6108	if (!data->ref) {
6109		if (data->name)
6110			del_named_trigger(data);
6111
6112		trigger_data_free(data);
6113
6114		remove_hist_vars(hist_data);
6115
6116		unregister_field_var_hists(hist_data);
6117
6118		destroy_hist_data(hist_data);
6119	}
6120}
6121
6122static struct event_trigger_ops event_hist_trigger_ops = {
6123	.trigger		= event_hist_trigger,
6124	.print			= event_hist_trigger_print,
6125	.init			= event_hist_trigger_init,
6126	.free			= event_hist_trigger_free,
6127};
6128
6129static int event_hist_trigger_named_init(struct event_trigger_data *data)
 
6130{
6131	data->ref++;
6132
6133	save_named_trigger(data->named_data->name, data);
6134
6135	event_hist_trigger_init(data->named_data);
6136
6137	return 0;
6138}
6139
6140static void event_hist_trigger_named_free(struct event_trigger_data *data)
 
6141{
6142	if (WARN_ON_ONCE(data->ref <= 0))
6143		return;
6144
6145	event_hist_trigger_free(data->named_data);
6146
6147	data->ref--;
6148	if (!data->ref) {
6149		del_named_trigger(data);
6150		trigger_data_free(data);
6151	}
6152}
6153
6154static struct event_trigger_ops event_hist_trigger_named_ops = {
6155	.trigger		= event_hist_trigger,
6156	.print			= event_hist_trigger_print,
6157	.init			= event_hist_trigger_named_init,
6158	.free			= event_hist_trigger_named_free,
6159};
6160
6161static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
6162							    char *param)
6163{
6164	return &event_hist_trigger_ops;
6165}
6166
6167static void hist_clear(struct event_trigger_data *data)
6168{
6169	struct hist_trigger_data *hist_data = data->private_data;
6170
6171	if (data->name)
6172		pause_named_trigger(data);
6173
6174	tracepoint_synchronize_unregister();
6175
6176	tracing_map_clear(hist_data->map);
6177
6178	if (data->name)
6179		unpause_named_trigger(data);
6180}
6181
6182static bool compatible_field(struct ftrace_event_field *field,
6183			     struct ftrace_event_field *test_field)
6184{
6185	if (field == test_field)
6186		return true;
6187	if (field == NULL || test_field == NULL)
6188		return false;
6189	if (strcmp(field->name, test_field->name) != 0)
6190		return false;
6191	if (strcmp(field->type, test_field->type) != 0)
6192		return false;
6193	if (field->size != test_field->size)
6194		return false;
6195	if (field->is_signed != test_field->is_signed)
6196		return false;
6197
6198	return true;
6199}
6200
6201static bool hist_trigger_match(struct event_trigger_data *data,
6202			       struct event_trigger_data *data_test,
6203			       struct event_trigger_data *named_data,
6204			       bool ignore_filter)
6205{
6206	struct tracing_map_sort_key *sort_key, *sort_key_test;
6207	struct hist_trigger_data *hist_data, *hist_data_test;
6208	struct hist_field *key_field, *key_field_test;
6209	unsigned int i;
6210
6211	if (named_data && (named_data != data_test) &&
6212	    (named_data != data_test->named_data))
6213		return false;
6214
6215	if (!named_data && is_named_trigger(data_test))
6216		return false;
6217
6218	hist_data = data->private_data;
6219	hist_data_test = data_test->private_data;
6220
6221	if (hist_data->n_vals != hist_data_test->n_vals ||
6222	    hist_data->n_fields != hist_data_test->n_fields ||
6223	    hist_data->n_sort_keys != hist_data_test->n_sort_keys)
6224		return false;
6225
6226	if (!ignore_filter) {
6227		if ((data->filter_str && !data_test->filter_str) ||
6228		   (!data->filter_str && data_test->filter_str))
6229			return false;
6230	}
6231
6232	for_each_hist_field(i, hist_data) {
6233		key_field = hist_data->fields[i];
6234		key_field_test = hist_data_test->fields[i];
6235
6236		if (key_field->flags != key_field_test->flags)
6237			return false;
6238		if (!compatible_field(key_field->field, key_field_test->field))
6239			return false;
6240		if (key_field->offset != key_field_test->offset)
6241			return false;
6242		if (key_field->size != key_field_test->size)
6243			return false;
6244		if (key_field->is_signed != key_field_test->is_signed)
6245			return false;
6246		if (!!key_field->var.name != !!key_field_test->var.name)
6247			return false;
6248		if (key_field->var.name &&
6249		    strcmp(key_field->var.name, key_field_test->var.name) != 0)
6250			return false;
6251	}
6252
6253	for (i = 0; i < hist_data->n_sort_keys; i++) {
6254		sort_key = &hist_data->sort_keys[i];
6255		sort_key_test = &hist_data_test->sort_keys[i];
6256
6257		if (sort_key->field_idx != sort_key_test->field_idx ||
6258		    sort_key->descending != sort_key_test->descending)
6259			return false;
6260	}
6261
6262	if (!ignore_filter && data->filter_str &&
6263	    (strcmp(data->filter_str, data_test->filter_str) != 0))
6264		return false;
6265
6266	if (!actions_match(hist_data, hist_data_test))
6267		return false;
6268
6269	return true;
6270}
6271
6272static bool existing_hist_update_only(char *glob,
6273				      struct event_trigger_data *data,
6274				      struct trace_event_file *file)
6275{
6276	struct hist_trigger_data *hist_data = data->private_data;
6277	struct event_trigger_data *test, *named_data = NULL;
6278	bool updated = false;
6279
6280	if (!hist_data->attrs->pause && !hist_data->attrs->cont &&
6281	    !hist_data->attrs->clear)
6282		goto out;
6283
6284	if (hist_data->attrs->name) {
6285		named_data = find_named_trigger(hist_data->attrs->name);
6286		if (named_data) {
6287			if (!hist_trigger_match(data, named_data, named_data,
6288						true))
6289				goto out;
6290		}
6291	}
6292
6293	if (hist_data->attrs->name && !named_data)
6294		goto out;
6295
6296	list_for_each_entry(test, &file->triggers, list) {
6297		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6298			if (!hist_trigger_match(data, test, named_data, false))
6299				continue;
6300			if (hist_data->attrs->pause)
6301				test->paused = true;
6302			else if (hist_data->attrs->cont)
6303				test->paused = false;
6304			else if (hist_data->attrs->clear)
6305				hist_clear(test);
6306			updated = true;
6307			goto out;
6308		}
6309	}
6310 out:
6311	return updated;
6312}
6313
6314static int hist_register_trigger(char *glob,
6315				 struct event_trigger_data *data,
6316				 struct trace_event_file *file)
6317{
6318	struct hist_trigger_data *hist_data = data->private_data;
6319	struct event_trigger_data *test, *named_data = NULL;
6320	struct trace_array *tr = file->tr;
6321	int ret = 0;
6322
6323	if (hist_data->attrs->name) {
6324		named_data = find_named_trigger(hist_data->attrs->name);
6325		if (named_data) {
6326			if (!hist_trigger_match(data, named_data, named_data,
6327						true)) {
6328				hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
6329				ret = -EINVAL;
6330				goto out;
6331			}
6332		}
6333	}
6334
6335	if (hist_data->attrs->name && !named_data)
6336		goto new;
6337
6338	lockdep_assert_held(&event_mutex);
6339
6340	list_for_each_entry(test, &file->triggers, list) {
6341		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6342			if (hist_trigger_match(data, test, named_data, false)) {
 
 
 
 
 
 
 
 
6343				hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
6344				ret = -EEXIST;
6345				goto out;
6346			}
 
6347		}
6348	}
6349 new:
6350	if (hist_data->attrs->cont || hist_data->attrs->clear) {
6351		hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
6352		ret = -ENOENT;
6353		goto out;
6354	}
6355
6356	if (hist_data->attrs->pause)
6357		data->paused = true;
6358
6359	if (named_data) {
6360		data->private_data = named_data->private_data;
6361		set_named_trigger_data(data, named_data);
6362		data->ops = &event_hist_trigger_named_ops;
6363	}
6364
6365	if (data->ops->init) {
6366		ret = data->ops->init(data);
6367		if (ret < 0)
6368			goto out;
6369	}
6370
6371	if (hist_data->enable_timestamps) {
6372		char *clock = hist_data->attrs->clock;
6373
6374		ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
6375		if (ret) {
6376			hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
6377			goto out;
6378		}
6379
6380		tracing_set_filter_buffering(file->tr, true);
6381	}
6382
6383	if (named_data)
6384		destroy_hist_data(hist_data);
 
 
6385 out:
6386	return ret;
6387}
6388
6389static int hist_trigger_enable(struct event_trigger_data *data,
6390			       struct trace_event_file *file)
6391{
6392	int ret = 0;
6393
6394	list_add_tail_rcu(&data->list, &file->triggers);
6395
6396	update_cond_flag(file);
6397
6398	if (trace_event_trigger_enable_disable(file, 1) < 0) {
6399		list_del_rcu(&data->list);
6400		update_cond_flag(file);
6401		ret--;
6402	}
6403
6404	return ret;
6405}
6406
6407static bool have_hist_trigger_match(struct event_trigger_data *data,
6408				    struct trace_event_file *file)
6409{
6410	struct hist_trigger_data *hist_data = data->private_data;
6411	struct event_trigger_data *test, *named_data = NULL;
6412	bool match = false;
6413
6414	lockdep_assert_held(&event_mutex);
6415
6416	if (hist_data->attrs->name)
6417		named_data = find_named_trigger(hist_data->attrs->name);
6418
6419	list_for_each_entry(test, &file->triggers, list) {
6420		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6421			if (hist_trigger_match(data, test, named_data, false)) {
6422				match = true;
6423				break;
6424			}
6425		}
6426	}
6427
6428	return match;
6429}
6430
6431static bool hist_trigger_check_refs(struct event_trigger_data *data,
6432				    struct trace_event_file *file)
6433{
6434	struct hist_trigger_data *hist_data = data->private_data;
6435	struct event_trigger_data *test, *named_data = NULL;
6436
6437	lockdep_assert_held(&event_mutex);
6438
6439	if (hist_data->attrs->name)
6440		named_data = find_named_trigger(hist_data->attrs->name);
6441
6442	list_for_each_entry(test, &file->triggers, list) {
6443		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6444			if (!hist_trigger_match(data, test, named_data, false))
6445				continue;
6446			hist_data = test->private_data;
6447			if (check_var_refs(hist_data))
6448				return true;
6449			break;
6450		}
6451	}
6452
6453	return false;
6454}
6455
6456static void hist_unregister_trigger(char *glob,
6457				    struct event_trigger_data *data,
6458				    struct trace_event_file *file)
6459{
6460	struct event_trigger_data *test = NULL, *iter, *named_data = NULL;
6461	struct hist_trigger_data *hist_data = data->private_data;
6462
6463	lockdep_assert_held(&event_mutex);
6464
6465	if (hist_data->attrs->name)
6466		named_data = find_named_trigger(hist_data->attrs->name);
6467
6468	list_for_each_entry(iter, &file->triggers, list) {
6469		if (iter->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6470			if (!hist_trigger_match(data, iter, named_data, false))
6471				continue;
6472			test = iter;
6473			list_del_rcu(&test->list);
6474			trace_event_trigger_enable_disable(file, 0);
6475			update_cond_flag(file);
6476			break;
6477		}
6478	}
6479
6480	if (test && test->ops->free)
6481		test->ops->free(test);
6482
6483	if (hist_data->enable_timestamps) {
6484		if (!hist_data->remove || test)
6485			tracing_set_filter_buffering(file->tr, false);
6486	}
6487}
6488
6489static bool hist_file_check_refs(struct trace_event_file *file)
6490{
6491	struct hist_trigger_data *hist_data;
6492	struct event_trigger_data *test;
6493
6494	lockdep_assert_held(&event_mutex);
6495
6496	list_for_each_entry(test, &file->triggers, list) {
6497		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6498			hist_data = test->private_data;
6499			if (check_var_refs(hist_data))
6500				return true;
6501		}
6502	}
6503
6504	return false;
6505}
6506
6507static void hist_unreg_all(struct trace_event_file *file)
6508{
6509	struct event_trigger_data *test, *n;
6510	struct hist_trigger_data *hist_data;
6511	struct synth_event *se;
6512	const char *se_name;
6513
6514	lockdep_assert_held(&event_mutex);
6515
6516	if (hist_file_check_refs(file))
6517		return;
6518
6519	list_for_each_entry_safe(test, n, &file->triggers, list) {
6520		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6521			hist_data = test->private_data;
6522			list_del_rcu(&test->list);
6523			trace_event_trigger_enable_disable(file, 0);
6524
6525			se_name = trace_event_name(file->event_call);
6526			se = find_synth_event(se_name);
6527			if (se)
6528				se->ref--;
6529
6530			update_cond_flag(file);
6531			if (hist_data->enable_timestamps)
6532				tracing_set_filter_buffering(file->tr, false);
6533			if (test->ops->free)
6534				test->ops->free(test);
6535		}
6536	}
6537}
6538
6539static int event_hist_trigger_parse(struct event_command *cmd_ops,
6540				    struct trace_event_file *file,
6541				    char *glob, char *cmd,
6542				    char *param_and_filter)
6543{
6544	unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6545	struct event_trigger_data *trigger_data;
6546	struct hist_trigger_attrs *attrs;
 
6547	struct hist_trigger_data *hist_data;
6548	char *param, *filter, *p, *start;
6549	struct synth_event *se;
6550	const char *se_name;
6551	bool remove;
 
6552	int ret = 0;
6553
6554	lockdep_assert_held(&event_mutex);
6555
6556	if (WARN_ON(!glob))
6557		return -EINVAL;
6558
6559	if (glob[0]) {
6560		hist_err_clear();
6561		last_cmd_set(file, param_and_filter);
6562	}
6563
6564	remove = event_trigger_check_remove(glob);
6565
6566	if (event_trigger_empty_param(param_and_filter))
6567		return -EINVAL;
6568
 
 
 
6569	/*
6570	 * separate the trigger from the filter (k:v [if filter])
6571	 * allowing for whitespace in the trigger
6572	 */
6573	p = param = param_and_filter;
6574	do {
6575		p = strstr(p, "if");
6576		if (!p)
6577			break;
6578		if (p == param_and_filter)
6579			return -EINVAL;
6580		if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6581			p++;
6582			continue;
6583		}
6584		if (p >= param_and_filter + strlen(param_and_filter) - (sizeof("if") - 1) - 1)
6585			return -EINVAL;
6586		if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6587			p++;
6588			continue;
6589		}
6590		break;
6591	} while (1);
6592
6593	if (!p)
6594		filter = NULL;
6595	else {
6596		*(p - 1) = '\0';
6597		filter = strstrip(p);
6598		param = strstrip(param);
6599	}
6600
6601	/*
6602	 * To simplify arithmetic expression parsing, replace occurrences of
6603	 * '.sym-offset' modifier with '.symXoffset'
6604	 */
6605	start = strstr(param, ".sym-offset");
6606	while (start) {
6607		*(start + 4) = 'X';
6608		start = strstr(start + 11, ".sym-offset");
6609	}
6610
6611	attrs = parse_hist_trigger_attrs(file->tr, param);
6612	if (IS_ERR(attrs))
6613		return PTR_ERR(attrs);
6614
6615	if (attrs->map_bits)
6616		hist_trigger_bits = attrs->map_bits;
6617
6618	hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6619	if (IS_ERR(hist_data)) {
6620		destroy_hist_trigger_attrs(attrs);
6621		return PTR_ERR(hist_data);
6622	}
6623
6624	trigger_data = event_trigger_alloc(cmd_ops, cmd, param, hist_data);
 
 
6625	if (!trigger_data) {
6626		ret = -ENOMEM;
6627		goto out_free;
6628	}
6629
6630	ret = event_trigger_set_filter(cmd_ops, file, filter, trigger_data);
6631	if (ret < 0)
6632		goto out_free;
 
 
 
 
 
 
 
 
 
 
 
 
6633
6634	if (remove) {
6635		if (!have_hist_trigger_match(trigger_data, file))
6636			goto out_free;
6637
6638		if (hist_trigger_check_refs(trigger_data, file)) {
6639			ret = -EBUSY;
6640			goto out_free;
6641		}
6642
6643		event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6644		se_name = trace_event_name(file->event_call);
6645		se = find_synth_event(se_name);
6646		if (se)
6647			se->ref--;
6648		ret = 0;
6649		goto out_free;
6650	}
6651
6652	if (existing_hist_update_only(glob, trigger_data, file))
 
 
 
 
 
 
 
 
6653		goto out_free;
6654
6655	ret = event_trigger_register(cmd_ops, file, glob, trigger_data);
6656	if (ret < 0)
6657		goto out_free;
6658
6659	if (get_named_trigger_data(trigger_data))
6660		goto enable;
6661
 
 
 
6662	ret = create_actions(hist_data);
6663	if (ret)
6664		goto out_unreg;
6665
6666	if (has_hist_vars(hist_data) || hist_data->n_var_refs) {
6667		ret = save_hist_vars(hist_data);
6668		if (ret)
6669			goto out_unreg;
6670	}
6671
6672	ret = tracing_map_init(hist_data->map);
6673	if (ret)
6674		goto out_unreg;
6675enable:
6676	ret = hist_trigger_enable(trigger_data, file);
6677	if (ret)
6678		goto out_unreg;
6679
6680	se_name = trace_event_name(file->event_call);
6681	se = find_synth_event(se_name);
6682	if (se)
6683		se->ref++;
 
 
6684 out:
6685	if (ret == 0 && glob[0])
6686		hist_err_clear();
6687
6688	return ret;
6689 out_unreg:
6690	event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6691 out_free:
6692	event_trigger_reset_filter(cmd_ops, trigger_data);
 
6693
6694	remove_hist_vars(hist_data);
6695
6696	kfree(trigger_data);
6697
6698	destroy_hist_data(hist_data);
6699	goto out;
6700}
6701
6702static struct event_command trigger_hist_cmd = {
6703	.name			= "hist",
6704	.trigger_type		= ETT_EVENT_HIST,
6705	.flags			= EVENT_CMD_FL_NEEDS_REC,
6706	.parse			= event_hist_trigger_parse,
6707	.reg			= hist_register_trigger,
6708	.unreg			= hist_unregister_trigger,
6709	.unreg_all		= hist_unreg_all,
6710	.get_trigger_ops	= event_hist_get_trigger_ops,
6711	.set_filter		= set_trigger_filter,
6712};
6713
6714__init int register_trigger_hist_cmd(void)
6715{
6716	int ret;
6717
6718	ret = register_event_command(&trigger_hist_cmd);
6719	WARN_ON(ret < 0);
6720
6721	return ret;
6722}
6723
6724static void
6725hist_enable_trigger(struct event_trigger_data *data,
6726		    struct trace_buffer *buffer,  void *rec,
6727		    struct ring_buffer_event *event)
6728{
6729	struct enable_trigger_data *enable_data = data->private_data;
6730	struct event_trigger_data *test;
6731
6732	list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6733				lockdep_is_held(&event_mutex)) {
6734		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6735			if (enable_data->enable)
6736				test->paused = false;
6737			else
6738				test->paused = true;
6739		}
6740	}
6741}
6742
6743static void
6744hist_enable_count_trigger(struct event_trigger_data *data,
6745			  struct trace_buffer *buffer,  void *rec,
6746			  struct ring_buffer_event *event)
6747{
6748	if (!data->count)
6749		return;
6750
6751	if (data->count != -1)
6752		(data->count)--;
6753
6754	hist_enable_trigger(data, buffer, rec, event);
6755}
6756
6757static struct event_trigger_ops hist_enable_trigger_ops = {
6758	.trigger		= hist_enable_trigger,
6759	.print			= event_enable_trigger_print,
6760	.init			= event_trigger_init,
6761	.free			= event_enable_trigger_free,
6762};
6763
6764static struct event_trigger_ops hist_enable_count_trigger_ops = {
6765	.trigger		= hist_enable_count_trigger,
6766	.print			= event_enable_trigger_print,
6767	.init			= event_trigger_init,
6768	.free			= event_enable_trigger_free,
6769};
6770
6771static struct event_trigger_ops hist_disable_trigger_ops = {
6772	.trigger		= hist_enable_trigger,
6773	.print			= event_enable_trigger_print,
6774	.init			= event_trigger_init,
6775	.free			= event_enable_trigger_free,
6776};
6777
6778static struct event_trigger_ops hist_disable_count_trigger_ops = {
6779	.trigger		= hist_enable_count_trigger,
6780	.print			= event_enable_trigger_print,
6781	.init			= event_trigger_init,
6782	.free			= event_enable_trigger_free,
6783};
6784
6785static struct event_trigger_ops *
6786hist_enable_get_trigger_ops(char *cmd, char *param)
6787{
6788	struct event_trigger_ops *ops;
6789	bool enable;
6790
6791	enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6792
6793	if (enable)
6794		ops = param ? &hist_enable_count_trigger_ops :
6795			&hist_enable_trigger_ops;
6796	else
6797		ops = param ? &hist_disable_count_trigger_ops :
6798			&hist_disable_trigger_ops;
6799
6800	return ops;
6801}
6802
6803static void hist_enable_unreg_all(struct trace_event_file *file)
6804{
6805	struct event_trigger_data *test, *n;
6806
6807	list_for_each_entry_safe(test, n, &file->triggers, list) {
6808		if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6809			list_del_rcu(&test->list);
6810			update_cond_flag(file);
6811			trace_event_trigger_enable_disable(file, 0);
6812			if (test->ops->free)
6813				test->ops->free(test);
6814		}
6815	}
6816}
6817
6818static struct event_command trigger_hist_enable_cmd = {
6819	.name			= ENABLE_HIST_STR,
6820	.trigger_type		= ETT_HIST_ENABLE,
6821	.parse			= event_enable_trigger_parse,
6822	.reg			= event_enable_register_trigger,
6823	.unreg			= event_enable_unregister_trigger,
6824	.unreg_all		= hist_enable_unreg_all,
6825	.get_trigger_ops	= hist_enable_get_trigger_ops,
6826	.set_filter		= set_trigger_filter,
6827};
6828
6829static struct event_command trigger_hist_disable_cmd = {
6830	.name			= DISABLE_HIST_STR,
6831	.trigger_type		= ETT_HIST_ENABLE,
6832	.parse			= event_enable_trigger_parse,
6833	.reg			= event_enable_register_trigger,
6834	.unreg			= event_enable_unregister_trigger,
6835	.unreg_all		= hist_enable_unreg_all,
6836	.get_trigger_ops	= hist_enable_get_trigger_ops,
6837	.set_filter		= set_trigger_filter,
6838};
6839
6840static __init void unregister_trigger_hist_enable_disable_cmds(void)
6841{
6842	unregister_event_command(&trigger_hist_enable_cmd);
6843	unregister_event_command(&trigger_hist_disable_cmd);
6844}
6845
6846__init int register_trigger_hist_enable_disable_cmds(void)
6847{
6848	int ret;
6849
6850	ret = register_event_command(&trigger_hist_enable_cmd);
6851	if (WARN_ON(ret < 0))
6852		return ret;
6853	ret = register_event_command(&trigger_hist_disable_cmd);
6854	if (WARN_ON(ret < 0))
6855		unregister_trigger_hist_enable_disable_cmds();
6856
6857	return ret;
6858}