1/*
   2 * trace_events_hist - trace event hist triggers
   3 *
   4 * This program is free software; you can redistribute it and/or modify
   5 * it under the terms of the GNU General Public License as published by
   6 * the Free Software Foundation; either version 2 of the License, or
   7 * (at your option) any later version.
   8 *
   9 * This program is distributed in the hope that it will be useful,
  10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 * GNU General Public License for more details.
  13 *
  14 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
  15 */
  16
  17#include <linux/module.h>
  18#include <linux/kallsyms.h>
  19#include <linux/mutex.h>
  20#include <linux/slab.h>
  21#include <linux/stacktrace.h>
  22#include <linux/rculist.h>
  23#include <linux/tracefs.h>
  24
  25#include "tracing_map.h"
  26#include "trace.h"
  27
  28#define SYNTH_SYSTEM		"synthetic"
  29#define SYNTH_FIELDS_MAX	16
  30
  31#define STR_VAR_LEN_MAX		32 /* must be multiple of sizeof(u64) */
  32
  33struct hist_field;
  34
  35typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  36				struct tracing_map_elt *elt,
  37				struct ring_buffer_event *rbe,
  38				void *event);
  39
  40#define HIST_FIELD_OPERANDS_MAX	2
  41#define HIST_FIELDS_MAX		(TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  42#define HIST_ACTIONS_MAX	8
  43
  44enum field_op_id {
  45	FIELD_OP_NONE,
  46	FIELD_OP_PLUS,
  47	FIELD_OP_MINUS,
  48	FIELD_OP_UNARY_MINUS,
  49};
  50
  51struct hist_var {
  52	char				*name;
  53	struct hist_trigger_data	*hist_data;
  54	unsigned int			idx;
  55};
  56
  57struct hist_field {
  58	struct ftrace_event_field	*field;
  59	unsigned long			flags;
  60	hist_field_fn_t			fn;
  61	unsigned int			size;
  62	unsigned int			offset;
  63	unsigned int                    is_signed;
  64	const char			*type;
  65	struct hist_field		*operands[HIST_FIELD_OPERANDS_MAX];
  66	struct hist_trigger_data	*hist_data;
  67	struct hist_var			var;
  68	enum field_op_id		operator;
  69	char				*system;
  70	char				*event_name;
  71	char				*name;
  72	unsigned int			var_idx;
  73	unsigned int			var_ref_idx;
  74	bool                            read_once;
  75};
  76
  77static u64 hist_field_none(struct hist_field *field,
  78			   struct tracing_map_elt *elt,
  79			   struct ring_buffer_event *rbe,
  80			   void *event)
  81{
  82	return 0;
  83}
  84
  85static u64 hist_field_counter(struct hist_field *field,
  86			      struct tracing_map_elt *elt,
  87			      struct ring_buffer_event *rbe,
  88			      void *event)
  89{
  90	return 1;
  91}
  92
  93static u64 hist_field_string(struct hist_field *hist_field,
  94			     struct tracing_map_elt *elt,
  95			     struct ring_buffer_event *rbe,
  96			     void *event)
  97{
  98	char *addr = (char *)(event + hist_field->field->offset);
  99
 100	return (u64)(unsigned long)addr;
 101}
 102
 103static u64 hist_field_dynstring(struct hist_field *hist_field,
 104				struct tracing_map_elt *elt,
 105				struct ring_buffer_event *rbe,
 106				void *event)
 107{
 108	u32 str_item = *(u32 *)(event + hist_field->field->offset);
 109	int str_loc = str_item & 0xffff;
 110	char *addr = (char *)(event + str_loc);
 111
 112	return (u64)(unsigned long)addr;
 113}
 114
 115static u64 hist_field_pstring(struct hist_field *hist_field,
 116			      struct tracing_map_elt *elt,
 117			      struct ring_buffer_event *rbe,
 118			      void *event)
 119{
 120	char **addr = (char **)(event + hist_field->field->offset);
 121
 122	return (u64)(unsigned long)*addr;
 123}
 124
 125static u64 hist_field_log2(struct hist_field *hist_field,
 126			   struct tracing_map_elt *elt,
 127			   struct ring_buffer_event *rbe,
 128			   void *event)
 129{
 130	struct hist_field *operand = hist_field->operands[0];
 131
 132	u64 val = operand->fn(operand, elt, rbe, event);
 133
 134	return (u64) ilog2(roundup_pow_of_two(val));
 135}
 136
 137static u64 hist_field_plus(struct hist_field *hist_field,
 138			   struct tracing_map_elt *elt,
 139			   struct ring_buffer_event *rbe,
 140			   void *event)
 141{
 142	struct hist_field *operand1 = hist_field->operands[0];
 143	struct hist_field *operand2 = hist_field->operands[1];
 144
 145	u64 val1 = operand1->fn(operand1, elt, rbe, event);
 146	u64 val2 = operand2->fn(operand2, elt, rbe, event);
 147
 148	return val1 + val2;
 149}
 150
 151static u64 hist_field_minus(struct hist_field *hist_field,
 152			    struct tracing_map_elt *elt,
 153			    struct ring_buffer_event *rbe,
 154			    void *event)
 155{
 156	struct hist_field *operand1 = hist_field->operands[0];
 157	struct hist_field *operand2 = hist_field->operands[1];
 158
 159	u64 val1 = operand1->fn(operand1, elt, rbe, event);
 160	u64 val2 = operand2->fn(operand2, elt, rbe, event);
 161
 162	return val1 - val2;
 163}
 164
 165static u64 hist_field_unary_minus(struct hist_field *hist_field,
 166				  struct tracing_map_elt *elt,
 167				  struct ring_buffer_event *rbe,
 168				  void *event)
 169{
 170	struct hist_field *operand = hist_field->operands[0];
 171
 172	s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 173	u64 val = (u64)-sval;
 174
 175	return val;
 176}
 177
 178#define DEFINE_HIST_FIELD_FN(type)					\
 179	static u64 hist_field_##type(struct hist_field *hist_field,	\
 180				     struct tracing_map_elt *elt,	\
 181				     struct ring_buffer_event *rbe,	\
 182				     void *event)			\
 183{									\
 184	type *addr = (type *)(event + hist_field->field->offset);	\
 185									\
 186	return (u64)(unsigned long)*addr;				\
 187}
 188
 189DEFINE_HIST_FIELD_FN(s64);
 190DEFINE_HIST_FIELD_FN(u64);
 191DEFINE_HIST_FIELD_FN(s32);
 192DEFINE_HIST_FIELD_FN(u32);
 193DEFINE_HIST_FIELD_FN(s16);
 194DEFINE_HIST_FIELD_FN(u16);
 195DEFINE_HIST_FIELD_FN(s8);
 196DEFINE_HIST_FIELD_FN(u8);
 197
 198#define for_each_hist_field(i, hist_data)	\
 199	for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 200
 201#define for_each_hist_val_field(i, hist_data)	\
 202	for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 203
 204#define for_each_hist_key_field(i, hist_data)	\
 205	for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 206
 207#define HIST_STACKTRACE_DEPTH	16
 208#define HIST_STACKTRACE_SIZE	(HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 209#define HIST_STACKTRACE_SKIP	5
 210
 211#define HITCOUNT_IDX		0
 212#define HIST_KEY_SIZE_MAX	(MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 213
 214enum hist_field_flags {
 215	HIST_FIELD_FL_HITCOUNT		= 1 << 0,
 216	HIST_FIELD_FL_KEY		= 1 << 1,
 217	HIST_FIELD_FL_STRING		= 1 << 2,
 218	HIST_FIELD_FL_HEX		= 1 << 3,
 219	HIST_FIELD_FL_SYM		= 1 << 4,
 220	HIST_FIELD_FL_SYM_OFFSET	= 1 << 5,
 221	HIST_FIELD_FL_EXECNAME		= 1 << 6,
 222	HIST_FIELD_FL_SYSCALL		= 1 << 7,
 223	HIST_FIELD_FL_STACKTRACE	= 1 << 8,
 224	HIST_FIELD_FL_LOG2		= 1 << 9,
 225	HIST_FIELD_FL_TIMESTAMP		= 1 << 10,
 226	HIST_FIELD_FL_TIMESTAMP_USECS	= 1 << 11,
 227	HIST_FIELD_FL_VAR		= 1 << 12,
 228	HIST_FIELD_FL_EXPR		= 1 << 13,
 229	HIST_FIELD_FL_VAR_REF		= 1 << 14,
 230	HIST_FIELD_FL_CPU		= 1 << 15,
 231	HIST_FIELD_FL_ALIAS		= 1 << 16,
 232};
 233
 234struct var_defs {
 235	unsigned int	n_vars;
 236	char		*name[TRACING_MAP_VARS_MAX];
 237	char		*expr[TRACING_MAP_VARS_MAX];
 238};
 239
 240struct hist_trigger_attrs {
 241	char		*keys_str;
 242	char		*vals_str;
 243	char		*sort_key_str;
 244	char		*name;
 245	char		*clock;
 246	bool		pause;
 247	bool		cont;
 248	bool		clear;
 249	bool		ts_in_usecs;
 250	unsigned int	map_bits;
 251
 252	char		*assignment_str[TRACING_MAP_VARS_MAX];
 253	unsigned int	n_assignments;
 254
 255	char		*action_str[HIST_ACTIONS_MAX];
 256	unsigned int	n_actions;
 257
 258	struct var_defs	var_defs;
 259};
 260
 261struct field_var {
 262	struct hist_field	*var;
 263	struct hist_field	*val;
 264};
 265
 266struct field_var_hist {
 267	struct hist_trigger_data	*hist_data;
 268	char				*cmd;
 269};
 270
 271struct hist_trigger_data {
 272	struct hist_field               *fields[HIST_FIELDS_MAX];
 273	unsigned int			n_vals;
 274	unsigned int			n_keys;
 275	unsigned int			n_fields;
 276	unsigned int			n_vars;
 277	unsigned int			key_size;
 278	struct tracing_map_sort_key	sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 279	unsigned int			n_sort_keys;
 280	struct trace_event_file		*event_file;
 281	struct hist_trigger_attrs	*attrs;
 282	struct tracing_map		*map;
 283	bool				enable_timestamps;
 284	bool				remove;
 285	struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 286	unsigned int			n_var_refs;
 287
 288	struct action_data		*actions[HIST_ACTIONS_MAX];
 289	unsigned int			n_actions;
 290
 291	struct hist_field               *synth_var_refs[SYNTH_FIELDS_MAX];
 292	unsigned int                    n_synth_var_refs;
 293	struct field_var		*field_vars[SYNTH_FIELDS_MAX];
 294	unsigned int			n_field_vars;
 295	unsigned int			n_field_var_str;
 296	struct field_var_hist		*field_var_hists[SYNTH_FIELDS_MAX];
 297	unsigned int			n_field_var_hists;
 298
 299	struct field_var		*max_vars[SYNTH_FIELDS_MAX];
 300	unsigned int			n_max_vars;
 301	unsigned int			n_max_var_str;
 302};
 303
 304struct synth_field {
 305	char *type;
 306	char *name;
 307	size_t size;
 308	bool is_signed;
 309	bool is_string;
 310};
 311
 312struct synth_event {
 313	struct list_head			list;
 314	int					ref;
 315	char					*name;
 316	struct synth_field			**fields;
 317	unsigned int				n_fields;
 318	unsigned int				n_u64;
 319	struct trace_event_class		class;
 320	struct trace_event_call			call;
 321	struct tracepoint			*tp;
 322};
 323
 324struct action_data;
 325
 326typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 327			     struct tracing_map_elt *elt, void *rec,
 328			     struct ring_buffer_event *rbe,
 329			     struct action_data *data, u64 *var_ref_vals);
 330
 331struct action_data {
 332	action_fn_t		fn;
 333	unsigned int		n_params;
 334	char			*params[SYNTH_FIELDS_MAX];
 335
 336	union {
 337		struct {
 338			unsigned int		var_ref_idx;
 339			char			*match_event;
 340			char			*match_event_system;
 341			char			*synth_event_name;
 342			struct synth_event	*synth_event;
 343		} onmatch;
 344
 345		struct {
 346			char			*var_str;
 347			char			*fn_name;
 348			unsigned int		max_var_ref_idx;
 349			struct hist_field	*max_var;
 350			struct hist_field	*var;
 351		} onmax;
 352	};
 353};
 354
 355
 356static char last_hist_cmd[MAX_FILTER_STR_VAL];
 357static char hist_err_str[MAX_FILTER_STR_VAL];
 358
 359static void last_cmd_set(char *str)
 360{
 361	if (!str)
 362		return;
 363
 364	strncpy(last_hist_cmd, str, MAX_FILTER_STR_VAL - 1);
 365}
 366
 367static void hist_err(char *str, char *var)
 368{
 369	int maxlen = MAX_FILTER_STR_VAL - 1;
 370
 371	if (!str)
 372		return;
 373
 374	if (strlen(hist_err_str))
 375		return;
 376
 377	if (!var)
 378		var = "";
 379
 380	if (strlen(hist_err_str) + strlen(str) + strlen(var) > maxlen)
 381		return;
 382
 383	strcat(hist_err_str, str);
 384	strcat(hist_err_str, var);
 385}
 386
 387static void hist_err_event(char *str, char *system, char *event, char *var)
 388{
 389	char err[MAX_FILTER_STR_VAL];
 390
 391	if (system && var)
 392		snprintf(err, MAX_FILTER_STR_VAL, "%s.%s.%s", system, event, var);
 393	else if (system)
 394		snprintf(err, MAX_FILTER_STR_VAL, "%s.%s", system, event);
 395	else
 396		strncpy(err, var, MAX_FILTER_STR_VAL);
 397
 398	hist_err(str, err);
 399}
 400
 401static void hist_err_clear(void)
 402{
 403	hist_err_str[0] = '\0';
 404}
 405
 406static bool have_hist_err(void)
 407{
 408	if (strlen(hist_err_str))
 409		return true;
 410
 411	return false;
 412}
 413
 414static LIST_HEAD(synth_event_list);
 415static DEFINE_MUTEX(synth_event_mutex);
 416
 417struct synth_trace_event {
 418	struct trace_entry	ent;
 419	u64			fields[];
 420};
 421
 422static int synth_event_define_fields(struct trace_event_call *call)
 423{
 424	struct synth_trace_event trace;
 425	int offset = offsetof(typeof(trace), fields);
 426	struct synth_event *event = call->data;
 427	unsigned int i, size, n_u64;
 428	char *name, *type;
 429	bool is_signed;
 430	int ret = 0;
 431
 432	for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 433		size = event->fields[i]->size;
 434		is_signed = event->fields[i]->is_signed;
 435		type = event->fields[i]->type;
 436		name = event->fields[i]->name;
 437		ret = trace_define_field(call, type, name, offset, size,
 438					 is_signed, FILTER_OTHER);
 439		if (ret)
 440			break;
 441
 442		if (event->fields[i]->is_string) {
 443			offset += STR_VAR_LEN_MAX;
 444			n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 445		} else {
 446			offset += sizeof(u64);
 447			n_u64++;
 448		}
 449	}
 450
 451	event->n_u64 = n_u64;
 452
 453	return ret;
 454}
 455
 456static bool synth_field_signed(char *type)
 457{
 458	if (strncmp(type, "u", 1) == 0)
 459		return false;
 460
 461	return true;
 462}
 463
 464static int synth_field_is_string(char *type)
 465{
 466	if (strstr(type, "char[") != NULL)
 467		return true;
 468
 469	return false;
 470}
 471
 472static int synth_field_string_size(char *type)
 473{
 474	char buf[4], *end, *start;
 475	unsigned int len;
 476	int size, err;
 477
 478	start = strstr(type, "char[");
 479	if (start == NULL)
 480		return -EINVAL;
 481	start += strlen("char[");
 482
 483	end = strchr(type, ']');
 484	if (!end || end < start)
 485		return -EINVAL;
 486
 487	len = end - start;
 488	if (len > 3)
 489		return -EINVAL;
 490
 491	strncpy(buf, start, len);
 492	buf[len] = '\0';
 493
 494	err = kstrtouint(buf, 0, &size);
 495	if (err)
 496		return err;
 497
 498	if (size > STR_VAR_LEN_MAX)
 499		return -EINVAL;
 500
 501	return size;
 502}
 503
 504static int synth_field_size(char *type)
 505{
 506	int size = 0;
 507
 508	if (strcmp(type, "s64") == 0)
 509		size = sizeof(s64);
 510	else if (strcmp(type, "u64") == 0)
 511		size = sizeof(u64);
 512	else if (strcmp(type, "s32") == 0)
 513		size = sizeof(s32);
 514	else if (strcmp(type, "u32") == 0)
 515		size = sizeof(u32);
 516	else if (strcmp(type, "s16") == 0)
 517		size = sizeof(s16);
 518	else if (strcmp(type, "u16") == 0)
 519		size = sizeof(u16);
 520	else if (strcmp(type, "s8") == 0)
 521		size = sizeof(s8);
 522	else if (strcmp(type, "u8") == 0)
 523		size = sizeof(u8);
 524	else if (strcmp(type, "char") == 0)
 525		size = sizeof(char);
 526	else if (strcmp(type, "unsigned char") == 0)
 527		size = sizeof(unsigned char);
 528	else if (strcmp(type, "int") == 0)
 529		size = sizeof(int);
 530	else if (strcmp(type, "unsigned int") == 0)
 531		size = sizeof(unsigned int);
 532	else if (strcmp(type, "long") == 0)
 533		size = sizeof(long);
 534	else if (strcmp(type, "unsigned long") == 0)
 535		size = sizeof(unsigned long);
 536	else if (strcmp(type, "pid_t") == 0)
 537		size = sizeof(pid_t);
 538	else if (synth_field_is_string(type))
 539		size = synth_field_string_size(type);
 540
 541	return size;
 542}
 543
 544static const char *synth_field_fmt(char *type)
 545{
 546	const char *fmt = "%llu";
 547
 548	if (strcmp(type, "s64") == 0)
 549		fmt = "%lld";
 550	else if (strcmp(type, "u64") == 0)
 551		fmt = "%llu";
 552	else if (strcmp(type, "s32") == 0)
 553		fmt = "%d";
 554	else if (strcmp(type, "u32") == 0)
 555		fmt = "%u";
 556	else if (strcmp(type, "s16") == 0)
 557		fmt = "%d";
 558	else if (strcmp(type, "u16") == 0)
 559		fmt = "%u";
 560	else if (strcmp(type, "s8") == 0)
 561		fmt = "%d";
 562	else if (strcmp(type, "u8") == 0)
 563		fmt = "%u";
 564	else if (strcmp(type, "char") == 0)
 565		fmt = "%d";
 566	else if (strcmp(type, "unsigned char") == 0)
 567		fmt = "%u";
 568	else if (strcmp(type, "int") == 0)
 569		fmt = "%d";
 570	else if (strcmp(type, "unsigned int") == 0)
 571		fmt = "%u";
 572	else if (strcmp(type, "long") == 0)
 573		fmt = "%ld";
 574	else if (strcmp(type, "unsigned long") == 0)
 575		fmt = "%lu";
 576	else if (strcmp(type, "pid_t") == 0)
 577		fmt = "%d";
 578	else if (synth_field_is_string(type))
 579		fmt = "%s";
 580
 581	return fmt;
 582}
 583
 584static enum print_line_t print_synth_event(struct trace_iterator *iter,
 585					   int flags,
 586					   struct trace_event *event)
 587{
 588	struct trace_array *tr = iter->tr;
 589	struct trace_seq *s = &iter->seq;
 590	struct synth_trace_event *entry;
 591	struct synth_event *se;
 592	unsigned int i, n_u64;
 593	char print_fmt[32];
 594	const char *fmt;
 595
 596	entry = (struct synth_trace_event *)iter->ent;
 597	se = container_of(event, struct synth_event, call.event);
 598
 599	trace_seq_printf(s, "%s: ", se->name);
 600
 601	for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
 602		if (trace_seq_has_overflowed(s))
 603			goto end;
 604
 605		fmt = synth_field_fmt(se->fields[i]->type);
 606
 607		/* parameter types */
 608		if (tr->trace_flags & TRACE_ITER_VERBOSE)
 609			trace_seq_printf(s, "%s ", fmt);
 610
 611		snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
 612
 613		/* parameter values */
 614		if (se->fields[i]->is_string) {
 615			trace_seq_printf(s, print_fmt, se->fields[i]->name,
 616					 (char *)&entry->fields[n_u64],
 617					 i == se->n_fields - 1 ? "" : " ");
 618			n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 619		} else {
 620			trace_seq_printf(s, print_fmt, se->fields[i]->name,
 621					 entry->fields[n_u64],
 622					 i == se->n_fields - 1 ? "" : " ");
 623			n_u64++;
 624		}
 625	}
 626end:
 627	trace_seq_putc(s, '\n');
 628
 629	return trace_handle_return(s);
 630}
 631
 632static struct trace_event_functions synth_event_funcs = {
 633	.trace		= print_synth_event
 634};
 635
 636static notrace void trace_event_raw_event_synth(void *__data,
 637						u64 *var_ref_vals,
 638						unsigned int var_ref_idx)
 639{
 640	struct trace_event_file *trace_file = __data;
 641	struct synth_trace_event *entry;
 642	struct trace_event_buffer fbuffer;
 643	struct ring_buffer *buffer;
 644	struct synth_event *event;
 645	unsigned int i, n_u64;
 646	int fields_size = 0;
 647
 648	event = trace_file->event_call->data;
 649
 650	if (trace_trigger_soft_disabled(trace_file))
 651		return;
 652
 653	fields_size = event->n_u64 * sizeof(u64);
 654
 655	/*
 656	 * Avoid ring buffer recursion detection, as this event
 657	 * is being performed within another event.
 658	 */
 659	buffer = trace_file->tr->trace_buffer.buffer;
 660	ring_buffer_nest_start(buffer);
 661
 662	entry = trace_event_buffer_reserve(&fbuffer, trace_file,
 663					   sizeof(*entry) + fields_size);
 664	if (!entry)
 665		goto out;
 666
 667	for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 668		if (event->fields[i]->is_string) {
 669			char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i];
 670			char *str_field = (char *)&entry->fields[n_u64];
 671
 672			strscpy(str_field, str_val, STR_VAR_LEN_MAX);
 673			n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 674		} else {
 675			entry->fields[n_u64] = var_ref_vals[var_ref_idx + i];
 676			n_u64++;
 677		}
 678	}
 679
 680	trace_event_buffer_commit(&fbuffer);
 681out:
 682	ring_buffer_nest_end(buffer);
 683}
 684
 685static void free_synth_event_print_fmt(struct trace_event_call *call)
 686{
 687	if (call) {
 688		kfree(call->print_fmt);
 689		call->print_fmt = NULL;
 690	}
 691}
 692
 693static int __set_synth_event_print_fmt(struct synth_event *event,
 694				       char *buf, int len)
 695{
 696	const char *fmt;
 697	int pos = 0;
 698	int i;
 699
 700	/* When len=0, we just calculate the needed length */
 701#define LEN_OR_ZERO (len ? len - pos : 0)
 702
 703	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 704	for (i = 0; i < event->n_fields; i++) {
 705		fmt = synth_field_fmt(event->fields[i]->type);
 706		pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
 707				event->fields[i]->name, fmt,
 708				i == event->n_fields - 1 ? "" : ", ");
 709	}
 710	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 711
 712	for (i = 0; i < event->n_fields; i++) {
 713		pos += snprintf(buf + pos, LEN_OR_ZERO,
 714				", REC->%s", event->fields[i]->name);
 715	}
 716
 717#undef LEN_OR_ZERO
 718
 719	/* return the length of print_fmt */
 720	return pos;
 721}
 722
 723static int set_synth_event_print_fmt(struct trace_event_call *call)
 724{
 725	struct synth_event *event = call->data;
 726	char *print_fmt;
 727	int len;
 728
 729	/* First: called with 0 length to calculate the needed length */
 730	len = __set_synth_event_print_fmt(event, NULL, 0);
 731
 732	print_fmt = kmalloc(len + 1, GFP_KERNEL);
 733	if (!print_fmt)
 734		return -ENOMEM;
 735
 736	/* Second: actually write the @print_fmt */
 737	__set_synth_event_print_fmt(event, print_fmt, len + 1);
 738	call->print_fmt = print_fmt;
 739
 740	return 0;
 741}
 742
 743static void free_synth_field(struct synth_field *field)
 744{
 745	kfree(field->type);
 746	kfree(field->name);
 747	kfree(field);
 748}
 749
 750static struct synth_field *parse_synth_field(char *field_type,
 751					     char *field_name)
 752{
 753	struct synth_field *field;
 754	int len, ret = 0;
 755	char *array;
 756
 757	if (field_type[0] == ';')
 758		field_type++;
 759
 760	len = strlen(field_name);
 761	if (field_name[len - 1] == ';')
 762		field_name[len - 1] = '\0';
 763
 764	field = kzalloc(sizeof(*field), GFP_KERNEL);
 765	if (!field)
 766		return ERR_PTR(-ENOMEM);
 767
 768	len = strlen(field_type) + 1;
 769	array = strchr(field_name, '[');
 770	if (array)
 771		len += strlen(array);
 772	field->type = kzalloc(len, GFP_KERNEL);
 773	if (!field->type) {
 774		ret = -ENOMEM;
 775		goto free;
 776	}
 777	strcat(field->type, field_type);
 778	if (array) {
 779		strcat(field->type, array);
 780		*array = '\0';
 781	}
 782
 783	field->size = synth_field_size(field->type);
 784	if (!field->size) {
 785		ret = -EINVAL;
 786		goto free;
 787	}
 788
 789	if (synth_field_is_string(field->type))
 790		field->is_string = true;
 791
 792	field->is_signed = synth_field_signed(field->type);
 793
 794	field->name = kstrdup(field_name, GFP_KERNEL);
 795	if (!field->name) {
 796		ret = -ENOMEM;
 797		goto free;
 798	}
 799 out:
 800	return field;
 801 free:
 802	free_synth_field(field);
 803	field = ERR_PTR(ret);
 804	goto out;
 805}
 806
 807static void free_synth_tracepoint(struct tracepoint *tp)
 808{
 809	if (!tp)
 810		return;
 811
 812	kfree(tp->name);
 813	kfree(tp);
 814}
 815
 816static struct tracepoint *alloc_synth_tracepoint(char *name)
 817{
 818	struct tracepoint *tp;
 819
 820	tp = kzalloc(sizeof(*tp), GFP_KERNEL);
 821	if (!tp)
 822		return ERR_PTR(-ENOMEM);
 823
 824	tp->name = kstrdup(name, GFP_KERNEL);
 825	if (!tp->name) {
 826		kfree(tp);
 827		return ERR_PTR(-ENOMEM);
 828	}
 829
 830	return tp;
 831}
 832
 833typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 834				    unsigned int var_ref_idx);
 835
 836static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 837			       unsigned int var_ref_idx)
 838{
 839	struct tracepoint *tp = event->tp;
 840
 841	if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 842		struct tracepoint_func *probe_func_ptr;
 843		synth_probe_func_t probe_func;
 844		void *__data;
 845
 846		if (!(cpu_online(raw_smp_processor_id())))
 847			return;
 848
 849		probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 850		if (probe_func_ptr) {
 851			do {
 852				probe_func = probe_func_ptr->func;
 853				__data = probe_func_ptr->data;
 854				probe_func(__data, var_ref_vals, var_ref_idx);
 855			} while ((++probe_func_ptr)->func);
 856		}
 857	}
 858}
 859
 860static struct synth_event *find_synth_event(const char *name)
 861{
 862	struct synth_event *event;
 863
 864	list_for_each_entry(event, &synth_event_list, list) {
 865		if (strcmp(event->name, name) == 0)
 866			return event;
 867	}
 868
 869	return NULL;
 870}
 871
 872static int register_synth_event(struct synth_event *event)
 873{
 874	struct trace_event_call *call = &event->call;
 875	int ret = 0;
 876
 877	event->call.class = &event->class;
 878	event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
 879	if (!event->class.system) {
 880		ret = -ENOMEM;
 881		goto out;
 882	}
 883
 884	event->tp = alloc_synth_tracepoint(event->name);
 885	if (IS_ERR(event->tp)) {
 886		ret = PTR_ERR(event->tp);
 887		event->tp = NULL;
 888		goto out;
 889	}
 890
 891	INIT_LIST_HEAD(&call->class->fields);
 892	call->event.funcs = &synth_event_funcs;
 893	call->class->define_fields = synth_event_define_fields;
 894
 895	ret = register_trace_event(&call->event);
 896	if (!ret) {
 897		ret = -ENODEV;
 898		goto out;
 899	}
 900	call->flags = TRACE_EVENT_FL_TRACEPOINT;
 901	call->class->reg = trace_event_reg;
 902	call->class->probe = trace_event_raw_event_synth;
 903	call->data = event;
 904	call->tp = event->tp;
 905
 906	ret = trace_add_event_call(call);
 907	if (ret) {
 908		pr_warn("Failed to register synthetic event: %s\n",
 909			trace_event_name(call));
 910		goto err;
 911	}
 912
 913	ret = set_synth_event_print_fmt(call);
 914	if (ret < 0) {
 915		trace_remove_event_call(call);
 916		goto err;
 917	}
 918 out:
 919	return ret;
 920 err:
 921	unregister_trace_event(&call->event);
 922	goto out;
 923}
 924
 925static int unregister_synth_event(struct synth_event *event)
 926{
 927	struct trace_event_call *call = &event->call;
 928	int ret;
 929
 930	ret = trace_remove_event_call(call);
 931
 932	return ret;
 933}
 934
 935static void free_synth_event(struct synth_event *event)
 936{
 937	unsigned int i;
 938
 939	if (!event)
 940		return;
 941
 942	for (i = 0; i < event->n_fields; i++)
 943		free_synth_field(event->fields[i]);
 944
 945	kfree(event->fields);
 946	kfree(event->name);
 947	kfree(event->class.system);
 948	free_synth_tracepoint(event->tp);
 949	free_synth_event_print_fmt(&event->call);
 950	kfree(event);
 951}
 952
 953static struct synth_event *alloc_synth_event(char *event_name, int n_fields,
 954					     struct synth_field **fields)
 955{
 956	struct synth_event *event;
 957	unsigned int i;
 958
 959	event = kzalloc(sizeof(*event), GFP_KERNEL);
 960	if (!event) {
 961		event = ERR_PTR(-ENOMEM);
 962		goto out;
 963	}
 964
 965	event->name = kstrdup(event_name, GFP_KERNEL);
 966	if (!event->name) {
 967		kfree(event);
 968		event = ERR_PTR(-ENOMEM);
 969		goto out;
 970	}
 971
 972	event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
 973	if (!event->fields) {
 974		free_synth_event(event);
 975		event = ERR_PTR(-ENOMEM);
 976		goto out;
 977	}
 978
 979	for (i = 0; i < n_fields; i++)
 980		event->fields[i] = fields[i];
 981
 982	event->n_fields = n_fields;
 983 out:
 984	return event;
 985}
 986
 987static void action_trace(struct hist_trigger_data *hist_data,
 988			 struct tracing_map_elt *elt, void *rec,
 989			 struct ring_buffer_event *rbe,
 990			 struct action_data *data, u64 *var_ref_vals)
 991{
 992	struct synth_event *event = data->onmatch.synth_event;
 993
 994	trace_synth(event, var_ref_vals, data->onmatch.var_ref_idx);
 995}
 996
 997struct hist_var_data {
 998	struct list_head list;
 999	struct hist_trigger_data *hist_data;
1000};
1001
1002static void add_or_delete_synth_event(struct synth_event *event, int delete)
1003{
1004	if (delete)
1005		free_synth_event(event);
1006	else {
1007		mutex_lock(&synth_event_mutex);
1008		if (!find_synth_event(event->name))
1009			list_add(&event->list, &synth_event_list);
1010		else
1011			free_synth_event(event);
1012		mutex_unlock(&synth_event_mutex);
1013	}
1014}
1015
1016static int create_synth_event(int argc, char **argv)
1017{
1018	struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1019	struct synth_event *event = NULL;
1020	bool delete_event = false;
1021	int i, n_fields = 0, ret = 0;
1022	char *name;
1023
1024	mutex_lock(&synth_event_mutex);
1025
1026	/*
1027	 * Argument syntax:
1028	 *  - Add synthetic event: <event_name> field[;field] ...
1029	 *  - Remove synthetic event: !<event_name> field[;field] ...
1030	 *      where 'field' = type field_name
1031	 */
1032	if (argc < 1) {
1033		ret = -EINVAL;
1034		goto out;
1035	}
1036
1037	name = argv[0];
1038	if (name[0] == '!') {
1039		delete_event = true;
1040		name++;
1041	}
1042
1043	event = find_synth_event(name);
1044	if (event) {
1045		if (delete_event) {
1046			if (event->ref) {
1047				event = NULL;
1048				ret = -EBUSY;
1049				goto out;
1050			}
1051			list_del(&event->list);
1052			goto out;
1053		}
1054		event = NULL;
1055		ret = -EEXIST;
1056		goto out;
1057	} else if (delete_event)
1058		goto out;
1059
1060	if (argc < 2) {
1061		ret = -EINVAL;
1062		goto out;
1063	}
1064
1065	for (i = 1; i < argc - 1; i++) {
1066		if (strcmp(argv[i], ";") == 0)
1067			continue;
1068		if (n_fields == SYNTH_FIELDS_MAX) {
1069			ret = -EINVAL;
1070			goto err;
1071		}
1072
1073		field = parse_synth_field(argv[i], argv[i + 1]);
1074		if (IS_ERR(field)) {
1075			ret = PTR_ERR(field);
1076			goto err;
1077		}
1078		fields[n_fields] = field;
1079		i++; n_fields++;
1080	}
1081
1082	if (i < argc) {
1083		ret = -EINVAL;
1084		goto err;
1085	}
1086
1087	event = alloc_synth_event(name, n_fields, fields);
1088	if (IS_ERR(event)) {
1089		ret = PTR_ERR(event);
1090		event = NULL;
1091		goto err;
1092	}
1093 out:
1094	mutex_unlock(&synth_event_mutex);
1095
1096	if (event) {
1097		if (delete_event) {
1098			ret = unregister_synth_event(event);
1099			add_or_delete_synth_event(event, !ret);
1100		} else {
1101			ret = register_synth_event(event);
1102			add_or_delete_synth_event(event, ret);
1103		}
1104	}
1105
1106	return ret;
1107 err:
1108	mutex_unlock(&synth_event_mutex);
1109
1110	for (i = 0; i < n_fields; i++)
1111		free_synth_field(fields[i]);
1112	free_synth_event(event);
1113
1114	return ret;
1115}
1116
1117static int release_all_synth_events(void)
1118{
1119	struct list_head release_events;
1120	struct synth_event *event, *e;
1121	int ret = 0;
1122
1123	INIT_LIST_HEAD(&release_events);
1124
1125	mutex_lock(&synth_event_mutex);
1126
1127	list_for_each_entry(event, &synth_event_list, list) {
1128		if (event->ref) {
1129			mutex_unlock(&synth_event_mutex);
1130			return -EBUSY;
1131		}
1132	}
1133
1134	list_splice_init(&event->list, &release_events);
1135
1136	mutex_unlock(&synth_event_mutex);
1137
1138	list_for_each_entry_safe(event, e, &release_events, list) {
1139		list_del(&event->list);
1140
1141		ret = unregister_synth_event(event);
1142		add_or_delete_synth_event(event, !ret);
1143	}
1144
1145	return ret;
1146}
1147
1148
1149static void *synth_events_seq_start(struct seq_file *m, loff_t *pos)
1150{
1151	mutex_lock(&synth_event_mutex);
1152
1153	return seq_list_start(&synth_event_list, *pos);
1154}
1155
1156static void *synth_events_seq_next(struct seq_file *m, void *v, loff_t *pos)
1157{
1158	return seq_list_next(v, &synth_event_list, pos);
1159}
1160
1161static void synth_events_seq_stop(struct seq_file *m, void *v)
1162{
1163	mutex_unlock(&synth_event_mutex);
1164}
1165
1166static int synth_events_seq_show(struct seq_file *m, void *v)
1167{
1168	struct synth_field *field;
1169	struct synth_event *event = v;
1170	unsigned int i;
1171
1172	seq_printf(m, "%s\t", event->name);
1173
1174	for (i = 0; i < event->n_fields; i++) {
1175		field = event->fields[i];
1176
1177		/* parameter values */
1178		seq_printf(m, "%s %s%s", field->type, field->name,
1179			   i == event->n_fields - 1 ? "" : "; ");
1180	}
1181
1182	seq_putc(m, '\n');
1183
1184	return 0;
1185}
1186
1187static const struct seq_operations synth_events_seq_op = {
1188	.start  = synth_events_seq_start,
1189	.next   = synth_events_seq_next,
1190	.stop   = synth_events_seq_stop,
1191	.show   = synth_events_seq_show
1192};
1193
1194static int synth_events_open(struct inode *inode, struct file *file)
1195{
1196	int ret;
1197
1198	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1199		ret = release_all_synth_events();
1200		if (ret < 0)
1201			return ret;
1202	}
1203
1204	return seq_open(file, &synth_events_seq_op);
1205}
1206
1207static ssize_t synth_events_write(struct file *file,
1208				  const char __user *buffer,
1209				  size_t count, loff_t *ppos)
1210{
1211	return trace_parse_run_command(file, buffer, count, ppos,
1212				       create_synth_event);
1213}
1214
1215static const struct file_operations synth_events_fops = {
1216	.open           = synth_events_open,
1217	.write		= synth_events_write,
1218	.read           = seq_read,
1219	.llseek         = seq_lseek,
1220	.release        = seq_release,
1221};
1222
1223static u64 hist_field_timestamp(struct hist_field *hist_field,
1224				struct tracing_map_elt *elt,
1225				struct ring_buffer_event *rbe,
1226				void *event)
1227{
1228	struct hist_trigger_data *hist_data = hist_field->hist_data;
1229	struct trace_array *tr = hist_data->event_file->tr;
1230
1231	u64 ts = ring_buffer_event_time_stamp(rbe);
1232
1233	if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
1234		ts = ns2usecs(ts);
1235
1236	return ts;
1237}
1238
1239static u64 hist_field_cpu(struct hist_field *hist_field,
1240			  struct tracing_map_elt *elt,
1241			  struct ring_buffer_event *rbe,
1242			  void *event)
1243{
1244	int cpu = smp_processor_id();
1245
1246	return cpu;
1247}
1248
1249static struct hist_field *
1250check_field_for_var_ref(struct hist_field *hist_field,
1251			struct hist_trigger_data *var_data,
1252			unsigned int var_idx)
1253{
1254	struct hist_field *found = NULL;
1255
1256	if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF) {
1257		if (hist_field->var.idx == var_idx &&
1258		    hist_field->var.hist_data == var_data) {
1259			found = hist_field;
1260		}
1261	}
1262
1263	return found;
1264}
1265
1266static struct hist_field *
1267check_field_for_var_refs(struct hist_trigger_data *hist_data,
1268			 struct hist_field *hist_field,
1269			 struct hist_trigger_data *var_data,
1270			 unsigned int var_idx,
1271			 unsigned int level)
1272{
1273	struct hist_field *found = NULL;
1274	unsigned int i;
1275
1276	if (level > 3)
1277		return found;
1278
1279	if (!hist_field)
1280		return found;
1281
1282	found = check_field_for_var_ref(hist_field, var_data, var_idx);
1283	if (found)
1284		return found;
1285
1286	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1287		struct hist_field *operand;
1288
1289		operand = hist_field->operands[i];
1290		found = check_field_for_var_refs(hist_data, operand, var_data,
1291						 var_idx, level + 1);
1292		if (found)
1293			return found;
1294	}
1295
1296	return found;
1297}
1298
1299static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1300				       struct hist_trigger_data *var_data,
1301				       unsigned int var_idx)
1302{
1303	struct hist_field *hist_field, *found = NULL;
1304	unsigned int i;
1305
1306	for_each_hist_field(i, hist_data) {
1307		hist_field = hist_data->fields[i];
1308		found = check_field_for_var_refs(hist_data, hist_field,
1309						 var_data, var_idx, 0);
1310		if (found)
1311			return found;
1312	}
1313
1314	for (i = 0; i < hist_data->n_synth_var_refs; i++) {
1315		hist_field = hist_data->synth_var_refs[i];
1316		found = check_field_for_var_refs(hist_data, hist_field,
1317						 var_data, var_idx, 0);
1318		if (found)
1319			return found;
1320	}
1321
1322	return found;
1323}
1324
1325static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1326					   unsigned int var_idx)
1327{
1328	struct trace_array *tr = hist_data->event_file->tr;
1329	struct hist_field *found = NULL;
1330	struct hist_var_data *var_data;
1331
1332	list_for_each_entry(var_data, &tr->hist_vars, list) {
1333		if (var_data->hist_data == hist_data)
1334			continue;
1335		found = find_var_ref(var_data->hist_data, hist_data, var_idx);
1336		if (found)
1337			break;
1338	}
1339
1340	return found;
1341}
1342
1343static bool check_var_refs(struct hist_trigger_data *hist_data)
1344{
1345	struct hist_field *field;
1346	bool found = false;
1347	int i;
1348
1349	for_each_hist_field(i, hist_data) {
1350		field = hist_data->fields[i];
1351		if (field && field->flags & HIST_FIELD_FL_VAR) {
1352			if (find_any_var_ref(hist_data, field->var.idx)) {
1353				found = true;
1354				break;
1355			}
1356		}
1357	}
1358
1359	return found;
1360}
1361
1362static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1363{
1364	struct trace_array *tr = hist_data->event_file->tr;
1365	struct hist_var_data *var_data, *found = NULL;
1366
1367	list_for_each_entry(var_data, &tr->hist_vars, list) {
1368		if (var_data->hist_data == hist_data) {
1369			found = var_data;
1370			break;
1371		}
1372	}
1373
1374	return found;
1375}
1376
1377static bool field_has_hist_vars(struct hist_field *hist_field,
1378				unsigned int level)
1379{
1380	int i;
1381
1382	if (level > 3)
1383		return false;
1384
1385	if (!hist_field)
1386		return false;
1387
1388	if (hist_field->flags & HIST_FIELD_FL_VAR ||
1389	    hist_field->flags & HIST_FIELD_FL_VAR_REF)
1390		return true;
1391
1392	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1393		struct hist_field *operand;
1394
1395		operand = hist_field->operands[i];
1396		if (field_has_hist_vars(operand, level + 1))
1397			return true;
1398	}
1399
1400	return false;
1401}
1402
1403static bool has_hist_vars(struct hist_trigger_data *hist_data)
1404{
1405	struct hist_field *hist_field;
1406	int i;
1407
1408	for_each_hist_field(i, hist_data) {
1409		hist_field = hist_data->fields[i];
1410		if (field_has_hist_vars(hist_field, 0))
1411			return true;
1412	}
1413
1414	return false;
1415}
1416
1417static int save_hist_vars(struct hist_trigger_data *hist_data)
1418{
1419	struct trace_array *tr = hist_data->event_file->tr;
1420	struct hist_var_data *var_data;
1421
1422	var_data = find_hist_vars(hist_data);
1423	if (var_data)
1424		return 0;
1425
1426	if (trace_array_get(tr) < 0)
1427		return -ENODEV;
1428
1429	var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1430	if (!var_data) {
1431		trace_array_put(tr);
1432		return -ENOMEM;
1433	}
1434
1435	var_data->hist_data = hist_data;
1436	list_add(&var_data->list, &tr->hist_vars);
1437
1438	return 0;
1439}
1440
1441static void remove_hist_vars(struct hist_trigger_data *hist_data)
1442{
1443	struct trace_array *tr = hist_data->event_file->tr;
1444	struct hist_var_data *var_data;
1445
1446	var_data = find_hist_vars(hist_data);
1447	if (!var_data)
1448		return;
1449
1450	if (WARN_ON(check_var_refs(hist_data)))
1451		return;
1452
1453	list_del(&var_data->list);
1454
1455	kfree(var_data);
1456
1457	trace_array_put(tr);
1458}
1459
1460static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1461					 const char *var_name)
1462{
1463	struct hist_field *hist_field, *found = NULL;
1464	int i;
1465
1466	for_each_hist_field(i, hist_data) {
1467		hist_field = hist_data->fields[i];
1468		if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1469		    strcmp(hist_field->var.name, var_name) == 0) {
1470			found = hist_field;
1471			break;
1472		}
1473	}
1474
1475	return found;
1476}
1477
1478static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1479				   struct trace_event_file *file,
1480				   const char *var_name)
1481{
1482	struct hist_trigger_data *test_data;
1483	struct event_trigger_data *test;
1484	struct hist_field *hist_field;
1485
1486	hist_field = find_var_field(hist_data, var_name);
1487	if (hist_field)
1488		return hist_field;
1489
1490	list_for_each_entry_rcu(test, &file->triggers, list) {
1491		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1492			test_data = test->private_data;
1493			hist_field = find_var_field(test_data, var_name);
1494			if (hist_field)
1495				return hist_field;
1496		}
1497	}
1498
1499	return NULL;
1500}
1501
1502static struct trace_event_file *find_var_file(struct trace_array *tr,
1503					      char *system,
1504					      char *event_name,
1505					      char *var_name)
1506{
1507	struct hist_trigger_data *var_hist_data;
1508	struct hist_var_data *var_data;
1509	struct trace_event_file *file, *found = NULL;
1510
1511	if (system)
1512		return find_event_file(tr, system, event_name);
1513
1514	list_for_each_entry(var_data, &tr->hist_vars, list) {
1515		var_hist_data = var_data->hist_data;
1516		file = var_hist_data->event_file;
1517		if (file == found)
1518			continue;
1519
1520		if (find_var_field(var_hist_data, var_name)) {
1521			if (found) {
1522				hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
1523				return NULL;
1524			}
1525
1526			found = file;
1527		}
1528	}
1529
1530	return found;
1531}
1532
1533static struct hist_field *find_file_var(struct trace_event_file *file,
1534					const char *var_name)
1535{
1536	struct hist_trigger_data *test_data;
1537	struct event_trigger_data *test;
1538	struct hist_field *hist_field;
1539
1540	list_for_each_entry_rcu(test, &file->triggers, list) {
1541		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1542			test_data = test->private_data;
1543			hist_field = find_var_field(test_data, var_name);
1544			if (hist_field)
1545				return hist_field;
1546		}
1547	}
1548
1549	return NULL;
1550}
1551
1552static struct hist_field *
1553find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1554{
1555	struct trace_array *tr = hist_data->event_file->tr;
1556	struct hist_field *hist_field, *found = NULL;
1557	struct trace_event_file *file;
1558	unsigned int i;
1559
1560	for (i = 0; i < hist_data->n_actions; i++) {
1561		struct action_data *data = hist_data->actions[i];
1562
1563		if (data->fn == action_trace) {
1564			char *system = data->onmatch.match_event_system;
1565			char *event_name = data->onmatch.match_event;
1566
1567			file = find_var_file(tr, system, event_name, var_name);
1568			if (!file)
1569				continue;
1570			hist_field = find_file_var(file, var_name);
1571			if (hist_field) {
1572				if (found) {
1573					hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
1574					return ERR_PTR(-EINVAL);
1575				}
1576
1577				found = hist_field;
1578			}
1579		}
1580	}
1581	return found;
1582}
1583
1584static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1585					 char *system,
1586					 char *event_name,
1587					 char *var_name)
1588{
1589	struct trace_array *tr = hist_data->event_file->tr;
1590	struct hist_field *hist_field = NULL;
1591	struct trace_event_file *file;
1592
1593	if (!system || !event_name) {
1594		hist_field = find_match_var(hist_data, var_name);
1595		if (IS_ERR(hist_field))
1596			return NULL;
1597		if (hist_field)
1598			return hist_field;
1599	}
1600
1601	file = find_var_file(tr, system, event_name, var_name);
1602	if (!file)
1603		return NULL;
1604
1605	hist_field = find_file_var(file, var_name);
1606
1607	return hist_field;
1608}
1609
1610struct hist_elt_data {
1611	char *comm;
1612	u64 *var_ref_vals;
1613	char *field_var_str[SYNTH_FIELDS_MAX];
1614};
1615
1616static u64 hist_field_var_ref(struct hist_field *hist_field,
1617			      struct tracing_map_elt *elt,
1618			      struct ring_buffer_event *rbe,
1619			      void *event)
1620{
1621	struct hist_elt_data *elt_data;
1622	u64 var_val = 0;
1623
1624	elt_data = elt->private_data;
1625	var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1626
1627	return var_val;
1628}
1629
1630static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1631			     u64 *var_ref_vals, bool self)
1632{
1633	struct hist_trigger_data *var_data;
1634	struct tracing_map_elt *var_elt;
1635	struct hist_field *hist_field;
1636	unsigned int i, var_idx;
1637	bool resolved = true;
1638	u64 var_val = 0;
1639
1640	for (i = 0; i < hist_data->n_var_refs; i++) {
1641		hist_field = hist_data->var_refs[i];
1642		var_idx = hist_field->var.idx;
1643		var_data = hist_field->var.hist_data;
1644
1645		if (var_data == NULL) {
1646			resolved = false;
1647			break;
1648		}
1649
1650		if ((self && var_data != hist_data) ||
1651		    (!self && var_data == hist_data))
1652			continue;
1653
1654		var_elt = tracing_map_lookup(var_data->map, key);
1655		if (!var_elt) {
1656			resolved = false;
1657			break;
1658		}
1659
1660		if (!tracing_map_var_set(var_elt, var_idx)) {
1661			resolved = false;
1662			break;
1663		}
1664
1665		if (self || !hist_field->read_once)
1666			var_val = tracing_map_read_var(var_elt, var_idx);
1667		else
1668			var_val = tracing_map_read_var_once(var_elt, var_idx);
1669
1670		var_ref_vals[i] = var_val;
1671	}
1672
1673	return resolved;
1674}
1675
1676static const char *hist_field_name(struct hist_field *field,
1677				   unsigned int level)
1678{
1679	const char *field_name = "";
1680
1681	if (level > 1)
1682		return field_name;
1683
1684	if (field->field)
1685		field_name = field->field->name;
1686	else if (field->flags & HIST_FIELD_FL_LOG2 ||
1687		 field->flags & HIST_FIELD_FL_ALIAS)
1688		field_name = hist_field_name(field->operands[0], ++level);
1689	else if (field->flags & HIST_FIELD_FL_CPU)
1690		field_name = "cpu";
1691	else if (field->flags & HIST_FIELD_FL_EXPR ||
1692		 field->flags & HIST_FIELD_FL_VAR_REF) {
1693		if (field->system) {
1694			static char full_name[MAX_FILTER_STR_VAL];
1695
1696			strcat(full_name, field->system);
1697			strcat(full_name, ".");
1698			strcat(full_name, field->event_name);
1699			strcat(full_name, ".");
1700			strcat(full_name, field->name);
1701			field_name = full_name;
1702		} else
1703			field_name = field->name;
1704	} else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1705		field_name = "common_timestamp";
1706
1707	if (field_name == NULL)
1708		field_name = "";
1709
1710	return field_name;
1711}
1712
1713static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1714{
1715	hist_field_fn_t fn = NULL;
1716
1717	switch (field_size) {
1718	case 8:
1719		if (field_is_signed)
1720			fn = hist_field_s64;
1721		else
1722			fn = hist_field_u64;
1723		break;
1724	case 4:
1725		if (field_is_signed)
1726			fn = hist_field_s32;
1727		else
1728			fn = hist_field_u32;
1729		break;
1730	case 2:
1731		if (field_is_signed)
1732			fn = hist_field_s16;
1733		else
1734			fn = hist_field_u16;
1735		break;
1736	case 1:
1737		if (field_is_signed)
1738			fn = hist_field_s8;
1739		else
1740			fn = hist_field_u8;
1741		break;
1742	}
1743
1744	return fn;
1745}
1746
1747static int parse_map_size(char *str)
1748{
1749	unsigned long size, map_bits;
1750	int ret;
1751
1752	strsep(&str, "=");
1753	if (!str) {
1754		ret = -EINVAL;
1755		goto out;
1756	}
1757
1758	ret = kstrtoul(str, 0, &size);
1759	if (ret)
1760		goto out;
1761
1762	map_bits = ilog2(roundup_pow_of_two(size));
1763	if (map_bits < TRACING_MAP_BITS_MIN ||
1764	    map_bits > TRACING_MAP_BITS_MAX)
1765		ret = -EINVAL;
1766	else
1767		ret = map_bits;
1768 out:
1769	return ret;
1770}
1771
1772static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1773{
1774	unsigned int i;
1775
1776	if (!attrs)
1777		return;
1778
1779	for (i = 0; i < attrs->n_assignments; i++)
1780		kfree(attrs->assignment_str[i]);
1781
1782	for (i = 0; i < attrs->n_actions; i++)
1783		kfree(attrs->action_str[i]);
1784
1785	kfree(attrs->name);
1786	kfree(attrs->sort_key_str);
1787	kfree(attrs->keys_str);
1788	kfree(attrs->vals_str);
1789	kfree(attrs->clock);
1790	kfree(attrs);
1791}
1792
1793static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1794{
1795	int ret = -EINVAL;
1796
1797	if (attrs->n_actions >= HIST_ACTIONS_MAX)
1798		return ret;
1799
1800	if ((strncmp(str, "onmatch(", strlen("onmatch(")) == 0) ||
1801	    (strncmp(str, "onmax(", strlen("onmax(")) == 0)) {
1802		attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1803		if (!attrs->action_str[attrs->n_actions]) {
1804			ret = -ENOMEM;
1805			return ret;
1806		}
1807		attrs->n_actions++;
1808		ret = 0;
1809	}
1810
1811	return ret;
1812}
1813
1814static int parse_assignment(char *str, struct hist_trigger_attrs *attrs)
1815{
1816	int ret = 0;
1817
1818	if ((strncmp(str, "key=", strlen("key=")) == 0) ||
1819	    (strncmp(str, "keys=", strlen("keys=")) == 0)) {
1820		attrs->keys_str = kstrdup(str, GFP_KERNEL);
1821		if (!attrs->keys_str) {
1822			ret = -ENOMEM;
1823			goto out;
1824		}
1825	} else if ((strncmp(str, "val=", strlen("val=")) == 0) ||
1826		 (strncmp(str, "vals=", strlen("vals=")) == 0) ||
1827		 (strncmp(str, "values=", strlen("values=")) == 0)) {
1828		attrs->vals_str = kstrdup(str, GFP_KERNEL);
1829		if (!attrs->vals_str) {
1830			ret = -ENOMEM;
1831			goto out;
1832		}
1833	} else if (strncmp(str, "sort=", strlen("sort=")) == 0) {
1834		attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
1835		if (!attrs->sort_key_str) {
1836			ret = -ENOMEM;
1837			goto out;
1838		}
1839	} else if (strncmp(str, "name=", strlen("name=")) == 0) {
1840		attrs->name = kstrdup(str, GFP_KERNEL);
1841		if (!attrs->name) {
1842			ret = -ENOMEM;
1843			goto out;
1844		}
1845	} else if (strncmp(str, "clock=", strlen("clock=")) == 0) {
1846		strsep(&str, "=");
1847		if (!str) {
1848			ret = -EINVAL;
1849			goto out;
1850		}
1851
1852		str = strstrip(str);
1853		attrs->clock = kstrdup(str, GFP_KERNEL);
1854		if (!attrs->clock) {
1855			ret = -ENOMEM;
1856			goto out;
1857		}
1858	} else if (strncmp(str, "size=", strlen("size=")) == 0) {
1859		int map_bits = parse_map_size(str);
1860
1861		if (map_bits < 0) {
1862			ret = map_bits;
1863			goto out;
1864		}
1865		attrs->map_bits = map_bits;
1866	} else {
1867		char *assignment;
1868
1869		if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1870			hist_err("Too many variables defined: ", str);
1871			ret = -EINVAL;
1872			goto out;
1873		}
1874
1875		assignment = kstrdup(str, GFP_KERNEL);
1876		if (!assignment) {
1877			ret = -ENOMEM;
1878			goto out;
1879		}
1880
1881		attrs->assignment_str[attrs->n_assignments++] = assignment;
1882	}
1883 out:
1884	return ret;
1885}
1886
1887static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
1888{
1889	struct hist_trigger_attrs *attrs;
1890	int ret = 0;
1891
1892	attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1893	if (!attrs)
1894		return ERR_PTR(-ENOMEM);
1895
1896	while (trigger_str) {
1897		char *str = strsep(&trigger_str, ":");
1898
1899		if (strchr(str, '=')) {
1900			ret = parse_assignment(str, attrs);
1901			if (ret)
1902				goto free;
1903		} else if (strcmp(str, "pause") == 0)
1904			attrs->pause = true;
1905		else if ((strcmp(str, "cont") == 0) ||
1906			 (strcmp(str, "continue") == 0))
1907			attrs->cont = true;
1908		else if (strcmp(str, "clear") == 0)
1909			attrs->clear = true;
1910		else {
1911			ret = parse_action(str, attrs);
1912			if (ret)
1913				goto free;
1914		}
1915	}
1916
1917	if (!attrs->keys_str) {
1918		ret = -EINVAL;
1919		goto free;
1920	}
1921
1922	if (!attrs->clock) {
1923		attrs->clock = kstrdup("global", GFP_KERNEL);
1924		if (!attrs->clock) {
1925			ret = -ENOMEM;
1926			goto free;
1927		}
1928	}
1929
1930	return attrs;
1931 free:
1932	destroy_hist_trigger_attrs(attrs);
1933
1934	return ERR_PTR(ret);
1935}
1936
1937static inline void save_comm(char *comm, struct task_struct *task)
1938{
1939	if (!task->pid) {
1940		strcpy(comm, "<idle>");
1941		return;
1942	}
1943
1944	if (WARN_ON_ONCE(task->pid < 0)) {
1945		strcpy(comm, "<XXX>");
1946		return;
1947	}
1948
1949	memcpy(comm, task->comm, TASK_COMM_LEN);
1950}
1951
1952static void hist_elt_data_free(struct hist_elt_data *elt_data)
1953{
1954	unsigned int i;
1955
1956	for (i = 0; i < SYNTH_FIELDS_MAX; i++)
1957		kfree(elt_data->field_var_str[i]);
1958
1959	kfree(elt_data->comm);
1960	kfree(elt_data);
1961}
1962
1963static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1964{
1965	struct hist_elt_data *elt_data = elt->private_data;
1966
1967	hist_elt_data_free(elt_data);
1968}
1969
1970static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1971{
1972	struct hist_trigger_data *hist_data = elt->map->private_data;
1973	unsigned int size = TASK_COMM_LEN;
1974	struct hist_elt_data *elt_data;
1975	struct hist_field *key_field;
1976	unsigned int i, n_str;
1977
1978	elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1979	if (!elt_data)
1980		return -ENOMEM;
1981
1982	for_each_hist_key_field(i, hist_data) {
1983		key_field = hist_data->fields[i];
1984
1985		if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
1986			elt_data->comm = kzalloc(size, GFP_KERNEL);
1987			if (!elt_data->comm) {
1988				kfree(elt_data);
1989				return -ENOMEM;
1990			}
1991			break;
1992		}
1993	}
1994
1995	n_str = hist_data->n_field_var_str + hist_data->n_max_var_str;
1996
1997	size = STR_VAR_LEN_MAX;
1998
1999	for (i = 0; i < n_str; i++) {
2000		elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
2001		if (!elt_data->field_var_str[i]) {
2002			hist_elt_data_free(elt_data);
2003			return -ENOMEM;
2004		}
2005	}
2006
2007	elt->private_data = elt_data;
2008
2009	return 0;
2010}
2011
2012static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
2013{
2014	struct hist_elt_data *elt_data = elt->private_data;
2015
2016	if (elt_data->comm)
2017		save_comm(elt_data->comm, current);
2018}
2019
2020static const struct tracing_map_ops hist_trigger_elt_data_ops = {
2021	.elt_alloc	= hist_trigger_elt_data_alloc,
2022	.elt_free	= hist_trigger_elt_data_free,
2023	.elt_init	= hist_trigger_elt_data_init,
2024};
2025
2026static const char *get_hist_field_flags(struct hist_field *hist_field)
2027{
2028	const char *flags_str = NULL;
2029
2030	if (hist_field->flags & HIST_FIELD_FL_HEX)
2031		flags_str = "hex";
2032	else if (hist_field->flags & HIST_FIELD_FL_SYM)
2033		flags_str = "sym";
2034	else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
2035		flags_str = "sym-offset";
2036	else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
2037		flags_str = "execname";
2038	else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
2039		flags_str = "syscall";
2040	else if (hist_field->flags & HIST_FIELD_FL_LOG2)
2041		flags_str = "log2";
2042	else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2043		flags_str = "usecs";
2044
2045	return flags_str;
2046}
2047
2048static void expr_field_str(struct hist_field *field, char *expr)
2049{
2050	if (field->flags & HIST_FIELD_FL_VAR_REF)
2051		strcat(expr, "$");
2052
2053	strcat(expr, hist_field_name(field, 0));
2054
2055	if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
2056		const char *flags_str = get_hist_field_flags(field);
2057
2058		if (flags_str) {
2059			strcat(expr, ".");
2060			strcat(expr, flags_str);
2061		}
2062	}
2063}
2064
2065static char *expr_str(struct hist_field *field, unsigned int level)
2066{
2067	char *expr;
2068
2069	if (level > 1)
2070		return NULL;
2071
2072	expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2073	if (!expr)
2074		return NULL;
2075
2076	if (!field->operands[0]) {
2077		expr_field_str(field, expr);
2078		return expr;
2079	}
2080
2081	if (field->operator == FIELD_OP_UNARY_MINUS) {
2082		char *subexpr;
2083
2084		strcat(expr, "-(");
2085		subexpr = expr_str(field->operands[0], ++level);
2086		if (!subexpr) {
2087			kfree(expr);
2088			return NULL;
2089		}
2090		strcat(expr, subexpr);
2091		strcat(expr, ")");
2092
2093		kfree(subexpr);
2094
2095		return expr;
2096	}
2097
2098	expr_field_str(field->operands[0], expr);
2099
2100	switch (field->operator) {
2101	case FIELD_OP_MINUS:
2102		strcat(expr, "-");
2103		break;
2104	case FIELD_OP_PLUS:
2105		strcat(expr, "+");
2106		break;
2107	default:
2108		kfree(expr);
2109		return NULL;
2110	}
2111
2112	expr_field_str(field->operands[1], expr);
2113
2114	return expr;
2115}
2116
2117static int contains_operator(char *str)
2118{
2119	enum field_op_id field_op = FIELD_OP_NONE;
2120	char *op;
2121
2122	op = strpbrk(str, "+-");
2123	if (!op)
2124		return FIELD_OP_NONE;
2125
2126	switch (*op) {
2127	case '-':
2128		if (*str == '-')
2129			field_op = FIELD_OP_UNARY_MINUS;
2130		else
2131			field_op = FIELD_OP_MINUS;
2132		break;
2133	case '+':
2134		field_op = FIELD_OP_PLUS;
2135		break;
2136	default:
2137		break;
2138	}
2139
2140	return field_op;
2141}
2142
2143static void destroy_hist_field(struct hist_field *hist_field,
2144			       unsigned int level)
2145{
2146	unsigned int i;
2147
2148	if (level > 3)
2149		return;
2150
2151	if (!hist_field)
2152		return;
2153
2154	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2155		destroy_hist_field(hist_field->operands[i], level + 1);
2156
2157	kfree(hist_field->var.name);
2158	kfree(hist_field->name);
2159	kfree(hist_field->type);
2160
2161	kfree(hist_field);
2162}
2163
2164static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
2165					    struct ftrace_event_field *field,
2166					    unsigned long flags,
2167					    char *var_name)
2168{
2169	struct hist_field *hist_field;
2170
2171	if (field && is_function_field(field))
2172		return NULL;
2173
2174	hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2175	if (!hist_field)
2176		return NULL;
2177
2178	hist_field->hist_data = hist_data;
2179
2180	if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
2181		goto out; /* caller will populate */
2182
2183	if (flags & HIST_FIELD_FL_VAR_REF) {
2184		hist_field->fn = hist_field_var_ref;
2185		goto out;
2186	}
2187
2188	if (flags & HIST_FIELD_FL_HITCOUNT) {
2189		hist_field->fn = hist_field_counter;
2190		hist_field->size = sizeof(u64);
2191		hist_field->type = kstrdup("u64", GFP_KERNEL);
2192		if (!hist_field->type)
2193			goto free;
2194		goto out;
2195	}
2196
2197	if (flags & HIST_FIELD_FL_STACKTRACE) {
2198		hist_field->fn = hist_field_none;
2199		goto out;
2200	}
2201
2202	if (flags & HIST_FIELD_FL_LOG2) {
2203		unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
2204		hist_field->fn = hist_field_log2;
2205		hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2206		hist_field->size = hist_field->operands[0]->size;
2207		hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
2208		if (!hist_field->type)
2209			goto free;
2210		goto out;
2211	}
2212
2213	if (flags & HIST_FIELD_FL_TIMESTAMP) {
2214		hist_field->fn = hist_field_timestamp;
2215		hist_field->size = sizeof(u64);
2216		hist_field->type = kstrdup("u64", GFP_KERNEL);
2217		if (!hist_field->type)
2218			goto free;
2219		goto out;
2220	}
2221
2222	if (flags & HIST_FIELD_FL_CPU) {
2223		hist_field->fn = hist_field_cpu;
2224		hist_field->size = sizeof(int);
2225		hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
2226		if (!hist_field->type)
2227			goto free;
2228		goto out;
2229	}
2230
2231	if (WARN_ON_ONCE(!field))
2232		goto out;
2233
2234	if (is_string_field(field)) {
2235		flags |= HIST_FIELD_FL_STRING;
2236
2237		hist_field->size = MAX_FILTER_STR_VAL;
2238		hist_field->type = kstrdup(field->type, GFP_KERNEL);
2239		if (!hist_field->type)
2240			goto free;
2241
2242		if (field->filter_type == FILTER_STATIC_STRING)
2243			hist_field->fn = hist_field_string;
2244		else if (field->filter_type == FILTER_DYN_STRING)
2245			hist_field->fn = hist_field_dynstring;
2246		else
2247			hist_field->fn = hist_field_pstring;
2248	} else {
2249		hist_field->size = field->size;
2250		hist_field->is_signed = field->is_signed;
2251		hist_field->type = kstrdup(field->type, GFP_KERNEL);
2252		if (!hist_field->type)
2253			goto free;
2254
2255		hist_field->fn = select_value_fn(field->size,
2256						 field->is_signed);
2257		if (!hist_field->fn) {
2258			destroy_hist_field(hist_field, 0);
2259			return NULL;
2260		}
2261	}
2262 out:
2263	hist_field->field = field;
2264	hist_field->flags = flags;
2265
2266	if (var_name) {
2267		hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2268		if (!hist_field->var.name)
2269			goto free;
2270	}
2271
2272	return hist_field;
2273 free:
2274	destroy_hist_field(hist_field, 0);
2275	return NULL;
2276}
2277
2278static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2279{
2280	unsigned int i;
2281
2282	for (i = 0; i < HIST_FIELDS_MAX; i++) {
2283		if (hist_data->fields[i]) {
2284			destroy_hist_field(hist_data->fields[i], 0);
2285			hist_data->fields[i] = NULL;
2286		}
2287	}
2288}
2289
2290static int init_var_ref(struct hist_field *ref_field,
2291			struct hist_field *var_field,
2292			char *system, char *event_name)
2293{
2294	int err = 0;
2295
2296	ref_field->var.idx = var_field->var.idx;
2297	ref_field->var.hist_data = var_field->hist_data;
2298	ref_field->size = var_field->size;
2299	ref_field->is_signed = var_field->is_signed;
2300	ref_field->flags |= var_field->flags &
2301		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2302
2303	if (system) {
2304		ref_field->system = kstrdup(system, GFP_KERNEL);
2305		if (!ref_field->system)
2306			return -ENOMEM;
2307	}
2308
2309	if (event_name) {
2310		ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2311		if (!ref_field->event_name) {
2312			err = -ENOMEM;
2313			goto free;
2314		}
2315	}
2316
2317	if (var_field->var.name) {
2318		ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2319		if (!ref_field->name) {
2320			err = -ENOMEM;
2321			goto free;
2322		}
2323	} else if (var_field->name) {
2324		ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2325		if (!ref_field->name) {
2326			err = -ENOMEM;
2327			goto free;
2328		}
2329	}
2330
2331	ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
2332	if (!ref_field->type) {
2333		err = -ENOMEM;
2334		goto free;
2335	}
2336 out:
2337	return err;
2338 free:
2339	kfree(ref_field->system);
2340	kfree(ref_field->event_name);
2341	kfree(ref_field->name);
2342
2343	goto out;
2344}
2345
2346static struct hist_field *create_var_ref(struct hist_field *var_field,
2347					 char *system, char *event_name)
2348{
2349	unsigned long flags = HIST_FIELD_FL_VAR_REF;
2350	struct hist_field *ref_field;
2351
2352	ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2353	if (ref_field) {
2354		if (init_var_ref(ref_field, var_field, system, event_name)) {
2355			destroy_hist_field(ref_field, 0);
2356			return NULL;
2357		}
2358	}
2359
2360	return ref_field;
2361}
2362
2363static bool is_var_ref(char *var_name)
2364{
2365	if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2366		return false;
2367
2368	return true;
2369}
2370
2371static char *field_name_from_var(struct hist_trigger_data *hist_data,
2372				 char *var_name)
2373{
2374	char *name, *field;
2375	unsigned int i;
2376
2377	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2378		name = hist_data->attrs->var_defs.name[i];
2379
2380		if (strcmp(var_name, name) == 0) {
2381			field = hist_data->attrs->var_defs.expr[i];
2382			if (contains_operator(field) || is_var_ref(field))
2383				continue;
2384			return field;
2385		}
2386	}
2387
2388	return NULL;
2389}
2390
2391static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2392				 char *system, char *event_name,
2393				 char *var_name)
2394{
2395	struct trace_event_call *call;
2396
2397	if (system && event_name) {
2398		call = hist_data->event_file->event_call;
2399
2400		if (strcmp(system, call->class->system) != 0)
2401			return NULL;
2402
2403		if (strcmp(event_name, trace_event_name(call)) != 0)
2404			return NULL;
2405	}
2406
2407	if (!!system != !!event_name)
2408		return NULL;
2409
2410	if (!is_var_ref(var_name))
2411		return NULL;
2412
2413	var_name++;
2414
2415	return field_name_from_var(hist_data, var_name);
2416}
2417
2418static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2419					char *system, char *event_name,
2420					char *var_name)
2421{
2422	struct hist_field *var_field = NULL, *ref_field = NULL;
2423
2424	if (!is_var_ref(var_name))
2425		return NULL;
2426
2427	var_name++;
2428
2429	var_field = find_event_var(hist_data, system, event_name, var_name);
2430	if (var_field)
2431		ref_field = create_var_ref(var_field, system, event_name);
2432
2433	if (!ref_field)
2434		hist_err_event("Couldn't find variable: $",
2435			       system, event_name, var_name);
2436
2437	return ref_field;
2438}
2439
2440static struct ftrace_event_field *
2441parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2442	    char *field_str, unsigned long *flags)
2443{
2444	struct ftrace_event_field *field = NULL;
2445	char *field_name, *modifier, *str;
2446
2447	modifier = str = kstrdup(field_str, GFP_KERNEL);
2448	if (!modifier)
2449		return ERR_PTR(-ENOMEM);
2450
2451	field_name = strsep(&modifier, ".");
2452	if (modifier) {
2453		if (strcmp(modifier, "hex") == 0)
2454			*flags |= HIST_FIELD_FL_HEX;
2455		else if (strcmp(modifier, "sym") == 0)
2456			*flags |= HIST_FIELD_FL_SYM;
2457		else if (strcmp(modifier, "sym-offset") == 0)
2458			*flags |= HIST_FIELD_FL_SYM_OFFSET;
2459		else if ((strcmp(modifier, "execname") == 0) &&
2460			 (strcmp(field_name, "common_pid") == 0))
2461			*flags |= HIST_FIELD_FL_EXECNAME;
2462		else if (strcmp(modifier, "syscall") == 0)
2463			*flags |= HIST_FIELD_FL_SYSCALL;
2464		else if (strcmp(modifier, "log2") == 0)
2465			*flags |= HIST_FIELD_FL_LOG2;
2466		else if (strcmp(modifier, "usecs") == 0)
2467			*flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2468		else {
2469			hist_err("Invalid field modifier: ", modifier);
2470			field = ERR_PTR(-EINVAL);
2471			goto out;
2472		}
2473	}
2474
2475	if (strcmp(field_name, "common_timestamp") == 0) {
2476		*flags |= HIST_FIELD_FL_TIMESTAMP;
2477		hist_data->enable_timestamps = true;
2478		if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2479			hist_data->attrs->ts_in_usecs = true;
2480	} else if (strcmp(field_name, "cpu") == 0)
2481		*flags |= HIST_FIELD_FL_CPU;
2482	else {
2483		field = trace_find_event_field(file->event_call, field_name);
2484		if (!field || !field->size) {
2485			hist_err("Couldn't find field: ", field_name);
2486			field = ERR_PTR(-EINVAL);
2487			goto out;
2488		}
2489	}
2490 out:
2491	kfree(str);
2492
2493	return field;
2494}
2495
2496static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2497				       struct hist_field *var_ref,
2498				       char *var_name)
2499{
2500	struct hist_field *alias = NULL;
2501	unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2502
2503	alias = create_hist_field(hist_data, NULL, flags, var_name);
2504	if (!alias)
2505		return NULL;
2506
2507	alias->fn = var_ref->fn;
2508	alias->operands[0] = var_ref;
2509
2510	if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2511		destroy_hist_field(alias, 0);
2512		return NULL;
2513	}
2514
2515	return alias;
2516}
2517
2518static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2519				     struct trace_event_file *file, char *str,
2520				     unsigned long *flags, char *var_name)
2521{
2522	char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2523	struct ftrace_event_field *field = NULL;
2524	struct hist_field *hist_field = NULL;
2525	int ret = 0;
2526
2527	s = strchr(str, '.');
2528	if (s) {
2529		s = strchr(++s, '.');
2530		if (s) {
2531			ref_system = strsep(&str, ".");
2532			if (!str) {
2533				ret = -EINVAL;
2534				goto out;
2535			}
2536			ref_event = strsep(&str, ".");
2537			if (!str) {
2538				ret = -EINVAL;
2539				goto out;
2540			}
2541			ref_var = str;
2542		}
2543	}
2544
2545	s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2546	if (!s) {
2547		hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var);
2548		if (hist_field) {
2549			hist_data->var_refs[hist_data->n_var_refs] = hist_field;
2550			hist_field->var_ref_idx = hist_data->n_var_refs++;
2551			if (var_name) {
2552				hist_field = create_alias(hist_data, hist_field, var_name);
2553				if (!hist_field) {
2554					ret = -ENOMEM;
2555					goto out;
2556				}
2557			}
2558			return hist_field;
2559		}
2560	} else
2561		str = s;
2562
2563	field = parse_field(hist_data, file, str, flags);
2564	if (IS_ERR(field)) {
2565		ret = PTR_ERR(field);
2566		goto out;
2567	}
2568
2569	hist_field = create_hist_field(hist_data, field, *flags, var_name);
2570	if (!hist_field) {
2571		ret = -ENOMEM;
2572		goto out;
2573	}
2574
2575	return hist_field;
2576 out:
2577	return ERR_PTR(ret);
2578}
2579
2580static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2581				     struct trace_event_file *file,
2582				     char *str, unsigned long flags,
2583				     char *var_name, unsigned int level);
2584
2585static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2586				      struct trace_event_file *file,
2587				      char *str, unsigned long flags,
2588				      char *var_name, unsigned int level)
2589{
2590	struct hist_field *operand1, *expr = NULL;
2591	unsigned long operand_flags;
2592	int ret = 0;
2593	char *s;
2594
2595	/* we support only -(xxx) i.e. explicit parens required */
2596
2597	if (level > 3) {
2598		hist_err("Too many subexpressions (3 max): ", str);
2599		ret = -EINVAL;
2600		goto free;
2601	}
2602
2603	str++; /* skip leading '-' */
2604
2605	s = strchr(str, '(');
2606	if (s)
2607		str++;
2608	else {
2609		ret = -EINVAL;
2610		goto free;
2611	}
2612
2613	s = strrchr(str, ')');
2614	if (s)
2615		*s = '\0';
2616	else {
2617		ret = -EINVAL; /* no closing ')' */
2618		goto free;
2619	}
2620
2621	flags |= HIST_FIELD_FL_EXPR;
2622	expr = create_hist_field(hist_data, NULL, flags, var_name);
2623	if (!expr) {
2624		ret = -ENOMEM;
2625		goto free;
2626	}
2627
2628	operand_flags = 0;
2629	operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2630	if (IS_ERR(operand1)) {
2631		ret = PTR_ERR(operand1);
2632		goto free;
2633	}
2634
2635	expr->flags |= operand1->flags &
2636		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2637	expr->fn = hist_field_unary_minus;
2638	expr->operands[0] = operand1;
2639	expr->operator = FIELD_OP_UNARY_MINUS;
2640	expr->name = expr_str(expr, 0);
2641	expr->type = kstrdup(operand1->type, GFP_KERNEL);
2642	if (!expr->type) {
2643		ret = -ENOMEM;
2644		goto free;
2645	}
2646
2647	return expr;
2648 free:
2649	destroy_hist_field(expr, 0);
2650	return ERR_PTR(ret);
2651}
2652
2653static int check_expr_operands(struct hist_field *operand1,
2654			       struct hist_field *operand2)
2655{
2656	unsigned long operand1_flags = operand1->flags;
2657	unsigned long operand2_flags = operand2->flags;
2658
2659	if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2660	    (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2661		struct hist_field *var;
2662
2663		var = find_var_field(operand1->var.hist_data, operand1->name);
2664		if (!var)
2665			return -EINVAL;
2666		operand1_flags = var->flags;
2667	}
2668
2669	if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2670	    (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2671		struct hist_field *var;
2672
2673		var = find_var_field(operand2->var.hist_data, operand2->name);
2674		if (!var)
2675			return -EINVAL;
2676		operand2_flags = var->flags;
2677	}
2678
2679	if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2680	    (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2681		hist_err("Timestamp units in expression don't match", NULL);
2682		return -EINVAL;
2683	}
2684
2685	return 0;
2686}
2687
2688static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2689				     struct trace_event_file *file,
2690				     char *str, unsigned long flags,
2691				     char *var_name, unsigned int level)
2692{
2693	struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2694	unsigned long operand_flags;
2695	int field_op, ret = -EINVAL;
2696	char *sep, *operand1_str;
2697
2698	if (level > 3) {
2699		hist_err("Too many subexpressions (3 max): ", str);
2700		return ERR_PTR(-EINVAL);
2701	}
2702
2703	field_op = contains_operator(str);
2704
2705	if (field_op == FIELD_OP_NONE)
2706		return parse_atom(hist_data, file, str, &flags, var_name);
2707
2708	if (field_op == FIELD_OP_UNARY_MINUS)
2709		return parse_unary(hist_data, file, str, flags, var_name, ++level);
2710
2711	switch (field_op) {
2712	case FIELD_OP_MINUS:
2713		sep = "-";
2714		break;
2715	case FIELD_OP_PLUS:
2716		sep = "+";
2717		break;
2718	default:
2719		goto free;
2720	}
2721
2722	operand1_str = strsep(&str, sep);
2723	if (!operand1_str || !str)
2724		goto free;
2725
2726	operand_flags = 0;
2727	operand1 = parse_atom(hist_data, file, operand1_str,
2728			      &operand_flags, NULL);
2729	if (IS_ERR(operand1)) {
2730		ret = PTR_ERR(operand1);
2731		operand1 = NULL;
2732		goto free;
2733	}
2734
2735	/* rest of string could be another expression e.g. b+c in a+b+c */
2736	operand_flags = 0;
2737	operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2738	if (IS_ERR(operand2)) {
2739		ret = PTR_ERR(operand2);
2740		operand2 = NULL;
2741		goto free;
2742	}
2743
2744	ret = check_expr_operands(operand1, operand2);
2745	if (ret)
2746		goto free;
2747
2748	flags |= HIST_FIELD_FL_EXPR;
2749
2750	flags |= operand1->flags &
2751		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2752
2753	expr = create_hist_field(hist_data, NULL, flags, var_name);
2754	if (!expr) {
2755		ret = -ENOMEM;
2756		goto free;
2757	}
2758
2759	operand1->read_once = true;
2760	operand2->read_once = true;
2761
2762	expr->operands[0] = operand1;
2763	expr->operands[1] = operand2;
2764	expr->operator = field_op;
2765	expr->name = expr_str(expr, 0);
2766	expr->type = kstrdup(operand1->type, GFP_KERNEL);
2767	if (!expr->type) {
2768		ret = -ENOMEM;
2769		goto free;
2770	}
2771
2772	switch (field_op) {
2773	case FIELD_OP_MINUS:
2774		expr->fn = hist_field_minus;
2775		break;
2776	case FIELD_OP_PLUS:
2777		expr->fn = hist_field_plus;
2778		break;
2779	default:
2780		ret = -EINVAL;
2781		goto free;
2782	}
2783
2784	return expr;
2785 free:
2786	destroy_hist_field(operand1, 0);
2787	destroy_hist_field(operand2, 0);
2788	destroy_hist_field(expr, 0);
2789
2790	return ERR_PTR(ret);
2791}
2792
2793static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2794				 struct trace_event_file *file)
2795{
2796	struct event_trigger_data *test;
2797
2798	list_for_each_entry_rcu(test, &file->triggers, list) {
2799		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2800			if (test->private_data == hist_data)
2801				return test->filter_str;
2802		}
2803	}
2804
2805	return NULL;
2806}
2807
2808static struct event_command trigger_hist_cmd;
2809static int event_hist_trigger_func(struct event_command *cmd_ops,
2810				   struct trace_event_file *file,
2811				   char *glob, char *cmd, char *param);
2812
2813static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2814			    struct hist_trigger_data *hist_data,
2815			    unsigned int n_keys)
2816{
2817	struct hist_field *target_hist_field, *hist_field;
2818	unsigned int n, i, j;
2819
2820	if (hist_data->n_fields - hist_data->n_vals != n_keys)
2821		return false;
2822
2823	i = hist_data->n_vals;
2824	j = target_hist_data->n_vals;
2825
2826	for (n = 0; n < n_keys; n++) {
2827		hist_field = hist_data->fields[i + n];
2828		target_hist_field = target_hist_data->fields[j + n];
2829
2830		if (strcmp(hist_field->type, target_hist_field->type) != 0)
2831			return false;
2832		if (hist_field->size != target_hist_field->size)
2833			return false;
2834		if (hist_field->is_signed != target_hist_field->is_signed)
2835			return false;
2836	}
2837
2838	return true;
2839}
2840
2841static struct hist_trigger_data *
2842find_compatible_hist(struct hist_trigger_data *target_hist_data,
2843		     struct trace_event_file *file)
2844{
2845	struct hist_trigger_data *hist_data;
2846	struct event_trigger_data *test;
2847	unsigned int n_keys;
2848
2849	n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2850
2851	list_for_each_entry_rcu(test, &file->triggers, list) {
2852		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2853			hist_data = test->private_data;
2854
2855			if (compatible_keys(target_hist_data, hist_data, n_keys))
2856				return hist_data;
2857		}
2858	}
2859
2860	return NULL;
2861}
2862
2863static struct trace_event_file *event_file(struct trace_array *tr,
2864					   char *system, char *event_name)
2865{
2866	struct trace_event_file *file;
2867
2868	file = find_event_file(tr, system, event_name);
2869	if (!file)
2870		return ERR_PTR(-EINVAL);
2871
2872	return file;
2873}
2874
2875static struct hist_field *
2876find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2877			 char *system, char *event_name, char *field_name)
2878{
2879	struct hist_field *event_var;
2880	char *synthetic_name;
2881
2882	synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2883	if (!synthetic_name)
2884		return ERR_PTR(-ENOMEM);
2885
2886	strcpy(synthetic_name, "synthetic_");
2887	strcat(synthetic_name, field_name);
2888
2889	event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2890
2891	kfree(synthetic_name);
2892
2893	return event_var;
2894}
2895
2896/**
2897 * create_field_var_hist - Automatically create a histogram and var for a field
2898 * @target_hist_data: The target hist trigger
2899 * @subsys_name: Optional subsystem name
2900 * @event_name: Optional event name
2901 * @field_name: The name of the field (and the resulting variable)
2902 *
2903 * Hist trigger actions fetch data from variables, not directly from
2904 * events.  However, for convenience, users are allowed to directly
2905 * specify an event field in an action, which will be automatically
2906 * converted into a variable on their behalf.
2907
2908 * If a user specifies a field on an event that isn't the event the
2909 * histogram currently being defined (the target event histogram), the
2910 * only way that can be accomplished is if a new hist trigger is
2911 * created and the field variable defined on that.
2912 *
2913 * This function creates a new histogram compatible with the target
2914 * event (meaning a histogram with the same key as the target
2915 * histogram), and creates a variable for the specified field, but
2916 * with 'synthetic_' prepended to the variable name in order to avoid
2917 * collision with normal field variables.
2918 *
2919 * Return: The variable created for the field.
2920 */
2921static struct hist_field *
2922create_field_var_hist(struct hist_trigger_data *target_hist_data,
2923		      char *subsys_name, char *event_name, char *field_name)
2924{
2925	struct trace_array *tr = target_hist_data->event_file->tr;
2926	struct hist_field *event_var = ERR_PTR(-EINVAL);
2927	struct hist_trigger_data *hist_data;
2928	unsigned int i, n, first = true;
2929	struct field_var_hist *var_hist;
2930	struct trace_event_file *file;
2931	struct hist_field *key_field;
2932	char *saved_filter;
2933	char *cmd;
2934	int ret;
2935
2936	if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2937		hist_err_event("onmatch: Too many field variables defined: ",
2938			       subsys_name, event_name, field_name);
2939		return ERR_PTR(-EINVAL);
2940	}
2941
2942	file = event_file(tr, subsys_name, event_name);
2943
2944	if (IS_ERR(file)) {
2945		hist_err_event("onmatch: Event file not found: ",
2946			       subsys_name, event_name, field_name);
2947		ret = PTR_ERR(file);
2948		return ERR_PTR(ret);
2949	}
2950
2951	/*
2952	 * Look for a histogram compatible with target.  We'll use the
2953	 * found histogram specification to create a new matching
2954	 * histogram with our variable on it.  target_hist_data is not
2955	 * yet a registered histogram so we can't use that.
2956	 */
2957	hist_data = find_compatible_hist(target_hist_data, file);
2958	if (!hist_data) {
2959		hist_err_event("onmatch: Matching event histogram not found: ",
2960			       subsys_name, event_name, field_name);
2961		return ERR_PTR(-EINVAL);
2962	}
2963
2964	/* See if a synthetic field variable has already been created */
2965	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2966					     event_name, field_name);
2967	if (!IS_ERR_OR_NULL(event_var))
2968		return event_var;
2969
2970	var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2971	if (!var_hist)
2972		return ERR_PTR(-ENOMEM);
2973
2974	cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2975	if (!cmd) {
2976		kfree(var_hist);
2977		return ERR_PTR(-ENOMEM);
2978	}
2979
2980	/* Use the same keys as the compatible histogram */
2981	strcat(cmd, "keys=");
2982
2983	for_each_hist_key_field(i, hist_data) {
2984		key_field = hist_data->fields[i];
2985		if (!first)
2986			strcat(cmd, ",");
2987		strcat(cmd, key_field->field->name);
2988		first = false;
2989	}
2990
2991	/* Create the synthetic field variable specification */
2992	strcat(cmd, ":synthetic_");
2993	strcat(cmd, field_name);
2994	strcat(cmd, "=");
2995	strcat(cmd, field_name);
2996
2997	/* Use the same filter as the compatible histogram */
2998	saved_filter = find_trigger_filter(hist_data, file);
2999	if (saved_filter) {
3000		strcat(cmd, " if ");
3001		strcat(cmd, saved_filter);
3002	}
3003
3004	var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3005	if (!var_hist->cmd) {
3006		kfree(cmd);
3007		kfree(var_hist);
3008		return ERR_PTR(-ENOMEM);
3009	}
3010
3011	/* Save the compatible histogram information */
3012	var_hist->hist_data = hist_data;
3013
3014	/* Create the new histogram with our variable */
3015	ret = event_hist_trigger_func(&trigger_hist_cmd, file,
3016				      "", "hist", cmd);
3017	if (ret) {
3018		kfree(cmd);
3019		kfree(var_hist->cmd);
3020		kfree(var_hist);
3021		hist_err_event("onmatch: Couldn't create histogram for field: ",
3022			       subsys_name, event_name, field_name);
3023		return ERR_PTR(ret);
3024	}
3025
3026	kfree(cmd);
3027
3028	/* If we can't find the variable, something went wrong */
3029	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3030					     event_name, field_name);
3031	if (IS_ERR_OR_NULL(event_var)) {
3032		kfree(var_hist->cmd);
3033		kfree(var_hist);
3034		hist_err_event("onmatch: Couldn't find synthetic variable: ",
3035			       subsys_name, event_name, field_name);
3036		return ERR_PTR(-EINVAL);
3037	}
3038
3039	n = target_hist_data->n_field_var_hists;
3040	target_hist_data->field_var_hists[n] = var_hist;
3041	target_hist_data->n_field_var_hists++;
3042
3043	return event_var;
3044}
3045
3046static struct hist_field *
3047find_target_event_var(struct hist_trigger_data *hist_data,
3048		      char *subsys_name, char *event_name, char *var_name)
3049{
3050	struct trace_event_file *file = hist_data->event_file;
3051	struct hist_field *hist_field = NULL;
3052
3053	if (subsys_name) {
3054		struct trace_event_call *call;
3055
3056		if (!event_name)
3057			return NULL;
3058
3059		call = file->event_call;
3060
3061		if (strcmp(subsys_name, call->class->system) != 0)
3062			return NULL;
3063
3064		if (strcmp(event_name, trace_event_name(call)) != 0)
3065			return NULL;
3066	}
3067
3068	hist_field = find_var_field(hist_data, var_name);
3069
3070	return hist_field;
3071}
3072
3073static inline void __update_field_vars(struct tracing_map_elt *elt,
3074				       struct ring_buffer_event *rbe,
3075				       void *rec,
3076				       struct field_var **field_vars,
3077				       unsigned int n_field_vars,
3078				       unsigned int field_var_str_start)
3079{
3080	struct hist_elt_data *elt_data = elt->private_data;
3081	unsigned int i, j, var_idx;
3082	u64 var_val;
3083
3084	for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3085		struct field_var *field_var = field_vars[i];
3086		struct hist_field *var = field_var->var;
3087		struct hist_field *val = field_var->val;
3088
3089		var_val = val->fn(val, elt, rbe, rec);
3090		var_idx = var->var.idx;
3091
3092		if (val->flags & HIST_FIELD_FL_STRING) {
3093			char *str = elt_data->field_var_str[j++];
3094			char *val_str = (char *)(uintptr_t)var_val;
3095
3096			strscpy(str, val_str, STR_VAR_LEN_MAX);
3097			var_val = (u64)(uintptr_t)str;
3098		}
3099		tracing_map_set_var(elt, var_idx, var_val);
3100	}
3101}
3102
3103static void update_field_vars(struct hist_trigger_data *hist_data,
3104			      struct tracing_map_elt *elt,
3105			      struct ring_buffer_event *rbe,
3106			      void *rec)
3107{
3108	__update_field_vars(elt, rbe, rec, hist_data->field_vars,
3109			    hist_data->n_field_vars, 0);
3110}
3111
3112static void update_max_vars(struct hist_trigger_data *hist_data,
3113			    struct tracing_map_elt *elt,
3114			    struct ring_buffer_event *rbe,
3115			    void *rec)
3116{
3117	__update_field_vars(elt, rbe, rec, hist_data->max_vars,
3118			    hist_data->n_max_vars, hist_data->n_field_var_str);
3119}
3120
3121static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3122				     struct trace_event_file *file,
3123				     char *name, int size, const char *type)
3124{
3125	struct hist_field *var;
3126	int idx;
3127
3128	if (find_var(hist_data, file, name) && !hist_data->remove) {
3129		var = ERR_PTR(-EINVAL);
3130		goto out;
3131	}
3132
3133	var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3134	if (!var) {
3135		var = ERR_PTR(-ENOMEM);
3136		goto out;
3137	}
3138
3139	idx = tracing_map_add_var(hist_data->map);
3140	if (idx < 0) {
3141		kfree(var);
3142		var = ERR_PTR(-EINVAL);
3143		goto out;
3144	}
3145
3146	var->flags = HIST_FIELD_FL_VAR;
3147	var->var.idx = idx;
3148	var->var.hist_data = var->hist_data = hist_data;
3149	var->size = size;
3150	var->var.name = kstrdup(name, GFP_KERNEL);
3151	var->type = kstrdup(type, GFP_KERNEL);
3152	if (!var->var.name || !var->type) {
3153		kfree(var->var.name);
3154		kfree(var->type);
3155		kfree(var);
3156		var = ERR_PTR(-ENOMEM);
3157	}
3158 out:
3159	return var;
3160}
3161
3162static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3163					  struct trace_event_file *file,
3164					  char *field_name)
3165{
3166	struct hist_field *val = NULL, *var = NULL;
3167	unsigned long flags = HIST_FIELD_FL_VAR;
3168	struct field_var *field_var;
3169	int ret = 0;
3170
3171	if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3172		hist_err("Too many field variables defined: ", field_name);
3173		ret = -EINVAL;
3174		goto err;
3175	}
3176
3177	val = parse_atom(hist_data, file, field_name, &flags, NULL);
3178	if (IS_ERR(val)) {
3179		hist_err("Couldn't parse field variable: ", field_name);
3180		ret = PTR_ERR(val);
3181		goto err;
3182	}
3183
3184	var = create_var(hist_data, file, field_name, val->size, val->type);
3185	if (IS_ERR(var)) {
3186		hist_err("Couldn't create or find variable: ", field_name);
3187		kfree(val);
3188		ret = PTR_ERR(var);
3189		goto err;
3190	}
3191
3192	field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3193	if (!field_var) {
3194		kfree(val);
3195		kfree(var);
3196		ret =  -ENOMEM;
3197		goto err;
3198	}
3199
3200	field_var->var = var;
3201	field_var->val = val;
3202 out:
3203	return field_var;
3204 err:
3205	field_var = ERR_PTR(ret);
3206	goto out;
3207}
3208
3209/**
3210 * create_target_field_var - Automatically create a variable for a field
3211 * @target_hist_data: The target hist trigger
3212 * @subsys_name: Optional subsystem name
3213 * @event_name: Optional event name
3214 * @var_name: The name of the field (and the resulting variable)
3215 *
3216 * Hist trigger actions fetch data from variables, not directly from
3217 * events.  However, for convenience, users are allowed to directly
3218 * specify an event field in an action, which will be automatically
3219 * converted into a variable on their behalf.
3220
3221 * This function creates a field variable with the name var_name on
3222 * the hist trigger currently being defined on the target event.  If
3223 * subsys_name and event_name are specified, this function simply
3224 * verifies that they do in fact match the target event subsystem and
3225 * event name.
3226 *
3227 * Return: The variable created for the field.
3228 */
3229static struct field_var *
3230create_target_field_var(struct hist_trigger_data *target_hist_data,
3231			char *subsys_name, char *event_name, char *var_name)
3232{
3233	struct trace_event_file *file = target_hist_data->event_file;
3234
3235	if (subsys_name) {
3236		struct trace_event_call *call;
3237
3238		if (!event_name)
3239			return NULL;
3240
3241		call = file->event_call;
3242
3243		if (strcmp(subsys_name, call->class->system) != 0)
3244			return NULL;
3245
3246		if (strcmp(event_name, trace_event_name(call)) != 0)
3247			return NULL;
3248	}
3249
3250	return create_field_var(target_hist_data, file, var_name);
3251}
3252
3253static void onmax_print(struct seq_file *m,
3254			struct hist_trigger_data *hist_data,
3255			struct tracing_map_elt *elt,
3256			struct action_data *data)
3257{
3258	unsigned int i, save_var_idx, max_idx = data->onmax.max_var->var.idx;
3259
3260	seq_printf(m, "\n\tmax: %10llu", tracing_map_read_var(elt, max_idx));
3261
3262	for (i = 0; i < hist_data->n_max_vars; i++) {
3263		struct hist_field *save_val = hist_data->max_vars[i]->val;
3264		struct hist_field *save_var = hist_data->max_vars[i]->var;
3265		u64 val;
3266
3267		save_var_idx = save_var->var.idx;
3268
3269		val = tracing_map_read_var(elt, save_var_idx);
3270
3271		if (save_val->flags & HIST_FIELD_FL_STRING) {
3272			seq_printf(m, "  %s: %-32s", save_var->var.name,
3273				   (char *)(uintptr_t)(val));
3274		} else
3275			seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3276	}
3277}
3278
3279static void onmax_save(struct hist_trigger_data *hist_data,
3280		       struct tracing_map_elt *elt, void *rec,
3281		       struct ring_buffer_event *rbe,
3282		       struct action_data *data, u64 *var_ref_vals)
3283{
3284	unsigned int max_idx = data->onmax.max_var->var.idx;
3285	unsigned int max_var_ref_idx = data->onmax.max_var_ref_idx;
3286
3287	u64 var_val, max_val;
3288
3289	var_val = var_ref_vals[max_var_ref_idx];
3290	max_val = tracing_map_read_var(elt, max_idx);
3291
3292	if (var_val <= max_val)
3293		return;
3294
3295	tracing_map_set_var(elt, max_idx, var_val);
3296
3297	update_max_vars(hist_data, elt, rbe, rec);
3298}
3299
3300static void onmax_destroy(struct action_data *data)
3301{
3302	unsigned int i;
3303
3304	destroy_hist_field(data->onmax.max_var, 0);
3305	destroy_hist_field(data->onmax.var, 0);
3306
3307	kfree(data->onmax.var_str);
3308	kfree(data->onmax.fn_name);
3309
3310	for (i = 0; i < data->n_params; i++)
3311		kfree(data->params[i]);
3312
3313	kfree(data);
3314}
3315
3316static int onmax_create(struct hist_trigger_data *hist_data,
3317			struct action_data *data)
3318{
3319	struct trace_event_file *file = hist_data->event_file;
3320	struct hist_field *var_field, *ref_field, *max_var;
3321	unsigned int var_ref_idx = hist_data->n_var_refs;
3322	struct field_var *field_var;
3323	char *onmax_var_str, *param;
3324	unsigned long flags;
3325	unsigned int i;
3326	int ret = 0;
3327
3328	onmax_var_str = data->onmax.var_str;
3329	if (onmax_var_str[0] != '$') {
3330		hist_err("onmax: For onmax(x), x must be a variable: ", onmax_var_str);
3331		return -EINVAL;
3332	}
3333	onmax_var_str++;
3334
3335	var_field = find_target_event_var(hist_data, NULL, NULL, onmax_var_str);
3336	if (!var_field) {
3337		hist_err("onmax: Couldn't find onmax variable: ", onmax_var_str);
3338		return -EINVAL;
3339	}
3340
3341	flags = HIST_FIELD_FL_VAR_REF;
3342	ref_field = create_hist_field(hist_data, NULL, flags, NULL);
3343	if (!ref_field)
3344		return -ENOMEM;
3345
3346	if (init_var_ref(ref_field, var_field, NULL, NULL)) {
3347		destroy_hist_field(ref_field, 0);
3348		ret = -ENOMEM;
3349		goto out;
3350	}
3351	hist_data->var_refs[hist_data->n_var_refs] = ref_field;
3352	ref_field->var_ref_idx = hist_data->n_var_refs++;
3353	data->onmax.var = ref_field;
3354
3355	data->fn = onmax_save;
3356	data->onmax.max_var_ref_idx = var_ref_idx;
3357	max_var = create_var(hist_data, file, "max", sizeof(u64), "u64");
3358	if (IS_ERR(max_var)) {
3359		hist_err("onmax: Couldn't create onmax variable: ", "max");
3360		ret = PTR_ERR(max_var);
3361		goto out;
3362	}
3363	data->onmax.max_var = max_var;
3364
3365	for (i = 0; i < data->n_params; i++) {
3366		param = kstrdup(data->params[i], GFP_KERNEL);
3367		if (!param) {
3368			ret = -ENOMEM;
3369			goto out;
3370		}
3371
3372		field_var = create_target_field_var(hist_data, NULL, NULL, param);
3373		if (IS_ERR(field_var)) {
3374			hist_err("onmax: Couldn't create field variable: ", param);
3375			ret = PTR_ERR(field_var);
3376			kfree(param);
3377			goto out;
3378		}
3379
3380		hist_data->max_vars[hist_data->n_max_vars++] = field_var;
3381		if (field_var->val->flags & HIST_FIELD_FL_STRING)
3382			hist_data->n_max_var_str++;
3383
3384		kfree(param);
3385	}
3386 out:
3387	return ret;
3388}
3389
3390static int parse_action_params(char *params, struct action_data *data)
3391{
3392	char *param, *saved_param;
3393	int ret = 0;
3394
3395	while (params) {
3396		if (data->n_params >= SYNTH_FIELDS_MAX)
3397			goto out;
3398
3399		param = strsep(&params, ",");
3400		if (!param) {
3401			ret = -EINVAL;
3402			goto out;
3403		}
3404
3405		param = strstrip(param);
3406		if (strlen(param) < 2) {
3407			hist_err("Invalid action param: ", param);
3408			ret = -EINVAL;
3409			goto out;
3410		}
3411
3412		saved_param = kstrdup(param, GFP_KERNEL);
3413		if (!saved_param) {
3414			ret = -ENOMEM;
3415			goto out;
3416		}
3417
3418		data->params[data->n_params++] = saved_param;
3419	}
3420 out:
3421	return ret;
3422}
3423
3424static struct action_data *onmax_parse(char *str)
3425{
3426	char *onmax_fn_name, *onmax_var_str;
3427	struct action_data *data;
3428	int ret = -EINVAL;
3429
3430	data = kzalloc(sizeof(*data), GFP_KERNEL);
3431	if (!data)
3432		return ERR_PTR(-ENOMEM);
3433
3434	onmax_var_str = strsep(&str, ")");
3435	if (!onmax_var_str || !str) {
3436		ret = -EINVAL;
3437		goto free;
3438	}
3439
3440	data->onmax.var_str = kstrdup(onmax_var_str, GFP_KERNEL);
3441	if (!data->onmax.var_str) {
3442		ret = -ENOMEM;
3443		goto free;
3444	}
3445
3446	strsep(&str, ".");
3447	if (!str)
3448		goto free;
3449
3450	onmax_fn_name = strsep(&str, "(");
3451	if (!onmax_fn_name || !str)
3452		goto free;
3453
3454	if (strncmp(onmax_fn_name, "save", strlen("save")) == 0) {
3455		char *params = strsep(&str, ")");
3456
3457		if (!params) {
3458			ret = -EINVAL;
3459			goto free;
3460		}
3461
3462		ret = parse_action_params(params, data);
3463		if (ret)
3464			goto free;
3465	} else
3466		goto free;
3467
3468	data->onmax.fn_name = kstrdup(onmax_fn_name, GFP_KERNEL);
3469	if (!data->onmax.fn_name) {
3470		ret = -ENOMEM;
3471		goto free;
3472	}
3473 out:
3474	return data;
3475 free:
3476	onmax_destroy(data);
3477	data = ERR_PTR(ret);
3478	goto out;
3479}
3480
3481static void onmatch_destroy(struct action_data *data)
3482{
3483	unsigned int i;
3484
3485	mutex_lock(&synth_event_mutex);
3486
3487	kfree(data->onmatch.match_event);
3488	kfree(data->onmatch.match_event_system);
3489	kfree(data->onmatch.synth_event_name);
3490
3491	for (i = 0; i < data->n_params; i++)
3492		kfree(data->params[i]);
3493
3494	if (data->onmatch.synth_event)
3495		data->onmatch.synth_event->ref--;
3496
3497	kfree(data);
3498
3499	mutex_unlock(&synth_event_mutex);
3500}
3501
3502static void destroy_field_var(struct field_var *field_var)
3503{
3504	if (!field_var)
3505		return;
3506
3507	destroy_hist_field(field_var->var, 0);
3508	destroy_hist_field(field_var->val, 0);
3509
3510	kfree(field_var);
3511}
3512
3513static void destroy_field_vars(struct hist_trigger_data *hist_data)
3514{
3515	unsigned int i;
3516
3517	for (i = 0; i < hist_data->n_field_vars; i++)
3518		destroy_field_var(hist_data->field_vars[i]);
3519}
3520
3521static void save_field_var(struct hist_trigger_data *hist_data,
3522			   struct field_var *field_var)
3523{
3524	hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3525
3526	if (field_var->val->flags & HIST_FIELD_FL_STRING)
3527		hist_data->n_field_var_str++;
3528}
3529
3530
3531static void destroy_synth_var_refs(struct hist_trigger_data *hist_data)
3532{
3533	unsigned int i;
3534
3535	for (i = 0; i < hist_data->n_synth_var_refs; i++)
3536		destroy_hist_field(hist_data->synth_var_refs[i], 0);
3537}
3538
3539static void save_synth_var_ref(struct hist_trigger_data *hist_data,
3540			 struct hist_field *var_ref)
3541{
3542	hist_data->synth_var_refs[hist_data->n_synth_var_refs++] = var_ref;
3543
3544	hist_data->var_refs[hist_data->n_var_refs] = var_ref;
3545	var_ref->var_ref_idx = hist_data->n_var_refs++;
3546}
3547
3548static int check_synth_field(struct synth_event *event,
3549			     struct hist_field *hist_field,
3550			     unsigned int field_pos)
3551{
3552	struct synth_field *field;
3553
3554	if (field_pos >= event->n_fields)
3555		return -EINVAL;
3556
3557	field = event->fields[field_pos];
3558
3559	if (strcmp(field->type, hist_field->type) != 0)
3560		return -EINVAL;
3561
3562	return 0;
3563}
3564
3565static struct hist_field *
3566onmatch_find_var(struct hist_trigger_data *hist_data, struct action_data *data,
3567		 char *system, char *event, char *var)
3568{
3569	struct hist_field *hist_field;
3570
3571	var++; /* skip '$' */
3572
3573	hist_field = find_target_event_var(hist_data, system, event, var);
3574	if (!hist_field) {
3575		if (!system) {
3576			system = data->onmatch.match_event_system;
3577			event = data->onmatch.match_event;
3578		}
3579
3580		hist_field = find_event_var(hist_data, system, event, var);
3581	}
3582
3583	if (!hist_field)
3584		hist_err_event("onmatch: Couldn't find onmatch param: $", system, event, var);
3585
3586	return hist_field;
3587}
3588
3589static struct hist_field *
3590onmatch_create_field_var(struct hist_trigger_data *hist_data,
3591			 struct action_data *data, char *system,
3592			 char *event, char *var)
3593{
3594	struct hist_field *hist_field = NULL;
3595	struct field_var *field_var;
3596
3597	/*
3598	 * First try to create a field var on the target event (the
3599	 * currently being defined).  This will create a variable for
3600	 * unqualified fields on the target event, or if qualified,
3601	 * target fields that have qualified names matching the target.
3602	 */
3603	field_var = create_target_field_var(hist_data, system, event, var);
3604
3605	if (field_var && !IS_ERR(field_var)) {
3606		save_field_var(hist_data, field_var);
3607		hist_field = field_var->var;
3608	} else {
3609		field_var = NULL;
3610		/*
3611		 * If no explicit system.event is specfied, default to
3612		 * looking for fields on the onmatch(system.event.xxx)
3613		 * event.
3614		 */
3615		if (!system) {
3616			system = data->onmatch.match_event_system;
3617			event = data->onmatch.match_event;
3618		}
3619
3620		/*
3621		 * At this point, we're looking at a field on another
3622		 * event.  Because we can't modify a hist trigger on
3623		 * another event to add a variable for a field, we need
3624		 * to create a new trigger on that event and create the
3625		 * variable at the same time.
3626		 */
3627		hist_field = create_field_var_hist(hist_data, system, event, var);
3628		if (IS_ERR(hist_field))
3629			goto free;
3630	}
3631 out:
3632	return hist_field;
3633 free:
3634	destroy_field_var(field_var);
3635	hist_field = NULL;
3636	goto out;
3637}
3638
3639static int onmatch_create(struct hist_trigger_data *hist_data,
3640			  struct trace_event_file *file,
3641			  struct action_data *data)
3642{
3643	char *event_name, *param, *system = NULL;
3644	struct hist_field *hist_field, *var_ref;
3645	unsigned int i, var_ref_idx;
3646	unsigned int field_pos = 0;
3647	struct synth_event *event;
3648	int ret = 0;
3649
3650	mutex_lock(&synth_event_mutex);
3651	event = find_synth_event(data->onmatch.synth_event_name);
3652	if (!event) {
3653		hist_err("onmatch: Couldn't find synthetic event: ", data->onmatch.synth_event_name);
3654		mutex_unlock(&synth_event_mutex);
3655		return -EINVAL;
3656	}
3657	event->ref++;
3658	mutex_unlock(&synth_event_mutex);
3659
3660	var_ref_idx = hist_data->n_var_refs;
3661
3662	for (i = 0; i < data->n_params; i++) {
3663		char *p;
3664
3665		p = param = kstrdup(data->params[i], GFP_KERNEL);
3666		if (!param) {
3667			ret = -ENOMEM;
3668			goto err;
3669		}
3670
3671		system = strsep(&param, ".");
3672		if (!param) {
3673			param = (char *)system;
3674			system = event_name = NULL;
3675		} else {
3676			event_name = strsep(&param, ".");
3677			if (!param) {
3678				kfree(p);
3679				ret = -EINVAL;
3680				goto err;
3681			}
3682		}
3683
3684		if (param[0] == '$')
3685			hist_field = onmatch_find_var(hist_data, data, system,
3686						      event_name, param);
3687		else
3688			hist_field = onmatch_create_field_var(hist_data, data,
3689							      system,
3690							      event_name,
3691							      param);
3692
3693		if (!hist_field) {
3694			kfree(p);
3695			ret = -EINVAL;
3696			goto err;
3697		}
3698
3699		if (check_synth_field(event, hist_field, field_pos) == 0) {
3700			var_ref = create_var_ref(hist_field, system, event_name);
3701			if (!var_ref) {
3702				kfree(p);
3703				ret = -ENOMEM;
3704				goto err;
3705			}
3706
3707			save_synth_var_ref(hist_data, var_ref);
3708			field_pos++;
3709			kfree(p);
3710			continue;
3711		}
3712
3713		hist_err_event("onmatch: Param type doesn't match synthetic event field type: ",
3714			       system, event_name, param);
3715		kfree(p);
3716		ret = -EINVAL;
3717		goto err;
3718	}
3719
3720	if (field_pos != event->n_fields) {
3721		hist_err("onmatch: Param count doesn't match synthetic event field count: ", event->name);
3722		ret = -EINVAL;
3723		goto err;
3724	}
3725
3726	data->fn = action_trace;
3727	data->onmatch.synth_event = event;
3728	data->onmatch.var_ref_idx = var_ref_idx;
3729 out:
3730	return ret;
3731 err:
3732	mutex_lock(&synth_event_mutex);
3733	event->ref--;
3734	mutex_unlock(&synth_event_mutex);
3735
3736	goto out;
3737}
3738
3739static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3740{
3741	char *match_event, *match_event_system;
3742	char *synth_event_name, *params;
3743	struct action_data *data;
3744	int ret = -EINVAL;
3745
3746	data = kzalloc(sizeof(*data), GFP_KERNEL);
3747	if (!data)
3748		return ERR_PTR(-ENOMEM);
3749
3750	match_event = strsep(&str, ")");
3751	if (!match_event || !str) {
3752		hist_err("onmatch: Missing closing paren: ", match_event);
3753		goto free;
3754	}
3755
3756	match_event_system = strsep(&match_event, ".");
3757	if (!match_event) {
3758		hist_err("onmatch: Missing subsystem for match event: ", match_event_system);
3759		goto free;
3760	}
3761
3762	if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3763		hist_err_event("onmatch: Invalid subsystem or event name: ",
3764			       match_event_system, match_event, NULL);
3765		goto free;
3766	}
3767
3768	data->onmatch.match_event = kstrdup(match_event, GFP_KERNEL);
3769	if (!data->onmatch.match_event) {
3770		ret = -ENOMEM;
3771		goto free;
3772	}
3773
3774	data->onmatch.match_event_system = kstrdup(match_event_system, GFP_KERNEL);
3775	if (!data->onmatch.match_event_system) {
3776		ret = -ENOMEM;
3777		goto free;
3778	}
3779
3780	strsep(&str, ".");
3781	if (!str) {
3782		hist_err("onmatch: Missing . after onmatch(): ", str);
3783		goto free;
3784	}
3785
3786	synth_event_name = strsep(&str, "(");
3787	if (!synth_event_name || !str) {
3788		hist_err("onmatch: Missing opening paramlist paren: ", synth_event_name);
3789		goto free;
3790	}
3791
3792	data->onmatch.synth_event_name = kstrdup(synth_event_name, GFP_KERNEL);
3793	if (!data->onmatch.synth_event_name) {
3794		ret = -ENOMEM;
3795		goto free;
3796	}
3797
3798	params = strsep(&str, ")");
3799	if (!params || !str || (str && strlen(str))) {
3800		hist_err("onmatch: Missing closing paramlist paren: ", params);
3801		goto free;
3802	}
3803
3804	ret = parse_action_params(params, data);
3805	if (ret)
3806		goto free;
3807 out:
3808	return data;
3809 free:
3810	onmatch_destroy(data);
3811	data = ERR_PTR(ret);
3812	goto out;
3813}
3814
3815static int create_hitcount_val(struct hist_trigger_data *hist_data)
3816{
3817	hist_data->fields[HITCOUNT_IDX] =
3818		create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
3819	if (!hist_data->fields[HITCOUNT_IDX])
3820		return -ENOMEM;
3821
3822	hist_data->n_vals++;
3823	hist_data->n_fields++;
3824
3825	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3826		return -EINVAL;
3827
3828	return 0;
3829}
3830
3831static int __create_val_field(struct hist_trigger_data *hist_data,
3832			      unsigned int val_idx,
3833			      struct trace_event_file *file,
3834			      char *var_name, char *field_str,
3835			      unsigned long flags)
3836{
3837	struct hist_field *hist_field;
3838	int ret = 0;
3839
3840	hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3841	if (IS_ERR(hist_field)) {
3842		ret = PTR_ERR(hist_field);
3843		goto out;
3844	}
3845
3846	hist_data->fields[val_idx] = hist_field;
3847
3848	++hist_data->n_vals;
3849	++hist_data->n_fields;
3850
3851	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3852		ret = -EINVAL;
3853 out:
3854	return ret;
3855}
3856
3857static int create_val_field(struct hist_trigger_data *hist_data,
3858			    unsigned int val_idx,
3859			    struct trace_event_file *file,
3860			    char *field_str)
3861{
3862	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3863		return -EINVAL;
3864
3865	return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3866}
3867
3868static int create_var_field(struct hist_trigger_data *hist_data,
3869			    unsigned int val_idx,
3870			    struct trace_event_file *file,
3871			    char *var_name, char *expr_str)
3872{
3873	unsigned long flags = 0;
3874
3875	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3876		return -EINVAL;
3877
3878	if (find_var(hist_data, file, var_name) && !hist_data->remove) {
3879		hist_err("Variable already defined: ", var_name);
3880		return -EINVAL;
3881	}
3882
3883	flags |= HIST_FIELD_FL_VAR;
3884	hist_data->n_vars++;
3885	if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3886		return -EINVAL;
3887
3888	return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3889}
3890
3891static int create_val_fields(struct hist_trigger_data *hist_data,
3892			     struct trace_event_file *file)
3893{
3894	char *fields_str, *field_str;
3895	unsigned int i, j = 1;
3896	int ret;
3897
3898	ret = create_hitcount_val(hist_data);
3899	if (ret)
3900		goto out;
3901
3902	fields_str = hist_data->attrs->vals_str;
3903	if (!fields_str)
3904		goto out;
3905
3906	strsep(&fields_str, "=");
3907	if (!fields_str)
3908		goto out;
3909
3910	for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3911		     j < TRACING_MAP_VALS_MAX; i++) {
3912		field_str = strsep(&fields_str, ",");
3913		if (!field_str)
3914			break;
3915
3916		if (strcmp(field_str, "hitcount") == 0)
3917			continue;
3918
3919		ret = create_val_field(hist_data, j++, file, field_str);
3920		if (ret)
3921			goto out;
3922	}
3923
3924	if (fields_str && (strcmp(fields_str, "hitcount") != 0))
3925		ret = -EINVAL;
3926 out:
3927	return ret;
3928}
3929
3930static int create_key_field(struct hist_trigger_data *hist_data,
3931			    unsigned int key_idx,
3932			    unsigned int key_offset,
3933			    struct trace_event_file *file,
3934			    char *field_str)
3935{
3936	struct hist_field *hist_field = NULL;
3937
3938	unsigned long flags = 0;
3939	unsigned int key_size;
3940	int ret = 0;
3941
3942	if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
3943		return -EINVAL;
3944
3945	flags |= HIST_FIELD_FL_KEY;
3946
3947	if (strcmp(field_str, "stacktrace") == 0) {
3948		flags |= HIST_FIELD_FL_STACKTRACE;
3949		key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
3950		hist_field = create_hist_field(hist_data, NULL, flags, NULL);
3951	} else {
3952		hist_field = parse_expr(hist_data, file, field_str, flags,
3953					NULL, 0);
3954		if (IS_ERR(hist_field)) {
3955			ret = PTR_ERR(hist_field);
3956			goto out;
3957		}
3958
3959		if (hist_field->flags & HIST_FIELD_FL_VAR_REF) {
3960			hist_err("Using variable references as keys not supported: ", field_str);
3961			destroy_hist_field(hist_field, 0);
3962			ret = -EINVAL;
3963			goto out;
3964		}
3965
3966		key_size = hist_field->size;
3967	}
3968
3969	hist_data->fields[key_idx] = hist_field;
3970
3971	key_size = ALIGN(key_size, sizeof(u64));
3972	hist_data->fields[key_idx]->size = key_size;
3973	hist_data->fields[key_idx]->offset = key_offset;
3974
3975	hist_data->key_size += key_size;
3976
3977	if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
3978		ret = -EINVAL;
3979		goto out;
3980	}
3981
3982	hist_data->n_keys++;
3983	hist_data->n_fields++;
3984
3985	if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
3986		return -EINVAL;
3987
3988	ret = key_size;
3989 out:
3990	return ret;
3991}
3992
3993static int create_key_fields(struct hist_trigger_data *hist_data,
3994			     struct trace_event_file *file)
3995{
3996	unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
3997	char *fields_str, *field_str;
3998	int ret = -EINVAL;
3999
4000	fields_str = hist_data->attrs->keys_str;
4001	if (!fields_str)
4002		goto out;
4003
4004	strsep(&fields_str, "=");
4005	if (!fields_str)
4006		goto out;
4007
4008	for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4009		field_str = strsep(&fields_str, ",");
4010		if (!field_str)
4011			break;
4012		ret = create_key_field(hist_data, i, key_offset,
4013				       file, field_str);
4014		if (ret < 0)
4015			goto out;
4016		key_offset += ret;
4017	}
4018	if (fields_str) {
4019		ret = -EINVAL;
4020		goto out;
4021	}
4022	ret = 0;
4023 out:
4024	return ret;
4025}
4026
4027static int create_var_fields(struct hist_trigger_data *hist_data,
4028			     struct trace_event_file *file)
4029{
4030	unsigned int i, j = hist_data->n_vals;
4031	int ret = 0;
4032
4033	unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4034
4035	for (i = 0; i < n_vars; i++) {
4036		char *var_name = hist_data->attrs->var_defs.name[i];
4037		char *expr = hist_data->attrs->var_defs.expr[i];
4038
4039		ret = create_var_field(hist_data, j++, file, var_name, expr);
4040		if (ret)
4041			goto out;
4042	}
4043 out:
4044	return ret;
4045}
4046
4047static void free_var_defs(struct hist_trigger_data *hist_data)
4048{
4049	unsigned int i;
4050
4051	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4052		kfree(hist_data->attrs->var_defs.name[i]);
4053		kfree(hist_data->attrs->var_defs.expr[i]);
4054	}
4055
4056	hist_data->attrs->var_defs.n_vars = 0;
4057}
4058
4059static int parse_var_defs(struct hist_trigger_data *hist_data)
4060{
4061	char *s, *str, *var_name, *field_str;
4062	unsigned int i, j, n_vars = 0;
4063	int ret = 0;
4064
4065	for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4066		str = hist_data->attrs->assignment_str[i];
4067		for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4068			field_str = strsep(&str, ",");
4069			if (!field_str)
4070				break;
4071
4072			var_name = strsep(&field_str, "=");
4073			if (!var_name || !field_str) {
4074				hist_err("Malformed assignment: ", var_name);
4075				ret = -EINVAL;
4076				goto free;
4077			}
4078
4079			if (n_vars == TRACING_MAP_VARS_MAX) {
4080				hist_err("Too many variables defined: ", var_name);
4081				ret = -EINVAL;
4082				goto free;
4083			}
4084
4085			s = kstrdup(var_name, GFP_KERNEL);
4086			if (!s) {
4087				ret = -ENOMEM;
4088				goto free;
4089			}
4090			hist_data->attrs->var_defs.name[n_vars] = s;
4091
4092			s = kstrdup(field_str, GFP_KERNEL);
4093			if (!s) {
4094				kfree(hist_data->attrs->var_defs.name[n_vars]);
4095				ret = -ENOMEM;
4096				goto free;
4097			}
4098			hist_data->attrs->var_defs.expr[n_vars++] = s;
4099
4100			hist_data->attrs->var_defs.n_vars = n_vars;
4101		}
4102	}
4103
4104	return ret;
4105 free:
4106	free_var_defs(hist_data);
4107
4108	return ret;
4109}
4110
4111static int create_hist_fields(struct hist_trigger_data *hist_data,
4112			      struct trace_event_file *file)
4113{
4114	int ret;
4115
4116	ret = parse_var_defs(hist_data);
4117	if (ret)
4118		goto out;
4119
4120	ret = create_val_fields(hist_data, file);
4121	if (ret)
4122		goto out;
4123
4124	ret = create_var_fields(hist_data, file);
4125	if (ret)
4126		goto out;
4127
4128	ret = create_key_fields(hist_data, file);
4129	if (ret)
4130		goto out;
4131 out:
4132	free_var_defs(hist_data);
4133
4134	return ret;
4135}
4136
4137static int is_descending(const char *str)
4138{
4139	if (!str)
4140		return 0;
4141
4142	if (strcmp(str, "descending") == 0)
4143		return 1;
4144
4145	if (strcmp(str, "ascending") == 0)
4146		return 0;
4147
4148	return -EINVAL;
4149}
4150
4151static int create_sort_keys(struct hist_trigger_data *hist_data)
4152{
4153	char *fields_str = hist_data->attrs->sort_key_str;
4154	struct tracing_map_sort_key *sort_key;
4155	int descending, ret = 0;
4156	unsigned int i, j, k;
4157
4158	hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4159
4160	if (!fields_str)
4161		goto out;
4162
4163	strsep(&fields_str, "=");
4164	if (!fields_str) {
4165		ret = -EINVAL;
4166		goto out;
4167	}
4168
4169	for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4170		struct hist_field *hist_field;
4171		char *field_str, *field_name;
4172		const char *test_name;
4173
4174		sort_key = &hist_data->sort_keys[i];
4175
4176		field_str = strsep(&fields_str, ",");
4177		if (!field_str) {
4178			if (i == 0)
4179				ret = -EINVAL;
4180			break;
4181		}
4182
4183		if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4184			ret = -EINVAL;
4185			break;
4186		}
4187
4188		field_name = strsep(&field_str, ".");
4189		if (!field_name) {
4190			ret = -EINVAL;
4191			break;
4192		}
4193
4194		if (strcmp(field_name, "hitcount") == 0) {
4195			descending = is_descending(field_str);
4196			if (descending < 0) {
4197				ret = descending;
4198				break;
4199			}
4200			sort_key->descending = descending;
4201			continue;
4202		}
4203
4204		for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4205			unsigned int idx;
4206
4207			hist_field = hist_data->fields[j];
4208			if (hist_field->flags & HIST_FIELD_FL_VAR)
4209				continue;
4210
4211			idx = k++;
4212
4213			test_name = hist_field_name(hist_field, 0);
4214
4215			if (strcmp(field_name, test_name) == 0) {
4216				sort_key->field_idx = idx;
4217				descending = is_descending(field_str);
4218				if (descending < 0) {
4219					ret = descending;
4220					goto out;
4221				}
4222				sort_key->descending = descending;
4223				break;
4224			}
4225		}
4226		if (j == hist_data->n_fields) {
4227			ret = -EINVAL;
4228			break;
4229		}
4230	}
4231
4232	hist_data->n_sort_keys = i;
4233 out:
4234	return ret;
4235}
4236
4237static void destroy_actions(struct hist_trigger_data *hist_data)
4238{
4239	unsigned int i;
4240
4241	for (i = 0; i < hist_data->n_actions; i++) {
4242		struct action_data *data = hist_data->actions[i];
4243
4244		if (data->fn == action_trace)
4245			onmatch_destroy(data);
4246		else if (data->fn == onmax_save)
4247			onmax_destroy(data);
4248		else
4249			kfree(data);
4250	}
4251}
4252
4253static int parse_actions(struct hist_trigger_data *hist_data)
4254{
4255	struct trace_array *tr = hist_data->event_file->tr;
4256	struct action_data *data;
4257	unsigned int i;
4258	int ret = 0;
4259	char *str;
4260
4261	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4262		str = hist_data->attrs->action_str[i];
4263
4264		if (strncmp(str, "onmatch(", strlen("onmatch(")) == 0) {
4265			char *action_str = str + strlen("onmatch(");
4266
4267			data = onmatch_parse(tr, action_str);
4268			if (IS_ERR(data)) {
4269				ret = PTR_ERR(data);
4270				break;
4271			}
4272			data->fn = action_trace;
4273		} else if (strncmp(str, "onmax(", strlen("onmax(")) == 0) {
4274			char *action_str = str + strlen("onmax(");
4275
4276			data = onmax_parse(action_str);
4277			if (IS_ERR(data)) {
4278				ret = PTR_ERR(data);
4279				break;
4280			}
4281			data->fn = onmax_save;
4282		} else {
4283			ret = -EINVAL;
4284			break;
4285		}
4286
4287		hist_data->actions[hist_data->n_actions++] = data;
4288	}
4289
4290	return ret;
4291}
4292
4293static int create_actions(struct hist_trigger_data *hist_data,
4294			  struct trace_event_file *file)
4295{
4296	struct action_data *data;
4297	unsigned int i;
4298	int ret = 0;
4299
4300	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4301		data = hist_data->actions[i];
4302
4303		if (data->fn == action_trace) {
4304			ret = onmatch_create(hist_data, file, data);
4305			if (ret)
4306				return ret;
4307		} else if (data->fn == onmax_save) {
4308			ret = onmax_create(hist_data, data);
4309			if (ret)
4310				return ret;
4311		}
4312	}
4313
4314	return ret;
4315}
4316
4317static void print_actions(struct seq_file *m,
4318			  struct hist_trigger_data *hist_data,
4319			  struct tracing_map_elt *elt)
4320{
4321	unsigned int i;
4322
4323	for (i = 0; i < hist_data->n_actions; i++) {
4324		struct action_data *data = hist_data->actions[i];
4325
4326		if (data->fn == onmax_save)
4327			onmax_print(m, hist_data, elt, data);
4328	}
4329}
4330
4331static void print_onmax_spec(struct seq_file *m,
4332			     struct hist_trigger_data *hist_data,
4333			     struct action_data *data)
4334{
4335	unsigned int i;
4336
4337	seq_puts(m, ":onmax(");
4338	seq_printf(m, "%s", data->onmax.var_str);
4339	seq_printf(m, ").%s(", data->onmax.fn_name);
4340
4341	for (i = 0; i < hist_data->n_max_vars; i++) {
4342		seq_printf(m, "%s", hist_data->max_vars[i]->var->var.name);
4343		if (i < hist_data->n_max_vars - 1)
4344			seq_puts(m, ",");
4345	}
4346	seq_puts(m, ")");
4347}
4348
4349static void print_onmatch_spec(struct seq_file *m,
4350			       struct hist_trigger_data *hist_data,
4351			       struct action_data *data)
4352{
4353	unsigned int i;
4354
4355	seq_printf(m, ":onmatch(%s.%s).", data->onmatch.match_event_system,
4356		   data->onmatch.match_event);
4357
4358	seq_printf(m, "%s(", data->onmatch.synth_event->name);
4359
4360	for (i = 0; i < data->n_params; i++) {
4361		if (i)
4362			seq_puts(m, ",");
4363		seq_printf(m, "%s", data->params[i]);
4364	}
4365
4366	seq_puts(m, ")");
4367}
4368
4369static bool actions_match(struct hist_trigger_data *hist_data,
4370			  struct hist_trigger_data *hist_data_test)
4371{
4372	unsigned int i, j;
4373
4374	if (hist_data->n_actions != hist_data_test->n_actions)
4375		return false;
4376
4377	for (i = 0; i < hist_data->n_actions; i++) {
4378		struct action_data *data = hist_data->actions[i];
4379		struct action_data *data_test = hist_data_test->actions[i];
4380
4381		if (data->fn != data_test->fn)
4382			return false;
4383
4384		if (data->n_params != data_test->n_params)
4385			return false;
4386
4387		for (j = 0; j < data->n_params; j++) {
4388			if (strcmp(data->params[j], data_test->params[j]) != 0)
4389				return false;
4390		}
4391
4392		if (data->fn == action_trace) {
4393			if (strcmp(data->onmatch.synth_event_name,
4394				   data_test->onmatch.synth_event_name) != 0)
4395				return false;
4396			if (strcmp(data->onmatch.match_event_system,
4397				   data_test->onmatch.match_event_system) != 0)
4398				return false;
4399			if (strcmp(data->onmatch.match_event,
4400				   data_test->onmatch.match_event) != 0)
4401				return false;
4402		} else if (data->fn == onmax_save) {
4403			if (strcmp(data->onmax.var_str,
4404				   data_test->onmax.var_str) != 0)
4405				return false;
4406			if (strcmp(data->onmax.fn_name,
4407				   data_test->onmax.fn_name) != 0)
4408				return false;
4409		}
4410	}
4411
4412	return true;
4413}
4414
4415
4416static void print_actions_spec(struct seq_file *m,
4417			       struct hist_trigger_data *hist_data)
4418{
4419	unsigned int i;
4420
4421	for (i = 0; i < hist_data->n_actions; i++) {
4422		struct action_data *data = hist_data->actions[i];
4423
4424		if (data->fn == action_trace)
4425			print_onmatch_spec(m, hist_data, data);
4426		else if (data->fn == onmax_save)
4427			print_onmax_spec(m, hist_data, data);
4428	}
4429}
4430
4431static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4432{
4433	unsigned int i;
4434
4435	for (i = 0; i < hist_data->n_field_var_hists; i++) {
4436		kfree(hist_data->field_var_hists[i]->cmd);
4437		kfree(hist_data->field_var_hists[i]);
4438	}
4439}
4440
4441static void destroy_hist_data(struct hist_trigger_data *hist_data)
4442{
4443	if (!hist_data)
4444		return;
4445
4446	destroy_hist_trigger_attrs(hist_data->attrs);
4447	destroy_hist_fields(hist_data);
4448	tracing_map_destroy(hist_data->map);
4449
4450	destroy_actions(hist_data);
4451	destroy_field_vars(hist_data);
4452	destroy_field_var_hists(hist_data);
4453	destroy_synth_var_refs(hist_data);
4454
4455	kfree(hist_data);
4456}
4457
4458static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4459{
4460	struct tracing_map *map = hist_data->map;
4461	struct ftrace_event_field *field;
4462	struct hist_field *hist_field;
4463	int i, idx = 0;
4464
4465	for_each_hist_field(i, hist_data) {
4466		hist_field = hist_data->fields[i];
4467		if (hist_field->flags & HIST_FIELD_FL_KEY) {
4468			tracing_map_cmp_fn_t cmp_fn;
4469
4470			field = hist_field->field;
4471
4472			if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4473				cmp_fn = tracing_map_cmp_none;
4474			else if (!field)
4475				cmp_fn = tracing_map_cmp_num(hist_field->size,
4476							     hist_field->is_signed);
4477			else if (is_string_field(field))
4478				cmp_fn = tracing_map_cmp_string;
4479			else
4480				cmp_fn = tracing_map_cmp_num(field->size,
4481							     field->is_signed);
4482			idx = tracing_map_add_key_field(map,
4483							hist_field->offset,
4484							cmp_fn);
4485		} else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4486			idx = tracing_map_add_sum_field(map);
4487
4488		if (idx < 0)
4489			return idx;
4490
4491		if (hist_field->flags & HIST_FIELD_FL_VAR) {
4492			idx = tracing_map_add_var(map);
4493			if (idx < 0)
4494				return idx;
4495			hist_field->var.idx = idx;
4496			hist_field->var.hist_data = hist_data;
4497		}
4498	}
4499
4500	return 0;
4501}
4502
4503static struct hist_trigger_data *
4504create_hist_data(unsigned int map_bits,
4505		 struct hist_trigger_attrs *attrs,
4506		 struct trace_event_file *file,
4507		 bool remove)
4508{
4509	const struct tracing_map_ops *map_ops = NULL;
4510	struct hist_trigger_data *hist_data;
4511	int ret = 0;
4512
4513	hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4514	if (!hist_data)
4515		return ERR_PTR(-ENOMEM);
4516
4517	hist_data->attrs = attrs;
4518	hist_data->remove = remove;
4519	hist_data->event_file = file;
4520
4521	ret = parse_actions(hist_data);
4522	if (ret)
4523		goto free;
4524
4525	ret = create_hist_fields(hist_data, file);
4526	if (ret)
4527		goto free;
4528
4529	ret = create_sort_keys(hist_data);
4530	if (ret)
4531		goto free;
4532
4533	map_ops = &hist_trigger_elt_data_ops;
4534
4535	hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4536					    map_ops, hist_data);
4537	if (IS_ERR(hist_data->map)) {
4538		ret = PTR_ERR(hist_data->map);
4539		hist_data->map = NULL;
4540		goto free;
4541	}
4542
4543	ret = create_tracing_map_fields(hist_data);
4544	if (ret)
4545		goto free;
4546 out:
4547	return hist_data;
4548 free:
4549	hist_data->attrs = NULL;
4550
4551	destroy_hist_data(hist_data);
4552
4553	hist_data = ERR_PTR(ret);
4554
4555	goto out;
4556}
4557
4558static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4559				    struct tracing_map_elt *elt, void *rec,
4560				    struct ring_buffer_event *rbe,
4561				    u64 *var_ref_vals)
4562{
4563	struct hist_elt_data *elt_data;
4564	struct hist_field *hist_field;
4565	unsigned int i, var_idx;
4566	u64 hist_val;
4567
4568	elt_data = elt->private_data;
4569	elt_data->var_ref_vals = var_ref_vals;
4570
4571	for_each_hist_val_field(i, hist_data) {
4572		hist_field = hist_data->fields[i];
4573		hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4574		if (hist_field->flags & HIST_FIELD_FL_VAR) {
4575			var_idx = hist_field->var.idx;
4576			tracing_map_set_var(elt, var_idx, hist_val);
4577			continue;
4578		}
4579		tracing_map_update_sum(elt, i, hist_val);
4580	}
4581
4582	for_each_hist_key_field(i, hist_data) {
4583		hist_field = hist_data->fields[i];
4584		if (hist_field->flags & HIST_FIELD_FL_VAR) {
4585			hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4586			var_idx = hist_field->var.idx;
4587			tracing_map_set_var(elt, var_idx, hist_val);
4588		}
4589	}
4590
4591	update_field_vars(hist_data, elt, rbe, rec);
4592}
4593
4594static inline void add_to_key(char *compound_key, void *key,
4595			      struct hist_field *key_field, void *rec)
4596{
4597	size_t size = key_field->size;
4598
4599	if (key_field->flags & HIST_FIELD_FL_STRING) {
4600		struct ftrace_event_field *field;
4601
4602		field = key_field->field;
4603		if (field->filter_type == FILTER_DYN_STRING)
4604			size = *(u32 *)(rec + field->offset) >> 16;
4605		else if (field->filter_type == FILTER_PTR_STRING)
4606			size = strlen(key);
4607		else if (field->filter_type == FILTER_STATIC_STRING)
4608			size = field->size;
4609
4610		/* ensure NULL-termination */
4611		if (size > key_field->size - 1)
4612			size = key_field->size - 1;
4613	}
4614
4615	memcpy(compound_key + key_field->offset, key, size);
4616}
4617
4618static void
4619hist_trigger_actions(struct hist_trigger_data *hist_data,
4620		     struct tracing_map_elt *elt, void *rec,
4621		     struct ring_buffer_event *rbe, u64 *var_ref_vals)
4622{
4623	struct action_data *data;
4624	unsigned int i;
4625
4626	for (i = 0; i < hist_data->n_actions; i++) {
4627		data = hist_data->actions[i];
4628		data->fn(hist_data, elt, rec, rbe, data, var_ref_vals);
4629	}
4630}
4631
4632static void event_hist_trigger(struct event_trigger_data *data, void *rec,
4633			       struct ring_buffer_event *rbe)
4634{
4635	struct hist_trigger_data *hist_data = data->private_data;
4636	bool use_compound_key = (hist_data->n_keys > 1);
4637	unsigned long entries[HIST_STACKTRACE_DEPTH];
4638	u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4639	char compound_key[HIST_KEY_SIZE_MAX];
4640	struct tracing_map_elt *elt = NULL;
4641	struct stack_trace stacktrace;
4642	struct hist_field *key_field;
4643	u64 field_contents;
4644	void *key = NULL;
4645	unsigned int i;
4646
4647	memset(compound_key, 0, hist_data->key_size);
4648
4649	for_each_hist_key_field(i, hist_data) {
4650		key_field = hist_data->fields[i];
4651
4652		if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4653			stacktrace.max_entries = HIST_STACKTRACE_DEPTH;
4654			stacktrace.entries = entries;
4655			stacktrace.nr_entries = 0;
4656			stacktrace.skip = HIST_STACKTRACE_SKIP;
4657
4658			memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE);
4659			save_stack_trace(&stacktrace);
4660
4661			key = entries;
4662		} else {
4663			field_contents = key_field->fn(key_field, elt, rbe, rec);
4664			if (key_field->flags & HIST_FIELD_FL_STRING) {
4665				key = (void *)(unsigned long)field_contents;
4666				use_compound_key = true;
4667			} else
4668				key = (void *)&field_contents;
4669		}
4670
4671		if (use_compound_key)
4672			add_to_key(compound_key, key, key_field, rec);
4673	}
4674
4675	if (use_compound_key)
4676		key = compound_key;
4677
4678	if (hist_data->n_var_refs &&
4679	    !resolve_var_refs(hist_data, key, var_ref_vals, false))
4680		return;
4681
4682	elt = tracing_map_insert(hist_data->map, key);
4683	if (!elt)
4684		return;
4685
4686	hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
4687
4688	if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4689		hist_trigger_actions(hist_data, elt, rec, rbe, var_ref_vals);
4690}
4691
4692static void hist_trigger_stacktrace_print(struct seq_file *m,
4693					  unsigned long *stacktrace_entries,
4694					  unsigned int max_entries)
4695{
4696	char str[KSYM_SYMBOL_LEN];
4697	unsigned int spaces = 8;
4698	unsigned int i;
4699
4700	for (i = 0; i < max_entries; i++) {
4701		if (stacktrace_entries[i] == ULONG_MAX)
4702			return;
4703
4704		seq_printf(m, "%*c", 1 + spaces, ' ');
4705		sprint_symbol(str, stacktrace_entries[i]);
4706		seq_printf(m, "%s\n", str);
4707	}
4708}
4709
4710static void
4711hist_trigger_entry_print(struct seq_file *m,
4712			 struct hist_trigger_data *hist_data, void *key,
4713			 struct tracing_map_elt *elt)
4714{
4715	struct hist_field *key_field;
4716	char str[KSYM_SYMBOL_LEN];
4717	bool multiline = false;
4718	const char *field_name;
4719	unsigned int i;
4720	u64 uval;
4721
4722	seq_puts(m, "{ ");
4723
4724	for_each_hist_key_field(i, hist_data) {
4725		key_field = hist_data->fields[i];
4726
4727		if (i > hist_data->n_vals)
4728			seq_puts(m, ", ");
4729
4730		field_name = hist_field_name(key_field, 0);
4731
4732		if (key_field->flags & HIST_FIELD_FL_HEX) {
4733			uval = *(u64 *)(key + key_field->offset);
4734			seq_printf(m, "%s: %llx", field_name, uval);
4735		} else if (key_field->flags & HIST_FIELD_FL_SYM) {
4736			uval = *(u64 *)(key + key_field->offset);
4737			sprint_symbol_no_offset(str, uval);
4738			seq_printf(m, "%s: [%llx] %-45s", field_name,
4739				   uval, str);
4740		} else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4741			uval = *(u64 *)(key + key_field->offset);
4742			sprint_symbol(str, uval);
4743			seq_printf(m, "%s: [%llx] %-55s", field_name,
4744				   uval, str);
4745		} else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
4746			struct hist_elt_data *elt_data = elt->private_data;
4747			char *comm;
4748
4749			if (WARN_ON_ONCE(!elt_data))
4750				return;
4751
4752			comm = elt_data->comm;
4753
4754			uval = *(u64 *)(key + key_field->offset);
4755			seq_printf(m, "%s: %-16s[%10llu]", field_name,
4756				   comm, uval);
4757		} else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4758			const char *syscall_name;
4759
4760			uval = *(u64 *)(key + key_field->offset);
4761			syscall_name = get_syscall_name(uval);
4762			if (!syscall_name)
4763				syscall_name = "unknown_syscall";
4764
4765			seq_printf(m, "%s: %-30s[%3llu]", field_name,
4766				   syscall_name, uval);
4767		} else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4768			seq_puts(m, "stacktrace:\n");
4769			hist_trigger_stacktrace_print(m,
4770						      key + key_field->offset,
4771						      HIST_STACKTRACE_DEPTH);
4772			multiline = true;
4773		} else if (key_field->flags & HIST_FIELD_FL_LOG2) {
4774			seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4775				   *(u64 *)(key + key_field->offset));
4776		} else if (key_field->flags & HIST_FIELD_FL_STRING) {
4777			seq_printf(m, "%s: %-50s", field_name,
4778				   (char *)(key + key_field->offset));
4779		} else {
4780			uval = *(u64 *)(key + key_field->offset);
4781			seq_printf(m, "%s: %10llu", field_name, uval);
4782		}
4783	}
4784
4785	if (!multiline)
4786		seq_puts(m, " ");
4787
4788	seq_puts(m, "}");
4789
4790	seq_printf(m, " hitcount: %10llu",
4791		   tracing_map_read_sum(elt, HITCOUNT_IDX));
4792
4793	for (i = 1; i < hist_data->n_vals; i++) {
4794		field_name = hist_field_name(hist_data->fields[i], 0);
4795
4796		if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4797		    hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
4798			continue;
4799
4800		if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
4801			seq_printf(m, "  %s: %10llx", field_name,
4802				   tracing_map_read_sum(elt, i));
4803		} else {
4804			seq_printf(m, "  %s: %10llu", field_name,
4805				   tracing_map_read_sum(elt, i));
4806		}
4807	}
4808
4809	print_actions(m, hist_data, elt);
4810
4811	seq_puts(m, "\n");
4812}
4813
4814static int print_entries(struct seq_file *m,
4815			 struct hist_trigger_data *hist_data)
4816{
4817	struct tracing_map_sort_entry **sort_entries = NULL;
4818	struct tracing_map *map = hist_data->map;
4819	int i, n_entries;
4820
4821	n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4822					     hist_data->n_sort_keys,
4823					     &sort_entries);
4824	if (n_entries < 0)
4825		return n_entries;
4826
4827	for (i = 0; i < n_entries; i++)
4828		hist_trigger_entry_print(m, hist_data,
4829					 sort_entries[i]->key,
4830					 sort_entries[i]->elt);
4831
4832	tracing_map_destroy_sort_entries(sort_entries, n_entries);
4833
4834	return n_entries;
4835}
4836
4837static void hist_trigger_show(struct seq_file *m,
4838			      struct event_trigger_data *data, int n)
4839{
4840	struct hist_trigger_data *hist_data;
4841	int n_entries;
4842
4843	if (n > 0)
4844		seq_puts(m, "\n\n");
4845
4846	seq_puts(m, "# event histogram\n#\n# trigger info: ");
4847	data->ops->print(m, data->ops, data);
4848	seq_puts(m, "#\n\n");
4849
4850	hist_data = data->private_data;
4851	n_entries = print_entries(m, hist_data);
4852	if (n_entries < 0)
4853		n_entries = 0;
4854
4855	seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
4856		   (u64)atomic64_read(&hist_data->map->hits),
4857		   n_entries, (u64)atomic64_read(&hist_data->map->drops));
4858}
4859
4860static int hist_show(struct seq_file *m, void *v)
4861{
4862	struct event_trigger_data *data;
4863	struct trace_event_file *event_file;
4864	int n = 0, ret = 0;
4865
4866	mutex_lock(&event_mutex);
4867
4868	event_file = event_file_data(m->private);
4869	if (unlikely(!event_file)) {
4870		ret = -ENODEV;
4871		goto out_unlock;
4872	}
4873
4874	list_for_each_entry_rcu(data, &event_file->triggers, list) {
4875		if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4876			hist_trigger_show(m, data, n++);
4877	}
4878
4879	if (have_hist_err()) {
4880		seq_printf(m, "\nERROR: %s\n", hist_err_str);
4881		seq_printf(m, "  Last command: %s\n", last_hist_cmd);
4882	}
4883
4884 out_unlock:
4885	mutex_unlock(&event_mutex);
4886
4887	return ret;
4888}
4889
4890static int event_hist_open(struct inode *inode, struct file *file)
4891{
4892	return single_open(file, hist_show, file);
4893}
4894
4895const struct file_operations event_hist_fops = {
4896	.open = event_hist_open,
4897	.read = seq_read,
4898	.llseek = seq_lseek,
4899	.release = single_release,
4900};
4901
4902static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
4903{
4904	const char *field_name = hist_field_name(hist_field, 0);
4905
4906	if (hist_field->var.name)
4907		seq_printf(m, "%s=", hist_field->var.name);
4908
4909	if (hist_field->flags & HIST_FIELD_FL_CPU)
4910		seq_puts(m, "cpu");
4911	else if (field_name) {
4912		if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
4913		    hist_field->flags & HIST_FIELD_FL_ALIAS)
4914			seq_putc(m, '$');
4915		seq_printf(m, "%s", field_name);
4916	} else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
4917		seq_puts(m, "common_timestamp");
4918
4919	if (hist_field->flags) {
4920		if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
4921		    !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
4922			const char *flags = get_hist_field_flags(hist_field);
4923
4924			if (flags)
4925				seq_printf(m, ".%s", flags);
4926		}
4927	}
4928}
4929
4930static int event_hist_trigger_print(struct seq_file *m,
4931				    struct event_trigger_ops *ops,
4932				    struct event_trigger_data *data)
4933{
4934	struct hist_trigger_data *hist_data = data->private_data;
4935	struct hist_field *field;
4936	bool have_var = false;
4937	unsigned int i;
4938
4939	seq_puts(m, "hist:");
4940
4941	if (data->name)
4942		seq_printf(m, "%s:", data->name);
4943
4944	seq_puts(m, "keys=");
4945
4946	for_each_hist_key_field(i, hist_data) {
4947		field = hist_data->fields[i];
4948
4949		if (i > hist_data->n_vals)
4950			seq_puts(m, ",");
4951
4952		if (field->flags & HIST_FIELD_FL_STACKTRACE)
4953			seq_puts(m, "stacktrace");
4954		else
4955			hist_field_print(m, field);
4956	}
4957
4958	seq_puts(m, ":vals=");
4959
4960	for_each_hist_val_field(i, hist_data) {
4961		field = hist_data->fields[i];
4962		if (field->flags & HIST_FIELD_FL_VAR) {
4963			have_var = true;
4964			continue;
4965		}
4966
4967		if (i == HITCOUNT_IDX)
4968			seq_puts(m, "hitcount");
4969		else {
4970			seq_puts(m, ",");
4971			hist_field_print(m, field);
4972		}
4973	}
4974
4975	if (have_var) {
4976		unsigned int n = 0;
4977
4978		seq_puts(m, ":");
4979
4980		for_each_hist_val_field(i, hist_data) {
4981			field = hist_data->fields[i];
4982
4983			if (field->flags & HIST_FIELD_FL_VAR) {
4984				if (n++)
4985					seq_puts(m, ",");
4986				hist_field_print(m, field);
4987			}
4988		}
4989	}
4990
4991	seq_puts(m, ":sort=");
4992
4993	for (i = 0; i < hist_data->n_sort_keys; i++) {
4994		struct tracing_map_sort_key *sort_key;
4995		unsigned int idx, first_key_idx;
4996
4997		/* skip VAR vals */
4998		first_key_idx = hist_data->n_vals - hist_data->n_vars;
4999
5000		sort_key = &hist_data->sort_keys[i];
5001		idx = sort_key->field_idx;
5002
5003		if (WARN_ON(idx >= HIST_FIELDS_MAX))
5004			return -EINVAL;
5005
5006		if (i > 0)
5007			seq_puts(m, ",");
5008
5009		if (idx == HITCOUNT_IDX)
5010			seq_puts(m, "hitcount");
5011		else {
5012			if (idx >= first_key_idx)
5013				idx += hist_data->n_vars;
5014			hist_field_print(m, hist_data->fields[idx]);
5015		}
5016
5017		if (sort_key->descending)
5018			seq_puts(m, ".descending");
5019	}
5020	seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5021	if (hist_data->enable_timestamps)
5022		seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5023
5024	print_actions_spec(m, hist_data);
5025
5026	if (data->filter_str)
5027		seq_printf(m, " if %s", data->filter_str);
5028
5029	if (data->paused)
5030		seq_puts(m, " [paused]");
5031	else
5032		seq_puts(m, " [active]");
5033
5034	seq_putc(m, '\n');
5035
5036	return 0;
5037}
5038
5039static int event_hist_trigger_init(struct event_trigger_ops *ops,
5040				   struct event_trigger_data *data)
5041{
5042	struct hist_trigger_data *hist_data = data->private_data;
5043
5044	if (!data->ref && hist_data->attrs->name)
5045		save_named_trigger(hist_data->attrs->name, data);
5046
5047	data->ref++;
5048
5049	return 0;
5050}
5051
5052static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5053{
5054	struct trace_event_file *file;
5055	unsigned int i;
5056	char *cmd;
5057	int ret;
5058
5059	for (i = 0; i < hist_data->n_field_var_hists; i++) {
5060		file = hist_data->field_var_hists[i]->hist_data->event_file;
5061		cmd = hist_data->field_var_hists[i]->cmd;
5062		ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5063					      "!hist", "hist", cmd);
5064	}
5065}
5066
5067static void event_hist_trigger_free(struct event_trigger_ops *ops,
5068				    struct event_trigger_data *data)
5069{
5070	struct hist_trigger_data *hist_data = data->private_data;
5071
5072	if (WARN_ON_ONCE(data->ref <= 0))
5073		return;
5074
5075	data->ref--;
5076	if (!data->ref) {
5077		if (data->name)
5078			del_named_trigger(data);
5079
5080		trigger_data_free(data);
5081
5082		remove_hist_vars(hist_data);
5083
5084		unregister_field_var_hists(hist_data);
5085
5086		destroy_hist_data(hist_data);
5087	}
5088}
5089
5090static struct event_trigger_ops event_hist_trigger_ops = {
5091	.func			= event_hist_trigger,
5092	.print			= event_hist_trigger_print,
5093	.init			= event_hist_trigger_init,
5094	.free			= event_hist_trigger_free,
5095};
5096
5097static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5098					 struct event_trigger_data *data)
5099{
5100	data->ref++;
5101
5102	save_named_trigger(data->named_data->name, data);
5103
5104	event_hist_trigger_init(ops, data->named_data);
5105
5106	return 0;
5107}
5108
5109static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5110					  struct event_trigger_data *data)
5111{
5112	if (WARN_ON_ONCE(data->ref <= 0))
5113		return;
5114
5115	event_hist_trigger_free(ops, data->named_data);
5116
5117	data->ref--;
5118	if (!data->ref) {
5119		del_named_trigger(data);
5120		trigger_data_free(data);
5121	}
5122}
5123
5124static struct event_trigger_ops event_hist_trigger_named_ops = {
5125	.func			= event_hist_trigger,
5126	.print			= event_hist_trigger_print,
5127	.init			= event_hist_trigger_named_init,
5128	.free			= event_hist_trigger_named_free,
5129};
5130
5131static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5132							    char *param)
5133{
5134	return &event_hist_trigger_ops;
5135}
5136
5137static void hist_clear(struct event_trigger_data *data)
5138{
5139	struct hist_trigger_data *hist_data = data->private_data;
5140
5141	if (data->name)
5142		pause_named_trigger(data);
5143
5144	synchronize_sched();
5145
5146	tracing_map_clear(hist_data->map);
5147
5148	if (data->name)
5149		unpause_named_trigger(data);
5150}
5151
5152static bool compatible_field(struct ftrace_event_field *field,
5153			     struct ftrace_event_field *test_field)
5154{
5155	if (field == test_field)
5156		return true;
5157	if (field == NULL || test_field == NULL)
5158		return false;
5159	if (strcmp(field->name, test_field->name) != 0)
5160		return false;
5161	if (strcmp(field->type, test_field->type) != 0)
5162		return false;
5163	if (field->size != test_field->size)
5164		return false;
5165	if (field->is_signed != test_field->is_signed)
5166		return false;
5167
5168	return true;
5169}
5170
5171static bool hist_trigger_match(struct event_trigger_data *data,
5172			       struct event_trigger_data *data_test,
5173			       struct event_trigger_data *named_data,
5174			       bool ignore_filter)
5175{
5176	struct tracing_map_sort_key *sort_key, *sort_key_test;
5177	struct hist_trigger_data *hist_data, *hist_data_test;
5178	struct hist_field *key_field, *key_field_test;
5179	unsigned int i;
5180
5181	if (named_data && (named_data != data_test) &&
5182	    (named_data != data_test->named_data))
5183		return false;
5184
5185	if (!named_data && is_named_trigger(data_test))
5186		return false;
5187
5188	hist_data = data->private_data;
5189	hist_data_test = data_test->private_data;
5190
5191	if (hist_data->n_vals != hist_data_test->n_vals ||
5192	    hist_data->n_fields != hist_data_test->n_fields ||
5193	    hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5194		return false;
5195
5196	if (!ignore_filter) {
5197		if ((data->filter_str && !data_test->filter_str) ||
5198		   (!data->filter_str && data_test->filter_str))
5199			return false;
5200	}
5201
5202	for_each_hist_field(i, hist_data) {
5203		key_field = hist_data->fields[i];
5204		key_field_test = hist_data_test->fields[i];
5205
5206		if (key_field->flags != key_field_test->flags)
5207			return false;
5208		if (!compatible_field(key_field->field, key_field_test->field))
5209			return false;
5210		if (key_field->offset != key_field_test->offset)
5211			return false;
5212		if (key_field->size != key_field_test->size)
5213			return false;
5214		if (key_field->is_signed != key_field_test->is_signed)
5215			return false;
5216		if (!!key_field->var.name != !!key_field_test->var.name)
5217			return false;
5218		if (key_field->var.name &&
5219		    strcmp(key_field->var.name, key_field_test->var.name) != 0)
5220			return false;
5221	}
5222
5223	for (i = 0; i < hist_data->n_sort_keys; i++) {
5224		sort_key = &hist_data->sort_keys[i];
5225		sort_key_test = &hist_data_test->sort_keys[i];
5226
5227		if (sort_key->field_idx != sort_key_test->field_idx ||
5228		    sort_key->descending != sort_key_test->descending)
5229			return false;
5230	}
5231
5232	if (!ignore_filter && data->filter_str &&
5233	    (strcmp(data->filter_str, data_test->filter_str) != 0))
5234		return false;
5235
5236	if (!actions_match(hist_data, hist_data_test))
5237		return false;
5238
5239	return true;
5240}
5241
5242static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5243				 struct event_trigger_data *data,
5244				 struct trace_event_file *file)
5245{
5246	struct hist_trigger_data *hist_data = data->private_data;
5247	struct event_trigger_data *test, *named_data = NULL;
5248	int ret = 0;
5249
5250	if (hist_data->attrs->name) {
5251		named_data = find_named_trigger(hist_data->attrs->name);
5252		if (named_data) {
5253			if (!hist_trigger_match(data, named_data, named_data,
5254						true)) {
5255				hist_err("Named hist trigger doesn't match existing named trigger (includes variables): ", hist_data->attrs->name);
5256				ret = -EINVAL;
5257				goto out;
5258			}
5259		}
5260	}
5261
5262	if (hist_data->attrs->name && !named_data)
5263		goto new;
5264
5265	list_for_each_entry_rcu(test, &file->triggers, list) {
5266		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5267			if (!hist_trigger_match(data, test, named_data, false))
5268				continue;
5269			if (hist_data->attrs->pause)
5270				test->paused = true;
5271			else if (hist_data->attrs->cont)
5272				test->paused = false;
5273			else if (hist_data->attrs->clear)
5274				hist_clear(test);
5275			else {
5276				hist_err("Hist trigger already exists", NULL);
5277				ret = -EEXIST;
5278			}
5279			goto out;
5280		}
5281	}
5282 new:
5283	if (hist_data->attrs->cont || hist_data->attrs->clear) {
5284		hist_err("Can't clear or continue a nonexistent hist trigger", NULL);
5285		ret = -ENOENT;
5286		goto out;
5287	}
5288
5289	if (hist_data->attrs->pause)
5290		data->paused = true;
5291
5292	if (named_data) {
5293		data->private_data = named_data->private_data;
5294		set_named_trigger_data(data, named_data);
5295		data->ops = &event_hist_trigger_named_ops;
5296	}
5297
5298	if (data->ops->init) {
5299		ret = data->ops->init(data->ops, data);
5300		if (ret < 0)
5301			goto out;
5302	}
5303
5304	if (hist_data->enable_timestamps) {
5305		char *clock = hist_data->attrs->clock;
5306
5307		ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5308		if (ret) {
5309			hist_err("Couldn't set trace_clock: ", clock);
5310			goto out;
5311		}
5312
5313		tracing_set_time_stamp_abs(file->tr, true);
5314	}
5315
5316	if (named_data)
5317		destroy_hist_data(hist_data);
5318
5319	ret++;
5320 out:
5321	return ret;
5322}
5323
5324static int hist_trigger_enable(struct event_trigger_data *data,
5325			       struct trace_event_file *file)
5326{
5327	int ret = 0;
5328
5329	list_add_tail_rcu(&data->list, &file->triggers);
5330
5331	update_cond_flag(file);
5332
5333	if (trace_event_trigger_enable_disable(file, 1) < 0) {
5334		list_del_rcu(&data->list);
5335		update_cond_flag(file);
5336		ret--;
5337	}
5338
5339	return ret;
5340}
5341
5342static bool have_hist_trigger_match(struct event_trigger_data *data,
5343				    struct trace_event_file *file)
5344{
5345	struct hist_trigger_data *hist_data = data->private_data;
5346	struct event_trigger_data *test, *named_data = NULL;
5347	bool match = false;
5348
5349	if (hist_data->attrs->name)
5350		named_data = find_named_trigger(hist_data->attrs->name);
5351
5352	list_for_each_entry_rcu(test, &file->triggers, list) {
5353		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5354			if (hist_trigger_match(data, test, named_data, false)) {
5355				match = true;
5356				break;
5357			}
5358		}
5359	}
5360
5361	return match;
5362}
5363
5364static bool hist_trigger_check_refs(struct event_trigger_data *data,
5365				    struct trace_event_file *file)
5366{
5367	struct hist_trigger_data *hist_data = data->private_data;
5368	struct event_trigger_data *test, *named_data = NULL;
5369
5370	if (hist_data->attrs->name)
5371		named_data = find_named_trigger(hist_data->attrs->name);
5372
5373	list_for_each_entry_rcu(test, &file->triggers, list) {
5374		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5375			if (!hist_trigger_match(data, test, named_data, false))
5376				continue;
5377			hist_data = test->private_data;
5378			if (check_var_refs(hist_data))
5379				return true;
5380			break;
5381		}
5382	}
5383
5384	return false;
5385}
5386
5387static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5388				    struct event_trigger_data *data,
5389				    struct trace_event_file *file)
5390{
5391	struct hist_trigger_data *hist_data = data->private_data;
5392	struct event_trigger_data *test, *named_data = NULL;
5393	bool unregistered = false;
5394
5395	if (hist_data->attrs->name)
5396		named_data = find_named_trigger(hist_data->attrs->name);
5397
5398	list_for_each_entry_rcu(test, &file->triggers, list) {
5399		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5400			if (!hist_trigger_match(data, test, named_data, false))
5401				continue;
5402			unregistered = true;
5403			list_del_rcu(&test->list);
5404			trace_event_trigger_enable_disable(file, 0);
5405			update_cond_flag(file);
5406			break;
5407		}
5408	}
5409
5410	if (unregistered && test->ops->free)
5411		test->ops->free(test->ops, test);
5412
5413	if (hist_data->enable_timestamps) {
5414		if (!hist_data->remove || unregistered)
5415			tracing_set_time_stamp_abs(file->tr, false);
5416	}
5417}
5418
5419static bool hist_file_check_refs(struct trace_event_file *file)
5420{
5421	struct hist_trigger_data *hist_data;
5422	struct event_trigger_data *test;
5423
5424	list_for_each_entry_rcu(test, &file->triggers, list) {
5425		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5426			hist_data = test->private_data;
5427			if (check_var_refs(hist_data))
5428				return true;
5429		}
5430	}
5431
5432	return false;
5433}
5434
5435static void hist_unreg_all(struct trace_event_file *file)
5436{
5437	struct event_trigger_data *test, *n;
5438	struct hist_trigger_data *hist_data;
5439	struct synth_event *se;
5440	const char *se_name;
5441
5442	if (hist_file_check_refs(file))
5443		return;
5444
5445	list_for_each_entry_safe(test, n, &file->triggers, list) {
5446		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5447			hist_data = test->private_data;
5448			list_del_rcu(&test->list);
5449			trace_event_trigger_enable_disable(file, 0);
5450
5451			mutex_lock(&synth_event_mutex);
5452			se_name = trace_event_name(file->event_call);
5453			se = find_synth_event(se_name);
5454			if (se)
5455				se->ref--;
5456			mutex_unlock(&synth_event_mutex);
5457
5458			update_cond_flag(file);
5459			if (hist_data->enable_timestamps)
5460				tracing_set_time_stamp_abs(file->tr, false);
5461			if (test->ops->free)
5462				test->ops->free(test->ops, test);
5463		}
5464	}
5465}
5466
5467static int event_hist_trigger_func(struct event_command *cmd_ops,
5468				   struct trace_event_file *file,
5469				   char *glob, char *cmd, char *param)
5470{
5471	unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5472	struct event_trigger_data *trigger_data;
5473	struct hist_trigger_attrs *attrs;
5474	struct event_trigger_ops *trigger_ops;
5475	struct hist_trigger_data *hist_data;
5476	struct synth_event *se;
5477	const char *se_name;
5478	bool remove = false;
5479	char *trigger, *p;
5480	int ret = 0;
5481
5482	if (glob && strlen(glob)) {
5483		last_cmd_set(param);
5484		hist_err_clear();
5485	}
5486
5487	if (!param)
5488		return -EINVAL;
5489
5490	if (glob[0] == '!')
5491		remove = true;
5492
5493	/*
5494	 * separate the trigger from the filter (k:v [if filter])
5495	 * allowing for whitespace in the trigger
5496	 */
5497	p = trigger = param;
5498	do {
5499		p = strstr(p, "if");
5500		if (!p)
5501			break;
5502		if (p == param)
5503			return -EINVAL;
5504		if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5505			p++;
5506			continue;
5507		}
5508		if (p >= param + strlen(param) - strlen("if") - 1)
5509			return -EINVAL;
5510		if (*(p + strlen("if")) != ' ' && *(p + strlen("if")) != '\t') {
5511			p++;
5512			continue;
5513		}
5514		break;
5515	} while (p);
5516
5517	if (!p)
5518		param = NULL;
5519	else {
5520		*(p - 1) = '\0';
5521		param = strstrip(p);
5522		trigger = strstrip(trigger);
5523	}
5524
5525	attrs = parse_hist_trigger_attrs(trigger);
5526	if (IS_ERR(attrs))
5527		return PTR_ERR(attrs);
5528
5529	if (attrs->map_bits)
5530		hist_trigger_bits = attrs->map_bits;
5531
5532	hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
5533	if (IS_ERR(hist_data)) {
5534		destroy_hist_trigger_attrs(attrs);
5535		return PTR_ERR(hist_data);
5536	}
5537
5538	trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5539
5540	trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
5541	if (!trigger_data) {
5542		ret = -ENOMEM;
5543		goto out_free;
5544	}
5545
5546	trigger_data->count = -1;
5547	trigger_data->ops = trigger_ops;
5548	trigger_data->cmd_ops = cmd_ops;
5549
5550	INIT_LIST_HEAD(&trigger_data->list);
5551	RCU_INIT_POINTER(trigger_data->filter, NULL);
5552
5553	trigger_data->private_data = hist_data;
5554
5555	/* if param is non-empty, it's supposed to be a filter */
5556	if (param && cmd_ops->set_filter) {
5557		ret = cmd_ops->set_filter(param, trigger_data, file);
5558		if (ret < 0)
5559			goto out_free;
5560	}
5561
5562	if (remove) {
5563		if (!have_hist_trigger_match(trigger_data, file))
5564			goto out_free;
5565
5566		if (hist_trigger_check_refs(trigger_data, file)) {
5567			ret = -EBUSY;
5568			goto out_free;
5569		}
5570
5571		cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5572
5573		mutex_lock(&synth_event_mutex);
5574		se_name = trace_event_name(file->event_call);
5575		se = find_synth_event(se_name);
5576		if (se)
5577			se->ref--;
5578		mutex_unlock(&synth_event_mutex);
5579
5580		ret = 0;
5581		goto out_free;
5582	}
5583
5584	ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5585	/*
5586	 * The above returns on success the # of triggers registered,
5587	 * but if it didn't register any it returns zero.  Consider no
5588	 * triggers registered a failure too.
5589	 */
5590	if (!ret) {
5591		if (!(attrs->pause || attrs->cont || attrs->clear))
5592			ret = -ENOENT;
5593		goto out_free;
5594	} else if (ret < 0)
5595		goto out_free;
5596
5597	if (get_named_trigger_data(trigger_data))
5598		goto enable;
5599
5600	if (has_hist_vars(hist_data))
5601		save_hist_vars(hist_data);
5602
5603	ret = create_actions(hist_data, file);
5604	if (ret)
5605		goto out_unreg;
5606
5607	ret = tracing_map_init(hist_data->map);
5608	if (ret)
5609		goto out_unreg;
5610enable:
5611	ret = hist_trigger_enable(trigger_data, file);
5612	if (ret)
5613		goto out_unreg;
5614
5615	mutex_lock(&synth_event_mutex);
5616	se_name = trace_event_name(file->event_call);
5617	se = find_synth_event(se_name);
5618	if (se)
5619		se->ref++;
5620	mutex_unlock(&synth_event_mutex);
5621
5622	/* Just return zero, not the number of registered triggers */
5623	ret = 0;
5624 out:
5625	if (ret == 0)
5626		hist_err_clear();
5627
5628	return ret;
5629 out_unreg:
5630	cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5631 out_free:
5632	if (cmd_ops->set_filter)
5633		cmd_ops->set_filter(NULL, trigger_data, NULL);
5634
5635	remove_hist_vars(hist_data);
5636
5637	kfree(trigger_data);
5638
5639	destroy_hist_data(hist_data);
5640	goto out;
5641}
5642
5643static struct event_command trigger_hist_cmd = {
5644	.name			= "hist",
5645	.trigger_type		= ETT_EVENT_HIST,
5646	.flags			= EVENT_CMD_FL_NEEDS_REC,
5647	.func			= event_hist_trigger_func,
5648	.reg			= hist_register_trigger,
5649	.unreg			= hist_unregister_trigger,
5650	.unreg_all		= hist_unreg_all,
5651	.get_trigger_ops	= event_hist_get_trigger_ops,
5652	.set_filter		= set_trigger_filter,
5653};
5654
5655__init int register_trigger_hist_cmd(void)
5656{
5657	int ret;
5658
5659	ret = register_event_command(&trigger_hist_cmd);
5660	WARN_ON(ret < 0);
5661
5662	return ret;
5663}
5664
5665static void
5666hist_enable_trigger(struct event_trigger_data *data, void *rec,
5667		    struct ring_buffer_event *event)
5668{
5669	struct enable_trigger_data *enable_data = data->private_data;
5670	struct event_trigger_data *test;
5671
5672	list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
5673		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5674			if (enable_data->enable)
5675				test->paused = false;
5676			else
5677				test->paused = true;
5678		}
5679	}
5680}
5681
5682static void
5683hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
5684			  struct ring_buffer_event *event)
5685{
5686	if (!data->count)
5687		return;
5688
5689	if (data->count != -1)
5690		(data->count)--;
5691
5692	hist_enable_trigger(data, rec, event);
5693}
5694
5695static struct event_trigger_ops hist_enable_trigger_ops = {
5696	.func			= hist_enable_trigger,
5697	.print			= event_enable_trigger_print,
5698	.init			= event_trigger_init,
5699	.free			= event_enable_trigger_free,
5700};
5701
5702static struct event_trigger_ops hist_enable_count_trigger_ops = {
5703	.func			= hist_enable_count_trigger,
5704	.print			= event_enable_trigger_print,
5705	.init			= event_trigger_init,
5706	.free			= event_enable_trigger_free,
5707};
5708
5709static struct event_trigger_ops hist_disable_trigger_ops = {
5710	.func			= hist_enable_trigger,
5711	.print			= event_enable_trigger_print,
5712	.init			= event_trigger_init,
5713	.free			= event_enable_trigger_free,
5714};
5715
5716static struct event_trigger_ops hist_disable_count_trigger_ops = {
5717	.func			= hist_enable_count_trigger,
5718	.print			= event_enable_trigger_print,
5719	.init			= event_trigger_init,
5720	.free			= event_enable_trigger_free,
5721};
5722
5723static struct event_trigger_ops *
5724hist_enable_get_trigger_ops(char *cmd, char *param)
5725{
5726	struct event_trigger_ops *ops;
5727	bool enable;
5728
5729	enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5730
5731	if (enable)
5732		ops = param ? &hist_enable_count_trigger_ops :
5733			&hist_enable_trigger_ops;
5734	else
5735		ops = param ? &hist_disable_count_trigger_ops :
5736			&hist_disable_trigger_ops;
5737
5738	return ops;
5739}
5740
5741static void hist_enable_unreg_all(struct trace_event_file *file)
5742{
5743	struct event_trigger_data *test, *n;
5744
5745	list_for_each_entry_safe(test, n, &file->triggers, list) {
5746		if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5747			list_del_rcu(&test->list);
5748			update_cond_flag(file);
5749			trace_event_trigger_enable_disable(file, 0);
5750			if (test->ops->free)
5751				test->ops->free(test->ops, test);
5752		}
5753	}
5754}
5755
5756static struct event_command trigger_hist_enable_cmd = {
5757	.name			= ENABLE_HIST_STR,
5758	.trigger_type		= ETT_HIST_ENABLE,
5759	.func			= event_enable_trigger_func,
5760	.reg			= event_enable_register_trigger,
5761	.unreg			= event_enable_unregister_trigger,
5762	.unreg_all		= hist_enable_unreg_all,
5763	.get_trigger_ops	= hist_enable_get_trigger_ops,
5764	.set_filter		= set_trigger_filter,
5765};
5766
5767static struct event_command trigger_hist_disable_cmd = {
5768	.name			= DISABLE_HIST_STR,
5769	.trigger_type		= ETT_HIST_ENABLE,
5770	.func			= event_enable_trigger_func,
5771	.reg			= event_enable_register_trigger,
5772	.unreg			= event_enable_unregister_trigger,
5773	.unreg_all		= hist_enable_unreg_all,
5774	.get_trigger_ops	= hist_enable_get_trigger_ops,
5775	.set_filter		= set_trigger_filter,
5776};
5777
5778static __init void unregister_trigger_hist_enable_disable_cmds(void)
5779{
5780	unregister_event_command(&trigger_hist_enable_cmd);
5781	unregister_event_command(&trigger_hist_disable_cmd);
5782}
5783
5784__init int register_trigger_hist_enable_disable_cmds(void)
5785{
5786	int ret;
5787
5788	ret = register_event_command(&trigger_hist_enable_cmd);
5789	if (WARN_ON(ret < 0))
5790		return ret;
5791	ret = register_event_command(&trigger_hist_disable_cmd);
5792	if (WARN_ON(ret < 0))
5793		unregister_trigger_hist_enable_disable_cmds();
5794
5795	return ret;
5796}
5797
5798static __init int trace_events_hist_init(void)
5799{
5800	struct dentry *entry = NULL;
5801	struct dentry *d_tracer;
5802	int err = 0;
5803
5804	d_tracer = tracing_init_dentry();
5805	if (IS_ERR(d_tracer)) {
5806		err = PTR_ERR(d_tracer);
5807		goto err;
5808	}
5809
5810	entry = tracefs_create_file("synthetic_events", 0644, d_tracer,
5811				    NULL, &synth_events_fops);
5812	if (!entry) {
5813		err = -ENODEV;
5814		goto err;
5815	}
5816
5817	return err;
5818 err:
5819	pr_warn("Could not create tracefs 'synthetic_events' entry\n");
5820
5821	return err;
5822}
5823
5824fs_initcall(trace_events_hist_init);