1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Common code for probe-based Dynamic events.
   4 *
   5 * This code was copied from kernel/trace/trace_kprobe.c written by
   6 * Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
   7 *
   8 * Updates to make this generic:
   9 * Copyright (C) IBM Corporation, 2010-2011
  10 * Author:     Srikar Dronamraju
  11 */
  12#define pr_fmt(fmt)	"trace_probe: " fmt
  13
  14#include <linux/bpf.h>
  15#include <linux/fs.h>
  16#include "trace_btf.h"
  17
  18#include "trace_probe.h"
  19
  20#undef C
  21#define C(a, b)		b
  22
  23static const char *trace_probe_err_text[] = { ERRORS };
  24
  25static const char *reserved_field_names[] = {
  26	"common_type",
  27	"common_flags",
  28	"common_preempt_count",
  29	"common_pid",
  30	"common_tgid",
  31	FIELD_STRING_IP,
  32	FIELD_STRING_RETIP,
  33	FIELD_STRING_FUNC,
  34};
  35
  36/* Printing  in basic type function template */
  37#define DEFINE_BASIC_PRINT_TYPE_FUNC(tname, type, fmt)			\
  38int PRINT_TYPE_FUNC_NAME(tname)(struct trace_seq *s, void *data, void *ent)\
  39{									\
  40	trace_seq_printf(s, fmt, *(type *)data);			\
  41	return !trace_seq_has_overflowed(s);				\
  42}									\
  43const char PRINT_TYPE_FMT_NAME(tname)[] = fmt;
  44
  45DEFINE_BASIC_PRINT_TYPE_FUNC(u8,  u8,  "%u")
  46DEFINE_BASIC_PRINT_TYPE_FUNC(u16, u16, "%u")
  47DEFINE_BASIC_PRINT_TYPE_FUNC(u32, u32, "%u")
  48DEFINE_BASIC_PRINT_TYPE_FUNC(u64, u64, "%Lu")
  49DEFINE_BASIC_PRINT_TYPE_FUNC(s8,  s8,  "%d")
  50DEFINE_BASIC_PRINT_TYPE_FUNC(s16, s16, "%d")
  51DEFINE_BASIC_PRINT_TYPE_FUNC(s32, s32, "%d")
  52DEFINE_BASIC_PRINT_TYPE_FUNC(s64, s64, "%Ld")
  53DEFINE_BASIC_PRINT_TYPE_FUNC(x8,  u8,  "0x%x")
  54DEFINE_BASIC_PRINT_TYPE_FUNC(x16, u16, "0x%x")
  55DEFINE_BASIC_PRINT_TYPE_FUNC(x32, u32, "0x%x")
  56DEFINE_BASIC_PRINT_TYPE_FUNC(x64, u64, "0x%Lx")
  57DEFINE_BASIC_PRINT_TYPE_FUNC(char, u8, "'%c'")
  58
  59int PRINT_TYPE_FUNC_NAME(symbol)(struct trace_seq *s, void *data, void *ent)
  60{
  61	trace_seq_printf(s, "%pS", (void *)*(unsigned long *)data);
  62	return !trace_seq_has_overflowed(s);
  63}
  64const char PRINT_TYPE_FMT_NAME(symbol)[] = "%pS";
  65
  66/* Print type function for string type */
  67int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, void *data, void *ent)
  68{
  69	int len = *(u32 *)data >> 16;
  70
  71	if (!len)
  72		trace_seq_puts(s, FAULT_STRING);
  73	else
  74		trace_seq_printf(s, "\"%s\"",
  75				 (const char *)get_loc_data(data, ent));
  76	return !trace_seq_has_overflowed(s);
  77}
  78
  79const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\"";
  80
  81/* Fetch type information table */
  82static const struct fetch_type probe_fetch_types[] = {
  83	/* Special types */
  84	__ASSIGN_FETCH_TYPE("string", string, string, sizeof(u32), 1, 1,
  85			    "__data_loc char[]"),
  86	__ASSIGN_FETCH_TYPE("ustring", string, string, sizeof(u32), 1, 1,
  87			    "__data_loc char[]"),
  88	__ASSIGN_FETCH_TYPE("symstr", string, string, sizeof(u32), 1, 1,
  89			    "__data_loc char[]"),
  90	/* Basic types */
  91	ASSIGN_FETCH_TYPE(u8,  u8,  0),
  92	ASSIGN_FETCH_TYPE(u16, u16, 0),
  93	ASSIGN_FETCH_TYPE(u32, u32, 0),
  94	ASSIGN_FETCH_TYPE(u64, u64, 0),
  95	ASSIGN_FETCH_TYPE(s8,  u8,  1),
  96	ASSIGN_FETCH_TYPE(s16, u16, 1),
  97	ASSIGN_FETCH_TYPE(s32, u32, 1),
  98	ASSIGN_FETCH_TYPE(s64, u64, 1),
  99	ASSIGN_FETCH_TYPE_ALIAS(x8,  u8,  u8,  0),
 100	ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
 101	ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
 102	ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
 103	ASSIGN_FETCH_TYPE_ALIAS(char, u8, u8,  0),
 104	ASSIGN_FETCH_TYPE_ALIAS(symbol, ADDR_FETCH_TYPE, ADDR_FETCH_TYPE, 0),
 105
 106	ASSIGN_FETCH_TYPE_END
 107};
 108
 109static const struct fetch_type *find_fetch_type(const char *type, unsigned long flags)
 110{
 111	int i;
 112
 113	/* Reject the symbol/symstr for uprobes */
 114	if (type && (flags & TPARG_FL_USER) &&
 115	    (!strcmp(type, "symbol") || !strcmp(type, "symstr")))
 116		return NULL;
 117
 118	if (!type)
 119		type = DEFAULT_FETCH_TYPE_STR;
 120
 121	/* Special case: bitfield */
 122	if (*type == 'b') {
 123		unsigned long bs;
 124
 125		type = strchr(type, '/');
 126		if (!type)
 127			goto fail;
 128
 129		type++;
 130		if (kstrtoul(type, 0, &bs))
 131			goto fail;
 132
 133		switch (bs) {
 134		case 8:
 135			return find_fetch_type("u8", flags);
 136		case 16:
 137			return find_fetch_type("u16", flags);
 138		case 32:
 139			return find_fetch_type("u32", flags);
 140		case 64:
 141			return find_fetch_type("u64", flags);
 142		default:
 143			goto fail;
 144		}
 145	}
 146
 147	for (i = 0; probe_fetch_types[i].name; i++) {
 148		if (strcmp(type, probe_fetch_types[i].name) == 0)
 149			return &probe_fetch_types[i];
 150	}
 151
 152fail:
 153	return NULL;
 154}
 155
 156static struct trace_probe_log trace_probe_log;
 157
 158void trace_probe_log_init(const char *subsystem, int argc, const char **argv)
 159{
 160	trace_probe_log.subsystem = subsystem;
 161	trace_probe_log.argc = argc;
 162	trace_probe_log.argv = argv;
 163	trace_probe_log.index = 0;
 164}
 165
 166void trace_probe_log_clear(void)
 167{
 168	memset(&trace_probe_log, 0, sizeof(trace_probe_log));
 169}
 170
 171void trace_probe_log_set_index(int index)
 172{
 173	trace_probe_log.index = index;
 174}
 175
 176void __trace_probe_log_err(int offset, int err_type)
 177{
 178	char *command, *p;
 179	int i, len = 0, pos = 0;
 180
 181	if (!trace_probe_log.argv)
 182		return;
 183
 184	/* Recalculate the length and allocate buffer */
 185	for (i = 0; i < trace_probe_log.argc; i++) {
 186		if (i == trace_probe_log.index)
 187			pos = len;
 188		len += strlen(trace_probe_log.argv[i]) + 1;
 189	}
 190	command = kzalloc(len, GFP_KERNEL);
 191	if (!command)
 192		return;
 193
 194	if (trace_probe_log.index >= trace_probe_log.argc) {
 195		/**
 196		 * Set the error position is next to the last arg + space.
 197		 * Note that len includes the terminal null and the cursor
 198		 * appears at pos + 1.
 199		 */
 200		pos = len;
 201		offset = 0;
 202	}
 203
 204	/* And make a command string from argv array */
 205	p = command;
 206	for (i = 0; i < trace_probe_log.argc; i++) {
 207		len = strlen(trace_probe_log.argv[i]);
 208		strcpy(p, trace_probe_log.argv[i]);
 209		p[len] = ' ';
 210		p += len + 1;
 211	}
 212	*(p - 1) = '\0';
 213
 214	tracing_log_err(NULL, trace_probe_log.subsystem, command,
 215			trace_probe_err_text, err_type, pos + offset);
 216
 217	kfree(command);
 218}
 219
 220/* Split symbol and offset. */
 221int traceprobe_split_symbol_offset(char *symbol, long *offset)
 222{
 223	char *tmp;
 224	int ret;
 225
 226	if (!offset)
 227		return -EINVAL;
 228
 229	tmp = strpbrk(symbol, "+-");
 230	if (tmp) {
 231		ret = kstrtol(tmp, 0, offset);
 232		if (ret)
 233			return ret;
 234		*tmp = '\0';
 235	} else
 236		*offset = 0;
 237
 238	return 0;
 239}
 240
 241/* @buf must has MAX_EVENT_NAME_LEN size */
 242int traceprobe_parse_event_name(const char **pevent, const char **pgroup,
 243				char *buf, int offset)
 244{
 245	const char *slash, *event = *pevent;
 246	int len;
 247
 248	slash = strchr(event, '/');
 249	if (!slash)
 250		slash = strchr(event, '.');
 251
 252	if (slash) {
 253		if (slash == event) {
 254			trace_probe_log_err(offset, NO_GROUP_NAME);
 255			return -EINVAL;
 256		}
 257		if (slash - event + 1 > MAX_EVENT_NAME_LEN) {
 258			trace_probe_log_err(offset, GROUP_TOO_LONG);
 259			return -EINVAL;
 260		}
 261		strscpy(buf, event, slash - event + 1);
 262		if (!is_good_system_name(buf)) {
 263			trace_probe_log_err(offset, BAD_GROUP_NAME);
 264			return -EINVAL;
 265		}
 266		*pgroup = buf;
 267		*pevent = slash + 1;
 268		offset += slash - event + 1;
 269		event = *pevent;
 270	}
 271	len = strlen(event);
 272	if (len == 0) {
 273		if (slash) {
 274			*pevent = NULL;
 275			return 0;
 276		}
 277		trace_probe_log_err(offset, NO_EVENT_NAME);
 278		return -EINVAL;
 279	} else if (len >= MAX_EVENT_NAME_LEN) {
 280		trace_probe_log_err(offset, EVENT_TOO_LONG);
 281		return -EINVAL;
 282	}
 283	if (!is_good_name(event)) {
 284		trace_probe_log_err(offset, BAD_EVENT_NAME);
 285		return -EINVAL;
 286	}
 287	return 0;
 288}
 289
 290static int parse_trace_event_arg(char *arg, struct fetch_insn *code,
 291				 struct traceprobe_parse_context *ctx)
 292{
 293	struct ftrace_event_field *field;
 294	struct list_head *head;
 295
 296	head = trace_get_fields(ctx->event);
 297	list_for_each_entry(field, head, link) {
 298		if (!strcmp(arg, field->name)) {
 299			code->op = FETCH_OP_TP_ARG;
 300			code->data = field;
 301			return 0;
 302		}
 303	}
 304	return -ENOENT;
 305}
 306
 307#ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
 308
 309static u32 btf_type_int(const struct btf_type *t)
 310{
 311	return *(u32 *)(t + 1);
 312}
 313
 314static bool btf_type_is_char_ptr(struct btf *btf, const struct btf_type *type)
 315{
 316	const struct btf_type *real_type;
 317	u32 intdata;
 318	s32 tid;
 319
 320	real_type = btf_type_skip_modifiers(btf, type->type, &tid);
 321	if (!real_type)
 322		return false;
 323
 324	if (BTF_INFO_KIND(real_type->info) != BTF_KIND_INT)
 325		return false;
 326
 327	intdata = btf_type_int(real_type);
 328	return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
 329		&& BTF_INT_BITS(intdata) == 8;
 330}
 331
 332static bool btf_type_is_char_array(struct btf *btf, const struct btf_type *type)
 333{
 334	const struct btf_type *real_type;
 335	const struct btf_array *array;
 336	u32 intdata;
 337	s32 tid;
 338
 339	if (BTF_INFO_KIND(type->info) != BTF_KIND_ARRAY)
 340		return false;
 341
 342	array = (const struct btf_array *)(type + 1);
 343
 344	real_type = btf_type_skip_modifiers(btf, array->type, &tid);
 345
 346	intdata = btf_type_int(real_type);
 347	return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
 348		&& BTF_INT_BITS(intdata) == 8;
 349}
 350
 351static int check_prepare_btf_string_fetch(char *typename,
 352				struct fetch_insn **pcode,
 353				struct traceprobe_parse_context *ctx)
 354{
 355	struct btf *btf = ctx->btf;
 356
 357	if (!btf || !ctx->last_type)
 358		return 0;
 359
 360	/* char [] does not need any change. */
 361	if (btf_type_is_char_array(btf, ctx->last_type))
 362		return 0;
 363
 364	/* char * requires dereference the pointer. */
 365	if (btf_type_is_char_ptr(btf, ctx->last_type)) {
 366		struct fetch_insn *code = *pcode + 1;
 367
 368		if (code->op == FETCH_OP_END) {
 369			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
 370			return -E2BIG;
 371		}
 372		if (typename[0] == 'u')
 373			code->op = FETCH_OP_UDEREF;
 374		else
 375			code->op = FETCH_OP_DEREF;
 376		code->offset = 0;
 377		*pcode = code;
 378		return 0;
 379	}
 380	/* Other types are not available for string */
 381	trace_probe_log_err(ctx->offset, BAD_TYPE4STR);
 382	return -EINVAL;
 383}
 384
 385static const char *fetch_type_from_btf_type(struct btf *btf,
 386					const struct btf_type *type,
 387					struct traceprobe_parse_context *ctx)
 388{
 389	u32 intdata;
 390
 391	/* TODO: const char * could be converted as a string */
 392	switch (BTF_INFO_KIND(type->info)) {
 393	case BTF_KIND_ENUM:
 394		/* enum is "int", so convert to "s32" */
 395		return "s32";
 396	case BTF_KIND_ENUM64:
 397		return "s64";
 398	case BTF_KIND_PTR:
 399		/* pointer will be converted to "x??" */
 400		if (IS_ENABLED(CONFIG_64BIT))
 401			return "x64";
 402		else
 403			return "x32";
 404	case BTF_KIND_INT:
 405		intdata = btf_type_int(type);
 406		if (BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED) {
 407			switch (BTF_INT_BITS(intdata)) {
 408			case 8:
 409				return "s8";
 410			case 16:
 411				return "s16";
 412			case 32:
 413				return "s32";
 414			case 64:
 415				return "s64";
 416			}
 417		} else {	/* unsigned */
 418			switch (BTF_INT_BITS(intdata)) {
 419			case 8:
 420				return "u8";
 421			case 16:
 422				return "u16";
 423			case 32:
 424				return "u32";
 425			case 64:
 426				return "u64";
 427			}
 428			/* bitfield, size is encoded in the type */
 429			ctx->last_bitsize = BTF_INT_BITS(intdata);
 430			ctx->last_bitoffs += BTF_INT_OFFSET(intdata);
 431			return "u64";
 432		}
 433	}
 434	/* TODO: support other types */
 435
 436	return NULL;
 437}
 438
 439static int query_btf_context(struct traceprobe_parse_context *ctx)
 440{
 441	const struct btf_param *param;
 442	const struct btf_type *type;
 443	struct btf *btf;
 444	s32 nr;
 445
 446	if (ctx->btf)
 447		return 0;
 448
 449	if (!ctx->funcname)
 450		return -EINVAL;
 451
 452	type = btf_find_func_proto(ctx->funcname, &btf);
 453	if (!type)
 454		return -ENOENT;
 455
 456	ctx->btf = btf;
 457	ctx->proto = type;
 458
 459	/* ctx->params is optional, since func(void) will not have params. */
 460	nr = 0;
 461	param = btf_get_func_param(type, &nr);
 462	if (!IS_ERR_OR_NULL(param)) {
 463		/* Hide the first 'data' argument of tracepoint */
 464		if (ctx->flags & TPARG_FL_TPOINT) {
 465			nr--;
 466			param++;
 467		}
 468	}
 469
 470	if (nr > 0) {
 471		ctx->nr_params = nr;
 472		ctx->params = param;
 473	} else {
 474		ctx->nr_params = 0;
 475		ctx->params = NULL;
 476	}
 477
 478	return 0;
 479}
 480
 481static void clear_btf_context(struct traceprobe_parse_context *ctx)
 482{
 483	if (ctx->btf) {
 484		btf_put(ctx->btf);
 485		ctx->btf = NULL;
 486		ctx->proto = NULL;
 487		ctx->params = NULL;
 488		ctx->nr_params = 0;
 489	}
 490}
 491
 492/* Return 1 if the field separater is arrow operator ('->') */
 493static int split_next_field(char *varname, char **next_field,
 494			    struct traceprobe_parse_context *ctx)
 495{
 496	char *field;
 497	int ret = 0;
 498
 499	field = strpbrk(varname, ".-");
 500	if (field) {
 501		if (field[0] == '-' && field[1] == '>') {
 502			field[0] = '\0';
 503			field += 2;
 504			ret = 1;
 505		} else if (field[0] == '.') {
 506			field[0] = '\0';
 507			field += 1;
 508		} else {
 509			trace_probe_log_err(ctx->offset + field - varname, BAD_HYPHEN);
 510			return -EINVAL;
 511		}
 512		*next_field = field;
 513	}
 514
 515	return ret;
 516}
 517
 518/*
 519 * Parse the field of data structure. The @type must be a pointer type
 520 * pointing the target data structure type.
 521 */
 522static int parse_btf_field(char *fieldname, const struct btf_type *type,
 523			   struct fetch_insn **pcode, struct fetch_insn *end,
 524			   struct traceprobe_parse_context *ctx)
 525{
 526	struct fetch_insn *code = *pcode;
 527	const struct btf_member *field;
 528	u32 bitoffs, anon_offs;
 529	char *next;
 530	int is_ptr;
 531	s32 tid;
 532
 533	do {
 534		/* Outer loop for solving arrow operator ('->') */
 535		if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) {
 536			trace_probe_log_err(ctx->offset, NO_PTR_STRCT);
 537			return -EINVAL;
 538		}
 539		/* Convert a struct pointer type to a struct type */
 540		type = btf_type_skip_modifiers(ctx->btf, type->type, &tid);
 541		if (!type) {
 542			trace_probe_log_err(ctx->offset, BAD_BTF_TID);
 543			return -EINVAL;
 544		}
 545
 546		bitoffs = 0;
 547		do {
 548			/* Inner loop for solving dot operator ('.') */
 549			next = NULL;
 550			is_ptr = split_next_field(fieldname, &next, ctx);
 551			if (is_ptr < 0)
 552				return is_ptr;
 553
 554			anon_offs = 0;
 555			field = btf_find_struct_member(ctx->btf, type, fieldname,
 556						       &anon_offs);
 557			if (IS_ERR(field)) {
 558				trace_probe_log_err(ctx->offset, BAD_BTF_TID);
 559				return PTR_ERR(field);
 560			}
 561			if (!field) {
 562				trace_probe_log_err(ctx->offset, NO_BTF_FIELD);
 563				return -ENOENT;
 564			}
 565			/* Add anonymous structure/union offset */
 566			bitoffs += anon_offs;
 567
 568			/* Accumulate the bit-offsets of the dot-connected fields */
 569			if (btf_type_kflag(type)) {
 570				bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset);
 571				ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset);
 572			} else {
 573				bitoffs += field->offset;
 574				ctx->last_bitsize = 0;
 575			}
 576
 577			type = btf_type_skip_modifiers(ctx->btf, field->type, &tid);
 578			if (!type) {
 579				trace_probe_log_err(ctx->offset, BAD_BTF_TID);
 580				return -EINVAL;
 581			}
 582
 583			ctx->offset += next - fieldname;
 584			fieldname = next;
 585		} while (!is_ptr && fieldname);
 586
 587		if (++code == end) {
 588			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
 589			return -EINVAL;
 590		}
 591		code->op = FETCH_OP_DEREF;	/* TODO: user deref support */
 592		code->offset = bitoffs / 8;
 593		*pcode = code;
 594
 595		ctx->last_bitoffs = bitoffs % 8;
 596		ctx->last_type = type;
 597	} while (fieldname);
 598
 599	return 0;
 600}
 601
 602static int __store_entry_arg(struct trace_probe *tp, int argnum);
 603
 604static int parse_btf_arg(char *varname,
 605			 struct fetch_insn **pcode, struct fetch_insn *end,
 606			 struct traceprobe_parse_context *ctx)
 607{
 608	struct fetch_insn *code = *pcode;
 609	const struct btf_param *params;
 610	const struct btf_type *type;
 611	char *field = NULL;
 612	int i, is_ptr, ret;
 613	u32 tid;
 614
 615	if (WARN_ON_ONCE(!ctx->funcname))
 616		return -EINVAL;
 617
 618	is_ptr = split_next_field(varname, &field, ctx);
 619	if (is_ptr < 0)
 620		return is_ptr;
 621	if (!is_ptr && field) {
 622		/* dot-connected field on an argument is not supported. */
 623		trace_probe_log_err(ctx->offset + field - varname,
 624				    NOSUP_DAT_ARG);
 625		return -EOPNOTSUPP;
 626	}
 627
 628	if (ctx->flags & TPARG_FL_RETURN && !strcmp(varname, "$retval")) {
 629		code->op = FETCH_OP_RETVAL;
 630		/* Check whether the function return type is not void */
 631		if (query_btf_context(ctx) == 0) {
 632			if (ctx->proto->type == 0) {
 633				trace_probe_log_err(ctx->offset, NO_RETVAL);
 634				return -ENOENT;
 635			}
 636			tid = ctx->proto->type;
 637			goto found;
 638		}
 639		if (field) {
 640			trace_probe_log_err(ctx->offset + field - varname,
 641					    NO_BTF_ENTRY);
 642			return -ENOENT;
 643		}
 644		return 0;
 645	}
 646
 647	if (!ctx->btf) {
 648		ret = query_btf_context(ctx);
 649		if (ret < 0 || ctx->nr_params == 0) {
 650			trace_probe_log_err(ctx->offset, NO_BTF_ENTRY);
 651			return PTR_ERR(params);
 652		}
 653	}
 654	params = ctx->params;
 655
 656	for (i = 0; i < ctx->nr_params; i++) {
 657		const char *name = btf_name_by_offset(ctx->btf, params[i].name_off);
 658
 659		if (name && !strcmp(name, varname)) {
 660			if (tparg_is_function_entry(ctx->flags)) {
 661				code->op = FETCH_OP_ARG;
 662				if (ctx->flags & TPARG_FL_TPOINT)
 663					code->param = i + 1;
 664				else
 665					code->param = i;
 666			} else if (tparg_is_function_return(ctx->flags)) {
 667				code->op = FETCH_OP_EDATA;
 668				ret = __store_entry_arg(ctx->tp, i);
 669				if (ret < 0) {
 670					/* internal error */
 671					return ret;
 672				}
 673				code->offset = ret;
 674			}
 675			tid = params[i].type;
 676			goto found;
 677		}
 678	}
 679	trace_probe_log_err(ctx->offset, NO_BTFARG);
 680	return -ENOENT;
 681
 682found:
 683	type = btf_type_skip_modifiers(ctx->btf, tid, &tid);
 684	if (!type) {
 685		trace_probe_log_err(ctx->offset, BAD_BTF_TID);
 686		return -EINVAL;
 687	}
 688	/* Initialize the last type information */
 689	ctx->last_type = type;
 690	ctx->last_bitoffs = 0;
 691	ctx->last_bitsize = 0;
 692	if (field) {
 693		ctx->offset += field - varname;
 694		return parse_btf_field(field, type, pcode, end, ctx);
 695	}
 696	return 0;
 697}
 698
 699static const struct fetch_type *find_fetch_type_from_btf_type(
 700					struct traceprobe_parse_context *ctx)
 701{
 702	struct btf *btf = ctx->btf;
 703	const char *typestr = NULL;
 704
 705	if (btf && ctx->last_type)
 706		typestr = fetch_type_from_btf_type(btf, ctx->last_type, ctx);
 707
 708	return find_fetch_type(typestr, ctx->flags);
 709}
 710
 711static int parse_btf_bitfield(struct fetch_insn **pcode,
 712			      struct traceprobe_parse_context *ctx)
 713{
 714	struct fetch_insn *code = *pcode;
 715
 716	if ((ctx->last_bitsize % 8 == 0) && ctx->last_bitoffs == 0)
 717		return 0;
 718
 719	code++;
 720	if (code->op != FETCH_OP_NOP) {
 721		trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
 722		return -EINVAL;
 723	}
 724	*pcode = code;
 725
 726	code->op = FETCH_OP_MOD_BF;
 727	code->lshift = 64 - (ctx->last_bitsize + ctx->last_bitoffs);
 728	code->rshift = 64 - ctx->last_bitsize;
 729	code->basesize = 64 / 8;
 730	return 0;
 731}
 732
 733#else
 734static void clear_btf_context(struct traceprobe_parse_context *ctx)
 735{
 736	ctx->btf = NULL;
 737}
 738
 739static int query_btf_context(struct traceprobe_parse_context *ctx)
 740{
 741	return -EOPNOTSUPP;
 742}
 743
 744static int parse_btf_arg(char *varname,
 745			 struct fetch_insn **pcode, struct fetch_insn *end,
 746			 struct traceprobe_parse_context *ctx)
 747{
 748	trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
 749	return -EOPNOTSUPP;
 750}
 751
 752static int parse_btf_bitfield(struct fetch_insn **pcode,
 753			      struct traceprobe_parse_context *ctx)
 754{
 755	trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
 756	return -EOPNOTSUPP;
 757}
 758
 759#define find_fetch_type_from_btf_type(ctx)		\
 760	find_fetch_type(NULL, ctx->flags)
 761
 762static int check_prepare_btf_string_fetch(char *typename,
 763				struct fetch_insn **pcode,
 764				struct traceprobe_parse_context *ctx)
 765{
 766	return 0;
 767}
 768
 769#endif
 770
 771#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
 772
 773static int __store_entry_arg(struct trace_probe *tp, int argnum)
 774{
 775	struct probe_entry_arg *earg = tp->entry_arg;
 776	bool match = false;
 777	int i, offset;
 778
 779	if (!earg) {
 780		earg = kzalloc(sizeof(*tp->entry_arg), GFP_KERNEL);
 781		if (!earg)
 782			return -ENOMEM;
 783		earg->size = 2 * tp->nr_args + 1;
 784		earg->code = kcalloc(earg->size, sizeof(struct fetch_insn),
 785				     GFP_KERNEL);
 786		if (!earg->code) {
 787			kfree(earg);
 788			return -ENOMEM;
 789		}
 790		/* Fill the code buffer with 'end' to simplify it */
 791		for (i = 0; i < earg->size; i++)
 792			earg->code[i].op = FETCH_OP_END;
 793		tp->entry_arg = earg;
 794	}
 795
 796	offset = 0;
 797	for (i = 0; i < earg->size - 1; i++) {
 798		switch (earg->code[i].op) {
 799		case FETCH_OP_END:
 800			earg->code[i].op = FETCH_OP_ARG;
 801			earg->code[i].param = argnum;
 802			earg->code[i + 1].op = FETCH_OP_ST_EDATA;
 803			earg->code[i + 1].offset = offset;
 804			return offset;
 805		case FETCH_OP_ARG:
 806			match = (earg->code[i].param == argnum);
 807			break;
 808		case FETCH_OP_ST_EDATA:
 809			offset = earg->code[i].offset;
 810			if (match)
 811				return offset;
 812			offset += sizeof(unsigned long);
 813			break;
 814		default:
 815			break;
 816		}
 817	}
 818	return -ENOSPC;
 819}
 820
 821int traceprobe_get_entry_data_size(struct trace_probe *tp)
 822{
 823	struct probe_entry_arg *earg = tp->entry_arg;
 824	int i, size = 0;
 825
 826	if (!earg)
 827		return 0;
 828
 829	for (i = 0; i < earg->size; i++) {
 830		switch (earg->code[i].op) {
 831		case FETCH_OP_END:
 832			goto out;
 833		case FETCH_OP_ST_EDATA:
 834			size = earg->code[i].offset + sizeof(unsigned long);
 835			break;
 836		default:
 837			break;
 838		}
 839	}
 840out:
 841	return size;
 842}
 843
 844void store_trace_entry_data(void *edata, struct trace_probe *tp, struct pt_regs *regs)
 845{
 846	struct probe_entry_arg *earg = tp->entry_arg;
 847	unsigned long val = 0;
 848	int i;
 849
 850	if (!earg)
 851		return;
 852
 853	for (i = 0; i < earg->size; i++) {
 854		struct fetch_insn *code = &earg->code[i];
 855
 856		switch (code->op) {
 857		case FETCH_OP_ARG:
 858			val = regs_get_kernel_argument(regs, code->param);
 859			break;
 860		case FETCH_OP_ST_EDATA:
 861			*(unsigned long *)((unsigned long)edata + code->offset) = val;
 862			break;
 863		case FETCH_OP_END:
 864			goto end;
 865		default:
 866			break;
 867		}
 868	}
 869end:
 870	return;
 871}
 872NOKPROBE_SYMBOL(store_trace_entry_data)
 873#endif
 874
 875#define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
 876
 877/* Parse $vars. @orig_arg points '$', which syncs to @ctx->offset */
 878static int parse_probe_vars(char *orig_arg, const struct fetch_type *t,
 879			    struct fetch_insn **pcode,
 880			    struct fetch_insn *end,
 881			    struct traceprobe_parse_context *ctx)
 882{
 883	struct fetch_insn *code = *pcode;
 884	int err = TP_ERR_BAD_VAR;
 885	char *arg = orig_arg + 1;
 886	unsigned long param;
 887	int ret = 0;
 888	int len;
 889
 890	if (ctx->flags & TPARG_FL_TEVENT) {
 891		if (code->data)
 892			return -EFAULT;
 893		ret = parse_trace_event_arg(arg, code, ctx);
 894		if (!ret)
 895			return 0;
 896		if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
 897			code->op = FETCH_OP_COMM;
 898			return 0;
 899		}
 900		/* backward compatibility */
 901		ctx->offset = 0;
 902		goto inval;
 903	}
 904
 905	if (str_has_prefix(arg, "retval")) {
 906		if (!(ctx->flags & TPARG_FL_RETURN)) {
 907			err = TP_ERR_RETVAL_ON_PROBE;
 908			goto inval;
 909		}
 910		if (!(ctx->flags & TPARG_FL_KERNEL) ||
 911		    !IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) {
 912			code->op = FETCH_OP_RETVAL;
 913			return 0;
 
 
 914		}
 915		return parse_btf_arg(orig_arg, pcode, end, ctx);
 916	}
 917
 918	len = str_has_prefix(arg, "stack");
 919	if (len) {
 920
 921		if (arg[len] == '\0') {
 922			code->op = FETCH_OP_STACKP;
 923			return 0;
 924		}
 925
 926		if (isdigit(arg[len])) {
 927			ret = kstrtoul(arg + len, 10, &param);
 928			if (ret)
 929				goto inval;
 930
 931			if ((ctx->flags & TPARG_FL_KERNEL) &&
 932			    param > PARAM_MAX_STACK) {
 933				err = TP_ERR_BAD_STACK_NUM;
 934				goto inval;
 
 
 935			}
 936			code->op = FETCH_OP_STACK;
 937			code->param = (unsigned int)param;
 938			return 0;
 939		}
 940		goto inval;
 941	}
 942
 943	if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
 944		code->op = FETCH_OP_COMM;
 945		return 0;
 946	}
 947
 948#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
 949	len = str_has_prefix(arg, "arg");
 950	if (len) {
 
 951		ret = kstrtoul(arg + len, 10, &param);
 952		if (ret)
 953			goto inval;
 954
 955		if (!param || param > PARAM_MAX_STACK) {
 956			err = TP_ERR_BAD_ARG_NUM;
 957			goto inval;
 958		}
 959		param--; /* argN starts from 1, but internal arg[N] starts from 0 */
 960
 961		if (tparg_is_function_entry(ctx->flags)) {
 962			code->op = FETCH_OP_ARG;
 963			code->param = (unsigned int)param;
 964			/*
 965			 * The tracepoint probe will probe a stub function, and the
 966			 * first parameter of the stub is a dummy and should be ignored.
 967			 */
 968			if (ctx->flags & TPARG_FL_TPOINT)
 969				code->param++;
 970		} else if (tparg_is_function_return(ctx->flags)) {
 971			/* function entry argument access from return probe */
 972			ret = __store_entry_arg(ctx->tp, param);
 973			if (ret < 0)	/* This error should be an internal error */
 974				return ret;
 975
 976			code->op = FETCH_OP_EDATA;
 977			code->offset = ret;
 978		} else {
 979			err = TP_ERR_NOFENTRY_ARGS;
 980			goto inval;
 981		}
 982		return 0;
 983	}
 984#endif
 
 
 985
 986inval:
 987	__trace_probe_log_err(ctx->offset, err);
 
 
 988	return -EINVAL;
 989}
 990
 991static int str_to_immediate(char *str, unsigned long *imm)
 992{
 993	if (isdigit(str[0]))
 994		return kstrtoul(str, 0, imm);
 995	else if (str[0] == '-')
 996		return kstrtol(str, 0, (long *)imm);
 997	else if (str[0] == '+')
 998		return kstrtol(str + 1, 0, (long *)imm);
 999	return -EINVAL;
1000}
1001
1002static int __parse_imm_string(char *str, char **pbuf, int offs)
1003{
1004	size_t len = strlen(str);
1005
1006	if (str[len - 1] != '"') {
1007		trace_probe_log_err(offs + len, IMMSTR_NO_CLOSE);
1008		return -EINVAL;
1009	}
1010	*pbuf = kstrndup(str, len - 1, GFP_KERNEL);
1011	if (!*pbuf)
1012		return -ENOMEM;
1013	return 0;
1014}
1015
1016/* Recursive argument parser */
1017static int
1018parse_probe_arg(char *arg, const struct fetch_type *type,
1019		struct fetch_insn **pcode, struct fetch_insn *end,
1020		struct traceprobe_parse_context *ctx)
1021{
1022	struct fetch_insn *code = *pcode;
1023	unsigned long param;
1024	int deref = FETCH_OP_DEREF;
1025	long offset = 0;
1026	char *tmp;
1027	int ret = 0;
1028
1029	switch (arg[0]) {
1030	case '$':
1031		ret = parse_probe_vars(arg, type, pcode, end, ctx);
1032		break;
1033
1034	case '%':	/* named register */
1035		if (ctx->flags & (TPARG_FL_TEVENT | TPARG_FL_FPROBE)) {
1036			/* eprobe and fprobe do not handle registers */
1037			trace_probe_log_err(ctx->offset, BAD_VAR);
1038			break;
1039		}
1040		ret = regs_query_register_offset(arg + 1);
1041		if (ret >= 0) {
1042			code->op = FETCH_OP_REG;
1043			code->param = (unsigned int)ret;
1044			ret = 0;
1045		} else
1046			trace_probe_log_err(ctx->offset, BAD_REG_NAME);
1047		break;
1048
1049	case '@':	/* memory, file-offset or symbol */
1050		if (isdigit(arg[1])) {
1051			ret = kstrtoul(arg + 1, 0, &param);
1052			if (ret) {
1053				trace_probe_log_err(ctx->offset, BAD_MEM_ADDR);
1054				break;
1055			}
1056			/* load address */
1057			code->op = FETCH_OP_IMM;
1058			code->immediate = param;
1059		} else if (arg[1] == '+') {
1060			/* kprobes don't support file offsets */
1061			if (ctx->flags & TPARG_FL_KERNEL) {
1062				trace_probe_log_err(ctx->offset, FILE_ON_KPROBE);
1063				return -EINVAL;
1064			}
1065			ret = kstrtol(arg + 2, 0, &offset);
1066			if (ret) {
1067				trace_probe_log_err(ctx->offset, BAD_FILE_OFFS);
1068				break;
1069			}
1070
1071			code->op = FETCH_OP_FOFFS;
1072			code->immediate = (unsigned long)offset;  // imm64?
1073		} else {
1074			/* uprobes don't support symbols */
1075			if (!(ctx->flags & TPARG_FL_KERNEL)) {
1076				trace_probe_log_err(ctx->offset, SYM_ON_UPROBE);
1077				return -EINVAL;
1078			}
1079			/* Preserve symbol for updating */
1080			code->op = FETCH_NOP_SYMBOL;
1081			code->data = kstrdup(arg + 1, GFP_KERNEL);
1082			if (!code->data)
1083				return -ENOMEM;
1084			if (++code == end) {
1085				trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1086				return -EINVAL;
1087			}
1088			code->op = FETCH_OP_IMM;
1089			code->immediate = 0;
1090		}
1091		/* These are fetching from memory */
1092		if (++code == end) {
1093			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1094			return -EINVAL;
1095		}
1096		*pcode = code;
1097		code->op = FETCH_OP_DEREF;
1098		code->offset = offset;
1099		break;
1100
1101	case '+':	/* deref memory */
1102	case '-':
1103		if (arg[1] == 'u') {
1104			deref = FETCH_OP_UDEREF;
1105			arg[1] = arg[0];
1106			arg++;
1107		}
1108		if (arg[0] == '+')
1109			arg++;	/* Skip '+', because kstrtol() rejects it. */
1110		tmp = strchr(arg, '(');
1111		if (!tmp) {
1112			trace_probe_log_err(ctx->offset, DEREF_NEED_BRACE);
1113			return -EINVAL;
1114		}
1115		*tmp = '\0';
1116		ret = kstrtol(arg, 0, &offset);
1117		if (ret) {
1118			trace_probe_log_err(ctx->offset, BAD_DEREF_OFFS);
1119			break;
1120		}
1121		ctx->offset += (tmp + 1 - arg) + (arg[0] != '-' ? 1 : 0);
1122		arg = tmp + 1;
1123		tmp = strrchr(arg, ')');
1124		if (!tmp) {
1125			trace_probe_log_err(ctx->offset + strlen(arg),
1126					    DEREF_OPEN_BRACE);
1127			return -EINVAL;
1128		} else {
1129			const struct fetch_type *t2 = find_fetch_type(NULL, ctx->flags);
1130			int cur_offs = ctx->offset;
1131
1132			*tmp = '\0';
1133			ret = parse_probe_arg(arg, t2, &code, end, ctx);
1134			if (ret)
1135				break;
1136			ctx->offset = cur_offs;
1137			if (code->op == FETCH_OP_COMM ||
1138			    code->op == FETCH_OP_DATA) {
1139				trace_probe_log_err(ctx->offset, COMM_CANT_DEREF);
1140				return -EINVAL;
1141			}
1142			if (++code == end) {
1143				trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1144				return -EINVAL;
1145			}
1146			*pcode = code;
1147
1148			code->op = deref;
1149			code->offset = offset;
1150			/* Reset the last type if used */
1151			ctx->last_type = NULL;
1152		}
1153		break;
1154	case '\\':	/* Immediate value */
1155		if (arg[1] == '"') {	/* Immediate string */
1156			ret = __parse_imm_string(arg + 2, &tmp, ctx->offset + 2);
1157			if (ret)
1158				break;
1159			code->op = FETCH_OP_DATA;
1160			code->data = tmp;
1161		} else {
1162			ret = str_to_immediate(arg + 1, &code->immediate);
1163			if (ret)
1164				trace_probe_log_err(ctx->offset + 1, BAD_IMM);
1165			else
1166				code->op = FETCH_OP_IMM;
1167		}
1168		break;
1169	default:
1170		if (isalpha(arg[0]) || arg[0] == '_') {	/* BTF variable */
1171			if (!tparg_is_function_entry(ctx->flags) &&
1172			    !tparg_is_function_return(ctx->flags)) {
1173				trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
1174				return -EINVAL;
1175			}
1176			ret = parse_btf_arg(arg, pcode, end, ctx);
1177			break;
1178		}
1179	}
1180	if (!ret && code->op == FETCH_OP_NOP) {
1181		/* Parsed, but do not find fetch method */
1182		trace_probe_log_err(ctx->offset, BAD_FETCH_ARG);
1183		ret = -EINVAL;
1184	}
1185	return ret;
1186}
1187
 
 
1188/* Bitfield type needs to be parsed into a fetch function */
1189static int __parse_bitfield_probe_arg(const char *bf,
1190				      const struct fetch_type *t,
1191				      struct fetch_insn **pcode)
1192{
1193	struct fetch_insn *code = *pcode;
1194	unsigned long bw, bo;
1195	char *tail;
1196
1197	if (*bf != 'b')
1198		return 0;
1199
1200	bw = simple_strtoul(bf + 1, &tail, 0);	/* Use simple one */
1201
1202	if (bw == 0 || *tail != '@')
1203		return -EINVAL;
1204
1205	bf = tail + 1;
1206	bo = simple_strtoul(bf, &tail, 0);
1207
1208	if (tail == bf || *tail != '/')
1209		return -EINVAL;
1210	code++;
1211	if (code->op != FETCH_OP_NOP)
1212		return -EINVAL;
1213	*pcode = code;
1214
1215	code->op = FETCH_OP_MOD_BF;
1216	code->lshift = BYTES_TO_BITS(t->size) - (bw + bo);
1217	code->rshift = BYTES_TO_BITS(t->size) - bw;
1218	code->basesize = t->size;
1219
1220	return (BYTES_TO_BITS(t->size) < (bw + bo)) ? -EINVAL : 0;
1221}
1222
1223/* Split type part from @arg and return it. */
1224static char *parse_probe_arg_type(char *arg, struct probe_arg *parg,
1225				  struct traceprobe_parse_context *ctx)
1226{
1227	char *t = NULL, *t2, *t3;
1228	int offs;
 
 
1229
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1230	t = strchr(arg, ':');
1231	if (t) {
1232		*t++ = '\0';
1233		t2 = strchr(t, '[');
1234		if (t2) {
1235			*t2++ = '\0';
1236			t3 = strchr(t2, ']');
1237			if (!t3) {
1238				offs = t2 + strlen(t2) - arg;
1239
1240				trace_probe_log_err(ctx->offset + offs,
1241						    ARRAY_NO_CLOSE);
1242				return ERR_PTR(-EINVAL);
1243			} else if (t3[1] != '\0') {
1244				trace_probe_log_err(ctx->offset + t3 + 1 - arg,
1245						    BAD_ARRAY_SUFFIX);
1246				return ERR_PTR(-EINVAL);
1247			}
1248			*t3 = '\0';
1249			if (kstrtouint(t2, 0, &parg->count) || !parg->count) {
1250				trace_probe_log_err(ctx->offset + t2 - arg,
1251						    BAD_ARRAY_NUM);
1252				return ERR_PTR(-EINVAL);
1253			}
1254			if (parg->count > MAX_ARRAY_LEN) {
1255				trace_probe_log_err(ctx->offset + t2 - arg,
1256						    ARRAY_TOO_BIG);
1257				return ERR_PTR(-EINVAL);
1258			}
1259		}
1260	}
1261	offs = t ? t - arg : 0;
1262
1263	/*
1264	 * Since $comm and immediate string can not be dereferenced,
1265	 * we can find those by strcmp. But ignore for eprobes.
1266	 */
1267	if (!(ctx->flags & TPARG_FL_TEVENT) &&
1268	    (strcmp(arg, "$comm") == 0 || strcmp(arg, "$COMM") == 0 ||
1269	     strncmp(arg, "\\\"", 2) == 0)) {
1270		/* The type of $comm must be "string", and not an array type. */
1271		if (parg->count || (t && strcmp(t, "string"))) {
1272			trace_probe_log_err(ctx->offset + offs, NEED_STRING_TYPE);
1273			return ERR_PTR(-EINVAL);
1274		}
1275		parg->type = find_fetch_type("string", ctx->flags);
1276	} else
1277		parg->type = find_fetch_type(t, ctx->flags);
1278
1279	if (!parg->type) {
1280		trace_probe_log_err(ctx->offset + offs, BAD_TYPE);
1281		return ERR_PTR(-EINVAL);
1282	}
 
 
1283
1284	return t;
1285}
 
 
 
 
 
 
 
1286
1287/* After parsing, adjust the fetch_insn according to the probe_arg */
1288static int finalize_fetch_insn(struct fetch_insn *code,
1289			       struct probe_arg *parg,
1290			       char *type,
1291			       int type_offset,
1292			       struct traceprobe_parse_context *ctx)
1293{
1294	struct fetch_insn *scode;
1295	int ret;
1296
 
1297	/* Store operation */
1298	if (parg->type->is_string) {
1299		/* Check bad combination of the type and the last fetch_insn. */
1300		if (!strcmp(parg->type->name, "symstr")) {
1301			if (code->op != FETCH_OP_REG && code->op != FETCH_OP_STACK &&
1302			    code->op != FETCH_OP_RETVAL && code->op != FETCH_OP_ARG &&
1303			    code->op != FETCH_OP_DEREF && code->op != FETCH_OP_TP_ARG) {
1304				trace_probe_log_err(ctx->offset + type_offset,
1305						    BAD_SYMSTRING);
1306				return -EINVAL;
1307			}
1308		} else {
1309			if (code->op != FETCH_OP_DEREF && code->op != FETCH_OP_UDEREF &&
1310			    code->op != FETCH_OP_IMM && code->op != FETCH_OP_COMM &&
1311			    code->op != FETCH_OP_DATA && code->op != FETCH_OP_TP_ARG) {
1312				trace_probe_log_err(ctx->offset + type_offset,
1313						    BAD_STRING);
1314				return -EINVAL;
1315			}
1316		}
1317
1318		if (!strcmp(parg->type->name, "symstr") ||
1319		    (code->op == FETCH_OP_IMM || code->op == FETCH_OP_COMM ||
1320		     code->op == FETCH_OP_DATA) || code->op == FETCH_OP_TP_ARG ||
1321		     parg->count) {
1322			/*
1323			 * IMM, DATA and COMM is pointing actual address, those
1324			 * must be kept, and if parg->count != 0, this is an
1325			 * array of string pointers instead of string address
1326			 * itself.
1327			 * For the symstr, it doesn't need to dereference, thus
1328			 * it just get the value.
1329			 */
1330			code++;
1331			if (code->op != FETCH_OP_NOP) {
1332				trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1333				return -EINVAL;
1334			}
1335		}
1336
1337		/* If op == DEREF, replace it with STRING */
1338		if (!strcmp(parg->type->name, "ustring") ||
1339		    code->op == FETCH_OP_UDEREF)
1340			code->op = FETCH_OP_ST_USTRING;
1341		else if (!strcmp(parg->type->name, "symstr"))
1342			code->op = FETCH_OP_ST_SYMSTR;
1343		else
1344			code->op = FETCH_OP_ST_STRING;
1345		code->size = parg->type->size;
1346		parg->dynamic = true;
1347	} else if (code->op == FETCH_OP_DEREF) {
1348		code->op = FETCH_OP_ST_MEM;
1349		code->size = parg->type->size;
1350	} else if (code->op == FETCH_OP_UDEREF) {
1351		code->op = FETCH_OP_ST_UMEM;
1352		code->size = parg->type->size;
1353	} else {
1354		code++;
1355		if (code->op != FETCH_OP_NOP) {
1356			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1357			return -E2BIG;
1358		}
1359		code->op = FETCH_OP_ST_RAW;
1360		code->size = parg->type->size;
1361	}
1362
1363	/* Save storing fetch_insn. */
1364	scode = code;
1365
1366	/* Modify operation */
1367	if (type != NULL) {
1368		/* Bitfield needs a special fetch_insn. */
1369		ret = __parse_bitfield_probe_arg(type, parg->type, &code);
1370		if (ret) {
1371			trace_probe_log_err(ctx->offset + type_offset, BAD_BITFIELD);
1372			return ret;
1373		}
1374	} else if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) &&
1375		   ctx->last_type) {
1376		/* If user not specified the type, try parsing BTF bitfield. */
1377		ret = parse_btf_bitfield(&code, ctx);
1378		if (ret)
1379			return ret;
1380	}
1381
1382	/* Loop(Array) operation */
1383	if (parg->count) {
1384		if (scode->op != FETCH_OP_ST_MEM &&
1385		    scode->op != FETCH_OP_ST_STRING &&
1386		    scode->op != FETCH_OP_ST_USTRING) {
1387			trace_probe_log_err(ctx->offset + type_offset, BAD_STRING);
1388			return -EINVAL;
 
1389		}
1390		code++;
1391		if (code->op != FETCH_OP_NOP) {
1392			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1393			return -E2BIG;
1394		}
1395		code->op = FETCH_OP_LP_ARRAY;
1396		code->param = parg->count;
1397	}
1398
1399	/* Finalize the fetch_insn array. */
1400	code++;
1401	code->op = FETCH_OP_END;
1402
1403	return 0;
1404}
1405
1406/* String length checking wrapper */
1407static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size,
1408					   struct probe_arg *parg,
1409					   struct traceprobe_parse_context *ctx)
1410{
1411	struct fetch_insn *code, *tmp = NULL;
1412	char *type, *arg;
1413	int ret, len;
1414
1415	len = strlen(argv);
1416	if (len > MAX_ARGSTR_LEN) {
1417		trace_probe_log_err(ctx->offset, ARG_TOO_LONG);
1418		return -E2BIG;
1419	} else if (len == 0) {
1420		trace_probe_log_err(ctx->offset, NO_ARG_BODY);
1421		return -EINVAL;
1422	}
1423
1424	arg = kstrdup(argv, GFP_KERNEL);
1425	if (!arg)
1426		return -ENOMEM;
1427
1428	parg->comm = kstrdup(arg, GFP_KERNEL);
1429	if (!parg->comm) {
1430		ret = -ENOMEM;
1431		goto out;
1432	}
1433
1434	type = parse_probe_arg_type(arg, parg, ctx);
1435	if (IS_ERR(type)) {
1436		ret = PTR_ERR(type);
1437		goto out;
1438	}
1439
1440	code = tmp = kcalloc(FETCH_INSN_MAX, sizeof(*code), GFP_KERNEL);
1441	if (!code) {
1442		ret = -ENOMEM;
1443		goto out;
1444	}
1445	code[FETCH_INSN_MAX - 1].op = FETCH_OP_END;
1446
1447	ctx->last_type = NULL;
1448	ret = parse_probe_arg(arg, parg->type, &code, &code[FETCH_INSN_MAX - 1],
1449			      ctx);
1450	if (ret < 0)
1451		goto fail;
1452
1453	/* Update storing type if BTF is available */
1454	if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) &&
1455	    ctx->last_type) {
1456		if (!type) {
1457			parg->type = find_fetch_type_from_btf_type(ctx);
1458		} else if (strstr(type, "string")) {
1459			ret = check_prepare_btf_string_fetch(type, &code, ctx);
1460			if (ret)
1461				goto fail;
1462		}
1463	}
1464	parg->offset = *size;
1465	*size += parg->type->size * (parg->count ?: 1);
1466
1467	if (parg->count) {
1468		len = strlen(parg->type->fmttype) + 6;
1469		parg->fmt = kmalloc(len, GFP_KERNEL);
1470		if (!parg->fmt) {
1471			ret = -ENOMEM;
1472			goto fail;
1473		}
1474		snprintf(parg->fmt, len, "%s[%d]", parg->type->fmttype,
1475			 parg->count);
1476	}
1477
1478	ret = finalize_fetch_insn(code, parg, type, type ? type - arg : 0, ctx);
1479	if (ret < 0)
1480		goto fail;
1481
1482	for (; code < tmp + FETCH_INSN_MAX; code++)
1483		if (code->op == FETCH_OP_END)
1484			break;
1485	/* Shrink down the code buffer */
1486	parg->code = kcalloc(code - tmp + 1, sizeof(*code), GFP_KERNEL);
1487	if (!parg->code)
1488		ret = -ENOMEM;
1489	else
1490		memcpy(parg->code, tmp, sizeof(*code) * (code - tmp + 1));
1491
1492fail:
1493	if (ret < 0) {
1494		for (code = tmp; code < tmp + FETCH_INSN_MAX; code++)
1495			if (code->op == FETCH_NOP_SYMBOL ||
1496			    code->op == FETCH_OP_DATA)
1497				kfree(code->data);
1498	}
1499	kfree(tmp);
1500out:
1501	kfree(arg);
1502
1503	return ret;
1504}
1505
1506/* Return 1 if name is reserved or already used by another argument */
1507static int traceprobe_conflict_field_name(const char *name,
1508					  struct probe_arg *args, int narg)
1509{
1510	int i;
1511
1512	for (i = 0; i < ARRAY_SIZE(reserved_field_names); i++)
1513		if (strcmp(reserved_field_names[i], name) == 0)
1514			return 1;
1515
1516	for (i = 0; i < narg; i++)
1517		if (strcmp(args[i].name, name) == 0)
1518			return 1;
1519
1520	return 0;
1521}
1522
1523static char *generate_probe_arg_name(const char *arg, int idx)
1524{
1525	char *name = NULL;
1526	const char *end;
1527
1528	/*
1529	 * If argument name is omitted, try arg as a name (BTF variable)
1530	 * or "argN".
1531	 */
1532	if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) {
1533		end = strchr(arg, ':');
1534		if (!end)
1535			end = arg + strlen(arg);
1536
1537		name = kmemdup_nul(arg, end - arg, GFP_KERNEL);
1538		if (!name || !is_good_name(name)) {
1539			kfree(name);
1540			name = NULL;
1541		}
1542	}
1543
1544	if (!name)
1545		name = kasprintf(GFP_KERNEL, "arg%d", idx + 1);
1546
1547	return name;
1548}
1549
1550int traceprobe_parse_probe_arg(struct trace_probe *tp, int i, const char *arg,
1551			       struct traceprobe_parse_context *ctx)
1552{
1553	struct probe_arg *parg = &tp->args[i];
1554	const char *body;
1555
1556	ctx->tp = tp;
 
 
1557	body = strchr(arg, '=');
1558	if (body) {
1559		if (body - arg > MAX_ARG_NAME_LEN) {
1560			trace_probe_log_err(0, ARG_NAME_TOO_LONG);
1561			return -EINVAL;
1562		} else if (body == arg) {
1563			trace_probe_log_err(0, NO_ARG_NAME);
1564			return -EINVAL;
1565		}
1566		parg->name = kmemdup_nul(arg, body - arg, GFP_KERNEL);
1567		body++;
1568	} else {
1569		parg->name = generate_probe_arg_name(arg, i);
 
1570		body = arg;
1571	}
1572	if (!parg->name)
1573		return -ENOMEM;
1574
1575	if (!is_good_name(parg->name)) {
1576		trace_probe_log_err(0, BAD_ARG_NAME);
1577		return -EINVAL;
1578	}
1579	if (traceprobe_conflict_field_name(parg->name, tp->args, i)) {
1580		trace_probe_log_err(0, USED_ARG_NAME);
1581		return -EINVAL;
1582	}
1583	ctx->offset = body - arg;
1584	/* Parse fetch argument */
1585	return traceprobe_parse_probe_arg_body(body, &tp->size, parg, ctx);
 
1586}
1587
1588void traceprobe_free_probe_arg(struct probe_arg *arg)
1589{
1590	struct fetch_insn *code = arg->code;
1591
1592	while (code && code->op != FETCH_OP_END) {
1593		if (code->op == FETCH_NOP_SYMBOL ||
1594		    code->op == FETCH_OP_DATA)
1595			kfree(code->data);
1596		code++;
1597	}
1598	kfree(arg->code);
1599	kfree(arg->name);
1600	kfree(arg->comm);
1601	kfree(arg->fmt);
1602}
1603
1604static int argv_has_var_arg(int argc, const char *argv[], int *args_idx,
1605			    struct traceprobe_parse_context *ctx)
1606{
1607	int i, found = 0;
1608
1609	for (i = 0; i < argc; i++)
1610		if (str_has_prefix(argv[i], "$arg")) {
1611			trace_probe_log_set_index(i + 2);
1612
1613			if (!tparg_is_function_entry(ctx->flags) &&
1614			    !tparg_is_function_return(ctx->flags)) {
1615				trace_probe_log_err(0, NOFENTRY_ARGS);
1616				return -EINVAL;
1617			}
1618
1619			if (isdigit(argv[i][4])) {
1620				found = 1;
1621				continue;
1622			}
1623
1624			if (argv[i][4] != '*') {
1625				trace_probe_log_err(0, BAD_VAR);
1626				return -EINVAL;
1627			}
1628
1629			if (*args_idx >= 0 && *args_idx < argc) {
1630				trace_probe_log_err(0, DOUBLE_ARGS);
1631				return -EINVAL;
1632			}
1633			found = 1;
1634			*args_idx = i;
1635		}
1636
1637	return found;
1638}
1639
1640static int sprint_nth_btf_arg(int idx, const char *type,
1641			      char *buf, int bufsize,
1642			      struct traceprobe_parse_context *ctx)
1643{
1644	const char *name;
1645	int ret;
1646
1647	if (idx >= ctx->nr_params) {
1648		trace_probe_log_err(0, NO_BTFARG);
1649		return -ENOENT;
1650	}
1651	name = btf_name_by_offset(ctx->btf, ctx->params[idx].name_off);
1652	if (!name) {
1653		trace_probe_log_err(0, NO_BTF_ENTRY);
1654		return -ENOENT;
1655	}
1656	ret = snprintf(buf, bufsize, "%s%s", name, type);
1657	if (ret >= bufsize) {
1658		trace_probe_log_err(0, ARGS_2LONG);
1659		return -E2BIG;
1660	}
1661	return ret;
1662}
1663
1664/* Return new_argv which must be freed after use */
1665const char **traceprobe_expand_meta_args(int argc, const char *argv[],
1666					 int *new_argc, char *buf, int bufsize,
1667					 struct traceprobe_parse_context *ctx)
1668{
1669	const struct btf_param *params = NULL;
1670	int i, j, n, used, ret, args_idx = -1;
1671	const char **new_argv = NULL;
1672
1673	ret = argv_has_var_arg(argc, argv, &args_idx, ctx);
1674	if (ret < 0)
1675		return ERR_PTR(ret);
1676
1677	if (!ret) {
1678		*new_argc = argc;
1679		return NULL;
1680	}
1681
1682	ret = query_btf_context(ctx);
1683	if (ret < 0 || ctx->nr_params == 0) {
1684		if (args_idx != -1) {
1685			/* $arg* requires BTF info */
1686			trace_probe_log_err(0, NOSUP_BTFARG);
1687			return (const char **)params;
1688		}
1689		*new_argc = argc;
1690		return NULL;
1691	}
1692
1693	if (args_idx >= 0)
1694		*new_argc = argc + ctx->nr_params - 1;
1695	else
1696		*new_argc = argc;
1697
1698	new_argv = kcalloc(*new_argc, sizeof(char *), GFP_KERNEL);
1699	if (!new_argv)
1700		return ERR_PTR(-ENOMEM);
1701
1702	used = 0;
1703	for (i = 0, j = 0; i < argc; i++) {
1704		trace_probe_log_set_index(i + 2);
1705		if (i == args_idx) {
1706			for (n = 0; n < ctx->nr_params; n++) {
1707				ret = sprint_nth_btf_arg(n, "", buf + used,
1708							 bufsize - used, ctx);
1709				if (ret < 0)
1710					goto error;
1711
1712				new_argv[j++] = buf + used;
1713				used += ret + 1;
1714			}
1715			continue;
1716		}
1717
1718		if (str_has_prefix(argv[i], "$arg")) {
1719			char *type = NULL;
1720
1721			n = simple_strtoul(argv[i] + 4, &type, 10);
1722			if (type && !(*type == ':' || *type == '\0')) {
1723				trace_probe_log_err(0, BAD_VAR);
1724				ret = -ENOENT;
1725				goto error;
1726			}
1727			/* Note: $argN starts from $arg1 */
1728			ret = sprint_nth_btf_arg(n - 1, type, buf + used,
1729						 bufsize - used, ctx);
1730			if (ret < 0)
1731				goto error;
1732			new_argv[j++] = buf + used;
1733			used += ret + 1;
1734		} else
1735			new_argv[j++] = argv[i];
1736	}
1737
1738	return new_argv;
1739
1740error:
1741	kfree(new_argv);
1742	return ERR_PTR(ret);
1743}
1744
1745/* @buf: *buf must be equal to NULL. Caller must to free *buf */
1746int traceprobe_expand_dentry_args(int argc, const char *argv[], char **buf)
1747{
1748	int i, used, ret;
1749	const int bufsize = MAX_DENTRY_ARGS_LEN;
1750	char *tmpbuf = NULL;
1751
1752	if (*buf)
1753		return -EINVAL;
1754
1755	used = 0;
1756	for (i = 0; i < argc; i++) {
1757		char *tmp;
1758		char *equal;
1759		size_t arg_len;
1760
1761		if (!glob_match("*:%p[dD]", argv[i]))
1762			continue;
1763
1764		if (!tmpbuf) {
1765			tmpbuf = kmalloc(bufsize, GFP_KERNEL);
1766			if (!tmpbuf)
1767				return -ENOMEM;
1768		}
1769
1770		tmp = kstrdup(argv[i], GFP_KERNEL);
1771		if (!tmp)
1772			goto nomem;
1773
1774		equal = strchr(tmp, '=');
1775		if (equal)
1776			*equal = '\0';
1777		arg_len = strlen(argv[i]);
1778		tmp[arg_len - 4] = '\0';
1779		if (argv[i][arg_len - 1] == 'd')
1780			ret = snprintf(tmpbuf + used, bufsize - used,
1781				       "%s%s+0x0(+0x%zx(%s)):string",
1782				       equal ? tmp : "", equal ? "=" : "",
1783				       offsetof(struct dentry, d_name.name),
1784				       equal ? equal + 1 : tmp);
1785		else
1786			ret = snprintf(tmpbuf + used, bufsize - used,
1787				       "%s%s+0x0(+0x%zx(+0x%zx(%s))):string",
1788				       equal ? tmp : "", equal ? "=" : "",
1789				       offsetof(struct dentry, d_name.name),
1790				       offsetof(struct file, f_path.dentry),
1791				       equal ? equal + 1 : tmp);
1792
1793		kfree(tmp);
1794		if (ret >= bufsize - used)
1795			goto nomem;
1796		argv[i] = tmpbuf + used;
1797		used += ret + 1;
1798	}
1799
1800	*buf = tmpbuf;
1801	return 0;
1802nomem:
1803	kfree(tmpbuf);
1804	return -ENOMEM;
1805}
1806
1807void traceprobe_finish_parse(struct traceprobe_parse_context *ctx)
1808{
1809	clear_btf_context(ctx);
1810}
1811
1812int traceprobe_update_arg(struct probe_arg *arg)
1813{
1814	struct fetch_insn *code = arg->code;
1815	long offset;
1816	char *tmp;
1817	char c;
1818	int ret = 0;
1819
1820	while (code && code->op != FETCH_OP_END) {
1821		if (code->op == FETCH_NOP_SYMBOL) {
1822			if (code[1].op != FETCH_OP_IMM)
1823				return -EINVAL;
1824
1825			tmp = strpbrk(code->data, "+-");
1826			if (tmp)
1827				c = *tmp;
1828			ret = traceprobe_split_symbol_offset(code->data,
1829							     &offset);
1830			if (ret)
1831				return ret;
1832
1833			code[1].immediate =
1834				(unsigned long)kallsyms_lookup_name(code->data);
1835			if (tmp)
1836				*tmp = c;
1837			if (!code[1].immediate)
1838				return -ENOENT;
1839			code[1].immediate += offset;
1840		}
1841		code++;
1842	}
1843	return 0;
1844}
1845
1846/* When len=0, we just calculate the needed length */
1847#define LEN_OR_ZERO (len ? len - pos : 0)
1848static int __set_print_fmt(struct trace_probe *tp, char *buf, int len,
1849			   enum probe_print_type ptype)
1850{
1851	struct probe_arg *parg;
1852	int i, j;
1853	int pos = 0;
1854	const char *fmt, *arg;
1855
1856	switch (ptype) {
1857	case PROBE_PRINT_NORMAL:
1858		fmt = "(%lx)";
1859		arg = ", REC->" FIELD_STRING_IP;
1860		break;
1861	case PROBE_PRINT_RETURN:
1862		fmt = "(%lx <- %lx)";
1863		arg = ", REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP;
1864		break;
1865	case PROBE_PRINT_EVENT:
1866		fmt = "";
1867		arg = "";
1868		break;
1869	default:
1870		WARN_ON_ONCE(1);
1871		return 0;
1872	}
1873
1874	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt);
1875
1876	for (i = 0; i < tp->nr_args; i++) {
1877		parg = tp->args + i;
1878		pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=", parg->name);
1879		if (parg->count) {
1880			pos += snprintf(buf + pos, LEN_OR_ZERO, "{%s",
1881					parg->type->fmt);
1882			for (j = 1; j < parg->count; j++)
1883				pos += snprintf(buf + pos, LEN_OR_ZERO, ",%s",
1884						parg->type->fmt);
1885			pos += snprintf(buf + pos, LEN_OR_ZERO, "}");
1886		} else
1887			pos += snprintf(buf + pos, LEN_OR_ZERO, "%s",
1888					parg->type->fmt);
1889	}
1890
1891	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", arg);
1892
1893	for (i = 0; i < tp->nr_args; i++) {
1894		parg = tp->args + i;
1895		if (parg->count) {
1896			if (parg->type->is_string)
1897				fmt = ", __get_str(%s[%d])";
1898			else
1899				fmt = ", REC->%s[%d]";
1900			for (j = 0; j < parg->count; j++)
1901				pos += snprintf(buf + pos, LEN_OR_ZERO,
1902						fmt, parg->name, j);
1903		} else {
1904			if (parg->type->is_string)
1905				fmt = ", __get_str(%s)";
1906			else
1907				fmt = ", REC->%s";
1908			pos += snprintf(buf + pos, LEN_OR_ZERO,
1909					fmt, parg->name);
1910		}
1911	}
1912
1913	/* return the length of print_fmt */
1914	return pos;
1915}
1916#undef LEN_OR_ZERO
1917
1918int traceprobe_set_print_fmt(struct trace_probe *tp, enum probe_print_type ptype)
1919{
1920	struct trace_event_call *call = trace_probe_event_call(tp);
1921	int len;
1922	char *print_fmt;
1923
1924	/* First: called with 0 length to calculate the needed length */
1925	len = __set_print_fmt(tp, NULL, 0, ptype);
1926	print_fmt = kmalloc(len + 1, GFP_KERNEL);
1927	if (!print_fmt)
1928		return -ENOMEM;
1929
1930	/* Second: actually write the @print_fmt */
1931	__set_print_fmt(tp, print_fmt, len + 1, ptype);
1932	call->print_fmt = print_fmt;
1933
1934	return 0;
1935}
1936
1937int traceprobe_define_arg_fields(struct trace_event_call *event_call,
1938				 size_t offset, struct trace_probe *tp)
1939{
1940	int ret, i;
1941
1942	/* Set argument names as fields */
1943	for (i = 0; i < tp->nr_args; i++) {
1944		struct probe_arg *parg = &tp->args[i];
1945		const char *fmt = parg->type->fmttype;
1946		int size = parg->type->size;
1947
1948		if (parg->fmt)
1949			fmt = parg->fmt;
1950		if (parg->count)
1951			size *= parg->count;
1952		ret = trace_define_field(event_call, fmt, parg->name,
1953					 offset + parg->offset, size,
1954					 parg->type->is_signed,
1955					 FILTER_OTHER);
1956		if (ret)
1957			return ret;
1958	}
1959	return 0;
1960}
1961
1962static void trace_probe_event_free(struct trace_probe_event *tpe)
1963{
1964	kfree(tpe->class.system);
1965	kfree(tpe->call.name);
1966	kfree(tpe->call.print_fmt);
1967	kfree(tpe);
1968}
1969
1970int trace_probe_append(struct trace_probe *tp, struct trace_probe *to)
1971{
1972	if (trace_probe_has_sibling(tp))
1973		return -EBUSY;
1974
1975	list_del_init(&tp->list);
1976	trace_probe_event_free(tp->event);
1977
1978	tp->event = to->event;
1979	list_add_tail(&tp->list, trace_probe_probe_list(to));
1980
1981	return 0;
1982}
1983
1984void trace_probe_unlink(struct trace_probe *tp)
1985{
1986	list_del_init(&tp->list);
1987	if (list_empty(trace_probe_probe_list(tp)))
1988		trace_probe_event_free(tp->event);
1989	tp->event = NULL;
1990}
1991
1992void trace_probe_cleanup(struct trace_probe *tp)
1993{
1994	int i;
1995
1996	for (i = 0; i < tp->nr_args; i++)
1997		traceprobe_free_probe_arg(&tp->args[i]);
1998
1999	if (tp->entry_arg) {
2000		kfree(tp->entry_arg->code);
2001		kfree(tp->entry_arg);
2002		tp->entry_arg = NULL;
2003	}
2004
2005	if (tp->event)
2006		trace_probe_unlink(tp);
2007}
2008
2009int trace_probe_init(struct trace_probe *tp, const char *event,
2010		     const char *group, bool alloc_filter, int nargs)
2011{
2012	struct trace_event_call *call;
2013	size_t size = sizeof(struct trace_probe_event);
2014	int ret = 0;
2015
2016	if (!event || !group)
2017		return -EINVAL;
2018
2019	if (alloc_filter)
2020		size += sizeof(struct trace_uprobe_filter);
2021
2022	tp->event = kzalloc(size, GFP_KERNEL);
2023	if (!tp->event)
2024		return -ENOMEM;
2025
2026	INIT_LIST_HEAD(&tp->event->files);
2027	INIT_LIST_HEAD(&tp->event->class.fields);
2028	INIT_LIST_HEAD(&tp->event->probes);
2029	INIT_LIST_HEAD(&tp->list);
2030	list_add(&tp->list, &tp->event->probes);
2031
2032	call = trace_probe_event_call(tp);
2033	call->class = &tp->event->class;
2034	call->name = kstrdup(event, GFP_KERNEL);
2035	if (!call->name) {
2036		ret = -ENOMEM;
2037		goto error;
2038	}
2039
2040	tp->event->class.system = kstrdup(group, GFP_KERNEL);
2041	if (!tp->event->class.system) {
2042		ret = -ENOMEM;
2043		goto error;
2044	}
2045
2046	tp->nr_args = nargs;
2047	/* Make sure pointers in args[] are NULL */
2048	if (nargs)
2049		memset(tp->args, 0, sizeof(tp->args[0]) * nargs);
2050
2051	return 0;
2052
2053error:
2054	trace_probe_cleanup(tp);
2055	return ret;
2056}
2057
2058static struct trace_event_call *
2059find_trace_event_call(const char *system, const char *event_name)
2060{
2061	struct trace_event_call *tp_event;
2062	const char *name;
2063
2064	list_for_each_entry(tp_event, &ftrace_events, list) {
2065		if (!tp_event->class->system ||
2066		    strcmp(system, tp_event->class->system))
2067			continue;
2068		name = trace_event_name(tp_event);
2069		if (!name || strcmp(event_name, name))
2070			continue;
2071		return tp_event;
2072	}
2073
2074	return NULL;
2075}
2076
2077int trace_probe_register_event_call(struct trace_probe *tp)
2078{
2079	struct trace_event_call *call = trace_probe_event_call(tp);
2080	int ret;
2081
2082	lockdep_assert_held(&event_mutex);
2083
2084	if (find_trace_event_call(trace_probe_group_name(tp),
2085				  trace_probe_name(tp)))
2086		return -EEXIST;
2087
2088	ret = register_trace_event(&call->event);
2089	if (!ret)
2090		return -ENODEV;
2091
2092	ret = trace_add_event_call(call);
2093	if (ret)
2094		unregister_trace_event(&call->event);
2095
2096	return ret;
2097}
2098
2099int trace_probe_add_file(struct trace_probe *tp, struct trace_event_file *file)
2100{
2101	struct event_file_link *link;
2102
2103	link = kmalloc(sizeof(*link), GFP_KERNEL);
2104	if (!link)
2105		return -ENOMEM;
2106
2107	link->file = file;
2108	INIT_LIST_HEAD(&link->list);
2109	list_add_tail_rcu(&link->list, &tp->event->files);
2110	trace_probe_set_flag(tp, TP_FLAG_TRACE);
2111	return 0;
2112}
2113
2114struct event_file_link *trace_probe_get_file_link(struct trace_probe *tp,
2115						  struct trace_event_file *file)
2116{
2117	struct event_file_link *link;
2118
2119	trace_probe_for_each_link(link, tp) {
2120		if (link->file == file)
2121			return link;
2122	}
2123
2124	return NULL;
2125}
2126
2127int trace_probe_remove_file(struct trace_probe *tp,
2128			    struct trace_event_file *file)
2129{
2130	struct event_file_link *link;
2131
2132	link = trace_probe_get_file_link(tp, file);
2133	if (!link)
2134		return -ENOENT;
2135
2136	list_del_rcu(&link->list);
2137	kvfree_rcu_mightsleep(link);
2138
2139	if (list_empty(&tp->event->files))
2140		trace_probe_clear_flag(tp, TP_FLAG_TRACE);
2141
2142	return 0;
2143}
2144
2145/*
2146 * Return the smallest index of different type argument (start from 1).
2147 * If all argument types and name are same, return 0.
2148 */
2149int trace_probe_compare_arg_type(struct trace_probe *a, struct trace_probe *b)
2150{
2151	int i;
2152
2153	/* In case of more arguments */
2154	if (a->nr_args < b->nr_args)
2155		return a->nr_args + 1;
2156	if (a->nr_args > b->nr_args)
2157		return b->nr_args + 1;
2158
2159	for (i = 0; i < a->nr_args; i++) {
2160		if ((b->nr_args <= i) ||
2161		    ((a->args[i].type != b->args[i].type) ||
2162		     (a->args[i].count != b->args[i].count) ||
2163		     strcmp(a->args[i].name, b->args[i].name)))
2164			return i + 1;
2165	}
2166
2167	return 0;
2168}
2169
2170bool trace_probe_match_command_args(struct trace_probe *tp,
2171				    int argc, const char **argv)
2172{
2173	char buf[MAX_ARGSTR_LEN + 1];
2174	int i;
2175
2176	if (tp->nr_args < argc)
2177		return false;
2178
2179	for (i = 0; i < argc; i++) {
2180		snprintf(buf, sizeof(buf), "%s=%s",
2181			 tp->args[i].name, tp->args[i].comm);
2182		if (strcmp(buf, argv[i]))
2183			return false;
2184	}
2185	return true;
2186}
2187
2188int trace_probe_create(const char *raw_command, int (*createfn)(int, const char **))
2189{
2190	int argc = 0, ret = 0;
2191	char **argv;
2192
2193	argv = argv_split(GFP_KERNEL, raw_command, &argc);
2194	if (!argv)
2195		return -ENOMEM;
2196
2197	if (argc)
2198		ret = createfn(argc, (const char **)argv);
2199
2200	argv_free(argv);
2201
2202	return ret;
2203}
2204
2205int trace_probe_print_args(struct trace_seq *s, struct probe_arg *args, int nr_args,
2206		 u8 *data, void *field)
2207{
2208	void *p;
2209	int i, j;
2210
2211	for (i = 0; i < nr_args; i++) {
2212		struct probe_arg *a = args + i;
2213
2214		trace_seq_printf(s, " %s=", a->name);
2215		if (likely(!a->count)) {
2216			if (!a->type->print(s, data + a->offset, field))
2217				return -ENOMEM;
2218			continue;
2219		}
2220		trace_seq_putc(s, '{');
2221		p = data + a->offset;
2222		for (j = 0; j < a->count; j++) {
2223			if (!a->type->print(s, p, field))
2224				return -ENOMEM;
2225			trace_seq_putc(s, j == a->count - 1 ? '}' : ',');
2226			p += a->type->size;
2227		}
2228	}
2229	return 0;
2230}