1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * builtin-inject.c
   4 *
   5 * Builtin inject command: Examine the live mode (stdin) event stream
   6 * and repipe it to stdout while optionally injecting additional
   7 * events into it.
   8 */
   9#include "builtin.h"
  10
  11#include "util/color.h"
  12#include "util/dso.h"
  13#include "util/vdso.h"
  14#include "util/evlist.h"
  15#include "util/evsel.h"
  16#include "util/map.h"
  17#include "util/session.h"
  18#include "util/tool.h"
  19#include "util/debug.h"
  20#include "util/build-id.h"
  21#include "util/data.h"
  22#include "util/auxtrace.h"
  23#include "util/jit.h"
  24#include "util/string2.h"
  25#include "util/symbol.h"
  26#include "util/synthetic-events.h"
  27#include "util/thread.h"
  28#include "util/namespaces.h"
  29#include "util/util.h"
  30#include "util/tsc.h"
  31
  32#include <internal/lib.h>
  33
  34#include <linux/err.h>
  35#include <subcmd/parse-options.h>
  36#include <uapi/linux/mman.h> /* To get things like MAP_HUGETLB even on older libc headers */
  37
  38#include <linux/list.h>
  39#include <linux/string.h>
  40#include <linux/zalloc.h>
  41#include <linux/hash.h>
  42#include <ctype.h>
  43#include <errno.h>
  44#include <signal.h>
  45#include <inttypes.h>
  46
  47struct guest_event {
  48	struct perf_sample		sample;
  49	union perf_event		*event;
  50	char				*event_buf;
  51};
  52
  53struct guest_id {
  54	/* hlist_node must be first, see free_hlist() */
  55	struct hlist_node		node;
  56	u64				id;
  57	u64				host_id;
  58	u32				vcpu;
  59};
  60
  61struct guest_tid {
  62	/* hlist_node must be first, see free_hlist() */
  63	struct hlist_node		node;
  64	/* Thread ID of QEMU thread */
  65	u32				tid;
  66	u32				vcpu;
  67};
  68
  69struct guest_vcpu {
  70	/* Current host CPU */
  71	u32				cpu;
  72	/* Thread ID of QEMU thread */
  73	u32				tid;
  74};
  75
  76struct guest_session {
  77	char				*perf_data_file;
  78	u32				machine_pid;
  79	u64				time_offset;
  80	double				time_scale;
  81	struct perf_tool		tool;
  82	struct perf_data		data;
  83	struct perf_session		*session;
  84	char				*tmp_file_name;
  85	int				tmp_fd;
  86	struct perf_tsc_conversion	host_tc;
  87	struct perf_tsc_conversion	guest_tc;
  88	bool				copy_kcore_dir;
  89	bool				have_tc;
  90	bool				fetched;
  91	bool				ready;
  92	u16				dflt_id_hdr_size;
  93	u64				dflt_id;
  94	u64				highest_id;
  95	/* Array of guest_vcpu */
  96	struct guest_vcpu		*vcpu;
  97	size_t				vcpu_cnt;
  98	/* Hash table for guest_id */
  99	struct hlist_head		heads[PERF_EVLIST__HLIST_SIZE];
 100	/* Hash table for guest_tid */
 101	struct hlist_head		tids[PERF_EVLIST__HLIST_SIZE];
 102	/* Place to stash next guest event */
 103	struct guest_event		ev;
 104};
 105
 106struct perf_inject {
 107	struct perf_tool	tool;
 108	struct perf_session	*session;
 109	bool			build_ids;
 110	bool			build_id_all;
 111	bool			sched_stat;
 112	bool			have_auxtrace;
 113	bool			strip;
 114	bool			jit_mode;
 115	bool			in_place_update;
 116	bool			in_place_update_dry_run;
 117	bool			is_pipe;
 118	bool			copy_kcore_dir;
 119	const char		*input_name;
 120	struct perf_data	output;
 121	u64			bytes_written;
 122	u64			aux_id;
 123	struct list_head	samples;
 124	struct itrace_synth_opts itrace_synth_opts;
 125	char			*event_copy;
 126	struct perf_file_section secs[HEADER_FEAT_BITS];
 127	struct guest_session	guest_session;
 128	struct strlist		*known_build_ids;
 129};
 130
 131struct event_entry {
 132	struct list_head node;
 133	u32		 tid;
 134	union perf_event event[];
 135};
 136
 137static int dso__inject_build_id(struct dso *dso, struct perf_tool *tool,
 138				struct machine *machine, u8 cpumode, u32 flags);
 139
 140static int output_bytes(struct perf_inject *inject, void *buf, size_t sz)
 141{
 142	ssize_t size;
 143
 144	size = perf_data__write(&inject->output, buf, sz);
 145	if (size < 0)
 146		return -errno;
 147
 148	inject->bytes_written += size;
 149	return 0;
 150}
 151
 152static int perf_event__repipe_synth(struct perf_tool *tool,
 153				    union perf_event *event)
 154{
 155	struct perf_inject *inject = container_of(tool, struct perf_inject,
 156						  tool);
 157
 158	return output_bytes(inject, event, event->header.size);
 159}
 160
 161static int perf_event__repipe_oe_synth(struct perf_tool *tool,
 162				       union perf_event *event,
 163				       struct ordered_events *oe __maybe_unused)
 164{
 165	return perf_event__repipe_synth(tool, event);
 166}
 167
 168#ifdef HAVE_JITDUMP
 169static int perf_event__drop_oe(struct perf_tool *tool __maybe_unused,
 170			       union perf_event *event __maybe_unused,
 171			       struct ordered_events *oe __maybe_unused)
 172{
 173	return 0;
 174}
 175#endif
 176
 177static int perf_event__repipe_op2_synth(struct perf_session *session,
 178					union perf_event *event)
 179{
 180	return perf_event__repipe_synth(session->tool, event);
 181}
 182
 183static int perf_event__repipe_op4_synth(struct perf_session *session,
 184					union perf_event *event,
 185					u64 data __maybe_unused,
 186					const char *str __maybe_unused)
 187{
 188	return perf_event__repipe_synth(session->tool, event);
 189}
 190
 191static int perf_event__repipe_attr(struct perf_tool *tool,
 192				   union perf_event *event,
 193				   struct evlist **pevlist)
 194{
 195	struct perf_inject *inject = container_of(tool, struct perf_inject,
 196						  tool);
 197	int ret;
 198
 199	ret = perf_event__process_attr(tool, event, pevlist);
 200	if (ret)
 201		return ret;
 202
 203	if (!inject->is_pipe)
 204		return 0;
 205
 206	return perf_event__repipe_synth(tool, event);
 207}
 208
 209static int perf_event__repipe_event_update(struct perf_tool *tool,
 210					   union perf_event *event,
 211					   struct evlist **pevlist __maybe_unused)
 212{
 213	return perf_event__repipe_synth(tool, event);
 214}
 215
 216#ifdef HAVE_AUXTRACE_SUPPORT
 217
 218static int copy_bytes(struct perf_inject *inject, struct perf_data *data, off_t size)
 219{
 220	char buf[4096];
 221	ssize_t ssz;
 222	int ret;
 223
 224	while (size > 0) {
 225		ssz = perf_data__read(data, buf, min(size, (off_t)sizeof(buf)));
 226		if (ssz < 0)
 227			return -errno;
 228		ret = output_bytes(inject, buf, ssz);
 229		if (ret)
 230			return ret;
 231		size -= ssz;
 232	}
 233
 234	return 0;
 235}
 236
 237static s64 perf_event__repipe_auxtrace(struct perf_session *session,
 238				       union perf_event *event)
 239{
 240	struct perf_tool *tool = session->tool;
 241	struct perf_inject *inject = container_of(tool, struct perf_inject,
 242						  tool);
 243	int ret;
 244
 245	inject->have_auxtrace = true;
 246
 247	if (!inject->output.is_pipe) {
 248		off_t offset;
 249
 250		offset = lseek(inject->output.file.fd, 0, SEEK_CUR);
 251		if (offset == -1)
 252			return -errno;
 253		ret = auxtrace_index__auxtrace_event(&session->auxtrace_index,
 254						     event, offset);
 255		if (ret < 0)
 256			return ret;
 257	}
 258
 259	if (perf_data__is_pipe(session->data) || !session->one_mmap) {
 260		ret = output_bytes(inject, event, event->header.size);
 261		if (ret < 0)
 262			return ret;
 263		ret = copy_bytes(inject, session->data,
 264				 event->auxtrace.size);
 265	} else {
 266		ret = output_bytes(inject, event,
 267				   event->header.size + event->auxtrace.size);
 268	}
 269	if (ret < 0)
 270		return ret;
 271
 272	return event->auxtrace.size;
 273}
 274
 275#else
 276
 277static s64
 278perf_event__repipe_auxtrace(struct perf_session *session __maybe_unused,
 279			    union perf_event *event __maybe_unused)
 280{
 281	pr_err("AUX area tracing not supported\n");
 282	return -EINVAL;
 283}
 284
 285#endif
 286
 287static int perf_event__repipe(struct perf_tool *tool,
 288			      union perf_event *event,
 289			      struct perf_sample *sample __maybe_unused,
 290			      struct machine *machine __maybe_unused)
 291{
 292	return perf_event__repipe_synth(tool, event);
 293}
 294
 295static int perf_event__drop(struct perf_tool *tool __maybe_unused,
 296			    union perf_event *event __maybe_unused,
 297			    struct perf_sample *sample __maybe_unused,
 298			    struct machine *machine __maybe_unused)
 299{
 300	return 0;
 301}
 302
 303static int perf_event__drop_aux(struct perf_tool *tool,
 304				union perf_event *event __maybe_unused,
 305				struct perf_sample *sample,
 306				struct machine *machine __maybe_unused)
 307{
 308	struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
 309
 310	if (!inject->aux_id)
 311		inject->aux_id = sample->id;
 312
 313	return 0;
 314}
 315
 316static union perf_event *
 317perf_inject__cut_auxtrace_sample(struct perf_inject *inject,
 318				 union perf_event *event,
 319				 struct perf_sample *sample)
 320{
 321	size_t sz1 = sample->aux_sample.data - (void *)event;
 322	size_t sz2 = event->header.size - sample->aux_sample.size - sz1;
 323	union perf_event *ev;
 324
 325	if (inject->event_copy == NULL) {
 326		inject->event_copy = malloc(PERF_SAMPLE_MAX_SIZE);
 327		if (!inject->event_copy)
 328			return ERR_PTR(-ENOMEM);
 329	}
 330	ev = (union perf_event *)inject->event_copy;
 331	if (sz1 > event->header.size || sz2 > event->header.size ||
 332	    sz1 + sz2 > event->header.size ||
 333	    sz1 < sizeof(struct perf_event_header) + sizeof(u64))
 334		return event;
 335
 336	memcpy(ev, event, sz1);
 337	memcpy((void *)ev + sz1, (void *)event + event->header.size - sz2, sz2);
 338	ev->header.size = sz1 + sz2;
 339	((u64 *)((void *)ev + sz1))[-1] = 0;
 340
 341	return ev;
 342}
 343
 344typedef int (*inject_handler)(struct perf_tool *tool,
 345			      union perf_event *event,
 346			      struct perf_sample *sample,
 347			      struct evsel *evsel,
 348			      struct machine *machine);
 349
 350static int perf_event__repipe_sample(struct perf_tool *tool,
 351				     union perf_event *event,
 352				     struct perf_sample *sample,
 353				     struct evsel *evsel,
 354				     struct machine *machine)
 355{
 356	struct perf_inject *inject = container_of(tool, struct perf_inject,
 357						  tool);
 358
 359	if (evsel && evsel->handler) {
 360		inject_handler f = evsel->handler;
 361		return f(tool, event, sample, evsel, machine);
 362	}
 363
 364	build_id__mark_dso_hit(tool, event, sample, evsel, machine);
 365
 366	if (inject->itrace_synth_opts.set && sample->aux_sample.size) {
 367		event = perf_inject__cut_auxtrace_sample(inject, event, sample);
 368		if (IS_ERR(event))
 369			return PTR_ERR(event);
 370	}
 371
 372	return perf_event__repipe_synth(tool, event);
 373}
 374
 375static int perf_event__repipe_mmap(struct perf_tool *tool,
 376				   union perf_event *event,
 377				   struct perf_sample *sample,
 378				   struct machine *machine)
 379{
 380	int err;
 381
 382	err = perf_event__process_mmap(tool, event, sample, machine);
 383	perf_event__repipe(tool, event, sample, machine);
 384
 385	return err;
 386}
 387
 388#ifdef HAVE_JITDUMP
 389static int perf_event__jit_repipe_mmap(struct perf_tool *tool,
 390				       union perf_event *event,
 391				       struct perf_sample *sample,
 392				       struct machine *machine)
 393{
 394	struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
 395	u64 n = 0;
 396	int ret;
 397
 398	/*
 399	 * if jit marker, then inject jit mmaps and generate ELF images
 400	 */
 401	ret = jit_process(inject->session, &inject->output, machine,
 402			  event->mmap.filename, event->mmap.pid, event->mmap.tid, &n);
 403	if (ret < 0)
 404		return ret;
 405	if (ret) {
 406		inject->bytes_written += n;
 407		return 0;
 408	}
 409	return perf_event__repipe_mmap(tool, event, sample, machine);
 410}
 411#endif
 412
 413static struct dso *findnew_dso(int pid, int tid, const char *filename,
 414			       struct dso_id *id, struct machine *machine)
 415{
 416	struct thread *thread;
 417	struct nsinfo *nsi = NULL;
 418	struct nsinfo *nnsi;
 419	struct dso *dso;
 420	bool vdso;
 421
 422	thread = machine__findnew_thread(machine, pid, tid);
 423	if (thread == NULL) {
 424		pr_err("cannot find or create a task %d/%d.\n", tid, pid);
 425		return NULL;
 426	}
 427
 428	vdso = is_vdso_map(filename);
 429	nsi = nsinfo__get(thread__nsinfo(thread));
 430
 431	if (vdso) {
 432		/* The vdso maps are always on the host and not the
 433		 * container.  Ensure that we don't use setns to look
 434		 * them up.
 435		 */
 436		nnsi = nsinfo__copy(nsi);
 437		if (nnsi) {
 438			nsinfo__put(nsi);
 439			nsinfo__clear_need_setns(nnsi);
 440			nsi = nnsi;
 441		}
 442		dso = machine__findnew_vdso(machine, thread);
 443	} else {
 444		dso = machine__findnew_dso_id(machine, filename, id);
 445	}
 446
 447	if (dso) {
 448		mutex_lock(&dso->lock);
 449		nsinfo__put(dso->nsinfo);
 450		dso->nsinfo = nsi;
 451		mutex_unlock(&dso->lock);
 452	} else
 453		nsinfo__put(nsi);
 454
 455	thread__put(thread);
 456	return dso;
 457}
 458
 459static int perf_event__repipe_buildid_mmap(struct perf_tool *tool,
 460					   union perf_event *event,
 461					   struct perf_sample *sample,
 462					   struct machine *machine)
 463{
 464	struct dso *dso;
 465
 466	dso = findnew_dso(event->mmap.pid, event->mmap.tid,
 467			  event->mmap.filename, NULL, machine);
 468
 469	if (dso && !dso->hit) {
 470		dso->hit = 1;
 471		dso__inject_build_id(dso, tool, machine, sample->cpumode, 0);
 472	}
 473	dso__put(dso);
 474
 475	return perf_event__repipe(tool, event, sample, machine);
 476}
 477
 478static int perf_event__repipe_mmap2(struct perf_tool *tool,
 479				   union perf_event *event,
 480				   struct perf_sample *sample,
 481				   struct machine *machine)
 482{
 483	int err;
 484
 485	err = perf_event__process_mmap2(tool, event, sample, machine);
 486	perf_event__repipe(tool, event, sample, machine);
 487
 488	if (event->header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID) {
 489		struct dso *dso;
 490
 491		dso = findnew_dso(event->mmap2.pid, event->mmap2.tid,
 492				  event->mmap2.filename, NULL, machine);
 493		if (dso) {
 494			/* mark it not to inject build-id */
 495			dso->hit = 1;
 496		}
 497		dso__put(dso);
 498	}
 499
 500	return err;
 501}
 502
 503#ifdef HAVE_JITDUMP
 504static int perf_event__jit_repipe_mmap2(struct perf_tool *tool,
 505					union perf_event *event,
 506					struct perf_sample *sample,
 507					struct machine *machine)
 508{
 509	struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
 510	u64 n = 0;
 511	int ret;
 512
 513	/*
 514	 * if jit marker, then inject jit mmaps and generate ELF images
 515	 */
 516	ret = jit_process(inject->session, &inject->output, machine,
 517			  event->mmap2.filename, event->mmap2.pid, event->mmap2.tid, &n);
 518	if (ret < 0)
 519		return ret;
 520	if (ret) {
 521		inject->bytes_written += n;
 522		return 0;
 523	}
 524	return perf_event__repipe_mmap2(tool, event, sample, machine);
 525}
 526#endif
 527
 528static int perf_event__repipe_buildid_mmap2(struct perf_tool *tool,
 529					    union perf_event *event,
 530					    struct perf_sample *sample,
 531					    struct machine *machine)
 532{
 533	struct dso_id dso_id = {
 534		.maj = event->mmap2.maj,
 535		.min = event->mmap2.min,
 536		.ino = event->mmap2.ino,
 537		.ino_generation = event->mmap2.ino_generation,
 538	};
 539	struct dso *dso;
 540
 541	if (event->header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID) {
 542		/* cannot use dso_id since it'd have invalid info */
 543		dso = findnew_dso(event->mmap2.pid, event->mmap2.tid,
 544				  event->mmap2.filename, NULL, machine);
 545		if (dso) {
 546			/* mark it not to inject build-id */
 547			dso->hit = 1;
 548		}
 549		dso__put(dso);
 550		perf_event__repipe(tool, event, sample, machine);
 551		return 0;
 552	}
 553
 554	dso = findnew_dso(event->mmap2.pid, event->mmap2.tid,
 555			  event->mmap2.filename, &dso_id, machine);
 556
 557	if (dso && !dso->hit) {
 558		dso->hit = 1;
 559		dso__inject_build_id(dso, tool, machine, sample->cpumode,
 560				     event->mmap2.flags);
 561	}
 562	dso__put(dso);
 563
 564	perf_event__repipe(tool, event, sample, machine);
 565
 566	return 0;
 567}
 568
 569static int perf_event__repipe_fork(struct perf_tool *tool,
 570				   union perf_event *event,
 571				   struct perf_sample *sample,
 572				   struct machine *machine)
 573{
 574	int err;
 575
 576	err = perf_event__process_fork(tool, event, sample, machine);
 577	perf_event__repipe(tool, event, sample, machine);
 578
 579	return err;
 580}
 581
 582static int perf_event__repipe_comm(struct perf_tool *tool,
 583				   union perf_event *event,
 584				   struct perf_sample *sample,
 585				   struct machine *machine)
 586{
 587	int err;
 588
 589	err = perf_event__process_comm(tool, event, sample, machine);
 590	perf_event__repipe(tool, event, sample, machine);
 591
 592	return err;
 593}
 594
 595static int perf_event__repipe_namespaces(struct perf_tool *tool,
 596					 union perf_event *event,
 597					 struct perf_sample *sample,
 598					 struct machine *machine)
 599{
 600	int err = perf_event__process_namespaces(tool, event, sample, machine);
 601
 602	perf_event__repipe(tool, event, sample, machine);
 603
 604	return err;
 605}
 606
 607static int perf_event__repipe_exit(struct perf_tool *tool,
 608				   union perf_event *event,
 609				   struct perf_sample *sample,
 610				   struct machine *machine)
 611{
 612	int err;
 613
 614	err = perf_event__process_exit(tool, event, sample, machine);
 615	perf_event__repipe(tool, event, sample, machine);
 616
 617	return err;
 618}
 619
 620#ifdef HAVE_LIBTRACEEVENT
 621static int perf_event__repipe_tracing_data(struct perf_session *session,
 622					   union perf_event *event)
 623{
 
 
 624	perf_event__repipe_synth(session->tool, event);
 
 625
 626	return perf_event__process_tracing_data(session, event);
 627}
 628#endif
 629
 630static int dso__read_build_id(struct dso *dso)
 631{
 632	struct nscookie nsc;
 633
 634	if (dso->has_build_id)
 635		return 0;
 636
 637	mutex_lock(&dso->lock);
 638	nsinfo__mountns_enter(dso->nsinfo, &nsc);
 639	if (filename__read_build_id(dso->long_name, &dso->bid) > 0)
 640		dso->has_build_id = true;
 641	else if (dso->nsinfo) {
 642		char *new_name = dso__filename_with_chroot(dso, dso->long_name);
 643
 644		if (new_name && filename__read_build_id(new_name, &dso->bid) > 0)
 645			dso->has_build_id = true;
 646		free(new_name);
 647	}
 648	nsinfo__mountns_exit(&nsc);
 649	mutex_unlock(&dso->lock);
 650
 651	return dso->has_build_id ? 0 : -1;
 652}
 653
 654static struct strlist *perf_inject__parse_known_build_ids(
 655	const char *known_build_ids_string)
 656{
 657	struct str_node *pos, *tmp;
 658	struct strlist *known_build_ids;
 659	int bid_len;
 660
 661	known_build_ids = strlist__new(known_build_ids_string, NULL);
 662	if (known_build_ids == NULL)
 663		return NULL;
 664	strlist__for_each_entry_safe(pos, tmp, known_build_ids) {
 665		const char *build_id, *dso_name;
 666
 667		build_id = skip_spaces(pos->s);
 668		dso_name = strchr(build_id, ' ');
 669		if (dso_name == NULL) {
 670			strlist__remove(known_build_ids, pos);
 671			continue;
 672		}
 673		bid_len = dso_name - pos->s;
 674		dso_name = skip_spaces(dso_name);
 675		if (bid_len % 2 != 0 || bid_len >= SBUILD_ID_SIZE) {
 676			strlist__remove(known_build_ids, pos);
 677			continue;
 678		}
 679		for (int ix = 0; 2 * ix + 1 < bid_len; ++ix) {
 680			if (!isxdigit(build_id[2 * ix]) ||
 681			    !isxdigit(build_id[2 * ix + 1])) {
 682				strlist__remove(known_build_ids, pos);
 683				break;
 684			}
 685		}
 686	}
 687	return known_build_ids;
 688}
 689
 690static bool perf_inject__lookup_known_build_id(struct perf_inject *inject,
 691					       struct dso *dso)
 692{
 693	struct str_node *pos;
 694	int bid_len;
 695
 696	strlist__for_each_entry(pos, inject->known_build_ids) {
 697		const char *build_id, *dso_name;
 698
 699		build_id = skip_spaces(pos->s);
 700		dso_name = strchr(build_id, ' ');
 701		bid_len = dso_name - pos->s;
 702		dso_name = skip_spaces(dso_name);
 703		if (strcmp(dso->long_name, dso_name))
 704			continue;
 705		for (int ix = 0; 2 * ix + 1 < bid_len; ++ix) {
 706			dso->bid.data[ix] = (hex(build_id[2 * ix]) << 4 |
 707					     hex(build_id[2 * ix + 1]));
 708		}
 709		dso->bid.size = bid_len / 2;
 710		dso->has_build_id = 1;
 711		return true;
 712	}
 713	return false;
 714}
 715
 716static int dso__inject_build_id(struct dso *dso, struct perf_tool *tool,
 717				struct machine *machine, u8 cpumode, u32 flags)
 718{
 719	struct perf_inject *inject = container_of(tool, struct perf_inject,
 720						  tool);
 721	int err;
 722
 723	if (is_anon_memory(dso->long_name) || flags & MAP_HUGETLB)
 724		return 0;
 725	if (is_no_dso_memory(dso->long_name))
 726		return 0;
 727
 728	if (inject->known_build_ids != NULL &&
 729	    perf_inject__lookup_known_build_id(inject, dso))
 730		return 1;
 731
 732	if (dso__read_build_id(dso) < 0) {
 733		pr_debug("no build_id found for %s\n", dso->long_name);
 734		return -1;
 735	}
 736
 737	err = perf_event__synthesize_build_id(tool, dso, cpumode,
 738					      perf_event__repipe, machine);
 739	if (err) {
 740		pr_err("Can't synthesize build_id event for %s\n", dso->long_name);
 741		return -1;
 742	}
 743
 744	return 0;
 745}
 746
 747int perf_event__inject_buildid(struct perf_tool *tool, union perf_event *event,
 748			       struct perf_sample *sample,
 749			       struct evsel *evsel __maybe_unused,
 750			       struct machine *machine)
 751{
 752	struct addr_location al;
 753	struct thread *thread;
 754
 755	addr_location__init(&al);
 756	thread = machine__findnew_thread(machine, sample->pid, sample->tid);
 757	if (thread == NULL) {
 758		pr_err("problem processing %d event, skipping it.\n",
 759		       event->header.type);
 760		goto repipe;
 761	}
 762
 763	if (thread__find_map(thread, sample->cpumode, sample->ip, &al)) {
 764		struct dso *dso = map__dso(al.map);
 765
 766		if (!dso->hit) {
 767			dso->hit = 1;
 768			dso__inject_build_id(dso, tool, machine,
 769					     sample->cpumode, map__flags(al.map));
 770		}
 771	}
 772
 773	thread__put(thread);
 774repipe:
 775	perf_event__repipe(tool, event, sample, machine);
 776	addr_location__exit(&al);
 777	return 0;
 778}
 779
 780static int perf_inject__sched_process_exit(struct perf_tool *tool,
 781					   union perf_event *event __maybe_unused,
 782					   struct perf_sample *sample,
 783					   struct evsel *evsel __maybe_unused,
 784					   struct machine *machine __maybe_unused)
 785{
 786	struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
 787	struct event_entry *ent;
 788
 789	list_for_each_entry(ent, &inject->samples, node) {
 790		if (sample->tid == ent->tid) {
 791			list_del_init(&ent->node);
 792			free(ent);
 793			break;
 794		}
 795	}
 796
 797	return 0;
 798}
 799
 800static int perf_inject__sched_switch(struct perf_tool *tool,
 801				     union perf_event *event,
 802				     struct perf_sample *sample,
 803				     struct evsel *evsel,
 804				     struct machine *machine)
 805{
 806	struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
 807	struct event_entry *ent;
 808
 809	perf_inject__sched_process_exit(tool, event, sample, evsel, machine);
 810
 811	ent = malloc(event->header.size + sizeof(struct event_entry));
 812	if (ent == NULL) {
 813		color_fprintf(stderr, PERF_COLOR_RED,
 814			     "Not enough memory to process sched switch event!");
 815		return -1;
 816	}
 817
 818	ent->tid = sample->tid;
 819	memcpy(&ent->event, event, event->header.size);
 820	list_add(&ent->node, &inject->samples);
 821	return 0;
 822}
 823
 824#ifdef HAVE_LIBTRACEEVENT
 825static int perf_inject__sched_stat(struct perf_tool *tool,
 826				   union perf_event *event __maybe_unused,
 827				   struct perf_sample *sample,
 828				   struct evsel *evsel,
 829				   struct machine *machine)
 830{
 831	struct event_entry *ent;
 832	union perf_event *event_sw;
 833	struct perf_sample sample_sw;
 834	struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
 835	u32 pid = evsel__intval(evsel, sample, "pid");
 836
 837	list_for_each_entry(ent, &inject->samples, node) {
 838		if (pid == ent->tid)
 839			goto found;
 840	}
 841
 842	return 0;
 843found:
 844	event_sw = &ent->event[0];
 845	evsel__parse_sample(evsel, event_sw, &sample_sw);
 846
 847	sample_sw.period = sample->period;
 848	sample_sw.time	 = sample->time;
 849	perf_event__synthesize_sample(event_sw, evsel->core.attr.sample_type,
 850				      evsel->core.attr.read_format, &sample_sw);
 851	build_id__mark_dso_hit(tool, event_sw, &sample_sw, evsel, machine);
 852	return perf_event__repipe(tool, event_sw, &sample_sw, machine);
 853}
 854#endif
 855
 856static struct guest_vcpu *guest_session__vcpu(struct guest_session *gs, u32 vcpu)
 857{
 858	if (realloc_array_as_needed(gs->vcpu, gs->vcpu_cnt, vcpu, NULL))
 859		return NULL;
 860	return &gs->vcpu[vcpu];
 861}
 862
 863static int guest_session__output_bytes(struct guest_session *gs, void *buf, size_t sz)
 864{
 865	ssize_t ret = writen(gs->tmp_fd, buf, sz);
 866
 867	return ret < 0 ? ret : 0;
 868}
 869
 870static int guest_session__repipe(struct perf_tool *tool,
 871				 union perf_event *event,
 872				 struct perf_sample *sample __maybe_unused,
 873				 struct machine *machine __maybe_unused)
 874{
 875	struct guest_session *gs = container_of(tool, struct guest_session, tool);
 876
 877	return guest_session__output_bytes(gs, event, event->header.size);
 878}
 879
 880static int guest_session__map_tid(struct guest_session *gs, u32 tid, u32 vcpu)
 881{
 882	struct guest_tid *guest_tid = zalloc(sizeof(*guest_tid));
 883	int hash;
 884
 885	if (!guest_tid)
 886		return -ENOMEM;
 887
 888	guest_tid->tid = tid;
 889	guest_tid->vcpu = vcpu;
 890	hash = hash_32(guest_tid->tid, PERF_EVLIST__HLIST_BITS);
 891	hlist_add_head(&guest_tid->node, &gs->tids[hash]);
 892
 893	return 0;
 894}
 895
 896static int host_peek_vm_comms_cb(struct perf_session *session __maybe_unused,
 897				 union perf_event *event,
 898				 u64 offset __maybe_unused, void *data)
 899{
 900	struct guest_session *gs = data;
 901	unsigned int vcpu;
 902	struct guest_vcpu *guest_vcpu;
 903	int ret;
 904
 905	if (event->header.type != PERF_RECORD_COMM ||
 906	    event->comm.pid != gs->machine_pid)
 907		return 0;
 908
 909	/*
 910	 * QEMU option -name debug-threads=on, causes thread names formatted as
 911	 * below, although it is not an ABI. Also libvirt seems to use this by
 912	 * default. Here we rely on it to tell us which thread is which VCPU.
 913	 */
 914	ret = sscanf(event->comm.comm, "CPU %u/KVM", &vcpu);
 915	if (ret <= 0)
 916		return ret;
 917	pr_debug("Found VCPU: tid %u comm %s vcpu %u\n",
 918		 event->comm.tid, event->comm.comm, vcpu);
 919	if (vcpu > INT_MAX) {
 920		pr_err("Invalid VCPU %u\n", vcpu);
 921		return -EINVAL;
 922	}
 923	guest_vcpu = guest_session__vcpu(gs, vcpu);
 924	if (!guest_vcpu)
 925		return -ENOMEM;
 926	if (guest_vcpu->tid && guest_vcpu->tid != event->comm.tid) {
 927		pr_err("Fatal error: Two threads found with the same VCPU\n");
 928		return -EINVAL;
 929	}
 930	guest_vcpu->tid = event->comm.tid;
 931
 932	return guest_session__map_tid(gs, event->comm.tid, vcpu);
 933}
 934
 935static int host_peek_vm_comms(struct perf_session *session, struct guest_session *gs)
 936{
 937	return perf_session__peek_events(session, session->header.data_offset,
 938					 session->header.data_size,
 939					 host_peek_vm_comms_cb, gs);
 940}
 941
 942static bool evlist__is_id_used(struct evlist *evlist, u64 id)
 943{
 944	return evlist__id2sid(evlist, id);
 945}
 946
 947static u64 guest_session__allocate_new_id(struct guest_session *gs, struct evlist *host_evlist)
 948{
 949	do {
 950		gs->highest_id += 1;
 951	} while (!gs->highest_id || evlist__is_id_used(host_evlist, gs->highest_id));
 952
 953	return gs->highest_id;
 954}
 955
 956static int guest_session__map_id(struct guest_session *gs, u64 id, u64 host_id, u32 vcpu)
 957{
 958	struct guest_id *guest_id = zalloc(sizeof(*guest_id));
 959	int hash;
 960
 961	if (!guest_id)
 962		return -ENOMEM;
 963
 964	guest_id->id = id;
 965	guest_id->host_id = host_id;
 966	guest_id->vcpu = vcpu;
 967	hash = hash_64(guest_id->id, PERF_EVLIST__HLIST_BITS);
 968	hlist_add_head(&guest_id->node, &gs->heads[hash]);
 969
 970	return 0;
 971}
 972
 973static u64 evlist__find_highest_id(struct evlist *evlist)
 974{
 975	struct evsel *evsel;
 976	u64 highest_id = 1;
 977
 978	evlist__for_each_entry(evlist, evsel) {
 979		u32 j;
 980
 981		for (j = 0; j < evsel->core.ids; j++) {
 982			u64 id = evsel->core.id[j];
 983
 984			if (id > highest_id)
 985				highest_id = id;
 986		}
 987	}
 988
 989	return highest_id;
 990}
 991
 992static int guest_session__map_ids(struct guest_session *gs, struct evlist *host_evlist)
 993{
 994	struct evlist *evlist = gs->session->evlist;
 995	struct evsel *evsel;
 996	int ret;
 997
 998	evlist__for_each_entry(evlist, evsel) {
 999		u32 j;
1000
1001		for (j = 0; j < evsel->core.ids; j++) {
1002			struct perf_sample_id *sid;
1003			u64 host_id;
1004			u64 id;
1005
1006			id = evsel->core.id[j];
1007			sid = evlist__id2sid(evlist, id);
1008			if (!sid || sid->cpu.cpu == -1)
1009				continue;
1010			host_id = guest_session__allocate_new_id(gs, host_evlist);
1011			ret = guest_session__map_id(gs, id, host_id, sid->cpu.cpu);
1012			if (ret)
1013				return ret;
1014		}
1015	}
1016
1017	return 0;
1018}
1019
1020static struct guest_id *guest_session__lookup_id(struct guest_session *gs, u64 id)
1021{
1022	struct hlist_head *head;
1023	struct guest_id *guest_id;
1024	int hash;
1025
1026	hash = hash_64(id, PERF_EVLIST__HLIST_BITS);
1027	head = &gs->heads[hash];
1028
1029	hlist_for_each_entry(guest_id, head, node)
1030		if (guest_id->id == id)
1031			return guest_id;
1032
1033	return NULL;
1034}
1035
1036static int process_attr(struct perf_tool *tool, union perf_event *event,
1037			struct perf_sample *sample __maybe_unused,
1038			struct machine *machine __maybe_unused)
1039{
1040	struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
1041
1042	return perf_event__process_attr(tool, event, &inject->session->evlist);
1043}
1044
1045static int guest_session__add_attr(struct guest_session *gs, struct evsel *evsel)
1046{
1047	struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session);
1048	struct perf_event_attr attr = evsel->core.attr;
1049	u64 *id_array;
1050	u32 *vcpu_array;
1051	int ret = -ENOMEM;
1052	u32 i;
1053
1054	id_array = calloc(evsel->core.ids, sizeof(*id_array));
1055	if (!id_array)
1056		return -ENOMEM;
1057
1058	vcpu_array = calloc(evsel->core.ids, sizeof(*vcpu_array));
1059	if (!vcpu_array)
1060		goto out;
1061
1062	for (i = 0; i < evsel->core.ids; i++) {
1063		u64 id = evsel->core.id[i];
1064		struct guest_id *guest_id = guest_session__lookup_id(gs, id);
1065
1066		if (!guest_id) {
1067			pr_err("Failed to find guest id %"PRIu64"\n", id);
1068			ret = -EINVAL;
1069			goto out;
1070		}
1071		id_array[i] = guest_id->host_id;
1072		vcpu_array[i] = guest_id->vcpu;
1073	}
1074
1075	attr.sample_type |= PERF_SAMPLE_IDENTIFIER;
1076	attr.exclude_host = 1;
1077	attr.exclude_guest = 0;
1078
1079	ret = perf_event__synthesize_attr(&inject->tool, &attr, evsel->core.ids,
1080					  id_array, process_attr);
1081	if (ret)
1082		pr_err("Failed to add guest attr.\n");
1083
1084	for (i = 0; i < evsel->core.ids; i++) {
1085		struct perf_sample_id *sid;
1086		u32 vcpu = vcpu_array[i];
1087
1088		sid = evlist__id2sid(inject->session->evlist, id_array[i]);
1089		/* Guest event is per-thread from the host point of view */
1090		sid->cpu.cpu = -1;
1091		sid->tid = gs->vcpu[vcpu].tid;
1092		sid->machine_pid = gs->machine_pid;
1093		sid->vcpu.cpu = vcpu;
1094	}
1095out:
1096	free(vcpu_array);
1097	free(id_array);
1098	return ret;
1099}
1100
1101static int guest_session__add_attrs(struct guest_session *gs)
1102{
1103	struct evlist *evlist = gs->session->evlist;
1104	struct evsel *evsel;
1105	int ret;
1106
1107	evlist__for_each_entry(evlist, evsel) {
1108		ret = guest_session__add_attr(gs, evsel);
1109		if (ret)
1110			return ret;
1111	}
1112
1113	return 0;
1114}
1115
1116static int synthesize_id_index(struct perf_inject *inject, size_t new_cnt)
1117{
1118	struct perf_session *session = inject->session;
1119	struct evlist *evlist = session->evlist;
1120	struct machine *machine = &session->machines.host;
1121	size_t from = evlist->core.nr_entries - new_cnt;
1122
1123	return __perf_event__synthesize_id_index(&inject->tool, perf_event__repipe,
1124						 evlist, machine, from);
1125}
1126
1127static struct guest_tid *guest_session__lookup_tid(struct guest_session *gs, u32 tid)
1128{
1129	struct hlist_head *head;
1130	struct guest_tid *guest_tid;
1131	int hash;
1132
1133	hash = hash_32(tid, PERF_EVLIST__HLIST_BITS);
1134	head = &gs->tids[hash];
1135
1136	hlist_for_each_entry(guest_tid, head, node)
1137		if (guest_tid->tid == tid)
1138			return guest_tid;
1139
1140	return NULL;
1141}
1142
1143static bool dso__is_in_kernel_space(struct dso *dso)
1144{
1145	if (dso__is_vdso(dso))
1146		return false;
1147
1148	return dso__is_kcore(dso) ||
1149	       dso->kernel ||
1150	       is_kernel_module(dso->long_name, PERF_RECORD_MISC_CPUMODE_UNKNOWN);
1151}
1152
1153static u64 evlist__first_id(struct evlist *evlist)
1154{
1155	struct evsel *evsel;
1156
1157	evlist__for_each_entry(evlist, evsel) {
1158		if (evsel->core.ids)
1159			return evsel->core.id[0];
1160	}
1161	return 0;
1162}
1163
1164static int process_build_id(struct perf_tool *tool,
1165			    union perf_event *event,
1166			    struct perf_sample *sample __maybe_unused,
1167			    struct machine *machine __maybe_unused)
1168{
1169	struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
1170
1171	return perf_event__process_build_id(inject->session, event);
1172}
1173
1174static int synthesize_build_id(struct perf_inject *inject, struct dso *dso, pid_t machine_pid)
1175{
1176	struct machine *machine = perf_session__findnew_machine(inject->session, machine_pid);
1177	u8 cpumode = dso__is_in_kernel_space(dso) ?
1178			PERF_RECORD_MISC_GUEST_KERNEL :
1179			PERF_RECORD_MISC_GUEST_USER;
1180
1181	if (!machine)
1182		return -ENOMEM;
1183
1184	dso->hit = 1;
1185
1186	return perf_event__synthesize_build_id(&inject->tool, dso, cpumode,
1187					       process_build_id, machine);
1188}
1189
1190static int guest_session__add_build_ids(struct guest_session *gs)
1191{
1192	struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session);
1193	struct machine *machine = &gs->session->machines.host;
1194	struct dso *dso;
1195	int ret;
1196
1197	/* Build IDs will be put in the Build ID feature section */
1198	perf_header__set_feat(&inject->session->header, HEADER_BUILD_ID);
1199
1200	dsos__for_each_with_build_id(dso, &machine->dsos.head) {
1201		ret = synthesize_build_id(inject, dso, gs->machine_pid);
1202		if (ret)
1203			return ret;
1204	}
1205
1206	return 0;
1207}
1208
1209static int guest_session__ksymbol_event(struct perf_tool *tool,
1210					union perf_event *event,
1211					struct perf_sample *sample __maybe_unused,
1212					struct machine *machine __maybe_unused)
1213{
1214	struct guest_session *gs = container_of(tool, struct guest_session, tool);
1215
1216	/* Only support out-of-line i.e. no BPF support */
1217	if (event->ksymbol.ksym_type != PERF_RECORD_KSYMBOL_TYPE_OOL)
1218		return 0;
1219
1220	return guest_session__output_bytes(gs, event, event->header.size);
1221}
1222
1223static int guest_session__start(struct guest_session *gs, const char *name, bool force)
1224{
1225	char tmp_file_name[] = "/tmp/perf-inject-guest_session-XXXXXX";
1226	struct perf_session *session;
1227	int ret;
1228
1229	/* Only these events will be injected */
1230	gs->tool.mmap		= guest_session__repipe;
1231	gs->tool.mmap2		= guest_session__repipe;
1232	gs->tool.comm		= guest_session__repipe;
1233	gs->tool.fork		= guest_session__repipe;
1234	gs->tool.exit		= guest_session__repipe;
1235	gs->tool.lost		= guest_session__repipe;
1236	gs->tool.context_switch	= guest_session__repipe;
1237	gs->tool.ksymbol	= guest_session__ksymbol_event;
1238	gs->tool.text_poke	= guest_session__repipe;
1239	/*
1240	 * Processing a build ID creates a struct dso with that build ID. Later,
1241	 * all guest dsos are iterated and the build IDs processed into the host
1242	 * session where they will be output to the Build ID feature section
1243	 * when the perf.data file header is written.
1244	 */
1245	gs->tool.build_id	= perf_event__process_build_id;
1246	/* Process the id index to know what VCPU an ID belongs to */
1247	gs->tool.id_index	= perf_event__process_id_index;
1248
1249	gs->tool.ordered_events	= true;
1250	gs->tool.ordering_requires_timestamps = true;
1251
1252	gs->data.path	= name;
1253	gs->data.force	= force;
1254	gs->data.mode	= PERF_DATA_MODE_READ;
1255
1256	session = perf_session__new(&gs->data, &gs->tool);
1257	if (IS_ERR(session))
1258		return PTR_ERR(session);
1259	gs->session = session;
1260
1261	/*
1262	 * Initial events have zero'd ID samples. Get default ID sample size
1263	 * used for removing them.
1264	 */
1265	gs->dflt_id_hdr_size = session->machines.host.id_hdr_size;
1266	/* And default ID for adding back a host-compatible ID sample */
1267	gs->dflt_id = evlist__first_id(session->evlist);
1268	if (!gs->dflt_id) {
1269		pr_err("Guest data has no sample IDs");
1270		return -EINVAL;
1271	}
1272
1273	/* Temporary file for guest events */
1274	gs->tmp_file_name = strdup(tmp_file_name);
1275	if (!gs->tmp_file_name)
1276		return -ENOMEM;
1277	gs->tmp_fd = mkstemp(gs->tmp_file_name);
1278	if (gs->tmp_fd < 0)
1279		return -errno;
1280
1281	if (zstd_init(&gs->session->zstd_data, 0) < 0)
1282		pr_warning("Guest session decompression initialization failed.\n");
1283
1284	/*
1285	 * perf does not support processing 2 sessions simultaneously, so output
1286	 * guest events to a temporary file.
1287	 */
1288	ret = perf_session__process_events(gs->session);
1289	if (ret)
1290		return ret;
1291
1292	if (lseek(gs->tmp_fd, 0, SEEK_SET))
1293		return -errno;
1294
1295	return 0;
1296}
1297
1298/* Free hlist nodes assuming hlist_node is the first member of hlist entries */
1299static void free_hlist(struct hlist_head *heads, size_t hlist_sz)
1300{
1301	struct hlist_node *pos, *n;
1302	size_t i;
1303
1304	for (i = 0; i < hlist_sz; ++i) {
1305		hlist_for_each_safe(pos, n, &heads[i]) {
1306			hlist_del(pos);
1307			free(pos);
1308		}
1309	}
1310}
1311
1312static void guest_session__exit(struct guest_session *gs)
1313{
1314	if (gs->session) {
1315		perf_session__delete(gs->session);
1316		free_hlist(gs->heads, PERF_EVLIST__HLIST_SIZE);
1317		free_hlist(gs->tids, PERF_EVLIST__HLIST_SIZE);
1318	}
1319	if (gs->tmp_file_name) {
1320		if (gs->tmp_fd >= 0)
1321			close(gs->tmp_fd);
1322		unlink(gs->tmp_file_name);
1323		zfree(&gs->tmp_file_name);
1324	}
1325	zfree(&gs->vcpu);
1326	zfree(&gs->perf_data_file);
1327}
1328
1329static void get_tsc_conv(struct perf_tsc_conversion *tc, struct perf_record_time_conv *time_conv)
1330{
1331	tc->time_shift		= time_conv->time_shift;
1332	tc->time_mult		= time_conv->time_mult;
1333	tc->time_zero		= time_conv->time_zero;
1334	tc->time_cycles		= time_conv->time_cycles;
1335	tc->time_mask		= time_conv->time_mask;
1336	tc->cap_user_time_zero	= time_conv->cap_user_time_zero;
1337	tc->cap_user_time_short	= time_conv->cap_user_time_short;
1338}
1339
1340static void guest_session__get_tc(struct guest_session *gs)
1341{
1342	struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session);
1343
1344	get_tsc_conv(&gs->host_tc, &inject->session->time_conv);
1345	get_tsc_conv(&gs->guest_tc, &gs->session->time_conv);
1346}
1347
1348static void guest_session__convert_time(struct guest_session *gs, u64 guest_time, u64 *host_time)
1349{
1350	u64 tsc;
1351
1352	if (!guest_time) {
1353		*host_time = 0;
1354		return;
1355	}
1356
1357	if (gs->guest_tc.cap_user_time_zero)
1358		tsc = perf_time_to_tsc(guest_time, &gs->guest_tc);
1359	else
1360		tsc = guest_time;
1361
1362	/*
1363	 * This is the correct order of operations for x86 if the TSC Offset and
1364	 * Multiplier values are used.
1365	 */
1366	tsc -= gs->time_offset;
1367	tsc /= gs->time_scale;
1368
1369	if (gs->host_tc.cap_user_time_zero)
1370		*host_time = tsc_to_perf_time(tsc, &gs->host_tc);
1371	else
1372		*host_time = tsc;
1373}
1374
1375static int guest_session__fetch(struct guest_session *gs)
1376{
1377	void *buf;
1378	struct perf_event_header *hdr;
1379	size_t hdr_sz = sizeof(*hdr);
1380	ssize_t ret;
1381
1382	buf = gs->ev.event_buf;
1383	if (!buf) {
1384		buf = malloc(PERF_SAMPLE_MAX_SIZE);
1385		if (!buf)
1386			return -ENOMEM;
1387		gs->ev.event_buf = buf;
1388	}
1389	hdr = buf;
1390	ret = readn(gs->tmp_fd, buf, hdr_sz);
1391	if (ret < 0)
1392		return ret;
1393
1394	if (!ret) {
1395		/* Zero size means EOF */
1396		hdr->size = 0;
1397		return 0;
1398	}
1399
1400	buf += hdr_sz;
1401
1402	ret = readn(gs->tmp_fd, buf, hdr->size - hdr_sz);
1403	if (ret < 0)
1404		return ret;
1405
1406	gs->ev.event = (union perf_event *)gs->ev.event_buf;
1407	gs->ev.sample.time = 0;
1408
1409	if (hdr->type >= PERF_RECORD_USER_TYPE_START) {
1410		pr_err("Unexpected type fetching guest event");
1411		return 0;
1412	}
1413
1414	ret = evlist__parse_sample(gs->session->evlist, gs->ev.event, &gs->ev.sample);
1415	if (ret) {
1416		pr_err("Parse failed fetching guest event");
1417		return ret;
1418	}
1419
1420	if (!gs->have_tc) {
1421		guest_session__get_tc(gs);
1422		gs->have_tc = true;
1423	}
1424
1425	guest_session__convert_time(gs, gs->ev.sample.time, &gs->ev.sample.time);
1426
1427	return 0;
1428}
1429
1430static int evlist__append_id_sample(struct evlist *evlist, union perf_event *ev,
1431				    const struct perf_sample *sample)
1432{
1433	struct evsel *evsel;
1434	void *array;
1435	int ret;
1436
1437	evsel = evlist__id2evsel(evlist, sample->id);
1438	array = ev;
1439
1440	if (!evsel) {
1441		pr_err("No evsel for id %"PRIu64"\n", sample->id);
1442		return -EINVAL;
1443	}
1444
1445	array += ev->header.size;
1446	ret = perf_event__synthesize_id_sample(array, evsel->core.attr.sample_type, sample);
1447	if (ret < 0)
1448		return ret;
1449
1450	if (ret & 7) {
1451		pr_err("Bad id sample size %d\n", ret);
1452		return -EINVAL;
1453	}
1454
1455	ev->header.size += ret;
1456
1457	return 0;
1458}
1459
1460static int guest_session__inject_events(struct guest_session *gs, u64 timestamp)
1461{
1462	struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session);
1463	int ret;
1464
1465	if (!gs->ready)
1466		return 0;
1467
1468	while (1) {
1469		struct perf_sample *sample;
1470		struct guest_id *guest_id;
1471		union perf_event *ev;
1472		u16 id_hdr_size;
1473		u8 cpumode;
1474		u64 id;
1475
1476		if (!gs->fetched) {
1477			ret = guest_session__fetch(gs);
1478			if (ret)
1479				return ret;
1480			gs->fetched = true;
1481		}
1482
1483		ev = gs->ev.event;
1484		sample = &gs->ev.sample;
1485
1486		if (!ev->header.size)
1487			return 0; /* EOF */
1488
1489		if (sample->time > timestamp)
1490			return 0;
1491
1492		/* Change cpumode to guest */
1493		cpumode = ev->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
1494		if (cpumode & PERF_RECORD_MISC_USER)
1495			cpumode = PERF_RECORD_MISC_GUEST_USER;
1496		else
1497			cpumode = PERF_RECORD_MISC_GUEST_KERNEL;
1498		ev->header.misc &= ~PERF_RECORD_MISC_CPUMODE_MASK;
1499		ev->header.misc |= cpumode;
1500
1501		id = sample->id;
1502		if (!id) {
1503			id = gs->dflt_id;
1504			id_hdr_size = gs->dflt_id_hdr_size;
1505		} else {
1506			struct evsel *evsel = evlist__id2evsel(gs->session->evlist, id);
1507
1508			id_hdr_size = evsel__id_hdr_size(evsel);
1509		}
1510
1511		if (id_hdr_size & 7) {
1512			pr_err("Bad id_hdr_size %u\n", id_hdr_size);
1513			return -EINVAL;
1514		}
1515
1516		if (ev->header.size & 7) {
1517			pr_err("Bad event size %u\n", ev->header.size);
1518			return -EINVAL;
1519		}
1520
1521		/* Remove guest id sample */
1522		ev->header.size -= id_hdr_size;
1523
1524		if (ev->header.size & 7) {
1525			pr_err("Bad raw event size %u\n", ev->header.size);
1526			return -EINVAL;
1527		}
1528
1529		guest_id = guest_session__lookup_id(gs, id);
1530		if (!guest_id) {
1531			pr_err("Guest event with unknown id %llu\n",
1532			       (unsigned long long)id);
1533			return -EINVAL;
1534		}
1535
1536		/* Change to host ID to avoid conflicting ID values */
1537		sample->id = guest_id->host_id;
1538		sample->stream_id = guest_id->host_id;
1539
1540		if (sample->cpu != (u32)-1) {
1541			if (sample->cpu >= gs->vcpu_cnt) {
1542				pr_err("Guest event with unknown VCPU %u\n",
1543				       sample->cpu);
1544				return -EINVAL;
1545			}
1546			/* Change to host CPU instead of guest VCPU */
1547			sample->cpu = gs->vcpu[sample->cpu].cpu;
1548		}
1549
1550		/* New id sample with new ID and CPU */
1551		ret = evlist__append_id_sample(inject->session->evlist, ev, sample);
1552		if (ret)
1553			return ret;
1554
1555		if (ev->header.size & 7) {
1556			pr_err("Bad new event size %u\n", ev->header.size);
1557			return -EINVAL;
1558		}
1559
1560		gs->fetched = false;
1561
1562		ret = output_bytes(inject, ev, ev->header.size);
1563		if (ret)
1564			return ret;
1565	}
1566}
1567
1568static int guest_session__flush_events(struct guest_session *gs)
1569{
1570	return guest_session__inject_events(gs, -1);
1571}
1572
1573static int host__repipe(struct perf_tool *tool,
1574			union perf_event *event,
1575			struct perf_sample *sample,
1576			struct machine *machine)
1577{
1578	struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
1579	int ret;
1580
1581	ret = guest_session__inject_events(&inject->guest_session, sample->time);
1582	if (ret)
1583		return ret;
1584
1585	return perf_event__repipe(tool, event, sample, machine);
1586}
1587
1588static int host__finished_init(struct perf_session *session, union perf_event *event)
1589{
1590	struct perf_inject *inject = container_of(session->tool, struct perf_inject, tool);
1591	struct guest_session *gs = &inject->guest_session;
1592	int ret;
1593
1594	/*
1595	 * Peek through host COMM events to find QEMU threads and the VCPU they
1596	 * are running.
1597	 */
1598	ret = host_peek_vm_comms(session, gs);
1599	if (ret)
1600		return ret;
1601
1602	if (!gs->vcpu_cnt) {
1603		pr_err("No VCPU threads found for pid %u\n", gs->machine_pid);
1604		return -EINVAL;
1605	}
1606
1607	/*
1608	 * Allocate new (unused) host sample IDs and map them to the guest IDs.
1609	 */
1610	gs->highest_id = evlist__find_highest_id(session->evlist);
1611	ret = guest_session__map_ids(gs, session->evlist);
1612	if (ret)
1613		return ret;
1614
1615	ret = guest_session__add_attrs(gs);
1616	if (ret)
1617		return ret;
1618
1619	ret = synthesize_id_index(inject, gs->session->evlist->core.nr_entries);
1620	if (ret) {
1621		pr_err("Failed to synthesize id_index\n");
1622		return ret;
1623	}
1624
1625	ret = guest_session__add_build_ids(gs);
1626	if (ret) {
1627		pr_err("Failed to add guest build IDs\n");
1628		return ret;
1629	}
1630
1631	gs->ready = true;
1632
1633	ret = guest_session__inject_events(gs, 0);
1634	if (ret)
1635		return ret;
1636
1637	return perf_event__repipe_op2_synth(session, event);
1638}
1639
1640/*
1641 * Obey finished-round ordering. The FINISHED_ROUND event is first processed
1642 * which flushes host events to file up until the last flush time. Then inject
1643 * guest events up to the same time. Finally write out the FINISHED_ROUND event
1644 * itself.
1645 */
1646static int host__finished_round(struct perf_tool *tool,
1647				union perf_event *event,
1648				struct ordered_events *oe)
1649{
1650	struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
1651	int ret = perf_event__process_finished_round(tool, event, oe);
1652	u64 timestamp = ordered_events__last_flush_time(oe);
1653
1654	if (ret)
1655		return ret;
1656
1657	ret = guest_session__inject_events(&inject->guest_session, timestamp);
1658	if (ret)
1659		return ret;
1660
1661	return perf_event__repipe_oe_synth(tool, event, oe);
1662}
1663
1664static int host__context_switch(struct perf_tool *tool,
1665				union perf_event *event,
1666				struct perf_sample *sample,
1667				struct machine *machine)
1668{
1669	struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
1670	bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
1671	struct guest_session *gs = &inject->guest_session;
1672	u32 pid = event->context_switch.next_prev_pid;
1673	u32 tid = event->context_switch.next_prev_tid;
1674	struct guest_tid *guest_tid;
1675	u32 vcpu;
1676
1677	if (out || pid != gs->machine_pid)
1678		goto out;
1679
1680	guest_tid = guest_session__lookup_tid(gs, tid);
1681	if (!guest_tid)
1682		goto out;
1683
1684	if (sample->cpu == (u32)-1) {
1685		pr_err("Switch event does not have CPU\n");
1686		return -EINVAL;
1687	}
1688
1689	vcpu = guest_tid->vcpu;
1690	if (vcpu >= gs->vcpu_cnt)
1691		return -EINVAL;
1692
1693	/* Guest is switching in, record which CPU the VCPU is now running on */
1694	gs->vcpu[vcpu].cpu = sample->cpu;
1695out:
1696	return host__repipe(tool, event, sample, machine);
1697}
1698
1699static void sig_handler(int sig __maybe_unused)
1700{
1701	session_done = 1;
1702}
1703
1704static int evsel__check_stype(struct evsel *evsel, u64 sample_type, const char *sample_msg)
1705{
1706	struct perf_event_attr *attr = &evsel->core.attr;
1707	const char *name = evsel__name(evsel);
1708
1709	if (!(attr->sample_type & sample_type)) {
1710		pr_err("Samples for %s event do not have %s attribute set.",
1711			name, sample_msg);
1712		return -EINVAL;
1713	}
1714
1715	return 0;
1716}
1717
1718static int drop_sample(struct perf_tool *tool __maybe_unused,
1719		       union perf_event *event __maybe_unused,
1720		       struct perf_sample *sample __maybe_unused,
1721		       struct evsel *evsel __maybe_unused,
1722		       struct machine *machine __maybe_unused)
1723{
1724	return 0;
1725}
1726
1727static void strip_init(struct perf_inject *inject)
1728{
1729	struct evlist *evlist = inject->session->evlist;
1730	struct evsel *evsel;
1731
1732	inject->tool.context_switch = perf_event__drop;
1733
1734	evlist__for_each_entry(evlist, evsel)
1735		evsel->handler = drop_sample;
1736}
1737
1738static int parse_vm_time_correlation(const struct option *opt, const char *str, int unset)
1739{
1740	struct perf_inject *inject = opt->value;
1741	const char *args;
1742	char *dry_run;
1743
1744	if (unset)
1745		return 0;
1746
1747	inject->itrace_synth_opts.set = true;
1748	inject->itrace_synth_opts.vm_time_correlation = true;
1749	inject->in_place_update = true;
1750
1751	if (!str)
1752		return 0;
1753
1754	dry_run = skip_spaces(str);
1755	if (!strncmp(dry_run, "dry-run", strlen("dry-run"))) {
1756		inject->itrace_synth_opts.vm_tm_corr_dry_run = true;
1757		inject->in_place_update_dry_run = true;
1758		args = dry_run + strlen("dry-run");
1759	} else {
1760		args = str;
1761	}
1762
1763	inject->itrace_synth_opts.vm_tm_corr_args = strdup(args);
1764
1765	return inject->itrace_synth_opts.vm_tm_corr_args ? 0 : -ENOMEM;
1766}
1767
1768static int parse_guest_data(const struct option *opt, const char *str, int unset)
1769{
1770	struct perf_inject *inject = opt->value;
1771	struct guest_session *gs = &inject->guest_session;
1772	char *tok;
1773	char *s;
1774
1775	if (unset)
1776		return 0;
1777
1778	if (!str)
1779		goto bad_args;
1780
1781	s = strdup(str);
1782	if (!s)
1783		return -ENOMEM;
1784
1785	gs->perf_data_file = strsep(&s, ",");
1786	if (!gs->perf_data_file)
1787		goto bad_args;
1788
1789	gs->copy_kcore_dir = has_kcore_dir(gs->perf_data_file);
1790	if (gs->copy_kcore_dir)
1791		inject->output.is_dir = true;
1792
1793	tok = strsep(&s, ",");
1794	if (!tok)
1795		goto bad_args;
1796	gs->machine_pid = strtoul(tok, NULL, 0);
1797	if (!inject->guest_session.machine_pid)
1798		goto bad_args;
1799
1800	gs->time_scale = 1;
1801
1802	tok = strsep(&s, ",");
1803	if (!tok)
1804		goto out;
1805	gs->time_offset = strtoull(tok, NULL, 0);
1806
1807	tok = strsep(&s, ",");
1808	if (!tok)
1809		goto out;
1810	gs->time_scale = strtod(tok, NULL);
1811	if (!gs->time_scale)
1812		goto bad_args;
1813out:
1814	return 0;
1815
1816bad_args:
1817	pr_err("--guest-data option requires guest perf.data file name, "
1818	       "guest machine PID, and optionally guest timestamp offset, "
1819	       "and guest timestamp scale factor, separated by commas.\n");
1820	return -1;
1821}
1822
1823static int save_section_info_cb(struct perf_file_section *section,
1824				struct perf_header *ph __maybe_unused,
1825				int feat, int fd __maybe_unused, void *data)
1826{
1827	struct perf_inject *inject = data;
1828
1829	inject->secs[feat] = *section;
1830	return 0;
1831}
1832
1833static int save_section_info(struct perf_inject *inject)
1834{
1835	struct perf_header *header = &inject->session->header;
1836	int fd = perf_data__fd(inject->session->data);
1837
1838	return perf_header__process_sections(header, fd, inject, save_section_info_cb);
1839}
1840
1841static bool keep_feat(int feat)
1842{
1843	switch (feat) {
1844	/* Keep original information that describes the machine or software */
1845	case HEADER_TRACING_DATA:
1846	case HEADER_HOSTNAME:
1847	case HEADER_OSRELEASE:
1848	case HEADER_VERSION:
1849	case HEADER_ARCH:
1850	case HEADER_NRCPUS:
1851	case HEADER_CPUDESC:
1852	case HEADER_CPUID:
1853	case HEADER_TOTAL_MEM:
1854	case HEADER_CPU_TOPOLOGY:
1855	case HEADER_NUMA_TOPOLOGY:
1856	case HEADER_PMU_MAPPINGS:
1857	case HEADER_CACHE:
1858	case HEADER_MEM_TOPOLOGY:
1859	case HEADER_CLOCKID:
1860	case HEADER_BPF_PROG_INFO:
1861	case HEADER_BPF_BTF:
1862	case HEADER_CPU_PMU_CAPS:
1863	case HEADER_CLOCK_DATA:
1864	case HEADER_HYBRID_TOPOLOGY:
1865	case HEADER_PMU_CAPS:
1866		return true;
1867	/* Information that can be updated */
1868	case HEADER_BUILD_ID:
1869	case HEADER_CMDLINE:
1870	case HEADER_EVENT_DESC:
1871	case HEADER_BRANCH_STACK:
1872	case HEADER_GROUP_DESC:
1873	case HEADER_AUXTRACE:
1874	case HEADER_STAT:
1875	case HEADER_SAMPLE_TIME:
1876	case HEADER_DIR_FORMAT:
1877	case HEADER_COMPRESSED:
1878	default:
1879		return false;
1880	};
1881}
1882
1883static int read_file(int fd, u64 offs, void *buf, size_t sz)
1884{
1885	ssize_t ret = preadn(fd, buf, sz, offs);
1886
1887	if (ret < 0)
1888		return -errno;
1889	if ((size_t)ret != sz)
1890		return -EINVAL;
1891	return 0;
1892}
1893
1894static int feat_copy(struct perf_inject *inject, int feat, struct feat_writer *fw)
1895{
1896	int fd = perf_data__fd(inject->session->data);
1897	u64 offs = inject->secs[feat].offset;
1898	size_t sz = inject->secs[feat].size;
1899	void *buf = malloc(sz);
1900	int ret;
1901
1902	if (!buf)
1903		return -ENOMEM;
1904
1905	ret = read_file(fd, offs, buf, sz);
1906	if (ret)
1907		goto out_free;
1908
1909	ret = fw->write(fw, buf, sz);
1910out_free:
1911	free(buf);
1912	return ret;
1913}
1914
1915struct inject_fc {
1916	struct feat_copier fc;
1917	struct perf_inject *inject;
1918};
1919
1920static int feat_copy_cb(struct feat_copier *fc, int feat, struct feat_writer *fw)
1921{
1922	struct inject_fc *inj_fc = container_of(fc, struct inject_fc, fc);
1923	struct perf_inject *inject = inj_fc->inject;
1924	int ret;
1925
1926	if (!inject->secs[feat].offset ||
1927	    !keep_feat(feat))
1928		return 0;
1929
1930	ret = feat_copy(inject, feat, fw);
1931	if (ret < 0)
1932		return ret;
1933
1934	return 1; /* Feature section copied */
1935}
1936
1937static int copy_kcore_dir(struct perf_inject *inject)
1938{
1939	char *cmd;
1940	int ret;
1941
1942	ret = asprintf(&cmd, "cp -r -n %s/kcore_dir* %s >/dev/null 2>&1",
1943		       inject->input_name, inject->output.path);
1944	if (ret < 0)
1945		return ret;
1946	pr_debug("%s\n", cmd);
1947	ret = system(cmd);
1948	free(cmd);
1949	return ret;
1950}
1951
1952static int guest_session__copy_kcore_dir(struct guest_session *gs)
1953{
1954	struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session);
1955	char *cmd;
1956	int ret;
1957
1958	ret = asprintf(&cmd, "cp -r -n %s/kcore_dir %s/kcore_dir__%u >/dev/null 2>&1",
1959		       gs->perf_data_file, inject->output.path, gs->machine_pid);
1960	if (ret < 0)
1961		return ret;
1962	pr_debug("%s\n", cmd);
1963	ret = system(cmd);
1964	free(cmd);
1965	return ret;
1966}
1967
1968static int output_fd(struct perf_inject *inject)
1969{
1970	return inject->in_place_update ? -1 : perf_data__fd(&inject->output);
1971}
1972
1973static int __cmd_inject(struct perf_inject *inject)
1974{
1975	int ret = -EINVAL;
1976	struct guest_session *gs = &inject->guest_session;
1977	struct perf_session *session = inject->session;
1978	int fd = output_fd(inject);
 
1979	u64 output_data_offset;
1980
1981	signal(SIGINT, sig_handler);
1982
1983	if (inject->build_ids || inject->sched_stat ||
1984	    inject->itrace_synth_opts.set || inject->build_id_all) {
1985		inject->tool.mmap	  = perf_event__repipe_mmap;
1986		inject->tool.mmap2	  = perf_event__repipe_mmap2;
1987		inject->tool.fork	  = perf_event__repipe_fork;
1988#ifdef HAVE_LIBTRACEEVENT
1989		inject->tool.tracing_data = perf_event__repipe_tracing_data;
1990#endif
1991	}
1992
1993	output_data_offset = perf_session__data_offset(session->evlist);
1994
1995	if (inject->build_id_all) {
1996		inject->tool.mmap	  = perf_event__repipe_buildid_mmap;
1997		inject->tool.mmap2	  = perf_event__repipe_buildid_mmap2;
1998	} else if (inject->build_ids) {
1999		inject->tool.sample = perf_event__inject_buildid;
2000	} else if (inject->sched_stat) {
2001		struct evsel *evsel;
2002
2003		evlist__for_each_entry(session->evlist, evsel) {
2004			const char *name = evsel__name(evsel);
2005
2006			if (!strcmp(name, "sched:sched_switch")) {
2007				if (evsel__check_stype(evsel, PERF_SAMPLE_TID, "TID"))
2008					return -EINVAL;
2009
2010				evsel->handler = perf_inject__sched_switch;
2011			} else if (!strcmp(name, "sched:sched_process_exit"))
2012				evsel->handler = perf_inject__sched_process_exit;
2013#ifdef HAVE_LIBTRACEEVENT
2014			else if (!strncmp(name, "sched:sched_stat_", 17))
2015				evsel->handler = perf_inject__sched_stat;
2016#endif
2017		}
2018	} else if (inject->itrace_synth_opts.vm_time_correlation) {
2019		session->itrace_synth_opts = &inject->itrace_synth_opts;
2020		memset(&inject->tool, 0, sizeof(inject->tool));
2021		inject->tool.id_index	    = perf_event__process_id_index;
2022		inject->tool.auxtrace_info  = perf_event__process_auxtrace_info;
2023		inject->tool.auxtrace	    = perf_event__process_auxtrace;
2024		inject->tool.auxtrace_error = perf_event__process_auxtrace_error;
2025		inject->tool.ordered_events = true;
2026		inject->tool.ordering_requires_timestamps = true;
2027	} else if (inject->itrace_synth_opts.set) {
2028		session->itrace_synth_opts = &inject->itrace_synth_opts;
2029		inject->itrace_synth_opts.inject = true;
2030		inject->tool.comm	    = perf_event__repipe_comm;
2031		inject->tool.namespaces	    = perf_event__repipe_namespaces;
2032		inject->tool.exit	    = perf_event__repipe_exit;
2033		inject->tool.id_index	    = perf_event__process_id_index;
2034		inject->tool.auxtrace_info  = perf_event__process_auxtrace_info;
2035		inject->tool.auxtrace	    = perf_event__process_auxtrace;
2036		inject->tool.aux	    = perf_event__drop_aux;
2037		inject->tool.itrace_start   = perf_event__drop_aux;
2038		inject->tool.aux_output_hw_id = perf_event__drop_aux;
2039		inject->tool.ordered_events = true;
2040		inject->tool.ordering_requires_timestamps = true;
2041		/* Allow space in the header for new attributes */
2042		output_data_offset = roundup(8192 + session->header.data_offset, 4096);
2043		if (inject->strip)
2044			strip_init(inject);
2045	} else if (gs->perf_data_file) {
2046		char *name = gs->perf_data_file;
2047
2048		/*
2049		 * Not strictly necessary, but keep these events in order wrt
2050		 * guest events.
2051		 */
2052		inject->tool.mmap		= host__repipe;
2053		inject->tool.mmap2		= host__repipe;
2054		inject->tool.comm		= host__repipe;
2055		inject->tool.fork		= host__repipe;
2056		inject->tool.exit		= host__repipe;
2057		inject->tool.lost		= host__repipe;
2058		inject->tool.context_switch	= host__repipe;
2059		inject->tool.ksymbol		= host__repipe;
2060		inject->tool.text_poke		= host__repipe;
2061		/*
2062		 * Once the host session has initialized, set up sample ID
2063		 * mapping and feed in guest attrs, build IDs and initial
2064		 * events.
2065		 */
2066		inject->tool.finished_init	= host__finished_init;
2067		/* Obey finished round ordering */
2068		inject->tool.finished_round	= host__finished_round,
2069		/* Keep track of which CPU a VCPU is runnng on */
2070		inject->tool.context_switch	= host__context_switch;
2071		/*
2072		 * Must order events to be able to obey finished round
2073		 * ordering.
2074		 */
2075		inject->tool.ordered_events	= true;
2076		inject->tool.ordering_requires_timestamps = true;
2077		/* Set up a separate session to process guest perf.data file */
2078		ret = guest_session__start(gs, name, session->data->force);
2079		if (ret) {
2080			pr_err("Failed to process %s, error %d\n", name, ret);
2081			return ret;
2082		}
2083		/* Allow space in the header for guest attributes */
2084		output_data_offset += gs->session->header.data_offset;
2085		output_data_offset = roundup(output_data_offset, 4096);
2086	}
2087
2088	if (!inject->itrace_synth_opts.set)
2089		auxtrace_index__free(&session->auxtrace_index);
2090
2091	if (!inject->is_pipe && !inject->in_place_update)
2092		lseek(fd, output_data_offset, SEEK_SET);
2093
2094	ret = perf_session__process_events(session);
2095	if (ret)
2096		return ret;
2097
2098	if (gs->session) {
2099		/*
2100		 * Remaining guest events have later timestamps. Flush them
2101		 * out to file.
2102		 */
2103		ret = guest_session__flush_events(gs);
2104		if (ret) {
2105			pr_err("Failed to flush guest events\n");
2106			return ret;
2107		}
2108	}
2109
2110	if (!inject->is_pipe && !inject->in_place_update) {
2111		struct inject_fc inj_fc = {
2112			.fc.copy = feat_copy_cb,
2113			.inject = inject,
2114		};
2115
2116		if (inject->build_ids)
2117			perf_header__set_feat(&session->header,
2118					      HEADER_BUILD_ID);
2119		/*
2120		 * Keep all buildids when there is unprocessed AUX data because
2121		 * it is not known which ones the AUX trace hits.
2122		 */
2123		if (perf_header__has_feat(&session->header, HEADER_BUILD_ID) &&
2124		    inject->have_auxtrace && !inject->itrace_synth_opts.set)
2125			dsos__hit_all(session);
2126		/*
2127		 * The AUX areas have been removed and replaced with
2128		 * synthesized hardware events, so clear the feature flag.
2129		 */
2130		if (inject->itrace_synth_opts.set) {
2131			perf_header__clear_feat(&session->header,
2132						HEADER_AUXTRACE);
2133			if (inject->itrace_synth_opts.last_branch ||
2134			    inject->itrace_synth_opts.add_last_branch)
2135				perf_header__set_feat(&session->header,
2136						      HEADER_BRANCH_STACK);
2137		}
2138		session->header.data_offset = output_data_offset;
2139		session->header.data_size = inject->bytes_written;
2140		perf_session__inject_header(session, session->evlist, fd, &inj_fc.fc);
2141
2142		if (inject->copy_kcore_dir) {
2143			ret = copy_kcore_dir(inject);
2144			if (ret) {
2145				pr_err("Failed to copy kcore\n");
2146				return ret;
2147			}
2148		}
2149		if (gs->copy_kcore_dir) {
2150			ret = guest_session__copy_kcore_dir(gs);
2151			if (ret) {
2152				pr_err("Failed to copy guest kcore\n");
2153				return ret;
2154			}
2155		}
2156	}
2157
2158	return ret;
2159}
2160
2161int cmd_inject(int argc, const char **argv)
2162{
2163	struct perf_inject inject = {
2164		.tool = {
2165			.sample		= perf_event__repipe_sample,
2166			.read		= perf_event__repipe_sample,
2167			.mmap		= perf_event__repipe,
2168			.mmap2		= perf_event__repipe,
2169			.comm		= perf_event__repipe,
2170			.namespaces	= perf_event__repipe,
2171			.cgroup		= perf_event__repipe,
2172			.fork		= perf_event__repipe,
2173			.exit		= perf_event__repipe,
2174			.lost		= perf_event__repipe,
2175			.lost_samples	= perf_event__repipe,
2176			.aux		= perf_event__repipe,
2177			.itrace_start	= perf_event__repipe,
2178			.aux_output_hw_id = perf_event__repipe,
2179			.context_switch	= perf_event__repipe,
2180			.throttle	= perf_event__repipe,
2181			.unthrottle	= perf_event__repipe,
2182			.ksymbol	= perf_event__repipe,
2183			.bpf		= perf_event__repipe,
2184			.text_poke	= perf_event__repipe,
2185			.attr		= perf_event__repipe_attr,
2186			.event_update	= perf_event__repipe_event_update,
2187			.tracing_data	= perf_event__repipe_op2_synth,
2188			.finished_round	= perf_event__repipe_oe_synth,
2189			.build_id	= perf_event__repipe_op2_synth,
2190			.id_index	= perf_event__repipe_op2_synth,
2191			.auxtrace_info	= perf_event__repipe_op2_synth,
2192			.auxtrace_error	= perf_event__repipe_op2_synth,
2193			.time_conv	= perf_event__repipe_op2_synth,
2194			.thread_map	= perf_event__repipe_op2_synth,
2195			.cpu_map	= perf_event__repipe_op2_synth,
2196			.stat_config	= perf_event__repipe_op2_synth,
2197			.stat		= perf_event__repipe_op2_synth,
2198			.stat_round	= perf_event__repipe_op2_synth,
2199			.feature	= perf_event__repipe_op2_synth,
2200			.finished_init	= perf_event__repipe_op2_synth,
2201			.compressed	= perf_event__repipe_op4_synth,
2202			.auxtrace	= perf_event__repipe_auxtrace,
2203		},
2204		.input_name  = "-",
2205		.samples = LIST_HEAD_INIT(inject.samples),
2206		.output = {
2207			.path = "-",
2208			.mode = PERF_DATA_MODE_WRITE,
2209			.use_stdio = true,
2210		},
2211	};
2212	struct perf_data data = {
2213		.mode = PERF_DATA_MODE_READ,
2214		.use_stdio = true,
2215	};
2216	int ret;
2217	bool repipe = true;
2218	const char *known_build_ids = NULL;
2219
2220	struct option options[] = {
2221		OPT_BOOLEAN('b', "build-ids", &inject.build_ids,
2222			    "Inject build-ids into the output stream"),
2223		OPT_BOOLEAN(0, "buildid-all", &inject.build_id_all,
2224			    "Inject build-ids of all DSOs into the output stream"),
2225		OPT_STRING(0, "known-build-ids", &known_build_ids,
2226			   "buildid path [,buildid path...]",
2227			   "build-ids to use for given paths"),
2228		OPT_STRING('i', "input", &inject.input_name, "file",
2229			   "input file name"),
2230		OPT_STRING('o', "output", &inject.output.path, "file",
2231			   "output file name"),
2232		OPT_BOOLEAN('s', "sched-stat", &inject.sched_stat,
2233			    "Merge sched-stat and sched-switch for getting events "
2234			    "where and how long tasks slept"),
2235#ifdef HAVE_JITDUMP
2236		OPT_BOOLEAN('j', "jit", &inject.jit_mode, "merge jitdump files into perf.data file"),
2237#endif
2238		OPT_INCR('v', "verbose", &verbose,
2239			 "be more verbose (show build ids, etc)"),
2240		OPT_STRING('k', "vmlinux", &symbol_conf.vmlinux_name,
2241			   "file", "vmlinux pathname"),
2242		OPT_BOOLEAN(0, "ignore-vmlinux", &symbol_conf.ignore_vmlinux,
2243			    "don't load vmlinux even if found"),
2244		OPT_STRING(0, "kallsyms", &symbol_conf.kallsyms_name, "file",
2245			   "kallsyms pathname"),
2246		OPT_BOOLEAN('f', "force", &data.force, "don't complain, do it"),
2247		OPT_CALLBACK_OPTARG(0, "itrace", &inject.itrace_synth_opts,
2248				    NULL, "opts", "Instruction Tracing options\n"
2249				    ITRACE_HELP,
2250				    itrace_parse_synth_opts),
2251		OPT_BOOLEAN(0, "strip", &inject.strip,
2252			    "strip non-synthesized events (use with --itrace)"),
2253		OPT_CALLBACK_OPTARG(0, "vm-time-correlation", &inject, NULL, "opts",
2254				    "correlate time between VM guests and the host",
2255				    parse_vm_time_correlation),
2256		OPT_CALLBACK_OPTARG(0, "guest-data", &inject, NULL, "opts",
2257				    "inject events from a guest perf.data file",
2258				    parse_guest_data),
2259		OPT_STRING(0, "guestmount", &symbol_conf.guestmount, "directory",
2260			   "guest mount directory under which every guest os"
2261			   " instance has a subdir"),
2262		OPT_END()
2263	};
2264	const char * const inject_usage[] = {
2265		"perf inject [<options>]",
2266		NULL
2267	};
2268
2269	if (!inject.itrace_synth_opts.set) {
2270		/* Disable eager loading of kernel symbols that adds overhead to perf inject. */
2271		symbol_conf.lazy_load_kernel_maps = true;
2272	}
2273
2274#ifndef HAVE_JITDUMP
2275	set_option_nobuild(options, 'j', "jit", "NO_LIBELF=1", true);
2276#endif
2277	argc = parse_options(argc, argv, options, inject_usage, 0);
2278
2279	/*
2280	 * Any (unrecognized) arguments left?
2281	 */
2282	if (argc)
2283		usage_with_options(inject_usage, options);
2284
2285	if (inject.strip && !inject.itrace_synth_opts.set) {
2286		pr_err("--strip option requires --itrace option\n");
2287		return -1;
2288	}
2289
2290	if (symbol__validate_sym_arguments())
2291		return -1;
2292
2293	if (inject.in_place_update) {
2294		if (!strcmp(inject.input_name, "-")) {
2295			pr_err("Input file name required for in-place updating\n");
2296			return -1;
2297		}
2298		if (strcmp(inject.output.path, "-")) {
2299			pr_err("Output file name must not be specified for in-place updating\n");
2300			return -1;
2301		}
2302		if (!data.force && !inject.in_place_update_dry_run) {
2303			pr_err("The input file would be updated in place, "
2304				"the --force option is required.\n");
2305			return -1;
2306		}
2307		if (!inject.in_place_update_dry_run)
2308			data.in_place_update = true;
2309	} else {
2310		if (strcmp(inject.output.path, "-") && !inject.strip &&
2311		    has_kcore_dir(inject.input_name)) {
2312			inject.output.is_dir = true;
2313			inject.copy_kcore_dir = true;
2314		}
2315		if (perf_data__open(&inject.output)) {
2316			perror("failed to create output file");
2317			return -1;
2318		}
2319	}
2320
2321	data.path = inject.input_name;
2322	if (!strcmp(inject.input_name, "-") || inject.output.is_pipe) {
2323		inject.is_pipe = true;
2324		/*
2325		 * Do not repipe header when input is a regular file
2326		 * since either it can rewrite the header at the end
2327		 * or write a new pipe header.
2328		 */
2329		if (strcmp(inject.input_name, "-"))
2330			repipe = false;
2331	}
2332
2333	inject.session = __perf_session__new(&data, repipe,
2334					     output_fd(&inject),
2335					     &inject.tool);
2336	if (IS_ERR(inject.session)) {
2337		ret = PTR_ERR(inject.session);
2338		goto out_close_output;
2339	}
2340
2341	if (zstd_init(&(inject.session->zstd_data), 0) < 0)
2342		pr_warning("Decompression initialization failed.\n");
2343
2344	/* Save original section info before feature bits change */
2345	ret = save_section_info(&inject);
2346	if (ret)
2347		goto out_delete;
2348
2349	if (!data.is_pipe && inject.output.is_pipe) {
2350		ret = perf_header__write_pipe(perf_data__fd(&inject.output));
2351		if (ret < 0) {
2352			pr_err("Couldn't write a new pipe header.\n");
2353			goto out_delete;
2354		}
2355
2356		ret = perf_event__synthesize_for_pipe(&inject.tool,
2357						      inject.session,
2358						      &inject.output,
2359						      perf_event__repipe);
2360		if (ret < 0)
2361			goto out_delete;
2362	}
2363
2364	if (inject.build_ids && !inject.build_id_all) {
2365		/*
2366		 * to make sure the mmap records are ordered correctly
2367		 * and so that the correct especially due to jitted code
2368		 * mmaps. We cannot generate the buildid hit list and
2369		 * inject the jit mmaps at the same time for now.
2370		 */
2371		inject.tool.ordered_events = true;
2372		inject.tool.ordering_requires_timestamps = true;
2373		if (known_build_ids != NULL) {
2374			inject.known_build_ids =
2375				perf_inject__parse_known_build_ids(known_build_ids);
2376
2377			if (inject.known_build_ids == NULL) {
2378				pr_err("Couldn't parse known build ids.\n");
2379				goto out_delete;
2380			}
2381		}
2382	}
2383
2384	if (inject.sched_stat) {
2385		inject.tool.ordered_events = true;
2386	}
2387
2388#ifdef HAVE_JITDUMP
2389	if (inject.jit_mode) {
2390		inject.tool.mmap2	   = perf_event__jit_repipe_mmap2;
2391		inject.tool.mmap	   = perf_event__jit_repipe_mmap;
2392		inject.tool.ordered_events = true;
2393		inject.tool.ordering_requires_timestamps = true;
2394		/*
2395		 * JIT MMAP injection injects all MMAP events in one go, so it
2396		 * does not obey finished_round semantics.
2397		 */
2398		inject.tool.finished_round = perf_event__drop_oe;
2399	}
2400#endif
2401	ret = symbol__init(&inject.session->header.env);
2402	if (ret < 0)
2403		goto out_delete;
2404
2405	ret = __cmd_inject(&inject);
2406
2407	guest_session__exit(&inject.guest_session);
2408
2409out_delete:
2410	strlist__delete(inject.known_build_ids);
2411	zstd_fini(&(inject.session->zstd_data));
2412	perf_session__delete(inject.session);
2413out_close_output:
2414	if (!inject.in_place_update)
2415		perf_data__close(&inject.output);
2416	free(inject.itrace_synth_opts.vm_tm_corr_args);
2417	free(inject.event_copy);
2418	free(inject.guest_session.ev.event_buf);
2419	return ret;
2420}