1/*
   2 * intel_pt.c: Intel Processor Trace support
   3 * Copyright (c) 2013-2015, Intel Corporation.
   4 *
   5 * This program is free software; you can redistribute it and/or modify it
   6 * under the terms and conditions of the GNU General Public License,
   7 * version 2, as published by the Free Software Foundation.
   8 *
   9 * This program is distributed in the hope it will be useful, but WITHOUT
  10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12 * more details.
  13 *
  14 */
  15
  16#include <inttypes.h>
  17#include <stdio.h>
  18#include <stdbool.h>
  19#include <errno.h>
  20#include <linux/kernel.h>
  21#include <linux/types.h>
  22
  23#include "../perf.h"
  24#include "session.h"
  25#include "machine.h"
  26#include "memswap.h"
  27#include "sort.h"
  28#include "tool.h"
  29#include "event.h"
  30#include "evlist.h"
  31#include "evsel.h"
  32#include "map.h"
  33#include "color.h"
  34#include "util.h"
  35#include "thread.h"
  36#include "thread-stack.h"
  37#include "symbol.h"
  38#include "callchain.h"
  39#include "dso.h"
  40#include "debug.h"
  41#include "auxtrace.h"
  42#include "tsc.h"
  43#include "intel-pt.h"
  44#include "config.h"
  45
  46#include "intel-pt-decoder/intel-pt-log.h"
  47#include "intel-pt-decoder/intel-pt-decoder.h"
  48#include "intel-pt-decoder/intel-pt-insn-decoder.h"
  49#include "intel-pt-decoder/intel-pt-pkt-decoder.h"
  50
  51#define MAX_TIMESTAMP (~0ULL)
  52
  53struct intel_pt {
  54	struct auxtrace auxtrace;
  55	struct auxtrace_queues queues;
  56	struct auxtrace_heap heap;
  57	u32 auxtrace_type;
  58	struct perf_session *session;
  59	struct machine *machine;
  60	struct perf_evsel *switch_evsel;
  61	struct thread *unknown_thread;
  62	bool timeless_decoding;
  63	bool sampling_mode;
  64	bool snapshot_mode;
  65	bool per_cpu_mmaps;
  66	bool have_tsc;
  67	bool data_queued;
  68	bool est_tsc;
  69	bool sync_switch;
  70	bool mispred_all;
  71	int have_sched_switch;
  72	u32 pmu_type;
  73	u64 kernel_start;
  74	u64 switch_ip;
  75	u64 ptss_ip;
  76
  77	struct perf_tsc_conversion tc;
  78	bool cap_user_time_zero;
  79
  80	struct itrace_synth_opts synth_opts;
  81
  82	bool sample_instructions;
  83	u64 instructions_sample_type;
 
  84	u64 instructions_id;
  85
  86	bool sample_branches;
  87	u32 branches_filter;
  88	u64 branches_sample_type;
  89	u64 branches_id;
  90
  91	bool sample_transactions;
  92	u64 transactions_sample_type;
  93	u64 transactions_id;
  94
  95	bool sample_ptwrites;
  96	u64 ptwrites_sample_type;
  97	u64 ptwrites_id;
  98
  99	bool sample_pwr_events;
 100	u64 pwr_events_sample_type;
 101	u64 mwait_id;
 102	u64 pwre_id;
 103	u64 exstop_id;
 104	u64 pwrx_id;
 105	u64 cbr_id;
 106
 107	u64 tsc_bit;
 108	u64 mtc_bit;
 109	u64 mtc_freq_bits;
 110	u32 tsc_ctc_ratio_n;
 111	u32 tsc_ctc_ratio_d;
 112	u64 cyc_bit;
 113	u64 noretcomp_bit;
 114	unsigned max_non_turbo_ratio;
 115	unsigned cbr2khz;
 116
 117	unsigned long num_events;
 118
 119	char *filter;
 120	struct addr_filters filts;
 121};
 122
 123enum switch_state {
 124	INTEL_PT_SS_NOT_TRACING,
 125	INTEL_PT_SS_UNKNOWN,
 126	INTEL_PT_SS_TRACING,
 127	INTEL_PT_SS_EXPECTING_SWITCH_EVENT,
 128	INTEL_PT_SS_EXPECTING_SWITCH_IP,
 129};
 130
 131struct intel_pt_queue {
 132	struct intel_pt *pt;
 133	unsigned int queue_nr;
 134	struct auxtrace_buffer *buffer;
 135	struct auxtrace_buffer *old_buffer;
 136	void *decoder;
 137	const struct intel_pt_state *state;
 138	struct ip_callchain *chain;
 139	struct branch_stack *last_branch;
 140	struct branch_stack *last_branch_rb;
 141	size_t last_branch_pos;
 142	union perf_event *event_buf;
 143	bool on_heap;
 144	bool stop;
 145	bool step_through_buffers;
 146	bool use_buffer_pid_tid;
 147	bool sync_switch;
 148	pid_t pid, tid;
 149	int cpu;
 150	int switch_state;
 151	pid_t next_tid;
 152	struct thread *thread;
 153	bool exclude_kernel;
 154	bool have_sample;
 155	u64 time;
 156	u64 timestamp;
 157	u32 flags;
 158	u16 insn_len;
 159	u64 last_insn_cnt;
 160	char insn[INTEL_PT_INSN_BUF_SZ];
 161};
 162
 163static void intel_pt_dump(struct intel_pt *pt __maybe_unused,
 164			  unsigned char *buf, size_t len)
 165{
 166	struct intel_pt_pkt packet;
 167	size_t pos = 0;
 168	int ret, pkt_len, i;
 169	char desc[INTEL_PT_PKT_DESC_MAX];
 170	const char *color = PERF_COLOR_BLUE;
 171
 172	color_fprintf(stdout, color,
 173		      ". ... Intel Processor Trace data: size %zu bytes\n",
 174		      len);
 175
 176	while (len) {
 177		ret = intel_pt_get_packet(buf, len, &packet);
 178		if (ret > 0)
 179			pkt_len = ret;
 180		else
 181			pkt_len = 1;
 182		printf(".");
 183		color_fprintf(stdout, color, "  %08x: ", pos);
 184		for (i = 0; i < pkt_len; i++)
 185			color_fprintf(stdout, color, " %02x", buf[i]);
 186		for (; i < 16; i++)
 187			color_fprintf(stdout, color, "   ");
 188		if (ret > 0) {
 189			ret = intel_pt_pkt_desc(&packet, desc,
 190						INTEL_PT_PKT_DESC_MAX);
 191			if (ret > 0)
 192				color_fprintf(stdout, color, " %s\n", desc);
 193		} else {
 194			color_fprintf(stdout, color, " Bad packet!\n");
 195		}
 196		pos += pkt_len;
 197		buf += pkt_len;
 198		len -= pkt_len;
 199	}
 200}
 201
 202static void intel_pt_dump_event(struct intel_pt *pt, unsigned char *buf,
 203				size_t len)
 204{
 205	printf(".\n");
 206	intel_pt_dump(pt, buf, len);
 207}
 208
 209static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a,
 210				   struct auxtrace_buffer *b)
 211{
 212	bool consecutive = false;
 213	void *start;
 214
 215	start = intel_pt_find_overlap(a->data, a->size, b->data, b->size,
 216				      pt->have_tsc, &consecutive);
 217	if (!start)
 218		return -EINVAL;
 219	b->use_size = b->data + b->size - start;
 220	b->use_data = start;
 221	if (b->use_size && consecutive)
 222		b->consecutive = true;
 223	return 0;
 224}
 225
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 226/* This function assumes data is processed sequentially only */
 227static int intel_pt_get_trace(struct intel_pt_buffer *b, void *data)
 228{
 229	struct intel_pt_queue *ptq = data;
 230	struct auxtrace_buffer *buffer = ptq->buffer;
 231	struct auxtrace_buffer *old_buffer = ptq->old_buffer;
 232	struct auxtrace_queue *queue;
 233	bool might_overlap;
 234
 235	if (ptq->stop) {
 236		b->len = 0;
 237		return 0;
 238	}
 239
 240	queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
 241
 242	buffer = auxtrace_buffer__next(queue, buffer);
 243	if (!buffer) {
 244		if (old_buffer)
 245			auxtrace_buffer__drop_data(old_buffer);
 246		b->len = 0;
 247		return 0;
 248	}
 249
 250	ptq->buffer = buffer;
 251
 252	if (!buffer->data) {
 253		int fd = perf_data__fd(ptq->pt->session->data);
 254
 255		buffer->data = auxtrace_buffer__get_data(buffer, fd);
 256		if (!buffer->data)
 257			return -ENOMEM;
 258	}
 259
 260	might_overlap = ptq->pt->snapshot_mode || ptq->pt->sampling_mode;
 261	if (might_overlap && !buffer->consecutive && old_buffer &&
 262	    intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))
 263		return -ENOMEM;
 264
 
 
 
 265	if (buffer->use_data) {
 266		b->len = buffer->use_size;
 267		b->buf = buffer->use_data;
 268	} else {
 269		b->len = buffer->size;
 270		b->buf = buffer->data;
 271	}
 272	b->ref_timestamp = buffer->reference;
 273
 274	if (!old_buffer || (might_overlap && !buffer->consecutive)) {
 
 275		b->consecutive = false;
 276		b->trace_nr = buffer->buffer_nr + 1;
 277	} else {
 278		b->consecutive = true;
 279	}
 280
 
 
 
 
 281	if (ptq->step_through_buffers)
 282		ptq->stop = true;
 283
 284	if (b->len) {
 285		if (old_buffer)
 286			auxtrace_buffer__drop_data(old_buffer);
 287		ptq->old_buffer = buffer;
 288	} else {
 289		auxtrace_buffer__drop_data(buffer);
 290		return intel_pt_get_trace(b, data);
 291	}
 292
 293	return 0;
 294}
 295
 296struct intel_pt_cache_entry {
 297	struct auxtrace_cache_entry	entry;
 298	u64				insn_cnt;
 299	u64				byte_cnt;
 300	enum intel_pt_insn_op		op;
 301	enum intel_pt_insn_branch	branch;
 302	int				length;
 303	int32_t				rel;
 304	char				insn[INTEL_PT_INSN_BUF_SZ];
 305};
 306
 307static int intel_pt_config_div(const char *var, const char *value, void *data)
 308{
 309	int *d = data;
 310	long val;
 311
 312	if (!strcmp(var, "intel-pt.cache-divisor")) {
 313		val = strtol(value, NULL, 0);
 314		if (val > 0 && val <= INT_MAX)
 315			*d = val;
 316	}
 317
 318	return 0;
 319}
 320
 321static int intel_pt_cache_divisor(void)
 322{
 323	static int d;
 324
 325	if (d)
 326		return d;
 327
 328	perf_config(intel_pt_config_div, &d);
 329
 330	if (!d)
 331		d = 64;
 332
 333	return d;
 334}
 335
 336static unsigned int intel_pt_cache_size(struct dso *dso,
 337					struct machine *machine)
 338{
 339	off_t size;
 340
 341	size = dso__data_size(dso, machine);
 342	size /= intel_pt_cache_divisor();
 343	if (size < 1000)
 344		return 10;
 345	if (size > (1 << 21))
 346		return 21;
 347	return 32 - __builtin_clz(size);
 348}
 349
 350static struct auxtrace_cache *intel_pt_cache(struct dso *dso,
 351					     struct machine *machine)
 352{
 353	struct auxtrace_cache *c;
 354	unsigned int bits;
 355
 356	if (dso->auxtrace_cache)
 357		return dso->auxtrace_cache;
 358
 359	bits = intel_pt_cache_size(dso, machine);
 360
 361	/* Ignoring cache creation failure */
 362	c = auxtrace_cache__new(bits, sizeof(struct intel_pt_cache_entry), 200);
 363
 364	dso->auxtrace_cache = c;
 365
 366	return c;
 367}
 368
 369static int intel_pt_cache_add(struct dso *dso, struct machine *machine,
 370			      u64 offset, u64 insn_cnt, u64 byte_cnt,
 371			      struct intel_pt_insn *intel_pt_insn)
 372{
 373	struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 374	struct intel_pt_cache_entry *e;
 375	int err;
 376
 377	if (!c)
 378		return -ENOMEM;
 379
 380	e = auxtrace_cache__alloc_entry(c);
 381	if (!e)
 382		return -ENOMEM;
 383
 384	e->insn_cnt = insn_cnt;
 385	e->byte_cnt = byte_cnt;
 386	e->op = intel_pt_insn->op;
 387	e->branch = intel_pt_insn->branch;
 388	e->length = intel_pt_insn->length;
 389	e->rel = intel_pt_insn->rel;
 390	memcpy(e->insn, intel_pt_insn->buf, INTEL_PT_INSN_BUF_SZ);
 391
 392	err = auxtrace_cache__add(c, offset, &e->entry);
 393	if (err)
 394		auxtrace_cache__free_entry(c, e);
 395
 396	return err;
 397}
 398
 399static struct intel_pt_cache_entry *
 400intel_pt_cache_lookup(struct dso *dso, struct machine *machine, u64 offset)
 401{
 402	struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 403
 404	if (!c)
 405		return NULL;
 406
 407	return auxtrace_cache__lookup(dso->auxtrace_cache, offset);
 408}
 409
 410static int intel_pt_walk_next_insn(struct intel_pt_insn *intel_pt_insn,
 411				   uint64_t *insn_cnt_ptr, uint64_t *ip,
 412				   uint64_t to_ip, uint64_t max_insn_cnt,
 413				   void *data)
 414{
 415	struct intel_pt_queue *ptq = data;
 416	struct machine *machine = ptq->pt->machine;
 417	struct thread *thread;
 418	struct addr_location al;
 419	unsigned char buf[INTEL_PT_INSN_BUF_SZ];
 
 420	ssize_t len;
 421	int x86_64;
 422	u8 cpumode;
 423	u64 offset, start_offset, start_ip;
 424	u64 insn_cnt = 0;
 425	bool one_map = true;
 426
 427	intel_pt_insn->length = 0;
 428
 429	if (to_ip && *ip == to_ip)
 430		goto out_no_cache;
 431
 
 
 432	if (*ip >= ptq->pt->kernel_start)
 433		cpumode = PERF_RECORD_MISC_KERNEL;
 434	else
 435		cpumode = PERF_RECORD_MISC_USER;
 436
 437	thread = ptq->thread;
 438	if (!thread) {
 439		if (cpumode != PERF_RECORD_MISC_KERNEL)
 440			return -EINVAL;
 441		thread = ptq->pt->unknown_thread;
 442	}
 443
 444	while (1) {
 445		thread__find_addr_map(thread, cpumode, MAP__FUNCTION, *ip, &al);
 446		if (!al.map || !al.map->dso)
 447			return -EINVAL;
 448
 449		if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
 450		    dso__data_status_seen(al.map->dso,
 451					  DSO_DATA_STATUS_SEEN_ITRACE))
 452			return -ENOENT;
 453
 454		offset = al.map->map_ip(al.map, *ip);
 455
 456		if (!to_ip && one_map) {
 457			struct intel_pt_cache_entry *e;
 458
 459			e = intel_pt_cache_lookup(al.map->dso, machine, offset);
 460			if (e &&
 461			    (!max_insn_cnt || e->insn_cnt <= max_insn_cnt)) {
 462				*insn_cnt_ptr = e->insn_cnt;
 463				*ip += e->byte_cnt;
 464				intel_pt_insn->op = e->op;
 465				intel_pt_insn->branch = e->branch;
 466				intel_pt_insn->length = e->length;
 467				intel_pt_insn->rel = e->rel;
 468				memcpy(intel_pt_insn->buf, e->insn,
 469				       INTEL_PT_INSN_BUF_SZ);
 470				intel_pt_log_insn_no_data(intel_pt_insn, *ip);
 471				return 0;
 472			}
 473		}
 474
 475		start_offset = offset;
 476		start_ip = *ip;
 477
 478		/* Load maps to ensure dso->is_64_bit has been updated */
 479		map__load(al.map);
 480
 481		x86_64 = al.map->dso->is_64_bit;
 482
 483		while (1) {
 484			len = dso__data_read_offset(al.map->dso, machine,
 485						    offset, buf,
 486						    INTEL_PT_INSN_BUF_SZ);
 487			if (len <= 0)
 488				return -EINVAL;
 489
 490			if (intel_pt_get_insn(buf, len, x86_64, intel_pt_insn))
 491				return -EINVAL;
 492
 493			intel_pt_log_insn(intel_pt_insn, *ip);
 494
 495			insn_cnt += 1;
 496
 497			if (intel_pt_insn->branch != INTEL_PT_BR_NO_BRANCH)
 498				goto out;
 499
 500			if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 501				goto out_no_cache;
 502
 503			*ip += intel_pt_insn->length;
 504
 505			if (to_ip && *ip == to_ip)
 506				goto out_no_cache;
 507
 508			if (*ip >= al.map->end)
 509				break;
 510
 511			offset += intel_pt_insn->length;
 512		}
 513		one_map = false;
 514	}
 515out:
 516	*insn_cnt_ptr = insn_cnt;
 517
 518	if (!one_map)
 519		goto out_no_cache;
 520
 521	/*
 522	 * Didn't lookup in the 'to_ip' case, so do it now to prevent duplicate
 523	 * entries.
 524	 */
 525	if (to_ip) {
 526		struct intel_pt_cache_entry *e;
 527
 528		e = intel_pt_cache_lookup(al.map->dso, machine, start_offset);
 529		if (e)
 530			return 0;
 531	}
 532
 533	/* Ignore cache errors */
 534	intel_pt_cache_add(al.map->dso, machine, start_offset, insn_cnt,
 535			   *ip - start_ip, intel_pt_insn);
 536
 537	return 0;
 538
 539out_no_cache:
 540	*insn_cnt_ptr = insn_cnt;
 541	return 0;
 542}
 543
 544static bool intel_pt_match_pgd_ip(struct intel_pt *pt, uint64_t ip,
 545				  uint64_t offset, const char *filename)
 546{
 547	struct addr_filter *filt;
 548	bool have_filter   = false;
 549	bool hit_tracestop = false;
 550	bool hit_filter    = false;
 551
 552	list_for_each_entry(filt, &pt->filts.head, list) {
 553		if (filt->start)
 554			have_filter = true;
 555
 556		if ((filename && !filt->filename) ||
 557		    (!filename && filt->filename) ||
 558		    (filename && strcmp(filename, filt->filename)))
 559			continue;
 560
 561		if (!(offset >= filt->addr && offset < filt->addr + filt->size))
 562			continue;
 563
 564		intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s hit filter: %s offset %#"PRIx64" size %#"PRIx64"\n",
 565			     ip, offset, filename ? filename : "[kernel]",
 566			     filt->start ? "filter" : "stop",
 567			     filt->addr, filt->size);
 568
 569		if (filt->start)
 570			hit_filter = true;
 571		else
 572			hit_tracestop = true;
 573	}
 574
 575	if (!hit_tracestop && !hit_filter)
 576		intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s is not in a filter region\n",
 577			     ip, offset, filename ? filename : "[kernel]");
 578
 579	return hit_tracestop || (have_filter && !hit_filter);
 580}
 581
 582static int __intel_pt_pgd_ip(uint64_t ip, void *data)
 583{
 584	struct intel_pt_queue *ptq = data;
 585	struct thread *thread;
 586	struct addr_location al;
 587	u8 cpumode;
 588	u64 offset;
 589
 590	if (ip >= ptq->pt->kernel_start)
 591		return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);
 592
 593	cpumode = PERF_RECORD_MISC_USER;
 594
 595	thread = ptq->thread;
 596	if (!thread)
 597		return -EINVAL;
 598
 599	thread__find_addr_map(thread, cpumode, MAP__FUNCTION, ip, &al);
 600	if (!al.map || !al.map->dso)
 601		return -EINVAL;
 602
 603	offset = al.map->map_ip(al.map, ip);
 604
 605	return intel_pt_match_pgd_ip(ptq->pt, ip, offset,
 606				     al.map->dso->long_name);
 607}
 608
 609static bool intel_pt_pgd_ip(uint64_t ip, void *data)
 610{
 611	return __intel_pt_pgd_ip(ip, data) > 0;
 612}
 613
 614static bool intel_pt_get_config(struct intel_pt *pt,
 615				struct perf_event_attr *attr, u64 *config)
 616{
 617	if (attr->type == pt->pmu_type) {
 618		if (config)
 619			*config = attr->config;
 620		return true;
 621	}
 622
 623	return false;
 624}
 625
 626static bool intel_pt_exclude_kernel(struct intel_pt *pt)
 627{
 628	struct perf_evsel *evsel;
 629
 630	evlist__for_each_entry(pt->session->evlist, evsel) {
 631		if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 632		    !evsel->attr.exclude_kernel)
 633			return false;
 634	}
 635	return true;
 636}
 637
 638static bool intel_pt_return_compression(struct intel_pt *pt)
 639{
 640	struct perf_evsel *evsel;
 641	u64 config;
 642
 643	if (!pt->noretcomp_bit)
 644		return true;
 645
 646	evlist__for_each_entry(pt->session->evlist, evsel) {
 647		if (intel_pt_get_config(pt, &evsel->attr, &config) &&
 648		    (config & pt->noretcomp_bit))
 649			return false;
 650	}
 651	return true;
 652}
 653
 654static bool intel_pt_branch_enable(struct intel_pt *pt)
 655{
 656	struct perf_evsel *evsel;
 657	u64 config;
 658
 659	evlist__for_each_entry(pt->session->evlist, evsel) {
 660		if (intel_pt_get_config(pt, &evsel->attr, &config) &&
 661		    (config & 1) && !(config & 0x2000))
 662			return false;
 663	}
 664	return true;
 665}
 666
 667static unsigned int intel_pt_mtc_period(struct intel_pt *pt)
 668{
 669	struct perf_evsel *evsel;
 670	unsigned int shift;
 671	u64 config;
 672
 673	if (!pt->mtc_freq_bits)
 674		return 0;
 675
 676	for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++)
 677		config >>= 1;
 678
 679	evlist__for_each_entry(pt->session->evlist, evsel) {
 680		if (intel_pt_get_config(pt, &evsel->attr, &config))
 681			return (config & pt->mtc_freq_bits) >> shift;
 682	}
 683	return 0;
 684}
 685
 686static bool intel_pt_timeless_decoding(struct intel_pt *pt)
 687{
 688	struct perf_evsel *evsel;
 689	bool timeless_decoding = true;
 690	u64 config;
 691
 692	if (!pt->tsc_bit || !pt->cap_user_time_zero)
 693		return true;
 694
 695	evlist__for_each_entry(pt->session->evlist, evsel) {
 696		if (!(evsel->attr.sample_type & PERF_SAMPLE_TIME))
 697			return true;
 698		if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 699			if (config & pt->tsc_bit)
 700				timeless_decoding = false;
 701			else
 702				return true;
 703		}
 704	}
 705	return timeless_decoding;
 706}
 707
 708static bool intel_pt_tracing_kernel(struct intel_pt *pt)
 709{
 710	struct perf_evsel *evsel;
 711
 712	evlist__for_each_entry(pt->session->evlist, evsel) {
 713		if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 714		    !evsel->attr.exclude_kernel)
 715			return true;
 716	}
 717	return false;
 718}
 719
 720static bool intel_pt_have_tsc(struct intel_pt *pt)
 721{
 722	struct perf_evsel *evsel;
 723	bool have_tsc = false;
 724	u64 config;
 725
 726	if (!pt->tsc_bit)
 727		return false;
 728
 729	evlist__for_each_entry(pt->session->evlist, evsel) {
 730		if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 731			if (config & pt->tsc_bit)
 732				have_tsc = true;
 733			else
 734				return false;
 735		}
 736	}
 737	return have_tsc;
 738}
 739
 740static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns)
 741{
 742	u64 quot, rem;
 743
 744	quot = ns / pt->tc.time_mult;
 745	rem  = ns % pt->tc.time_mult;
 746	return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) /
 747		pt->tc.time_mult;
 748}
 749
 750static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt,
 751						   unsigned int queue_nr)
 752{
 753	struct intel_pt_params params = { .get_trace = 0, };
 754	struct intel_pt_queue *ptq;
 755
 756	ptq = zalloc(sizeof(struct intel_pt_queue));
 757	if (!ptq)
 758		return NULL;
 759
 760	if (pt->synth_opts.callchain) {
 761		size_t sz = sizeof(struct ip_callchain);
 762
 763		sz += pt->synth_opts.callchain_sz * sizeof(u64);
 764		ptq->chain = zalloc(sz);
 765		if (!ptq->chain)
 766			goto out_free;
 767	}
 768
 769	if (pt->synth_opts.last_branch) {
 770		size_t sz = sizeof(struct branch_stack);
 771
 772		sz += pt->synth_opts.last_branch_sz *
 773		      sizeof(struct branch_entry);
 774		ptq->last_branch = zalloc(sz);
 775		if (!ptq->last_branch)
 776			goto out_free;
 777		ptq->last_branch_rb = zalloc(sz);
 778		if (!ptq->last_branch_rb)
 779			goto out_free;
 780	}
 781
 782	ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
 783	if (!ptq->event_buf)
 784		goto out_free;
 785
 786	ptq->pt = pt;
 787	ptq->queue_nr = queue_nr;
 788	ptq->exclude_kernel = intel_pt_exclude_kernel(pt);
 789	ptq->pid = -1;
 790	ptq->tid = -1;
 791	ptq->cpu = -1;
 792	ptq->next_tid = -1;
 793
 794	params.get_trace = intel_pt_get_trace;
 795	params.walk_insn = intel_pt_walk_next_insn;
 796	params.data = ptq;
 797	params.return_compression = intel_pt_return_compression(pt);
 798	params.branch_enable = intel_pt_branch_enable(pt);
 799	params.max_non_turbo_ratio = pt->max_non_turbo_ratio;
 800	params.mtc_period = intel_pt_mtc_period(pt);
 801	params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n;
 802	params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d;
 803
 804	if (pt->filts.cnt > 0)
 805		params.pgd_ip = intel_pt_pgd_ip;
 806
 807	if (pt->synth_opts.instructions) {
 808		if (pt->synth_opts.period) {
 809			switch (pt->synth_opts.period_type) {
 810			case PERF_ITRACE_PERIOD_INSTRUCTIONS:
 811				params.period_type =
 812						INTEL_PT_PERIOD_INSTRUCTIONS;
 813				params.period = pt->synth_opts.period;
 814				break;
 815			case PERF_ITRACE_PERIOD_TICKS:
 816				params.period_type = INTEL_PT_PERIOD_TICKS;
 817				params.period = pt->synth_opts.period;
 818				break;
 819			case PERF_ITRACE_PERIOD_NANOSECS:
 820				params.period_type = INTEL_PT_PERIOD_TICKS;
 821				params.period = intel_pt_ns_to_ticks(pt,
 822							pt->synth_opts.period);
 823				break;
 824			default:
 825				break;
 826			}
 827		}
 828
 829		if (!params.period) {
 830			params.period_type = INTEL_PT_PERIOD_INSTRUCTIONS;
 831			params.period = 1;
 832		}
 833	}
 834
 835	ptq->decoder = intel_pt_decoder_new(&params);
 836	if (!ptq->decoder)
 837		goto out_free;
 838
 839	return ptq;
 840
 841out_free:
 842	zfree(&ptq->event_buf);
 843	zfree(&ptq->last_branch);
 844	zfree(&ptq->last_branch_rb);
 845	zfree(&ptq->chain);
 846	free(ptq);
 847	return NULL;
 848}
 849
 850static void intel_pt_free_queue(void *priv)
 851{
 852	struct intel_pt_queue *ptq = priv;
 853
 854	if (!ptq)
 855		return;
 856	thread__zput(ptq->thread);
 857	intel_pt_decoder_free(ptq->decoder);
 858	zfree(&ptq->event_buf);
 859	zfree(&ptq->last_branch);
 860	zfree(&ptq->last_branch_rb);
 861	zfree(&ptq->chain);
 862	free(ptq);
 863}
 864
 865static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt,
 866				     struct auxtrace_queue *queue)
 867{
 868	struct intel_pt_queue *ptq = queue->priv;
 869
 870	if (queue->tid == -1 || pt->have_sched_switch) {
 871		ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);
 872		thread__zput(ptq->thread);
 873	}
 874
 875	if (!ptq->thread && ptq->tid != -1)
 876		ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);
 877
 878	if (ptq->thread) {
 879		ptq->pid = ptq->thread->pid_;
 880		if (queue->cpu == -1)
 881			ptq->cpu = ptq->thread->cpu;
 882	}
 883}
 884
 885static void intel_pt_sample_flags(struct intel_pt_queue *ptq)
 886{
 887	if (ptq->state->flags & INTEL_PT_ABORT_TX) {
 888		ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;
 889	} else if (ptq->state->flags & INTEL_PT_ASYNC) {
 890		if (ptq->state->to_ip)
 891			ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |
 892				     PERF_IP_FLAG_ASYNC |
 893				     PERF_IP_FLAG_INTERRUPT;
 894		else
 895			ptq->flags = PERF_IP_FLAG_BRANCH |
 896				     PERF_IP_FLAG_TRACE_END;
 897		ptq->insn_len = 0;
 898	} else {
 899		if (ptq->state->from_ip)
 900			ptq->flags = intel_pt_insn_type(ptq->state->insn_op);
 901		else
 902			ptq->flags = PERF_IP_FLAG_BRANCH |
 903				     PERF_IP_FLAG_TRACE_BEGIN;
 904		if (ptq->state->flags & INTEL_PT_IN_TX)
 905			ptq->flags |= PERF_IP_FLAG_IN_TX;
 906		ptq->insn_len = ptq->state->insn_len;
 907		memcpy(ptq->insn, ptq->state->insn, INTEL_PT_INSN_BUF_SZ);
 908	}
 909}
 910
 911static int intel_pt_setup_queue(struct intel_pt *pt,
 912				struct auxtrace_queue *queue,
 913				unsigned int queue_nr)
 914{
 915	struct intel_pt_queue *ptq = queue->priv;
 916
 917	if (list_empty(&queue->head))
 918		return 0;
 919
 920	if (!ptq) {
 921		ptq = intel_pt_alloc_queue(pt, queue_nr);
 922		if (!ptq)
 923			return -ENOMEM;
 924		queue->priv = ptq;
 925
 926		if (queue->cpu != -1)
 927			ptq->cpu = queue->cpu;
 928		ptq->tid = queue->tid;
 929
 930		if (pt->sampling_mode && !pt->snapshot_mode &&
 931		    pt->timeless_decoding)
 932			ptq->step_through_buffers = true;
 933
 934		ptq->sync_switch = pt->sync_switch;
 
 935	}
 936
 937	if (!ptq->on_heap &&
 938	    (!ptq->sync_switch ||
 939	     ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {
 940		const struct intel_pt_state *state;
 941		int ret;
 942
 943		if (pt->timeless_decoding)
 944			return 0;
 945
 946		intel_pt_log("queue %u getting timestamp\n", queue_nr);
 947		intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
 948			     queue_nr, ptq->cpu, ptq->pid, ptq->tid);
 949		while (1) {
 950			state = intel_pt_decode(ptq->decoder);
 951			if (state->err) {
 952				if (state->err == INTEL_PT_ERR_NODATA) {
 953					intel_pt_log("queue %u has no timestamp\n",
 954						     queue_nr);
 955					return 0;
 956				}
 957				continue;
 958			}
 959			if (state->timestamp)
 960				break;
 961		}
 962
 963		ptq->timestamp = state->timestamp;
 964		intel_pt_log("queue %u timestamp 0x%" PRIx64 "\n",
 965			     queue_nr, ptq->timestamp);
 966		ptq->state = state;
 967		ptq->have_sample = true;
 968		intel_pt_sample_flags(ptq);
 969		ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);
 970		if (ret)
 971			return ret;
 972		ptq->on_heap = true;
 973	}
 974
 975	return 0;
 976}
 977
 978static int intel_pt_setup_queues(struct intel_pt *pt)
 979{
 980	unsigned int i;
 981	int ret;
 982
 983	for (i = 0; i < pt->queues.nr_queues; i++) {
 984		ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i);
 985		if (ret)
 986			return ret;
 987	}
 988	return 0;
 989}
 990
 991static inline void intel_pt_copy_last_branch_rb(struct intel_pt_queue *ptq)
 992{
 993	struct branch_stack *bs_src = ptq->last_branch_rb;
 994	struct branch_stack *bs_dst = ptq->last_branch;
 995	size_t nr = 0;
 996
 997	bs_dst->nr = bs_src->nr;
 998
 999	if (!bs_src->nr)
1000		return;
1001
1002	nr = ptq->pt->synth_opts.last_branch_sz - ptq->last_branch_pos;
1003	memcpy(&bs_dst->entries[0],
1004	       &bs_src->entries[ptq->last_branch_pos],
1005	       sizeof(struct branch_entry) * nr);
1006
1007	if (bs_src->nr >= ptq->pt->synth_opts.last_branch_sz) {
1008		memcpy(&bs_dst->entries[nr],
1009		       &bs_src->entries[0],
1010		       sizeof(struct branch_entry) * ptq->last_branch_pos);
1011	}
1012}
1013
1014static inline void intel_pt_reset_last_branch_rb(struct intel_pt_queue *ptq)
1015{
1016	ptq->last_branch_pos = 0;
1017	ptq->last_branch_rb->nr = 0;
1018}
1019
1020static void intel_pt_update_last_branch_rb(struct intel_pt_queue *ptq)
1021{
1022	const struct intel_pt_state *state = ptq->state;
1023	struct branch_stack *bs = ptq->last_branch_rb;
1024	struct branch_entry *be;
1025
1026	if (!ptq->last_branch_pos)
1027		ptq->last_branch_pos = ptq->pt->synth_opts.last_branch_sz;
1028
1029	ptq->last_branch_pos -= 1;
1030
1031	be              = &bs->entries[ptq->last_branch_pos];
1032	be->from        = state->from_ip;
1033	be->to          = state->to_ip;
1034	be->flags.abort = !!(state->flags & INTEL_PT_ABORT_TX);
1035	be->flags.in_tx = !!(state->flags & INTEL_PT_IN_TX);
1036	/* No support for mispredict */
1037	be->flags.mispred = ptq->pt->mispred_all;
1038
1039	if (bs->nr < ptq->pt->synth_opts.last_branch_sz)
1040		bs->nr += 1;
1041}
1042
1043static inline bool intel_pt_skip_event(struct intel_pt *pt)
1044{
1045	return pt->synth_opts.initial_skip &&
1046	       pt->num_events++ < pt->synth_opts.initial_skip;
1047}
1048
1049static void intel_pt_prep_b_sample(struct intel_pt *pt,
1050				   struct intel_pt_queue *ptq,
1051				   union perf_event *event,
1052				   struct perf_sample *sample)
1053{
1054	event->sample.header.type = PERF_RECORD_SAMPLE;
1055	event->sample.header.misc = PERF_RECORD_MISC_USER;
1056	event->sample.header.size = sizeof(struct perf_event_header);
1057
1058	if (!pt->timeless_decoding)
1059		sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1060
1061	sample->cpumode = PERF_RECORD_MISC_USER;
1062	sample->ip = ptq->state->from_ip;
1063	sample->pid = ptq->pid;
1064	sample->tid = ptq->tid;
1065	sample->addr = ptq->state->to_ip;
1066	sample->period = 1;
1067	sample->cpu = ptq->cpu;
1068	sample->flags = ptq->flags;
1069	sample->insn_len = ptq->insn_len;
1070	memcpy(sample->insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);
1071}
1072
1073static int intel_pt_inject_event(union perf_event *event,
1074				 struct perf_sample *sample, u64 type)
 
1075{
1076	event->header.size = perf_event__sample_event_size(sample, type, 0);
1077	return perf_event__synthesize_sample(event, type, 0, sample);
1078}
1079
1080static inline int intel_pt_opt_inject(struct intel_pt *pt,
1081				      union perf_event *event,
1082				      struct perf_sample *sample, u64 type)
1083{
1084	if (!pt->synth_opts.inject)
1085		return 0;
1086
1087	return intel_pt_inject_event(event, sample, type);
1088}
1089
1090static int intel_pt_deliver_synth_b_event(struct intel_pt *pt,
1091					  union perf_event *event,
1092					  struct perf_sample *sample, u64 type)
1093{
1094	int ret;
1095
1096	ret = intel_pt_opt_inject(pt, event, sample, type);
1097	if (ret)
1098		return ret;
1099
1100	ret = perf_session__deliver_synth_event(pt->session, event, sample);
1101	if (ret)
1102		pr_err("Intel PT: failed to deliver event, error %d\n", ret);
1103
1104	return ret;
1105}
1106
1107static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)
1108{
 
1109	struct intel_pt *pt = ptq->pt;
1110	union perf_event *event = ptq->event_buf;
1111	struct perf_sample sample = { .ip = 0, };
1112	struct dummy_branch_stack {
1113		u64			nr;
1114		struct branch_entry	entries;
1115	} dummy_bs;
1116
1117	if (pt->branches_filter && !(pt->branches_filter & ptq->flags))
1118		return 0;
1119
1120	if (intel_pt_skip_event(pt))
1121		return 0;
 
1122
1123	intel_pt_prep_b_sample(pt, ptq, event, &sample);
 
1124
 
 
 
 
 
1125	sample.id = ptq->pt->branches_id;
1126	sample.stream_id = ptq->pt->branches_id;
 
 
 
 
1127
1128	/*
1129	 * perf report cannot handle events without a branch stack when using
1130	 * SORT_MODE__BRANCH so make a dummy one.
1131	 */
1132	if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) {
1133		dummy_bs = (struct dummy_branch_stack){
1134			.nr = 1,
1135			.entries = {
1136				.from = sample.ip,
1137				.to = sample.addr,
1138			},
1139		};
1140		sample.branch_stack = (struct branch_stack *)&dummy_bs;
1141	}
1142
1143	return intel_pt_deliver_synth_b_event(pt, event, &sample,
1144					      pt->branches_sample_type);
1145}
1146
1147static void intel_pt_prep_sample(struct intel_pt *pt,
1148				 struct intel_pt_queue *ptq,
1149				 union perf_event *event,
1150				 struct perf_sample *sample)
1151{
1152	intel_pt_prep_b_sample(pt, ptq, event, sample);
1153
1154	if (pt->synth_opts.callchain) {
1155		thread_stack__sample(ptq->thread, ptq->chain,
1156				     pt->synth_opts.callchain_sz, sample->ip);
1157		sample->callchain = ptq->chain;
1158	}
1159
1160	if (pt->synth_opts.last_branch) {
1161		intel_pt_copy_last_branch_rb(ptq);
1162		sample->branch_stack = ptq->last_branch;
1163	}
1164}
1165
1166static inline int intel_pt_deliver_synth_event(struct intel_pt *pt,
1167					       struct intel_pt_queue *ptq,
1168					       union perf_event *event,
1169					       struct perf_sample *sample,
1170					       u64 type)
1171{
1172	int ret;
1173
1174	ret = intel_pt_deliver_synth_b_event(pt, event, sample, type);
1175
1176	if (pt->synth_opts.last_branch)
1177		intel_pt_reset_last_branch_rb(ptq);
 
 
1178
1179	return ret;
1180}
1181
1182static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)
1183{
 
1184	struct intel_pt *pt = ptq->pt;
1185	union perf_event *event = ptq->event_buf;
1186	struct perf_sample sample = { .ip = 0, };
1187
1188	if (intel_pt_skip_event(pt))
1189		return 0;
 
1190
1191	intel_pt_prep_sample(pt, ptq, event, &sample);
 
1192
 
 
 
 
 
1193	sample.id = ptq->pt->instructions_id;
1194	sample.stream_id = ptq->pt->instructions_id;
1195	sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;
 
 
 
1196
1197	ptq->last_insn_cnt = ptq->state->tot_insn_cnt;
1198
1199	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1200					    pt->instructions_sample_type);
1201}
1202
1203static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)
1204{
1205	struct intel_pt *pt = ptq->pt;
1206	union perf_event *event = ptq->event_buf;
1207	struct perf_sample sample = { .ip = 0, };
1208
1209	if (intel_pt_skip_event(pt))
1210		return 0;
1211
1212	intel_pt_prep_sample(pt, ptq, event, &sample);
1213
1214	sample.id = ptq->pt->transactions_id;
1215	sample.stream_id = ptq->pt->transactions_id;
1216
1217	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1218					    pt->transactions_sample_type);
1219}
1220
1221static void intel_pt_prep_p_sample(struct intel_pt *pt,
1222				   struct intel_pt_queue *ptq,
1223				   union perf_event *event,
1224				   struct perf_sample *sample)
1225{
1226	intel_pt_prep_sample(pt, ptq, event, sample);
1227
1228	/*
1229	 * Zero IP is used to mean "trace start" but that is not the case for
1230	 * power or PTWRITE events with no IP, so clear the flags.
1231	 */
1232	if (!sample->ip)
1233		sample->flags = 0;
1234}
1235
1236static int intel_pt_synth_ptwrite_sample(struct intel_pt_queue *ptq)
1237{
1238	struct intel_pt *pt = ptq->pt;
1239	union perf_event *event = ptq->event_buf;
1240	struct perf_sample sample = { .ip = 0, };
1241	struct perf_synth_intel_ptwrite raw;
1242
1243	if (intel_pt_skip_event(pt))
1244		return 0;
1245
1246	intel_pt_prep_p_sample(pt, ptq, event, &sample);
1247
1248	sample.id = ptq->pt->ptwrites_id;
1249	sample.stream_id = ptq->pt->ptwrites_id;
1250
1251	raw.flags = 0;
1252	raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);
1253	raw.payload = cpu_to_le64(ptq->state->ptw_payload);
1254
1255	sample.raw_size = perf_synth__raw_size(raw);
1256	sample.raw_data = perf_synth__raw_data(&raw);
1257
1258	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1259					    pt->ptwrites_sample_type);
1260}
1261
1262static int intel_pt_synth_cbr_sample(struct intel_pt_queue *ptq)
1263{
1264	struct intel_pt *pt = ptq->pt;
1265	union perf_event *event = ptq->event_buf;
1266	struct perf_sample sample = { .ip = 0, };
1267	struct perf_synth_intel_cbr raw;
1268	u32 flags;
1269
1270	if (intel_pt_skip_event(pt))
1271		return 0;
1272
1273	intel_pt_prep_p_sample(pt, ptq, event, &sample);
1274
1275	sample.id = ptq->pt->cbr_id;
1276	sample.stream_id = ptq->pt->cbr_id;
1277
1278	flags = (u16)ptq->state->cbr_payload | (pt->max_non_turbo_ratio << 16);
1279	raw.flags = cpu_to_le32(flags);
1280	raw.freq = cpu_to_le32(raw.cbr * pt->cbr2khz);
1281	raw.reserved3 = 0;
1282
1283	sample.raw_size = perf_synth__raw_size(raw);
1284	sample.raw_data = perf_synth__raw_data(&raw);
1285
1286	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1287					    pt->pwr_events_sample_type);
1288}
1289
1290static int intel_pt_synth_mwait_sample(struct intel_pt_queue *ptq)
1291{
1292	struct intel_pt *pt = ptq->pt;
1293	union perf_event *event = ptq->event_buf;
1294	struct perf_sample sample = { .ip = 0, };
1295	struct perf_synth_intel_mwait raw;
1296
1297	if (intel_pt_skip_event(pt))
1298		return 0;
1299
1300	intel_pt_prep_p_sample(pt, ptq, event, &sample);
1301
1302	sample.id = ptq->pt->mwait_id;
1303	sample.stream_id = ptq->pt->mwait_id;
1304
1305	raw.reserved = 0;
1306	raw.payload = cpu_to_le64(ptq->state->mwait_payload);
1307
1308	sample.raw_size = perf_synth__raw_size(raw);
1309	sample.raw_data = perf_synth__raw_data(&raw);
1310
1311	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1312					    pt->pwr_events_sample_type);
1313}
1314
1315static int intel_pt_synth_pwre_sample(struct intel_pt_queue *ptq)
1316{
1317	struct intel_pt *pt = ptq->pt;
1318	union perf_event *event = ptq->event_buf;
1319	struct perf_sample sample = { .ip = 0, };
1320	struct perf_synth_intel_pwre raw;
1321
1322	if (intel_pt_skip_event(pt))
1323		return 0;
1324
1325	intel_pt_prep_p_sample(pt, ptq, event, &sample);
 
 
 
1326
1327	sample.id = ptq->pt->pwre_id;
1328	sample.stream_id = ptq->pt->pwre_id;
 
 
 
 
 
1329
1330	raw.reserved = 0;
1331	raw.payload = cpu_to_le64(ptq->state->pwre_payload);
 
 
1332
1333	sample.raw_size = perf_synth__raw_size(raw);
1334	sample.raw_data = perf_synth__raw_data(&raw);
1335
1336	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1337					    pt->pwr_events_sample_type);
1338}
1339
1340static int intel_pt_synth_exstop_sample(struct intel_pt_queue *ptq)
1341{
 
1342	struct intel_pt *pt = ptq->pt;
1343	union perf_event *event = ptq->event_buf;
1344	struct perf_sample sample = { .ip = 0, };
1345	struct perf_synth_intel_exstop raw;
1346
1347	if (intel_pt_skip_event(pt))
1348		return 0;
1349
1350	intel_pt_prep_p_sample(pt, ptq, event, &sample);
1351
1352	sample.id = ptq->pt->exstop_id;
1353	sample.stream_id = ptq->pt->exstop_id;
1354
1355	raw.flags = 0;
1356	raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);
1357
1358	sample.raw_size = perf_synth__raw_size(raw);
1359	sample.raw_data = perf_synth__raw_data(&raw);
1360
1361	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1362					    pt->pwr_events_sample_type);
1363}
1364
1365static int intel_pt_synth_pwrx_sample(struct intel_pt_queue *ptq)
1366{
1367	struct intel_pt *pt = ptq->pt;
1368	union perf_event *event = ptq->event_buf;
1369	struct perf_sample sample = { .ip = 0, };
1370	struct perf_synth_intel_pwrx raw;
 
 
 
 
 
1371
1372	if (intel_pt_skip_event(pt))
1373		return 0;
 
 
 
1374
1375	intel_pt_prep_p_sample(pt, ptq, event, &sample);
 
 
 
1376
1377	sample.id = ptq->pt->pwrx_id;
1378	sample.stream_id = ptq->pt->pwrx_id;
 
 
 
 
 
1379
1380	raw.reserved = 0;
1381	raw.payload = cpu_to_le64(ptq->state->pwrx_payload);
 
 
1382
1383	sample.raw_size = perf_synth__raw_size(raw);
1384	sample.raw_data = perf_synth__raw_data(&raw);
1385
1386	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1387					    pt->pwr_events_sample_type);
1388}
1389
1390static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu,
1391				pid_t pid, pid_t tid, u64 ip)
1392{
1393	union perf_event event;
1394	char msg[MAX_AUXTRACE_ERROR_MSG];
1395	int err;
1396
1397	intel_pt__strerror(code, msg, MAX_AUXTRACE_ERROR_MSG);
1398
1399	auxtrace_synth_error(&event.auxtrace_error, PERF_AUXTRACE_ERROR_ITRACE,
1400			     code, cpu, pid, tid, ip, msg);
1401
1402	err = perf_session__deliver_synth_event(pt->session, &event, NULL);
1403	if (err)
1404		pr_err("Intel Processor Trace: failed to deliver error event, error %d\n",
1405		       err);
1406
1407	return err;
1408}
1409
1410static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)
1411{
1412	struct auxtrace_queue *queue;
1413	pid_t tid = ptq->next_tid;
1414	int err;
1415
1416	if (tid == -1)
1417		return 0;
1418
1419	intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);
1420
1421	err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);
1422
1423	queue = &pt->queues.queue_array[ptq->queue_nr];
1424	intel_pt_set_pid_tid_cpu(pt, queue);
1425
1426	ptq->next_tid = -1;
1427
1428	return err;
1429}
1430
1431static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)
1432{
1433	struct intel_pt *pt = ptq->pt;
1434
1435	return ip == pt->switch_ip &&
1436	       (ptq->flags & PERF_IP_FLAG_BRANCH) &&
1437	       !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |
1438			       PERF_IP_FLAG_INTERRUPT | PERF_IP_FLAG_TX_ABORT));
1439}
1440
1441#define INTEL_PT_PWR_EVT (INTEL_PT_MWAIT_OP | INTEL_PT_PWR_ENTRY | \
1442			  INTEL_PT_EX_STOP | INTEL_PT_PWR_EXIT | \
1443			  INTEL_PT_CBR_CHG)
1444
1445static int intel_pt_sample(struct intel_pt_queue *ptq)
1446{
1447	const struct intel_pt_state *state = ptq->state;
1448	struct intel_pt *pt = ptq->pt;
1449	int err;
1450
1451	if (!ptq->have_sample)
1452		return 0;
1453
1454	ptq->have_sample = false;
1455
1456	if (pt->sample_pwr_events && (state->type & INTEL_PT_PWR_EVT)) {
1457		if (state->type & INTEL_PT_CBR_CHG) {
1458			err = intel_pt_synth_cbr_sample(ptq);
1459			if (err)
1460				return err;
1461		}
1462		if (state->type & INTEL_PT_MWAIT_OP) {
1463			err = intel_pt_synth_mwait_sample(ptq);
1464			if (err)
1465				return err;
1466		}
1467		if (state->type & INTEL_PT_PWR_ENTRY) {
1468			err = intel_pt_synth_pwre_sample(ptq);
1469			if (err)
1470				return err;
1471		}
1472		if (state->type & INTEL_PT_EX_STOP) {
1473			err = intel_pt_synth_exstop_sample(ptq);
1474			if (err)
1475				return err;
1476		}
1477		if (state->type & INTEL_PT_PWR_EXIT) {
1478			err = intel_pt_synth_pwrx_sample(ptq);
1479			if (err)
1480				return err;
1481		}
1482	}
1483
1484	if (pt->sample_instructions && (state->type & INTEL_PT_INSTRUCTION)) {
1485		err = intel_pt_synth_instruction_sample(ptq);
1486		if (err)
1487			return err;
1488	}
1489
1490	if (pt->sample_transactions && (state->type & INTEL_PT_TRANSACTION)) {
 
1491		err = intel_pt_synth_transaction_sample(ptq);
1492		if (err)
1493			return err;
1494	}
1495
1496	if (pt->sample_ptwrites && (state->type & INTEL_PT_PTW)) {
1497		err = intel_pt_synth_ptwrite_sample(ptq);
1498		if (err)
1499			return err;
1500	}
1501
1502	if (!(state->type & INTEL_PT_BRANCH))
1503		return 0;
1504
1505	if (pt->synth_opts.callchain || pt->synth_opts.thread_stack)
1506		thread_stack__event(ptq->thread, ptq->flags, state->from_ip,
1507				    state->to_ip, ptq->insn_len,
1508				    state->trace_nr);
1509	else
1510		thread_stack__set_trace_nr(ptq->thread, state->trace_nr);
1511
1512	if (pt->sample_branches) {
1513		err = intel_pt_synth_branch_sample(ptq);
1514		if (err)
1515			return err;
1516	}
1517
1518	if (pt->synth_opts.last_branch)
1519		intel_pt_update_last_branch_rb(ptq);
1520
1521	if (!ptq->sync_switch)
1522		return 0;
1523
1524	if (intel_pt_is_switch_ip(ptq, state->to_ip)) {
1525		switch (ptq->switch_state) {
1526		case INTEL_PT_SS_UNKNOWN:
1527		case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1528			err = intel_pt_next_tid(pt, ptq);
1529			if (err)
1530				return err;
1531			ptq->switch_state = INTEL_PT_SS_TRACING;
1532			break;
1533		default:
1534			ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;
1535			return 1;
1536		}
1537	} else if (!state->to_ip) {
1538		ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
1539	} else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {
1540		ptq->switch_state = INTEL_PT_SS_UNKNOWN;
1541	} else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1542		   state->to_ip == pt->ptss_ip &&
1543		   (ptq->flags & PERF_IP_FLAG_CALL)) {
1544		ptq->switch_state = INTEL_PT_SS_TRACING;
1545	}
1546
1547	return 0;
1548}
1549
1550static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip)
1551{
1552	struct machine *machine = pt->machine;
1553	struct map *map;
1554	struct symbol *sym, *start;
1555	u64 ip, switch_ip = 0;
1556	const char *ptss;
1557
1558	if (ptss_ip)
1559		*ptss_ip = 0;
1560
1561	map = machine__kernel_map(machine);
1562	if (!map)
1563		return 0;
1564
1565	if (map__load(map))
1566		return 0;
1567
1568	start = dso__first_symbol(map->dso, MAP__FUNCTION);
1569
1570	for (sym = start; sym; sym = dso__next_symbol(sym)) {
1571		if (sym->binding == STB_GLOBAL &&
1572		    !strcmp(sym->name, "__switch_to")) {
1573			ip = map->unmap_ip(map, sym->start);
1574			if (ip >= map->start && ip < map->end) {
1575				switch_ip = ip;
1576				break;
1577			}
1578		}
1579	}
1580
1581	if (!switch_ip || !ptss_ip)
1582		return 0;
1583
1584	if (pt->have_sched_switch == 1)
1585		ptss = "perf_trace_sched_switch";
1586	else
1587		ptss = "__perf_event_task_sched_out";
1588
1589	for (sym = start; sym; sym = dso__next_symbol(sym)) {
1590		if (!strcmp(sym->name, ptss)) {
1591			ip = map->unmap_ip(map, sym->start);
1592			if (ip >= map->start && ip < map->end) {
1593				*ptss_ip = ip;
1594				break;
1595			}
1596		}
1597	}
1598
1599	return switch_ip;
1600}
1601
1602static void intel_pt_enable_sync_switch(struct intel_pt *pt)
1603{
1604	unsigned int i;
1605
1606	pt->sync_switch = true;
1607
1608	for (i = 0; i < pt->queues.nr_queues; i++) {
1609		struct auxtrace_queue *queue = &pt->queues.queue_array[i];
1610		struct intel_pt_queue *ptq = queue->priv;
1611
1612		if (ptq)
1613			ptq->sync_switch = true;
1614	}
1615}
1616
1617static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)
1618{
1619	const struct intel_pt_state *state = ptq->state;
1620	struct intel_pt *pt = ptq->pt;
1621	int err;
1622
1623	if (!pt->kernel_start) {
1624		pt->kernel_start = machine__kernel_start(pt->machine);
1625		if (pt->per_cpu_mmaps &&
1626		    (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) &&
1627		    !pt->timeless_decoding && intel_pt_tracing_kernel(pt) &&
1628		    !pt->sampling_mode) {
1629			pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip);
1630			if (pt->switch_ip) {
1631				intel_pt_log("switch_ip: %"PRIx64" ptss_ip: %"PRIx64"\n",
1632					     pt->switch_ip, pt->ptss_ip);
1633				intel_pt_enable_sync_switch(pt);
1634			}
1635		}
1636	}
1637
1638	intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
1639		     ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
1640	while (1) {
1641		err = intel_pt_sample(ptq);
1642		if (err)
1643			return err;
1644
1645		state = intel_pt_decode(ptq->decoder);
1646		if (state->err) {
1647			if (state->err == INTEL_PT_ERR_NODATA)
1648				return 1;
1649			if (ptq->sync_switch &&
1650			    state->from_ip >= pt->kernel_start) {
1651				ptq->sync_switch = false;
1652				intel_pt_next_tid(pt, ptq);
1653			}
1654			if (pt->synth_opts.errors) {
1655				err = intel_pt_synth_error(pt, state->err,
1656							   ptq->cpu, ptq->pid,
1657							   ptq->tid,
1658							   state->from_ip);
1659				if (err)
1660					return err;
1661			}
1662			continue;
1663		}
1664
1665		ptq->state = state;
1666		ptq->have_sample = true;
1667		intel_pt_sample_flags(ptq);
1668
1669		/* Use estimated TSC upon return to user space */
1670		if (pt->est_tsc &&
1671		    (state->from_ip >= pt->kernel_start || !state->from_ip) &&
1672		    state->to_ip && state->to_ip < pt->kernel_start) {
1673			intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1674				     state->timestamp, state->est_timestamp);
1675			ptq->timestamp = state->est_timestamp;
1676		/* Use estimated TSC in unknown switch state */
1677		} else if (ptq->sync_switch &&
1678			   ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1679			   intel_pt_is_switch_ip(ptq, state->to_ip) &&
1680			   ptq->next_tid == -1) {
1681			intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1682				     state->timestamp, state->est_timestamp);
1683			ptq->timestamp = state->est_timestamp;
1684		} else if (state->timestamp > ptq->timestamp) {
1685			ptq->timestamp = state->timestamp;
1686		}
1687
1688		if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {
1689			*timestamp = ptq->timestamp;
1690			return 0;
1691		}
1692	}
1693	return 0;
1694}
1695
1696static inline int intel_pt_update_queues(struct intel_pt *pt)
1697{
1698	if (pt->queues.new_data) {
1699		pt->queues.new_data = false;
1700		return intel_pt_setup_queues(pt);
1701	}
1702	return 0;
1703}
1704
1705static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp)
1706{
1707	unsigned int queue_nr;
1708	u64 ts;
1709	int ret;
1710
1711	while (1) {
1712		struct auxtrace_queue *queue;
1713		struct intel_pt_queue *ptq;
1714
1715		if (!pt->heap.heap_cnt)
1716			return 0;
1717
1718		if (pt->heap.heap_array[0].ordinal >= timestamp)
1719			return 0;
1720
1721		queue_nr = pt->heap.heap_array[0].queue_nr;
1722		queue = &pt->queues.queue_array[queue_nr];
1723		ptq = queue->priv;
1724
1725		intel_pt_log("queue %u processing 0x%" PRIx64 " to 0x%" PRIx64 "\n",
1726			     queue_nr, pt->heap.heap_array[0].ordinal,
1727			     timestamp);
1728
1729		auxtrace_heap__pop(&pt->heap);
1730
1731		if (pt->heap.heap_cnt) {
1732			ts = pt->heap.heap_array[0].ordinal + 1;
1733			if (ts > timestamp)
1734				ts = timestamp;
1735		} else {
1736			ts = timestamp;
1737		}
1738
1739		intel_pt_set_pid_tid_cpu(pt, queue);
1740
1741		ret = intel_pt_run_decoder(ptq, &ts);
1742
1743		if (ret < 0) {
1744			auxtrace_heap__add(&pt->heap, queue_nr, ts);
1745			return ret;
1746		}
1747
1748		if (!ret) {
1749			ret = auxtrace_heap__add(&pt->heap, queue_nr, ts);
1750			if (ret < 0)
1751				return ret;
1752		} else {
1753			ptq->on_heap = false;
1754		}
1755	}
1756
1757	return 0;
1758}
1759
1760static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid,
1761					    u64 time_)
1762{
1763	struct auxtrace_queues *queues = &pt->queues;
1764	unsigned int i;
1765	u64 ts = 0;
1766
1767	for (i = 0; i < queues->nr_queues; i++) {
1768		struct auxtrace_queue *queue = &pt->queues.queue_array[i];
1769		struct intel_pt_queue *ptq = queue->priv;
1770
1771		if (ptq && (tid == -1 || ptq->tid == tid)) {
1772			ptq->time = time_;
1773			intel_pt_set_pid_tid_cpu(pt, queue);
1774			intel_pt_run_decoder(ptq, &ts);
1775		}
1776	}
1777	return 0;
1778}
1779
1780static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)
1781{
1782	return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,
1783				    sample->pid, sample->tid, 0);
1784}
1785
1786static struct intel_pt_queue *intel_pt_cpu_to_ptq(struct intel_pt *pt, int cpu)
1787{
1788	unsigned i, j;
1789
1790	if (cpu < 0 || !pt->queues.nr_queues)
1791		return NULL;
1792
1793	if ((unsigned)cpu >= pt->queues.nr_queues)
1794		i = pt->queues.nr_queues - 1;
1795	else
1796		i = cpu;
1797
1798	if (pt->queues.queue_array[i].cpu == cpu)
1799		return pt->queues.queue_array[i].priv;
1800
1801	for (j = 0; i > 0; j++) {
1802		if (pt->queues.queue_array[--i].cpu == cpu)
1803			return pt->queues.queue_array[i].priv;
1804	}
1805
1806	for (; j < pt->queues.nr_queues; j++) {
1807		if (pt->queues.queue_array[j].cpu == cpu)
1808			return pt->queues.queue_array[j].priv;
1809	}
1810
1811	return NULL;
1812}
1813
1814static int intel_pt_sync_switch(struct intel_pt *pt, int cpu, pid_t tid,
1815				u64 timestamp)
1816{
1817	struct intel_pt_queue *ptq;
1818	int err;
1819
1820	if (!pt->sync_switch)
1821		return 1;
1822
1823	ptq = intel_pt_cpu_to_ptq(pt, cpu);
1824	if (!ptq || !ptq->sync_switch)
1825		return 1;
1826
1827	switch (ptq->switch_state) {
1828	case INTEL_PT_SS_NOT_TRACING:
1829		ptq->next_tid = -1;
1830		break;
1831	case INTEL_PT_SS_UNKNOWN:
1832	case INTEL_PT_SS_TRACING:
1833		ptq->next_tid = tid;
1834		ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;
1835		return 0;
1836	case INTEL_PT_SS_EXPECTING_SWITCH_EVENT:
1837		if (!ptq->on_heap) {
1838			ptq->timestamp = perf_time_to_tsc(timestamp,
1839							  &pt->tc);
1840			err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,
1841						 ptq->timestamp);
1842			if (err)
1843				return err;
1844			ptq->on_heap = true;
1845		}
1846		ptq->switch_state = INTEL_PT_SS_TRACING;
1847		break;
1848	case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1849		ptq->next_tid = tid;
1850		intel_pt_log("ERROR: cpu %d expecting switch ip\n", cpu);
1851		break;
1852	default:
1853		break;
1854	}
1855
1856	return 1;
1857}
1858
1859static int intel_pt_process_switch(struct intel_pt *pt,
1860				   struct perf_sample *sample)
1861{
1862	struct perf_evsel *evsel;
1863	pid_t tid;
1864	int cpu, ret;
1865
1866	evsel = perf_evlist__id2evsel(pt->session->evlist, sample->id);
1867	if (evsel != pt->switch_evsel)
1868		return 0;
1869
1870	tid = perf_evsel__intval(evsel, sample, "next_pid");
1871	cpu = sample->cpu;
1872
1873	intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1874		     cpu, tid, sample->time, perf_time_to_tsc(sample->time,
1875		     &pt->tc));
1876
1877	ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1878	if (ret <= 0)
1879		return ret;
1880
1881	return machine__set_current_tid(pt->machine, cpu, -1, tid);
1882}
1883
1884static int intel_pt_context_switch(struct intel_pt *pt, union perf_event *event,
1885				   struct perf_sample *sample)
1886{
1887	bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
1888	pid_t pid, tid;
1889	int cpu, ret;
1890
1891	cpu = sample->cpu;
1892
1893	if (pt->have_sched_switch == 3) {
1894		if (!out)
1895			return 0;
1896		if (event->header.type != PERF_RECORD_SWITCH_CPU_WIDE) {
1897			pr_err("Expecting CPU-wide context switch event\n");
1898			return -EINVAL;
1899		}
1900		pid = event->context_switch.next_prev_pid;
1901		tid = event->context_switch.next_prev_tid;
1902	} else {
1903		if (out)
1904			return 0;
1905		pid = sample->pid;
1906		tid = sample->tid;
1907	}
1908
1909	if (tid == -1) {
1910		pr_err("context_switch event has no tid\n");
1911		return -EINVAL;
1912	}
1913
1914	intel_pt_log("context_switch: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1915		     cpu, pid, tid, sample->time, perf_time_to_tsc(sample->time,
1916		     &pt->tc));
1917
1918	ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1919	if (ret <= 0)
1920		return ret;
1921
1922	return machine__set_current_tid(pt->machine, cpu, pid, tid);
1923}
1924
1925static int intel_pt_process_itrace_start(struct intel_pt *pt,
1926					 union perf_event *event,
1927					 struct perf_sample *sample)
1928{
1929	if (!pt->per_cpu_mmaps)
1930		return 0;
1931
1932	intel_pt_log("itrace_start: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1933		     sample->cpu, event->itrace_start.pid,
1934		     event->itrace_start.tid, sample->time,
1935		     perf_time_to_tsc(sample->time, &pt->tc));
1936
1937	return machine__set_current_tid(pt->machine, sample->cpu,
1938					event->itrace_start.pid,
1939					event->itrace_start.tid);
1940}
1941
1942static int intel_pt_process_event(struct perf_session *session,
1943				  union perf_event *event,
1944				  struct perf_sample *sample,
1945				  struct perf_tool *tool)
1946{
1947	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1948					   auxtrace);
1949	u64 timestamp;
1950	int err = 0;
1951
1952	if (dump_trace)
1953		return 0;
1954
1955	if (!tool->ordered_events) {
1956		pr_err("Intel Processor Trace requires ordered events\n");
1957		return -EINVAL;
1958	}
1959
1960	if (sample->time && sample->time != (u64)-1)
1961		timestamp = perf_time_to_tsc(sample->time, &pt->tc);
1962	else
1963		timestamp = 0;
1964
1965	if (timestamp || pt->timeless_decoding) {
1966		err = intel_pt_update_queues(pt);
1967		if (err)
1968			return err;
1969	}
1970
1971	if (pt->timeless_decoding) {
1972		if (event->header.type == PERF_RECORD_EXIT) {
1973			err = intel_pt_process_timeless_queues(pt,
1974							       event->fork.tid,
1975							       sample->time);
1976		}
1977	} else if (timestamp) {
1978		err = intel_pt_process_queues(pt, timestamp);
1979	}
1980	if (err)
1981		return err;
1982
1983	if (event->header.type == PERF_RECORD_AUX &&
1984	    (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) &&
1985	    pt->synth_opts.errors) {
1986		err = intel_pt_lost(pt, sample);
1987		if (err)
1988			return err;
1989	}
1990
1991	if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE)
1992		err = intel_pt_process_switch(pt, sample);
1993	else if (event->header.type == PERF_RECORD_ITRACE_START)
1994		err = intel_pt_process_itrace_start(pt, event, sample);
1995	else if (event->header.type == PERF_RECORD_SWITCH ||
1996		 event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
1997		err = intel_pt_context_switch(pt, event, sample);
1998
1999	intel_pt_log("event %s (%u): cpu %d time %"PRIu64" tsc %#"PRIx64"\n",
2000		     perf_event__name(event->header.type), event->header.type,
2001		     sample->cpu, sample->time, timestamp);
2002
2003	return err;
2004}
2005
2006static int intel_pt_flush(struct perf_session *session, struct perf_tool *tool)
2007{
2008	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2009					   auxtrace);
2010	int ret;
2011
2012	if (dump_trace)
2013		return 0;
2014
2015	if (!tool->ordered_events)
2016		return -EINVAL;
2017
2018	ret = intel_pt_update_queues(pt);
2019	if (ret < 0)
2020		return ret;
2021
2022	if (pt->timeless_decoding)
2023		return intel_pt_process_timeless_queues(pt, -1,
2024							MAX_TIMESTAMP - 1);
2025
2026	return intel_pt_process_queues(pt, MAX_TIMESTAMP);
2027}
2028
2029static void intel_pt_free_events(struct perf_session *session)
2030{
2031	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2032					   auxtrace);
2033	struct auxtrace_queues *queues = &pt->queues;
2034	unsigned int i;
2035
2036	for (i = 0; i < queues->nr_queues; i++) {
2037		intel_pt_free_queue(queues->queue_array[i].priv);
2038		queues->queue_array[i].priv = NULL;
2039	}
2040	intel_pt_log_disable();
2041	auxtrace_queues__free(queues);
2042}
2043
2044static void intel_pt_free(struct perf_session *session)
2045{
2046	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2047					   auxtrace);
2048
2049	auxtrace_heap__free(&pt->heap);
2050	intel_pt_free_events(session);
2051	session->auxtrace = NULL;
2052	thread__put(pt->unknown_thread);
2053	addr_filters__exit(&pt->filts);
2054	zfree(&pt->filter);
2055	free(pt);
2056}
2057
2058static int intel_pt_process_auxtrace_event(struct perf_session *session,
2059					   union perf_event *event,
2060					   struct perf_tool *tool __maybe_unused)
2061{
2062	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2063					   auxtrace);
2064
 
 
 
2065	if (!pt->data_queued) {
2066		struct auxtrace_buffer *buffer;
2067		off_t data_offset;
2068		int fd = perf_data__fd(session->data);
2069		int err;
2070
2071		if (perf_data__is_pipe(session->data)) {
2072			data_offset = 0;
2073		} else {
2074			data_offset = lseek(fd, 0, SEEK_CUR);
2075			if (data_offset == -1)
2076				return -errno;
2077		}
2078
2079		err = auxtrace_queues__add_event(&pt->queues, session, event,
2080						 data_offset, &buffer);
2081		if (err)
2082			return err;
2083
2084		/* Dump here now we have copied a piped trace out of the pipe */
2085		if (dump_trace) {
2086			if (auxtrace_buffer__get_data(buffer, fd)) {
2087				intel_pt_dump_event(pt, buffer->data,
2088						    buffer->size);
2089				auxtrace_buffer__put_data(buffer);
2090			}
2091		}
2092	}
2093
2094	return 0;
2095}
2096
2097struct intel_pt_synth {
2098	struct perf_tool dummy_tool;
2099	struct perf_session *session;
2100};
2101
2102static int intel_pt_event_synth(struct perf_tool *tool,
2103				union perf_event *event,
2104				struct perf_sample *sample __maybe_unused,
2105				struct machine *machine __maybe_unused)
2106{
2107	struct intel_pt_synth *intel_pt_synth =
2108			container_of(tool, struct intel_pt_synth, dummy_tool);
2109
2110	return perf_session__deliver_synth_event(intel_pt_synth->session, event,
2111						 NULL);
2112}
2113
2114static int intel_pt_synth_event(struct perf_session *session, const char *name,
2115				struct perf_event_attr *attr, u64 id)
2116{
2117	struct intel_pt_synth intel_pt_synth;
2118	int err;
2119
2120	pr_debug("Synthesizing '%s' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
2121		 name, id, (u64)attr->sample_type);
2122
2123	memset(&intel_pt_synth, 0, sizeof(struct intel_pt_synth));
2124	intel_pt_synth.session = session;
2125
2126	err = perf_event__synthesize_attr(&intel_pt_synth.dummy_tool, attr, 1,
2127					  &id, intel_pt_event_synth);
2128	if (err)
2129		pr_err("%s: failed to synthesize '%s' event type\n",
2130		       __func__, name);
2131
2132	return err;
2133}
2134
2135static void intel_pt_set_event_name(struct perf_evlist *evlist, u64 id,
2136				    const char *name)
2137{
2138	struct perf_evsel *evsel;
2139
2140	evlist__for_each_entry(evlist, evsel) {
2141		if (evsel->id && evsel->id[0] == id) {
2142			if (evsel->name)
2143				zfree(&evsel->name);
2144			evsel->name = strdup(name);
2145			break;
2146		}
2147	}
2148}
2149
2150static struct perf_evsel *intel_pt_evsel(struct intel_pt *pt,
2151					 struct perf_evlist *evlist)
2152{
2153	struct perf_evsel *evsel;
2154
2155	evlist__for_each_entry(evlist, evsel) {
2156		if (evsel->attr.type == pt->pmu_type && evsel->ids)
2157			return evsel;
2158	}
2159
2160	return NULL;
2161}
2162
2163static int intel_pt_synth_events(struct intel_pt *pt,
2164				 struct perf_session *session)
2165{
2166	struct perf_evlist *evlist = session->evlist;
2167	struct perf_evsel *evsel = intel_pt_evsel(pt, evlist);
2168	struct perf_event_attr attr;
 
2169	u64 id;
2170	int err;
2171
2172	if (!evsel) {
 
 
 
 
 
 
 
2173		pr_debug("There are no selected events with Intel Processor Trace data\n");
2174		return 0;
2175	}
2176
2177	memset(&attr, 0, sizeof(struct perf_event_attr));
2178	attr.size = sizeof(struct perf_event_attr);
2179	attr.type = PERF_TYPE_HARDWARE;
2180	attr.sample_type = evsel->attr.sample_type & PERF_SAMPLE_MASK;
2181	attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
2182			    PERF_SAMPLE_PERIOD;
2183	if (pt->timeless_decoding)
2184		attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
2185	else
2186		attr.sample_type |= PERF_SAMPLE_TIME;
2187	if (!pt->per_cpu_mmaps)
2188		attr.sample_type &= ~(u64)PERF_SAMPLE_CPU;
2189	attr.exclude_user = evsel->attr.exclude_user;
2190	attr.exclude_kernel = evsel->attr.exclude_kernel;
2191	attr.exclude_hv = evsel->attr.exclude_hv;
2192	attr.exclude_host = evsel->attr.exclude_host;
2193	attr.exclude_guest = evsel->attr.exclude_guest;
2194	attr.sample_id_all = evsel->attr.sample_id_all;
2195	attr.read_format = evsel->attr.read_format;
2196
2197	id = evsel->id[0] + 1000000000;
2198	if (!id)
2199		id = 1;
2200
2201	if (pt->synth_opts.branches) {
2202		attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
2203		attr.sample_period = 1;
2204		attr.sample_type |= PERF_SAMPLE_ADDR;
2205		err = intel_pt_synth_event(session, "branches", &attr, id);
2206		if (err)
2207			return err;
2208		pt->sample_branches = true;
2209		pt->branches_sample_type = attr.sample_type;
2210		pt->branches_id = id;
2211		id += 1;
2212		attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR;
2213	}
2214
2215	if (pt->synth_opts.callchain)
2216		attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
2217	if (pt->synth_opts.last_branch)
2218		attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
2219
2220	if (pt->synth_opts.instructions) {
2221		attr.config = PERF_COUNT_HW_INSTRUCTIONS;
2222		if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)
2223			attr.sample_period =
2224				intel_pt_ns_to_ticks(pt, pt->synth_opts.period);
2225		else
2226			attr.sample_period = pt->synth_opts.period;
2227		err = intel_pt_synth_event(session, "instructions", &attr, id);
2228		if (err)
 
 
 
 
 
 
 
 
 
2229			return err;
 
2230		pt->sample_instructions = true;
2231		pt->instructions_sample_type = attr.sample_type;
2232		pt->instructions_id = id;
2233		id += 1;
2234	}
2235
2236	attr.sample_type &= ~(u64)PERF_SAMPLE_PERIOD;
2237	attr.sample_period = 1;
2238
2239	if (pt->synth_opts.transactions) {
2240		attr.config = PERF_COUNT_HW_INSTRUCTIONS;
2241		err = intel_pt_synth_event(session, "transactions", &attr, id);
2242		if (err)
 
 
 
 
 
 
 
 
 
2243			return err;
 
2244		pt->sample_transactions = true;
2245		pt->transactions_sample_type = attr.sample_type;
2246		pt->transactions_id = id;
2247		intel_pt_set_event_name(evlist, id, "transactions");
2248		id += 1;
2249	}
2250
2251	attr.type = PERF_TYPE_SYNTH;
2252	attr.sample_type |= PERF_SAMPLE_RAW;
2253
2254	if (pt->synth_opts.ptwrites) {
2255		attr.config = PERF_SYNTH_INTEL_PTWRITE;
2256		err = intel_pt_synth_event(session, "ptwrite", &attr, id);
2257		if (err)
2258			return err;
2259		pt->sample_ptwrites = true;
2260		pt->ptwrites_sample_type = attr.sample_type;
2261		pt->ptwrites_id = id;
2262		intel_pt_set_event_name(evlist, id, "ptwrite");
2263		id += 1;
 
 
 
 
 
 
 
 
2264	}
2265
2266	if (pt->synth_opts.pwr_events) {
2267		pt->sample_pwr_events = true;
2268		pt->pwr_events_sample_type = attr.sample_type;
2269
2270		attr.config = PERF_SYNTH_INTEL_CBR;
2271		err = intel_pt_synth_event(session, "cbr", &attr, id);
2272		if (err)
 
 
 
 
 
2273			return err;
2274		pt->cbr_id = id;
2275		intel_pt_set_event_name(evlist, id, "cbr");
2276		id += 1;
 
2277	}
2278
2279	if (pt->synth_opts.pwr_events && (evsel->attr.config & 0x10)) {
2280		attr.config = PERF_SYNTH_INTEL_MWAIT;
2281		err = intel_pt_synth_event(session, "mwait", &attr, id);
2282		if (err)
2283			return err;
2284		pt->mwait_id = id;
2285		intel_pt_set_event_name(evlist, id, "mwait");
2286		id += 1;
2287
2288		attr.config = PERF_SYNTH_INTEL_PWRE;
2289		err = intel_pt_synth_event(session, "pwre", &attr, id);
2290		if (err)
2291			return err;
2292		pt->pwre_id = id;
2293		intel_pt_set_event_name(evlist, id, "pwre");
2294		id += 1;
2295
2296		attr.config = PERF_SYNTH_INTEL_EXSTOP;
2297		err = intel_pt_synth_event(session, "exstop", &attr, id);
2298		if (err)
2299			return err;
2300		pt->exstop_id = id;
2301		intel_pt_set_event_name(evlist, id, "exstop");
2302		id += 1;
2303
2304		attr.config = PERF_SYNTH_INTEL_PWRX;
2305		err = intel_pt_synth_event(session, "pwrx", &attr, id);
2306		if (err)
2307			return err;
2308		pt->pwrx_id = id;
2309		intel_pt_set_event_name(evlist, id, "pwrx");
2310		id += 1;
2311	}
2312
2313	return 0;
2314}
2315
2316static struct perf_evsel *intel_pt_find_sched_switch(struct perf_evlist *evlist)
2317{
2318	struct perf_evsel *evsel;
2319
2320	evlist__for_each_entry_reverse(evlist, evsel) {
2321		const char *name = perf_evsel__name(evsel);
2322
2323		if (!strcmp(name, "sched:sched_switch"))
2324			return evsel;
2325	}
2326
2327	return NULL;
2328}
2329
2330static bool intel_pt_find_switch(struct perf_evlist *evlist)
2331{
2332	struct perf_evsel *evsel;
2333
2334	evlist__for_each_entry(evlist, evsel) {
2335		if (evsel->attr.context_switch)
2336			return true;
2337	}
2338
2339	return false;
2340}
2341
2342static int intel_pt_perf_config(const char *var, const char *value, void *data)
2343{
2344	struct intel_pt *pt = data;
2345
2346	if (!strcmp(var, "intel-pt.mispred-all"))
2347		pt->mispred_all = perf_config_bool(var, value);
2348
2349	return 0;
2350}
2351
2352static const char * const intel_pt_info_fmts[] = {
2353	[INTEL_PT_PMU_TYPE]		= "  PMU Type            %"PRId64"\n",
2354	[INTEL_PT_TIME_SHIFT]		= "  Time Shift          %"PRIu64"\n",
2355	[INTEL_PT_TIME_MULT]		= "  Time Muliplier      %"PRIu64"\n",
2356	[INTEL_PT_TIME_ZERO]		= "  Time Zero           %"PRIu64"\n",
2357	[INTEL_PT_CAP_USER_TIME_ZERO]	= "  Cap Time Zero       %"PRId64"\n",
2358	[INTEL_PT_TSC_BIT]		= "  TSC bit             %#"PRIx64"\n",
2359	[INTEL_PT_NORETCOMP_BIT]	= "  NoRETComp bit       %#"PRIx64"\n",
2360	[INTEL_PT_HAVE_SCHED_SWITCH]	= "  Have sched_switch   %"PRId64"\n",
2361	[INTEL_PT_SNAPSHOT_MODE]	= "  Snapshot mode       %"PRId64"\n",
2362	[INTEL_PT_PER_CPU_MMAPS]	= "  Per-cpu maps        %"PRId64"\n",
2363	[INTEL_PT_MTC_BIT]		= "  MTC bit             %#"PRIx64"\n",
2364	[INTEL_PT_TSC_CTC_N]		= "  TSC:CTC numerator   %"PRIu64"\n",
2365	[INTEL_PT_TSC_CTC_D]		= "  TSC:CTC denominator %"PRIu64"\n",
2366	[INTEL_PT_CYC_BIT]		= "  CYC bit             %#"PRIx64"\n",
2367	[INTEL_PT_MAX_NONTURBO_RATIO]	= "  Max non-turbo ratio %"PRIu64"\n",
2368	[INTEL_PT_FILTER_STR_LEN]	= "  Filter string len.  %"PRIu64"\n",
2369};
2370
2371static void intel_pt_print_info(u64 *arr, int start, int finish)
2372{
2373	int i;
2374
2375	if (!dump_trace)
2376		return;
2377
2378	for (i = start; i <= finish; i++)
2379		fprintf(stdout, intel_pt_info_fmts[i], arr[i]);
2380}
2381
2382static void intel_pt_print_info_str(const char *name, const char *str)
2383{
2384	if (!dump_trace)
2385		return;
2386
2387	fprintf(stdout, "  %-20s%s\n", name, str ? str : "");
2388}
2389
2390static bool intel_pt_has(struct auxtrace_info_event *auxtrace_info, int pos)
2391{
2392	return auxtrace_info->header.size >=
2393		sizeof(struct auxtrace_info_event) + (sizeof(u64) * (pos + 1));
2394}
2395
2396int intel_pt_process_auxtrace_info(union perf_event *event,
2397				   struct perf_session *session)
2398{
2399	struct auxtrace_info_event *auxtrace_info = &event->auxtrace_info;
2400	size_t min_sz = sizeof(u64) * INTEL_PT_PER_CPU_MMAPS;
2401	struct intel_pt *pt;
2402	void *info_end;
2403	u64 *info;
2404	int err;
2405
2406	if (auxtrace_info->header.size < sizeof(struct auxtrace_info_event) +
2407					min_sz)
2408		return -EINVAL;
2409
2410	pt = zalloc(sizeof(struct intel_pt));
2411	if (!pt)
2412		return -ENOMEM;
2413
2414	addr_filters__init(&pt->filts);
2415
2416	err = perf_config(intel_pt_perf_config, pt);
2417	if (err)
2418		goto err_free;
2419
2420	err = auxtrace_queues__init(&pt->queues);
2421	if (err)
2422		goto err_free;
2423
2424	intel_pt_log_set_name(INTEL_PT_PMU_NAME);
2425
2426	pt->session = session;
2427	pt->machine = &session->machines.host; /* No kvm support */
2428	pt->auxtrace_type = auxtrace_info->type;
2429	pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE];
2430	pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT];
2431	pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT];
2432	pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO];
2433	pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO];
2434	pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT];
2435	pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT];
2436	pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH];
2437	pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE];
2438	pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS];
2439	intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_PMU_TYPE,
2440			    INTEL_PT_PER_CPU_MMAPS);
2441
2442	if (intel_pt_has(auxtrace_info, INTEL_PT_CYC_BIT)) {
 
2443		pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT];
2444		pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS];
2445		pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N];
2446		pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D];
2447		pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT];
2448		intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_MTC_BIT,
2449				    INTEL_PT_CYC_BIT);
2450	}
2451
2452	if (intel_pt_has(auxtrace_info, INTEL_PT_MAX_NONTURBO_RATIO)) {
2453		pt->max_non_turbo_ratio =
2454			auxtrace_info->priv[INTEL_PT_MAX_NONTURBO_RATIO];
2455		intel_pt_print_info(&auxtrace_info->priv[0],
2456				    INTEL_PT_MAX_NONTURBO_RATIO,
2457				    INTEL_PT_MAX_NONTURBO_RATIO);
2458	}
2459
2460	info = &auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN] + 1;
2461	info_end = (void *)info + auxtrace_info->header.size;
2462
2463	if (intel_pt_has(auxtrace_info, INTEL_PT_FILTER_STR_LEN)) {
2464		size_t len;
2465
2466		len = auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN];
2467		intel_pt_print_info(&auxtrace_info->priv[0],
2468				    INTEL_PT_FILTER_STR_LEN,
2469				    INTEL_PT_FILTER_STR_LEN);
2470		if (len) {
2471			const char *filter = (const char *)info;
2472
2473			len = roundup(len + 1, 8);
2474			info += len >> 3;
2475			if ((void *)info > info_end) {
2476				pr_err("%s: bad filter string length\n", __func__);
2477				err = -EINVAL;
2478				goto err_free_queues;
2479			}
2480			pt->filter = memdup(filter, len);
2481			if (!pt->filter) {
2482				err = -ENOMEM;
2483				goto err_free_queues;
2484			}
2485			if (session->header.needs_swap)
2486				mem_bswap_64(pt->filter, len);
2487			if (pt->filter[len - 1]) {
2488				pr_err("%s: filter string not null terminated\n", __func__);
2489				err = -EINVAL;
2490				goto err_free_queues;
2491			}
2492			err = addr_filters__parse_bare_filter(&pt->filts,
2493							      filter);
2494			if (err)
2495				goto err_free_queues;
2496		}
2497		intel_pt_print_info_str("Filter string", pt->filter);
2498	}
2499
2500	pt->timeless_decoding = intel_pt_timeless_decoding(pt);
2501	pt->have_tsc = intel_pt_have_tsc(pt);
2502	pt->sampling_mode = false;
2503	pt->est_tsc = !pt->timeless_decoding;
2504
2505	pt->unknown_thread = thread__new(999999999, 999999999);
2506	if (!pt->unknown_thread) {
2507		err = -ENOMEM;
2508		goto err_free_queues;
2509	}
2510
2511	/*
2512	 * Since this thread will not be kept in any rbtree not in a
2513	 * list, initialize its list node so that at thread__put() the
2514	 * current thread lifetime assuption is kept and we don't segfault
2515	 * at list_del_init().
2516	 */
2517	INIT_LIST_HEAD(&pt->unknown_thread->node);
2518
2519	err = thread__set_comm(pt->unknown_thread, "unknown", 0);
2520	if (err)
2521		goto err_delete_thread;
2522	if (thread__init_map_groups(pt->unknown_thread, pt->machine)) {
2523		err = -ENOMEM;
2524		goto err_delete_thread;
2525	}
2526
2527	pt->auxtrace.process_event = intel_pt_process_event;
2528	pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event;
2529	pt->auxtrace.flush_events = intel_pt_flush;
2530	pt->auxtrace.free_events = intel_pt_free_events;
2531	pt->auxtrace.free = intel_pt_free;
2532	session->auxtrace = &pt->auxtrace;
2533
2534	if (dump_trace)
2535		return 0;
2536
2537	if (pt->have_sched_switch == 1) {
2538		pt->switch_evsel = intel_pt_find_sched_switch(session->evlist);
2539		if (!pt->switch_evsel) {
2540			pr_err("%s: missing sched_switch event\n", __func__);
2541			err = -EINVAL;
2542			goto err_delete_thread;
2543		}
2544	} else if (pt->have_sched_switch == 2 &&
2545		   !intel_pt_find_switch(session->evlist)) {
2546		pr_err("%s: missing context_switch attribute flag\n", __func__);
2547		err = -EINVAL;
2548		goto err_delete_thread;
2549	}
2550
2551	if (session->itrace_synth_opts && session->itrace_synth_opts->set) {
2552		pt->synth_opts = *session->itrace_synth_opts;
2553	} else {
2554		itrace_synth_opts__set_default(&pt->synth_opts);
2555		if (use_browser != -1) {
2556			pt->synth_opts.branches = false;
2557			pt->synth_opts.callchain = true;
2558		}
2559		if (session->itrace_synth_opts)
2560			pt->synth_opts.thread_stack =
2561				session->itrace_synth_opts->thread_stack;
2562	}
2563
2564	if (pt->synth_opts.log)
2565		intel_pt_log_enable();
2566
2567	/* Maximum non-turbo ratio is TSC freq / 100 MHz */
2568	if (pt->tc.time_mult) {
2569		u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000);
2570
2571		if (!pt->max_non_turbo_ratio)
2572			pt->max_non_turbo_ratio =
2573					(tsc_freq + 50000000) / 100000000;
2574		intel_pt_log("TSC frequency %"PRIu64"\n", tsc_freq);
2575		intel_pt_log("Maximum non-turbo ratio %u\n",
2576			     pt->max_non_turbo_ratio);
2577		pt->cbr2khz = tsc_freq / pt->max_non_turbo_ratio / 1000;
2578	}
2579
2580	if (pt->synth_opts.calls)
2581		pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |
2582				       PERF_IP_FLAG_TRACE_END;
2583	if (pt->synth_opts.returns)
2584		pt->branches_filter |= PERF_IP_FLAG_RETURN |
2585				       PERF_IP_FLAG_TRACE_BEGIN;
2586
2587	if (pt->synth_opts.callchain && !symbol_conf.use_callchain) {
2588		symbol_conf.use_callchain = true;
2589		if (callchain_register_param(&callchain_param) < 0) {
2590			symbol_conf.use_callchain = false;
2591			pt->synth_opts.callchain = false;
2592		}
2593	}
2594
2595	err = intel_pt_synth_events(pt, session);
2596	if (err)
2597		goto err_delete_thread;
2598
2599	err = auxtrace_queues__process_index(&pt->queues, session);
2600	if (err)
2601		goto err_delete_thread;
2602
2603	if (pt->queues.populated)
2604		pt->data_queued = true;
2605
2606	if (pt->timeless_decoding)
2607		pr_debug2("Intel PT decoding without timestamps\n");
2608
2609	return 0;
2610
2611err_delete_thread:
2612	thread__zput(pt->unknown_thread);
2613err_free_queues:
2614	intel_pt_log_disable();
2615	auxtrace_queues__free(&pt->queues);
2616	session->auxtrace = NULL;
2617err_free:
2618	addr_filters__exit(&pt->filts);
2619	zfree(&pt->filter);
2620	free(pt);
2621	return err;
2622}