1/*
   2 * intel_pt_decoder.c: Intel Processor Trace support
   3 * Copyright (c) 2013-2014, Intel Corporation.
   4 *
   5 * This program is free software; you can redistribute it and/or modify it
   6 * under the terms and conditions of the GNU General Public License,
   7 * version 2, as published by the Free Software Foundation.
   8 *
   9 * This program is distributed in the hope it will be useful, but WITHOUT
  10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12 * more details.
  13 *
  14 */
  15
  16#ifndef _GNU_SOURCE
  17#define _GNU_SOURCE
  18#endif
  19#include <stdlib.h>
  20#include <stdbool.h>
  21#include <string.h>
  22#include <errno.h>
  23#include <stdint.h>
  24#include <inttypes.h>
 
 
 
  25
  26#include "../cache.h"
  27#include "../util.h"
  28
  29#include "intel-pt-insn-decoder.h"
  30#include "intel-pt-pkt-decoder.h"
  31#include "intel-pt-decoder.h"
  32#include "intel-pt-log.h"
  33
  34#define INTEL_PT_BLK_SIZE 1024
  35
  36#define BIT63 (((uint64_t)1 << 63))
  37
  38#define INTEL_PT_RETURN 1
  39
  40/* Maximum number of loops with no packets consumed i.e. stuck in a loop */
  41#define INTEL_PT_MAX_LOOPS 10000
  42
  43struct intel_pt_blk {
  44	struct intel_pt_blk *prev;
  45	uint64_t ip[INTEL_PT_BLK_SIZE];
  46};
  47
  48struct intel_pt_stack {
  49	struct intel_pt_blk *blk;
  50	struct intel_pt_blk *spare;
  51	int pos;
  52};
  53
  54enum intel_pt_pkt_state {
  55	INTEL_PT_STATE_NO_PSB,
  56	INTEL_PT_STATE_NO_IP,
  57	INTEL_PT_STATE_ERR_RESYNC,
  58	INTEL_PT_STATE_IN_SYNC,
 
  59	INTEL_PT_STATE_TNT,
  60	INTEL_PT_STATE_TIP,
  61	INTEL_PT_STATE_TIP_PGD,
  62	INTEL_PT_STATE_FUP,
  63	INTEL_PT_STATE_FUP_NO_TIP,
  64};
  65
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  66#ifdef INTEL_PT_STRICT
  67#define INTEL_PT_STATE_ERR1	INTEL_PT_STATE_NO_PSB
  68#define INTEL_PT_STATE_ERR2	INTEL_PT_STATE_NO_PSB
  69#define INTEL_PT_STATE_ERR3	INTEL_PT_STATE_NO_PSB
  70#define INTEL_PT_STATE_ERR4	INTEL_PT_STATE_NO_PSB
  71#else
  72#define INTEL_PT_STATE_ERR1	(decoder->pkt_state)
  73#define INTEL_PT_STATE_ERR2	INTEL_PT_STATE_NO_IP
  74#define INTEL_PT_STATE_ERR3	INTEL_PT_STATE_ERR_RESYNC
  75#define INTEL_PT_STATE_ERR4	INTEL_PT_STATE_IN_SYNC
  76#endif
  77
  78struct intel_pt_decoder {
  79	int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
  80	int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
  81			 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
  82			 uint64_t max_insn_cnt, void *data);
 
 
  83	void *data;
  84	struct intel_pt_state state;
  85	const unsigned char *buf;
  86	size_t len;
  87	bool return_compression;
 
  88	bool mtc_insn;
  89	bool pge;
  90	bool have_tma;
  91	bool have_cyc;
 
 
 
 
  92	uint64_t pos;
  93	uint64_t last_ip;
  94	uint64_t ip;
  95	uint64_t cr3;
  96	uint64_t timestamp;
  97	uint64_t tsc_timestamp;
  98	uint64_t ref_timestamp;
 
 
  99	uint64_t ret_addr;
 100	uint64_t ctc_timestamp;
 101	uint64_t ctc_delta;
 102	uint64_t cycle_cnt;
 103	uint64_t cyc_ref_timestamp;
 104	uint32_t last_mtc;
 105	uint32_t tsc_ctc_ratio_n;
 106	uint32_t tsc_ctc_ratio_d;
 107	uint32_t tsc_ctc_mult;
 108	uint32_t tsc_slip;
 109	uint32_t ctc_rem_mask;
 110	int mtc_shift;
 111	struct intel_pt_stack stack;
 112	enum intel_pt_pkt_state pkt_state;
 
 
 
 
 113	struct intel_pt_pkt packet;
 114	struct intel_pt_pkt tnt;
 115	int pkt_step;
 116	int pkt_len;
 117	int last_packet_type;
 118	unsigned int cbr;
 
 119	unsigned int max_non_turbo_ratio;
 120	double max_non_turbo_ratio_fp;
 121	double cbr_cyc_to_tsc;
 122	double calc_cyc_to_tsc;
 123	bool have_calc_cyc_to_tsc;
 124	int exec_mode;
 125	unsigned int insn_bytes;
 126	uint64_t sign_bit;
 127	uint64_t sign_bits;
 128	uint64_t period;
 129	enum intel_pt_period_type period_type;
 130	uint64_t tot_insn_cnt;
 131	uint64_t period_insn_cnt;
 132	uint64_t period_mask;
 133	uint64_t period_ticks;
 134	uint64_t last_masked_timestamp;
 
 
 
 
 
 135	bool continuous_period;
 136	bool overflow;
 137	bool set_fup_tx_flags;
 
 
 
 
 
 
 138	unsigned int fup_tx_flags;
 139	unsigned int tx_flags;
 
 
 
 
 140	uint64_t timestamp_insn_cnt;
 
 141	uint64_t stuck_ip;
 142	int no_progress;
 143	int stuck_ip_prd;
 144	int stuck_ip_cnt;
 145	const unsigned char *next_buf;
 146	size_t next_len;
 147	unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
 148};
 149
 150static uint64_t intel_pt_lower_power_of_2(uint64_t x)
 151{
 152	int i;
 153
 154	for (i = 0; x != 1; i++)
 155		x >>= 1;
 156
 157	return x << i;
 158}
 159
 160static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
 161{
 162	if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
 163		uint64_t period;
 164
 165		period = intel_pt_lower_power_of_2(decoder->period);
 166		decoder->period_mask  = ~(period - 1);
 167		decoder->period_ticks = period;
 168	}
 169}
 170
 171static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
 172{
 173	if (!d)
 174		return 0;
 175	return (t / d) * n + ((t % d) * n) / d;
 176}
 177
 178struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
 179{
 180	struct intel_pt_decoder *decoder;
 181
 182	if (!params->get_trace || !params->walk_insn)
 183		return NULL;
 184
 185	decoder = zalloc(sizeof(struct intel_pt_decoder));
 186	if (!decoder)
 187		return NULL;
 188
 189	decoder->get_trace          = params->get_trace;
 190	decoder->walk_insn          = params->walk_insn;
 
 
 191	decoder->data               = params->data;
 192	decoder->return_compression = params->return_compression;
 
 193
 194	decoder->sign_bit           = (uint64_t)1 << 47;
 195	decoder->sign_bits          = ~(((uint64_t)1 << 48) - 1);
 196
 197	decoder->period             = params->period;
 198	decoder->period_type        = params->period_type;
 199
 200	decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
 201	decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
 202
 203	intel_pt_setup_period(decoder);
 204
 205	decoder->mtc_shift = params->mtc_period;
 206	decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
 207
 208	decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
 209	decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
 210
 211	if (!decoder->tsc_ctc_ratio_n)
 212		decoder->tsc_ctc_ratio_d = 0;
 213
 214	if (decoder->tsc_ctc_ratio_d) {
 215		if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
 216			decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
 217						decoder->tsc_ctc_ratio_d;
 218
 219		/*
 220		 * Allow for timestamps appearing to backwards because a TSC
 221		 * packet has slipped past a MTC packet, so allow 2 MTC ticks
 222		 * or ...
 223		 */
 224		decoder->tsc_slip = multdiv(2 << decoder->mtc_shift,
 225					decoder->tsc_ctc_ratio_n,
 226					decoder->tsc_ctc_ratio_d);
 227	}
 228	/* ... or 0x100 paranoia */
 229	if (decoder->tsc_slip < 0x100)
 230		decoder->tsc_slip = 0x100;
 
 
 
 
 
 231
 232	intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
 233	intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
 234	intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
 235	intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
 236	intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
 237
 238	return decoder;
 239}
 240
 241static void intel_pt_pop_blk(struct intel_pt_stack *stack)
 242{
 243	struct intel_pt_blk *blk = stack->blk;
 244
 245	stack->blk = blk->prev;
 246	if (!stack->spare)
 247		stack->spare = blk;
 248	else
 249		free(blk);
 250}
 251
 252static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
 253{
 254	if (!stack->pos) {
 255		if (!stack->blk)
 256			return 0;
 257		intel_pt_pop_blk(stack);
 258		if (!stack->blk)
 259			return 0;
 260		stack->pos = INTEL_PT_BLK_SIZE;
 261	}
 262	return stack->blk->ip[--stack->pos];
 263}
 264
 265static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
 266{
 267	struct intel_pt_blk *blk;
 268
 269	if (stack->spare) {
 270		blk = stack->spare;
 271		stack->spare = NULL;
 272	} else {
 273		blk = malloc(sizeof(struct intel_pt_blk));
 274		if (!blk)
 275			return -ENOMEM;
 276	}
 277
 278	blk->prev = stack->blk;
 279	stack->blk = blk;
 280	stack->pos = 0;
 281	return 0;
 282}
 283
 284static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
 285{
 286	int err;
 287
 288	if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
 289		err = intel_pt_alloc_blk(stack);
 290		if (err)
 291			return err;
 292	}
 293
 294	stack->blk->ip[stack->pos++] = ip;
 295	return 0;
 296}
 297
 298static void intel_pt_clear_stack(struct intel_pt_stack *stack)
 299{
 300	while (stack->blk)
 301		intel_pt_pop_blk(stack);
 302	stack->pos = 0;
 303}
 304
 305static void intel_pt_free_stack(struct intel_pt_stack *stack)
 306{
 307	intel_pt_clear_stack(stack);
 308	zfree(&stack->blk);
 309	zfree(&stack->spare);
 310}
 311
 312void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
 313{
 314	intel_pt_free_stack(&decoder->stack);
 315	free(decoder);
 316}
 317
 318static int intel_pt_ext_err(int code)
 319{
 320	switch (code) {
 321	case -ENOMEM:
 322		return INTEL_PT_ERR_NOMEM;
 323	case -ENOSYS:
 324		return INTEL_PT_ERR_INTERN;
 325	case -EBADMSG:
 326		return INTEL_PT_ERR_BADPKT;
 327	case -ENODATA:
 328		return INTEL_PT_ERR_NODATA;
 329	case -EILSEQ:
 330		return INTEL_PT_ERR_NOINSN;
 331	case -ENOENT:
 332		return INTEL_PT_ERR_MISMAT;
 333	case -EOVERFLOW:
 334		return INTEL_PT_ERR_OVR;
 335	case -ENOSPC:
 336		return INTEL_PT_ERR_LOST;
 337	case -ELOOP:
 338		return INTEL_PT_ERR_NELOOP;
 339	default:
 340		return INTEL_PT_ERR_UNK;
 341	}
 342}
 343
 344static const char *intel_pt_err_msgs[] = {
 345	[INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
 346	[INTEL_PT_ERR_INTERN] = "Internal error",
 347	[INTEL_PT_ERR_BADPKT] = "Bad packet",
 348	[INTEL_PT_ERR_NODATA] = "No more data",
 349	[INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
 350	[INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
 351	[INTEL_PT_ERR_OVR]    = "Overflow packet",
 352	[INTEL_PT_ERR_LOST]   = "Lost trace data",
 353	[INTEL_PT_ERR_UNK]    = "Unknown error!",
 354	[INTEL_PT_ERR_NELOOP] = "Never-ending loop",
 355};
 356
 357int intel_pt__strerror(int code, char *buf, size_t buflen)
 358{
 359	if (code < 1 || code > INTEL_PT_ERR_MAX)
 360		code = INTEL_PT_ERR_UNK;
 361	strlcpy(buf, intel_pt_err_msgs[code], buflen);
 362	return 0;
 363}
 364
 365static uint64_t intel_pt_calc_ip(struct intel_pt_decoder *decoder,
 366				 const struct intel_pt_pkt *packet,
 367				 uint64_t last_ip)
 368{
 369	uint64_t ip;
 370
 371	switch (packet->count) {
 372	case 2:
 373		ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
 374		     packet->payload;
 375		break;
 376	case 4:
 377		ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
 378		     packet->payload;
 379		break;
 
 
 
 
 
 
 
 
 
 
 380	case 6:
 381		ip = packet->payload;
 382		break;
 383	default:
 384		return 0;
 385	}
 386
 387	if (ip & decoder->sign_bit)
 388		return ip | decoder->sign_bits;
 389
 390	return ip;
 391}
 392
 393static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
 394{
 395	decoder->last_ip = intel_pt_calc_ip(decoder, &decoder->packet,
 396					    decoder->last_ip);
 397}
 398
 399static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
 400{
 401	intel_pt_set_last_ip(decoder);
 402	decoder->ip = decoder->last_ip;
 403}
 404
 405static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
 406{
 407	intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
 408			    decoder->buf);
 409}
 410
 411static int intel_pt_bug(struct intel_pt_decoder *decoder)
 412{
 413	intel_pt_log("ERROR: Internal error\n");
 414	decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 415	return -ENOSYS;
 416}
 417
 418static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
 419{
 420	decoder->tx_flags = 0;
 421}
 422
 423static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
 424{
 425	decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
 426}
 427
 428static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
 429{
 430	intel_pt_clear_tx_flags(decoder);
 431	decoder->have_tma = false;
 432	decoder->pkt_len = 1;
 433	decoder->pkt_step = 1;
 434	intel_pt_decoder_log_packet(decoder);
 435	if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
 436		intel_pt_log("ERROR: Bad packet\n");
 437		decoder->pkt_state = INTEL_PT_STATE_ERR1;
 438	}
 439	return -EBADMSG;
 440}
 441
 442static int intel_pt_get_data(struct intel_pt_decoder *decoder)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 443{
 444	struct intel_pt_buffer buffer = { .buf = 0, };
 445	int ret;
 446
 447	decoder->pkt_step = 0;
 448
 449	intel_pt_log("Getting more data\n");
 450	ret = decoder->get_trace(&buffer, decoder->data);
 451	if (ret)
 452		return ret;
 453	decoder->buf = buffer.buf;
 454	decoder->len = buffer.len;
 455	if (!decoder->len) {
 456		intel_pt_log("No more data\n");
 457		return -ENODATA;
 458	}
 459	if (!buffer.consecutive) {
 460		decoder->ip = 0;
 461		decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 462		decoder->ref_timestamp = buffer.ref_timestamp;
 463		decoder->timestamp = 0;
 464		decoder->have_tma = false;
 465		decoder->state.trace_nr = buffer.trace_nr;
 466		intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
 467			     decoder->ref_timestamp);
 468		return -ENOLINK;
 469	}
 470
 471	return 0;
 472}
 473
 474static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
 
 475{
 476	if (!decoder->next_buf)
 477		return intel_pt_get_data(decoder);
 478
 479	decoder->buf = decoder->next_buf;
 480	decoder->len = decoder->next_len;
 481	decoder->next_buf = 0;
 482	decoder->next_len = 0;
 483	return 0;
 484}
 485
 486static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
 487{
 488	unsigned char *buf = decoder->temp_buf;
 489	size_t old_len, len, n;
 490	int ret;
 491
 492	old_len = decoder->len;
 493	len = decoder->len;
 494	memcpy(buf, decoder->buf, len);
 495
 496	ret = intel_pt_get_data(decoder);
 497	if (ret) {
 498		decoder->pos += old_len;
 499		return ret < 0 ? ret : -EINVAL;
 500	}
 501
 502	n = INTEL_PT_PKT_MAX_SZ - len;
 503	if (n > decoder->len)
 504		n = decoder->len;
 505	memcpy(buf + len, decoder->buf, n);
 506	len += n;
 507
 508	ret = intel_pt_get_packet(buf, len, &decoder->packet);
 
 509	if (ret < (int)old_len) {
 510		decoder->next_buf = decoder->buf;
 511		decoder->next_len = decoder->len;
 512		decoder->buf = buf;
 513		decoder->len = old_len;
 514		return intel_pt_bad_packet(decoder);
 515	}
 516
 517	decoder->next_buf = decoder->buf + (ret - old_len);
 518	decoder->next_len = decoder->len - (ret - old_len);
 519
 520	decoder->buf = buf;
 521	decoder->len = ret;
 522
 523	return ret;
 524}
 525
 526struct intel_pt_pkt_info {
 527	struct intel_pt_decoder	  *decoder;
 528	struct intel_pt_pkt       packet;
 529	uint64_t                  pos;
 530	int                       pkt_len;
 531	int                       last_packet_type;
 532	void                      *data;
 533};
 534
 535typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
 536
 537/* Lookahead packets in current buffer */
 538static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
 539				  intel_pt_pkt_cb_t cb, void *data)
 540{
 541	struct intel_pt_pkt_info pkt_info;
 542	const unsigned char *buf = decoder->buf;
 
 543	size_t len = decoder->len;
 544	int ret;
 545
 546	pkt_info.decoder          = decoder;
 547	pkt_info.pos              = decoder->pos;
 548	pkt_info.pkt_len          = decoder->pkt_step;
 549	pkt_info.last_packet_type = decoder->last_packet_type;
 550	pkt_info.data             = data;
 551
 552	while (1) {
 553		do {
 554			pkt_info.pos += pkt_info.pkt_len;
 555			buf          += pkt_info.pkt_len;
 556			len          -= pkt_info.pkt_len;
 557
 558			if (!len)
 559				return INTEL_PT_NEED_MORE_BYTES;
 560
 561			ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
 
 562			if (!ret)
 563				return INTEL_PT_NEED_MORE_BYTES;
 564			if (ret < 0)
 565				return ret;
 566
 567			pkt_info.pkt_len = ret;
 568		} while (pkt_info.packet.type == INTEL_PT_PAD);
 569
 570		ret = cb(&pkt_info);
 571		if (ret)
 572			return 0;
 573
 574		pkt_info.last_packet_type = pkt_info.packet.type;
 575	}
 576}
 577
 578struct intel_pt_calc_cyc_to_tsc_info {
 579	uint64_t        cycle_cnt;
 580	unsigned int    cbr;
 581	uint32_t        last_mtc;
 582	uint64_t        ctc_timestamp;
 583	uint64_t        ctc_delta;
 584	uint64_t        tsc_timestamp;
 585	uint64_t        timestamp;
 586	bool            have_tma;
 
 587	bool            from_mtc;
 588	double          cbr_cyc_to_tsc;
 589};
 590
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 591static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
 592{
 593	struct intel_pt_decoder *decoder = pkt_info->decoder;
 594	struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
 595	uint64_t timestamp;
 596	double cyc_to_tsc;
 597	unsigned int cbr;
 598	uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
 599
 600	switch (pkt_info->packet.type) {
 601	case INTEL_PT_TNT:
 602	case INTEL_PT_TIP_PGE:
 603	case INTEL_PT_TIP:
 604	case INTEL_PT_FUP:
 605	case INTEL_PT_PSB:
 606	case INTEL_PT_PIP:
 607	case INTEL_PT_MODE_EXEC:
 608	case INTEL_PT_MODE_TSX:
 609	case INTEL_PT_PSBEND:
 610	case INTEL_PT_PAD:
 611	case INTEL_PT_VMCS:
 612	case INTEL_PT_MNT:
 
 
 
 
 
 
 613		return 0;
 614
 615	case INTEL_PT_MTC:
 616		if (!data->have_tma)
 617			return 0;
 618
 619		mtc = pkt_info->packet.payload;
 
 
 
 
 
 620		if (mtc > data->last_mtc)
 621			mtc_delta = mtc - data->last_mtc;
 622		else
 623			mtc_delta = mtc + 256 - data->last_mtc;
 624		data->ctc_delta += mtc_delta << decoder->mtc_shift;
 625		data->last_mtc = mtc;
 626
 627		if (decoder->tsc_ctc_mult) {
 628			timestamp = data->ctc_timestamp +
 629				data->ctc_delta * decoder->tsc_ctc_mult;
 630		} else {
 631			timestamp = data->ctc_timestamp +
 632				multdiv(data->ctc_delta,
 633					decoder->tsc_ctc_ratio_n,
 634					decoder->tsc_ctc_ratio_d);
 635		}
 636
 637		if (timestamp < data->timestamp)
 638			return 1;
 639
 640		if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 641			data->timestamp = timestamp;
 642			return 0;
 643		}
 644
 645		break;
 646
 647	case INTEL_PT_TSC:
 
 
 
 
 
 
 648		timestamp = pkt_info->packet.payload |
 649			    (data->timestamp & (0xffULL << 56));
 650		if (data->from_mtc && timestamp < data->timestamp &&
 651		    data->timestamp - timestamp < decoder->tsc_slip)
 652			return 1;
 653		if (timestamp < data->timestamp)
 654			timestamp += (1ULL << 56);
 655		if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 656			if (data->from_mtc)
 657				return 1;
 658			data->tsc_timestamp = timestamp;
 659			data->timestamp = timestamp;
 660			return 0;
 661		}
 662		break;
 663
 664	case INTEL_PT_TMA:
 665		if (data->from_mtc)
 666			return 1;
 667
 668		if (!decoder->tsc_ctc_ratio_d)
 669			return 0;
 670
 671		ctc = pkt_info->packet.payload;
 672		fc = pkt_info->packet.count;
 673		ctc_rem = ctc & decoder->ctc_rem_mask;
 674
 675		data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
 676
 677		data->ctc_timestamp = data->tsc_timestamp - fc;
 678		if (decoder->tsc_ctc_mult) {
 679			data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
 680		} else {
 681			data->ctc_timestamp -=
 682				multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
 683					decoder->tsc_ctc_ratio_d);
 684		}
 685
 686		data->ctc_delta = 0;
 687		data->have_tma = true;
 
 688
 689		return 0;
 690
 691	case INTEL_PT_CYC:
 692		data->cycle_cnt += pkt_info->packet.payload;
 693		return 0;
 694
 695	case INTEL_PT_CBR:
 696		cbr = pkt_info->packet.payload;
 697		if (data->cbr && data->cbr != cbr)
 698			return 1;
 699		data->cbr = cbr;
 700		data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
 701		return 0;
 702
 703	case INTEL_PT_TIP_PGD:
 704	case INTEL_PT_TRACESTOP:
 
 
 
 
 
 705	case INTEL_PT_OVF:
 706	case INTEL_PT_BAD: /* Does not happen */
 707	default:
 708		return 1;
 709	}
 710
 711	if (!data->cbr && decoder->cbr) {
 712		data->cbr = decoder->cbr;
 713		data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
 714	}
 715
 716	if (!data->cycle_cnt)
 717		return 1;
 718
 719	cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
 720
 721	if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
 722	    cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
 723		intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
 724			     cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 725		return 1;
 726	}
 727
 728	decoder->calc_cyc_to_tsc = cyc_to_tsc;
 729	decoder->have_calc_cyc_to_tsc = true;
 730
 731	if (data->cbr) {
 732		intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
 733			     cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 734	} else {
 735		intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
 736			     cyc_to_tsc, pkt_info->pos);
 737	}
 738
 739	return 1;
 740}
 741
 742static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
 743				     bool from_mtc)
 744{
 745	struct intel_pt_calc_cyc_to_tsc_info data = {
 746		.cycle_cnt      = 0,
 747		.cbr            = 0,
 748		.last_mtc       = decoder->last_mtc,
 749		.ctc_timestamp  = decoder->ctc_timestamp,
 750		.ctc_delta      = decoder->ctc_delta,
 751		.tsc_timestamp  = decoder->tsc_timestamp,
 752		.timestamp      = decoder->timestamp,
 753		.have_tma       = decoder->have_tma,
 
 754		.from_mtc       = from_mtc,
 755		.cbr_cyc_to_tsc = 0,
 756	};
 757
 
 
 
 
 
 
 
 
 758	intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
 759}
 760
 761static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
 762{
 763	int ret;
 764
 765	decoder->last_packet_type = decoder->packet.type;
 766
 767	do {
 768		decoder->pos += decoder->pkt_step;
 769		decoder->buf += decoder->pkt_step;
 770		decoder->len -= decoder->pkt_step;
 771
 772		if (!decoder->len) {
 773			ret = intel_pt_get_next_data(decoder);
 774			if (ret)
 775				return ret;
 776		}
 777
 
 778		ret = intel_pt_get_packet(decoder->buf, decoder->len,
 779					  &decoder->packet);
 780		if (ret == INTEL_PT_NEED_MORE_BYTES &&
 781		    decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
 782			ret = intel_pt_get_split_packet(decoder);
 783			if (ret < 0)
 784				return ret;
 785		}
 786		if (ret <= 0)
 787			return intel_pt_bad_packet(decoder);
 788
 789		decoder->pkt_len = ret;
 790		decoder->pkt_step = ret;
 791		intel_pt_decoder_log_packet(decoder);
 792	} while (decoder->packet.type == INTEL_PT_PAD);
 793
 794	return 0;
 795}
 796
 797static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
 798{
 799	uint64_t timestamp, masked_timestamp;
 800
 801	timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 802	masked_timestamp = timestamp & decoder->period_mask;
 803	if (decoder->continuous_period) {
 804		if (masked_timestamp != decoder->last_masked_timestamp)
 805			return 1;
 806	} else {
 807		timestamp += 1;
 808		masked_timestamp = timestamp & decoder->period_mask;
 809		if (masked_timestamp != decoder->last_masked_timestamp) {
 810			decoder->last_masked_timestamp = masked_timestamp;
 811			decoder->continuous_period = true;
 812		}
 813	}
 
 
 
 
 814	return decoder->period_ticks - (timestamp - masked_timestamp);
 815}
 816
 817static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
 818{
 819	switch (decoder->period_type) {
 820	case INTEL_PT_PERIOD_INSTRUCTIONS:
 821		return decoder->period - decoder->period_insn_cnt;
 822	case INTEL_PT_PERIOD_TICKS:
 823		return intel_pt_next_period(decoder);
 824	case INTEL_PT_PERIOD_NONE:
 825	case INTEL_PT_PERIOD_MTC:
 826	default:
 827		return 0;
 828	}
 829}
 830
 831static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
 832{
 833	uint64_t timestamp, masked_timestamp;
 834
 835	switch (decoder->period_type) {
 836	case INTEL_PT_PERIOD_INSTRUCTIONS:
 837		decoder->period_insn_cnt = 0;
 838		break;
 839	case INTEL_PT_PERIOD_TICKS:
 840		timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 841		masked_timestamp = timestamp & decoder->period_mask;
 842		decoder->last_masked_timestamp = masked_timestamp;
 
 
 
 843		break;
 844	case INTEL_PT_PERIOD_NONE:
 845	case INTEL_PT_PERIOD_MTC:
 846	default:
 847		break;
 848	}
 849
 850	decoder->state.type |= INTEL_PT_INSTRUCTION;
 851}
 852
 853static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
 854			      struct intel_pt_insn *intel_pt_insn, uint64_t ip)
 855{
 856	uint64_t max_insn_cnt, insn_cnt = 0;
 857	int err;
 858
 859	if (!decoder->mtc_insn)
 860		decoder->mtc_insn = true;
 861
 862	max_insn_cnt = intel_pt_next_sample(decoder);
 863
 864	err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
 865				 max_insn_cnt, decoder->data);
 866
 867	decoder->tot_insn_cnt += insn_cnt;
 868	decoder->timestamp_insn_cnt += insn_cnt;
 
 869	decoder->period_insn_cnt += insn_cnt;
 870
 871	if (err) {
 872		decoder->no_progress = 0;
 873		decoder->pkt_state = INTEL_PT_STATE_ERR2;
 874		intel_pt_log_at("ERROR: Failed to get instruction",
 875				decoder->ip);
 876		if (err == -ENOENT)
 877			return -ENOLINK;
 878		return -EILSEQ;
 879	}
 880
 881	if (ip && decoder->ip == ip) {
 882		err = -EAGAIN;
 883		goto out;
 884	}
 885
 886	if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 887		intel_pt_sample_insn(decoder);
 888
 889	if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
 890		decoder->state.type = INTEL_PT_INSTRUCTION;
 891		decoder->state.from_ip = decoder->ip;
 892		decoder->state.to_ip = 0;
 893		decoder->ip += intel_pt_insn->length;
 894		err = INTEL_PT_RETURN;
 895		goto out;
 896	}
 897
 898	if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
 899		/* Zero-length calls are excluded */
 900		if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
 901		    intel_pt_insn->rel) {
 902			err = intel_pt_push(&decoder->stack, decoder->ip +
 903					    intel_pt_insn->length);
 904			if (err)
 905				goto out;
 906		}
 907	} else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
 908		decoder->ret_addr = intel_pt_pop(&decoder->stack);
 909	}
 910
 911	if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
 912		int cnt = decoder->no_progress++;
 913
 914		decoder->state.from_ip = decoder->ip;
 915		decoder->ip += intel_pt_insn->length +
 916				intel_pt_insn->rel;
 917		decoder->state.to_ip = decoder->ip;
 918		err = INTEL_PT_RETURN;
 919
 920		/*
 921		 * Check for being stuck in a loop.  This can happen if a
 922		 * decoder error results in the decoder erroneously setting the
 923		 * ip to an address that is itself in an infinite loop that
 924		 * consumes no packets.  When that happens, there must be an
 925		 * unconditional branch.
 926		 */
 927		if (cnt) {
 928			if (cnt == 1) {
 929				decoder->stuck_ip = decoder->state.to_ip;
 930				decoder->stuck_ip_prd = 1;
 931				decoder->stuck_ip_cnt = 1;
 932			} else if (cnt > INTEL_PT_MAX_LOOPS ||
 933				   decoder->state.to_ip == decoder->stuck_ip) {
 934				intel_pt_log_at("ERROR: Never-ending loop",
 935						decoder->state.to_ip);
 936				decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
 937				err = -ELOOP;
 938				goto out;
 939			} else if (!--decoder->stuck_ip_cnt) {
 940				decoder->stuck_ip_prd += 1;
 941				decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
 942				decoder->stuck_ip = decoder->state.to_ip;
 943			}
 944		}
 945		goto out_no_progress;
 946	}
 947out:
 948	decoder->no_progress = 0;
 949out_no_progress:
 950	decoder->state.insn_op = intel_pt_insn->op;
 951	decoder->state.insn_len = intel_pt_insn->length;
 
 
 952
 953	if (decoder->tx_flags & INTEL_PT_IN_TX)
 954		decoder->state.flags |= INTEL_PT_IN_TX;
 955
 956	return err;
 957}
 958
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 959static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
 960{
 961	struct intel_pt_insn intel_pt_insn;
 962	uint64_t ip;
 963	int err;
 964
 965	ip = decoder->last_ip;
 966
 967	while (1) {
 968		err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
 969		if (err == INTEL_PT_RETURN)
 970			return 0;
 971		if (err == -EAGAIN) {
 972			if (decoder->set_fup_tx_flags) {
 973				decoder->set_fup_tx_flags = false;
 974				decoder->tx_flags = decoder->fup_tx_flags;
 975				decoder->state.type = INTEL_PT_TRANSACTION;
 976				decoder->state.from_ip = decoder->ip;
 977				decoder->state.to_ip = 0;
 978				decoder->state.flags = decoder->fup_tx_flags;
 979				return 0;
 980			}
 981			return err;
 982		}
 983		decoder->set_fup_tx_flags = false;
 984		if (err)
 985			return err;
 986
 987		if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
 988			intel_pt_log_at("ERROR: Unexpected indirect branch",
 989					decoder->ip);
 990			decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
 991			return -ENOENT;
 992		}
 993
 994		if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
 995			intel_pt_log_at("ERROR: Unexpected conditional branch",
 996					decoder->ip);
 997			decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
 998			return -ENOENT;
 999		}
1000
1001		intel_pt_bug(decoder);
1002	}
1003}
1004
1005static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1006{
1007	struct intel_pt_insn intel_pt_insn;
1008	int err;
1009
1010	err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
 
 
 
 
 
 
 
 
 
 
 
 
 
1011	if (err == INTEL_PT_RETURN)
1012		return 0;
1013	if (err)
1014		return err;
1015
1016	if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1017		if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1018			decoder->pge = false;
1019			decoder->continuous_period = false;
1020			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1021			decoder->state.from_ip = decoder->ip;
1022			decoder->state.to_ip = 0;
1023			if (decoder->packet.count != 0)
 
 
1024				decoder->ip = decoder->last_ip;
 
 
1025		} else {
1026			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1027			decoder->state.from_ip = decoder->ip;
1028			if (decoder->packet.count == 0) {
1029				decoder->state.to_ip = 0;
1030			} else {
1031				decoder->state.to_ip = decoder->last_ip;
1032				decoder->ip = decoder->last_ip;
1033			}
1034		}
1035		return 0;
1036	}
1037
1038	if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1039		intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1040				decoder->ip);
1041		decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1042		return -ENOENT;
1043	}
1044
1045	return intel_pt_bug(decoder);
1046}
1047
1048static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1049{
1050	struct intel_pt_insn intel_pt_insn;
1051	int err;
1052
1053	while (1) {
1054		err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1055		if (err == INTEL_PT_RETURN)
1056			return 0;
1057		if (err)
1058			return err;
1059
1060		if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1061			if (!decoder->return_compression) {
1062				intel_pt_log_at("ERROR: RET when expecting conditional branch",
1063						decoder->ip);
1064				decoder->pkt_state = INTEL_PT_STATE_ERR3;
1065				return -ENOENT;
1066			}
1067			if (!decoder->ret_addr) {
1068				intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1069						decoder->ip);
1070				decoder->pkt_state = INTEL_PT_STATE_ERR3;
1071				return -ENOENT;
1072			}
1073			if (!(decoder->tnt.payload & BIT63)) {
1074				intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1075						decoder->ip);
1076				decoder->pkt_state = INTEL_PT_STATE_ERR3;
1077				return -ENOENT;
1078			}
1079			decoder->tnt.count -= 1;
1080			if (!decoder->tnt.count)
 
 
1081				decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1082			decoder->tnt.payload <<= 1;
1083			decoder->state.from_ip = decoder->ip;
1084			decoder->ip = decoder->ret_addr;
1085			decoder->state.to_ip = decoder->ip;
1086			return 0;
1087		}
1088
1089		if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1090			/* Handle deferred TIPs */
1091			err = intel_pt_get_next_packet(decoder);
1092			if (err)
1093				return err;
1094			if (decoder->packet.type != INTEL_PT_TIP ||
1095			    decoder->packet.count == 0) {
1096				intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1097						decoder->ip);
1098				decoder->pkt_state = INTEL_PT_STATE_ERR3;
1099				decoder->pkt_step = 0;
1100				return -ENOENT;
1101			}
1102			intel_pt_set_last_ip(decoder);
1103			decoder->state.from_ip = decoder->ip;
1104			decoder->state.to_ip = decoder->last_ip;
1105			decoder->ip = decoder->last_ip;
1106			return 0;
1107		}
1108
1109		if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1110			decoder->tnt.count -= 1;
1111			if (!decoder->tnt.count)
 
 
1112				decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1113			if (decoder->tnt.payload & BIT63) {
1114				decoder->tnt.payload <<= 1;
1115				decoder->state.from_ip = decoder->ip;
1116				decoder->ip += intel_pt_insn.length +
1117					       intel_pt_insn.rel;
1118				decoder->state.to_ip = decoder->ip;
1119				return 0;
1120			}
1121			/* Instruction sample for a non-taken branch */
1122			if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1123				decoder->tnt.payload <<= 1;
1124				decoder->state.type = INTEL_PT_INSTRUCTION;
1125				decoder->state.from_ip = decoder->ip;
1126				decoder->state.to_ip = 0;
1127				decoder->ip += intel_pt_insn.length;
1128				return 0;
1129			}
 
1130			decoder->ip += intel_pt_insn.length;
1131			if (!decoder->tnt.count)
 
1132				return -EAGAIN;
 
1133			decoder->tnt.payload <<= 1;
1134			continue;
1135		}
1136
1137		return intel_pt_bug(decoder);
1138	}
1139}
1140
1141static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1142{
1143	unsigned int fup_tx_flags;
1144	int err;
1145
1146	fup_tx_flags = decoder->packet.payload &
1147		       (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1148	err = intel_pt_get_next_packet(decoder);
1149	if (err)
1150		return err;
1151	if (decoder->packet.type == INTEL_PT_FUP) {
1152		decoder->fup_tx_flags = fup_tx_flags;
1153		decoder->set_fup_tx_flags = true;
1154		if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1155			*no_tip = true;
1156	} else {
1157		intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1158				decoder->pos);
1159		intel_pt_update_in_tx(decoder);
1160	}
1161	return 0;
1162}
1163
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1164static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1165{
1166	uint64_t timestamp;
1167
1168	decoder->have_tma = false;
1169
1170	if (decoder->ref_timestamp) {
1171		timestamp = decoder->packet.payload |
1172			    (decoder->ref_timestamp & (0xffULL << 56));
1173		if (timestamp < decoder->ref_timestamp) {
1174			if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1175				timestamp += (1ULL << 56);
1176		} else {
1177			if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1178				timestamp -= (1ULL << 56);
1179		}
1180		decoder->tsc_timestamp = timestamp;
1181		decoder->timestamp = timestamp;
1182		decoder->ref_timestamp = 0;
1183		decoder->timestamp_insn_cnt = 0;
1184	} else if (decoder->timestamp) {
1185		timestamp = decoder->packet.payload |
1186			    (decoder->timestamp & (0xffULL << 56));
1187		decoder->tsc_timestamp = timestamp;
1188		if (timestamp < decoder->timestamp &&
1189		    decoder->timestamp - timestamp < decoder->tsc_slip) {
1190			intel_pt_log_to("Suppressing backwards timestamp",
1191					timestamp);
1192			timestamp = decoder->timestamp;
1193		}
1194		if (timestamp < decoder->timestamp) {
1195			intel_pt_log_to("Wraparound timestamp", timestamp);
1196			timestamp += (1ULL << 56);
1197			decoder->tsc_timestamp = timestamp;
1198		}
1199		decoder->timestamp = timestamp;
1200		decoder->timestamp_insn_cnt = 0;
1201	}
1202
1203	if (decoder->last_packet_type == INTEL_PT_CYC) {
1204		decoder->cyc_ref_timestamp = decoder->timestamp;
1205		decoder->cycle_cnt = 0;
1206		decoder->have_calc_cyc_to_tsc = false;
1207		intel_pt_calc_cyc_to_tsc(decoder, false);
1208	}
1209
1210	intel_pt_log_to("Setting timestamp", decoder->timestamp);
1211}
1212
1213static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1214{
1215	intel_pt_log("ERROR: Buffer overflow\n");
1216	intel_pt_clear_tx_flags(decoder);
1217	decoder->have_tma = false;
1218	decoder->cbr = 0;
1219	decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1220	decoder->overflow = true;
1221	return -EOVERFLOW;
1222}
1223
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1224static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1225{
1226	uint32_t ctc = decoder->packet.payload;
1227	uint32_t fc = decoder->packet.count;
1228	uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1229
1230	if (!decoder->tsc_ctc_ratio_d)
1231		return;
1232
 
 
 
 
 
1233	decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1234	decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1235	if (decoder->tsc_ctc_mult) {
1236		decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1237	} else {
1238		decoder->ctc_timestamp -= multdiv(ctc_rem,
1239						  decoder->tsc_ctc_ratio_n,
1240						  decoder->tsc_ctc_ratio_d);
1241	}
1242	decoder->ctc_delta = 0;
1243	decoder->have_tma = true;
 
1244	intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1245		     decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1246}
1247
1248static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1249{
1250	uint64_t timestamp;
1251	uint32_t mtc, mtc_delta;
1252
1253	if (!decoder->have_tma)
1254		return;
1255
1256	mtc = decoder->packet.payload;
1257
 
 
 
 
 
 
1258	if (mtc > decoder->last_mtc)
1259		mtc_delta = mtc - decoder->last_mtc;
1260	else
1261		mtc_delta = mtc + 256 - decoder->last_mtc;
1262
1263	decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1264
1265	if (decoder->tsc_ctc_mult) {
1266		timestamp = decoder->ctc_timestamp +
1267			    decoder->ctc_delta * decoder->tsc_ctc_mult;
1268	} else {
1269		timestamp = decoder->ctc_timestamp +
1270			    multdiv(decoder->ctc_delta,
1271				    decoder->tsc_ctc_ratio_n,
1272				    decoder->tsc_ctc_ratio_d);
1273	}
1274
1275	if (timestamp < decoder->timestamp)
1276		intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1277			     timestamp, decoder->timestamp);
1278	else
1279		decoder->timestamp = timestamp;
1280
 
 
1281	decoder->timestamp_insn_cnt = 0;
1282	decoder->last_mtc = mtc;
1283
1284	if (decoder->last_packet_type == INTEL_PT_CYC) {
1285		decoder->cyc_ref_timestamp = decoder->timestamp;
1286		decoder->cycle_cnt = 0;
1287		decoder->have_calc_cyc_to_tsc = false;
1288		intel_pt_calc_cyc_to_tsc(decoder, true);
1289	}
 
 
1290}
1291
1292static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1293{
1294	unsigned int cbr = decoder->packet.payload;
 
 
1295
1296	if (decoder->cbr == cbr)
1297		return;
1298
1299	decoder->cbr = cbr;
1300	decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
 
 
1301}
1302
1303static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1304{
1305	uint64_t timestamp = decoder->cyc_ref_timestamp;
1306
1307	decoder->have_cyc = true;
1308
1309	decoder->cycle_cnt += decoder->packet.payload;
 
 
 
1310
1311	if (!decoder->cyc_ref_timestamp)
1312		return;
1313
1314	if (decoder->have_calc_cyc_to_tsc)
1315		timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1316	else if (decoder->cbr)
1317		timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1318	else
1319		return;
1320
1321	if (timestamp < decoder->timestamp)
1322		intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1323			     timestamp, decoder->timestamp);
1324	else
1325		decoder->timestamp = timestamp;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1326}
1327
1328/* Walk PSB+ packets when already in sync. */
1329static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1330{
1331	int err;
1332
 
 
1333	while (1) {
1334		err = intel_pt_get_next_packet(decoder);
1335		if (err)
1336			return err;
1337
1338		switch (decoder->packet.type) {
1339		case INTEL_PT_PSBEND:
1340			return 0;
 
1341
1342		case INTEL_PT_TIP_PGD:
1343		case INTEL_PT_TIP_PGE:
1344		case INTEL_PT_TIP:
1345		case INTEL_PT_TNT:
1346		case INTEL_PT_TRACESTOP:
1347		case INTEL_PT_BAD:
1348		case INTEL_PT_PSB:
 
 
 
 
 
 
 
 
 
 
 
1349			decoder->have_tma = false;
1350			intel_pt_log("ERROR: Unexpected packet\n");
1351			return -EAGAIN;
 
1352
1353		case INTEL_PT_OVF:
1354			return intel_pt_overflow(decoder);
 
1355
1356		case INTEL_PT_TSC:
1357			intel_pt_calc_tsc_timestamp(decoder);
1358			break;
1359
1360		case INTEL_PT_TMA:
1361			intel_pt_calc_tma(decoder);
1362			break;
1363
1364		case INTEL_PT_CBR:
1365			intel_pt_calc_cbr(decoder);
1366			break;
1367
1368		case INTEL_PT_MODE_EXEC:
1369			decoder->exec_mode = decoder->packet.payload;
1370			break;
1371
1372		case INTEL_PT_PIP:
1373			decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1374			break;
1375
1376		case INTEL_PT_FUP:
1377			decoder->pge = true;
1378			intel_pt_set_last_ip(decoder);
 
1379			break;
1380
1381		case INTEL_PT_MODE_TSX:
1382			intel_pt_update_in_tx(decoder);
1383			break;
1384
1385		case INTEL_PT_MTC:
1386			intel_pt_calc_mtc_timestamp(decoder);
1387			if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1388				decoder->state.type |= INTEL_PT_INSTRUCTION;
1389			break;
1390
1391		case INTEL_PT_CYC:
1392		case INTEL_PT_VMCS:
1393		case INTEL_PT_MNT:
1394		case INTEL_PT_PAD:
1395		default:
1396			break;
1397		}
1398	}
 
 
 
 
1399}
1400
1401static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1402{
1403	int err;
1404
1405	if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1406		decoder->tx_flags = 0;
1407		decoder->state.flags &= ~INTEL_PT_IN_TX;
1408		decoder->state.flags |= INTEL_PT_ABORT_TX;
1409	} else {
1410		decoder->state.flags |= INTEL_PT_ASYNC;
1411	}
1412
1413	while (1) {
1414		err = intel_pt_get_next_packet(decoder);
1415		if (err)
1416			return err;
1417
1418		switch (decoder->packet.type) {
1419		case INTEL_PT_TNT:
1420		case INTEL_PT_FUP:
1421		case INTEL_PT_TRACESTOP:
1422		case INTEL_PT_PSB:
1423		case INTEL_PT_TSC:
1424		case INTEL_PT_TMA:
1425		case INTEL_PT_CBR:
1426		case INTEL_PT_MODE_TSX:
1427		case INTEL_PT_BAD:
1428		case INTEL_PT_PSBEND:
 
 
 
 
 
 
 
 
 
 
 
1429			intel_pt_log("ERROR: Missing TIP after FUP\n");
1430			decoder->pkt_state = INTEL_PT_STATE_ERR3;
 
1431			return -ENOENT;
1432
 
 
 
 
1433		case INTEL_PT_OVF:
1434			return intel_pt_overflow(decoder);
1435
1436		case INTEL_PT_TIP_PGD:
1437			decoder->state.from_ip = decoder->ip;
1438			decoder->state.to_ip = 0;
1439			if (decoder->packet.count != 0) {
 
1440				intel_pt_set_ip(decoder);
1441				intel_pt_log("Omitting PGD ip " x64_fmt "\n",
1442					     decoder->ip);
1443			}
1444			decoder->pge = false;
1445			decoder->continuous_period = false;
 
1446			return 0;
1447
1448		case INTEL_PT_TIP_PGE:
1449			decoder->pge = true;
1450			intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1451				     decoder->ip);
1452			decoder->state.from_ip = 0;
1453			if (decoder->packet.count == 0) {
1454				decoder->state.to_ip = 0;
1455			} else {
1456				intel_pt_set_ip(decoder);
1457				decoder->state.to_ip = decoder->ip;
1458			}
 
 
1459			return 0;
1460
1461		case INTEL_PT_TIP:
1462			decoder->state.from_ip = decoder->ip;
1463			if (decoder->packet.count == 0) {
1464				decoder->state.to_ip = 0;
1465			} else {
1466				intel_pt_set_ip(decoder);
1467				decoder->state.to_ip = decoder->ip;
1468			}
1469			return 0;
1470
1471		case INTEL_PT_PIP:
1472			decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1473			break;
1474
1475		case INTEL_PT_MTC:
1476			intel_pt_calc_mtc_timestamp(decoder);
1477			if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1478				decoder->state.type |= INTEL_PT_INSTRUCTION;
1479			break;
1480
1481		case INTEL_PT_CYC:
1482			intel_pt_calc_cyc_timestamp(decoder);
1483			break;
1484
1485		case INTEL_PT_MODE_EXEC:
1486			decoder->exec_mode = decoder->packet.payload;
1487			break;
1488
1489		case INTEL_PT_VMCS:
1490		case INTEL_PT_MNT:
1491		case INTEL_PT_PAD:
1492			break;
1493
1494		default:
1495			return intel_pt_bug(decoder);
1496		}
1497	}
1498}
1499
1500static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1501{
1502	bool no_tip = false;
1503	int err;
1504
1505	while (1) {
1506		err = intel_pt_get_next_packet(decoder);
1507		if (err)
1508			return err;
1509next:
1510		switch (decoder->packet.type) {
1511		case INTEL_PT_TNT:
1512			if (!decoder->packet.count)
1513				break;
1514			decoder->tnt = decoder->packet;
1515			decoder->pkt_state = INTEL_PT_STATE_TNT;
1516			err = intel_pt_walk_tnt(decoder);
1517			if (err == -EAGAIN)
1518				break;
1519			return err;
1520
1521		case INTEL_PT_TIP_PGD:
1522			if (decoder->packet.count != 0)
1523				intel_pt_set_last_ip(decoder);
1524			decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1525			return intel_pt_walk_tip(decoder);
1526
1527		case INTEL_PT_TIP_PGE: {
1528			decoder->pge = true;
 
1529			if (decoder->packet.count == 0) {
1530				intel_pt_log_at("Skipping zero TIP.PGE",
1531						decoder->pos);
1532				break;
1533			}
1534			intel_pt_set_ip(decoder);
1535			decoder->state.from_ip = 0;
1536			decoder->state.to_ip = decoder->ip;
 
1537			return 0;
1538		}
1539
1540		case INTEL_PT_OVF:
1541			return intel_pt_overflow(decoder);
1542
1543		case INTEL_PT_TIP:
1544			if (decoder->packet.count != 0)
1545				intel_pt_set_last_ip(decoder);
1546			decoder->pkt_state = INTEL_PT_STATE_TIP;
1547			return intel_pt_walk_tip(decoder);
1548
1549		case INTEL_PT_FUP:
1550			if (decoder->packet.count == 0) {
1551				intel_pt_log_at("Skipping zero FUP",
1552						decoder->pos);
1553				no_tip = false;
1554				break;
1555			}
1556			intel_pt_set_last_ip(decoder);
 
 
 
 
 
 
 
 
 
1557			err = intel_pt_walk_fup(decoder);
1558			if (err != -EAGAIN) {
1559				if (err)
1560					return err;
1561				if (no_tip)
1562					decoder->pkt_state =
1563						INTEL_PT_STATE_FUP_NO_TIP;
1564				else
1565					decoder->pkt_state = INTEL_PT_STATE_FUP;
1566				return 0;
1567			}
1568			if (no_tip) {
1569				no_tip = false;
1570				break;
1571			}
1572			return intel_pt_walk_fup_tip(decoder);
1573
1574		case INTEL_PT_TRACESTOP:
1575			decoder->pge = false;
1576			decoder->continuous_period = false;
1577			intel_pt_clear_tx_flags(decoder);
1578			decoder->have_tma = false;
1579			break;
1580
1581		case INTEL_PT_PSB:
 
 
1582			intel_pt_clear_stack(&decoder->stack);
1583			err = intel_pt_walk_psbend(decoder);
1584			if (err == -EAGAIN)
1585				goto next;
1586			if (err)
1587				return err;
 
 
 
 
 
 
 
1588			break;
1589
1590		case INTEL_PT_PIP:
1591			decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1592			break;
1593
1594		case INTEL_PT_MTC:
1595			intel_pt_calc_mtc_timestamp(decoder);
1596			if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1597				break;
1598			/*
1599			 * Ensure that there has been an instruction since the
1600			 * last MTC.
1601			 */
1602			if (!decoder->mtc_insn)
1603				break;
1604			decoder->mtc_insn = false;
1605			/* Ensure that there is a timestamp */
1606			if (!decoder->timestamp)
1607				break;
1608			decoder->state.type = INTEL_PT_INSTRUCTION;
1609			decoder->state.from_ip = decoder->ip;
1610			decoder->state.to_ip = 0;
1611			decoder->mtc_insn = false;
1612			return 0;
1613
1614		case INTEL_PT_TSC:
1615			intel_pt_calc_tsc_timestamp(decoder);
1616			break;
1617
1618		case INTEL_PT_TMA:
1619			intel_pt_calc_tma(decoder);
1620			break;
1621
1622		case INTEL_PT_CYC:
1623			intel_pt_calc_cyc_timestamp(decoder);
1624			break;
1625
1626		case INTEL_PT_CBR:
1627			intel_pt_calc_cbr(decoder);
 
 
1628			break;
1629
1630		case INTEL_PT_MODE_EXEC:
1631			decoder->exec_mode = decoder->packet.payload;
1632			break;
1633
1634		case INTEL_PT_MODE_TSX:
1635			/* MODE_TSX need not be followed by FUP */
1636			if (!decoder->pge) {
1637				intel_pt_update_in_tx(decoder);
1638				break;
1639			}
1640			err = intel_pt_mode_tsx(decoder, &no_tip);
1641			if (err)
1642				return err;
1643			goto next;
1644
1645		case INTEL_PT_BAD: /* Does not happen */
1646			return intel_pt_bug(decoder);
1647
1648		case INTEL_PT_PSBEND:
1649		case INTEL_PT_VMCS:
1650		case INTEL_PT_MNT:
1651		case INTEL_PT_PAD:
1652			break;
1653
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1654		default:
1655			return intel_pt_bug(decoder);
1656		}
1657	}
1658}
1659
 
 
 
 
 
 
 
1660/* Walk PSB+ packets to get in sync. */
1661static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
1662{
1663	int err;
1664
 
 
1665	while (1) {
1666		err = intel_pt_get_next_packet(decoder);
1667		if (err)
1668			return err;
1669
1670		switch (decoder->packet.type) {
1671		case INTEL_PT_TIP_PGD:
1672			decoder->continuous_period = false;
 
1673		case INTEL_PT_TIP_PGE:
1674		case INTEL_PT_TIP:
 
 
 
 
 
 
 
 
 
 
 
1675			intel_pt_log("ERROR: Unexpected packet\n");
1676			return -ENOENT;
 
1677
1678		case INTEL_PT_FUP:
1679			decoder->pge = true;
1680			if (decoder->last_ip || decoder->packet.count == 6 ||
1681			    decoder->packet.count == 0) {
1682				uint64_t current_ip = decoder->ip;
1683
1684				intel_pt_set_ip(decoder);
1685				if (current_ip)
1686					intel_pt_log_to("Setting IP",
1687							decoder->ip);
1688			}
1689			break;
1690
1691		case INTEL_PT_MTC:
1692			intel_pt_calc_mtc_timestamp(decoder);
1693			break;
1694
1695		case INTEL_PT_TSC:
1696			intel_pt_calc_tsc_timestamp(decoder);
1697			break;
1698
1699		case INTEL_PT_TMA:
1700			intel_pt_calc_tma(decoder);
1701			break;
1702
1703		case INTEL_PT_CYC:
1704			intel_pt_calc_cyc_timestamp(decoder);
1705			break;
1706
1707		case INTEL_PT_CBR:
1708			intel_pt_calc_cbr(decoder);
1709			break;
1710
1711		case INTEL_PT_PIP:
1712			decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1713			break;
1714
1715		case INTEL_PT_MODE_EXEC:
1716			decoder->exec_mode = decoder->packet.payload;
1717			break;
1718
1719		case INTEL_PT_MODE_TSX:
1720			intel_pt_update_in_tx(decoder);
1721			break;
1722
1723		case INTEL_PT_TRACESTOP:
1724			decoder->pge = false;
1725			decoder->continuous_period = false;
1726			intel_pt_clear_tx_flags(decoder);
 
 
1727		case INTEL_PT_TNT:
1728			decoder->have_tma = false;
1729			intel_pt_log("ERROR: Unexpected packet\n");
1730			if (decoder->ip)
1731				decoder->pkt_state = INTEL_PT_STATE_ERR4;
1732			else
1733				decoder->pkt_state = INTEL_PT_STATE_ERR3;
1734			return -ENOENT;
 
1735
1736		case INTEL_PT_BAD: /* Does not happen */
1737			return intel_pt_bug(decoder);
 
1738
1739		case INTEL_PT_OVF:
1740			return intel_pt_overflow(decoder);
 
1741
1742		case INTEL_PT_PSBEND:
1743			return 0;
 
1744
1745		case INTEL_PT_PSB:
1746		case INTEL_PT_VMCS:
1747		case INTEL_PT_MNT:
1748		case INTEL_PT_PAD:
1749		default:
1750			break;
1751		}
1752	}
 
 
 
 
1753}
1754
1755static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
1756{
1757	int err;
1758
1759	while (1) {
1760		err = intel_pt_get_next_packet(decoder);
1761		if (err)
1762			return err;
1763
1764		switch (decoder->packet.type) {
1765		case INTEL_PT_TIP_PGD:
1766			decoder->continuous_period = false;
 
 
 
 
 
 
 
 
1767		case INTEL_PT_TIP_PGE:
 
 
 
 
 
 
 
 
 
1768		case INTEL_PT_TIP:
1769			decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
1770			if (decoder->last_ip || decoder->packet.count == 6 ||
1771			    decoder->packet.count == 0)
1772				intel_pt_set_ip(decoder);
1773			if (decoder->ip)
1774				return 0;
1775			break;
1776
1777		case INTEL_PT_FUP:
1778			if (decoder->overflow) {
1779				if (decoder->last_ip ||
1780				    decoder->packet.count == 6 ||
1781				    decoder->packet.count == 0)
1782					intel_pt_set_ip(decoder);
1783				if (decoder->ip)
1784					return 0;
1785			}
1786			if (decoder->packet.count)
1787				intel_pt_set_last_ip(decoder);
1788			break;
1789
1790		case INTEL_PT_MTC:
1791			intel_pt_calc_mtc_timestamp(decoder);
1792			break;
1793
1794		case INTEL_PT_TSC:
1795			intel_pt_calc_tsc_timestamp(decoder);
1796			break;
1797
1798		case INTEL_PT_TMA:
1799			intel_pt_calc_tma(decoder);
1800			break;
1801
1802		case INTEL_PT_CYC:
1803			intel_pt_calc_cyc_timestamp(decoder);
1804			break;
1805
1806		case INTEL_PT_CBR:
1807			intel_pt_calc_cbr(decoder);
1808			break;
1809
1810		case INTEL_PT_PIP:
1811			decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1812			break;
1813
1814		case INTEL_PT_MODE_EXEC:
1815			decoder->exec_mode = decoder->packet.payload;
1816			break;
1817
1818		case INTEL_PT_MODE_TSX:
1819			intel_pt_update_in_tx(decoder);
1820			break;
1821
1822		case INTEL_PT_OVF:
1823			return intel_pt_overflow(decoder);
1824
1825		case INTEL_PT_BAD: /* Does not happen */
1826			return intel_pt_bug(decoder);
1827
1828		case INTEL_PT_TRACESTOP:
1829			decoder->pge = false;
1830			decoder->continuous_period = false;
1831			intel_pt_clear_tx_flags(decoder);
1832			decoder->have_tma = false;
1833			break;
1834
1835		case INTEL_PT_PSB:
 
 
 
1836			err = intel_pt_walk_psb(decoder);
1837			if (err)
1838				return err;
1839			if (decoder->ip) {
1840				/* Do not have a sample */
1841				decoder->state.type = 0;
1842				return 0;
1843			}
1844			break;
1845
1846		case INTEL_PT_TNT:
1847		case INTEL_PT_PSBEND:
1848		case INTEL_PT_VMCS:
1849		case INTEL_PT_MNT:
1850		case INTEL_PT_PAD:
 
 
 
 
 
 
 
 
 
 
 
1851		default:
1852			break;
1853		}
1854	}
1855}
1856
1857static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
1858{
1859	int err;
1860
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1861	intel_pt_log("Scanning for full IP\n");
1862	err = intel_pt_walk_to_ip(decoder);
1863	if (err)
1864		return err;
1865
1866	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1867	decoder->overflow = false;
1868
1869	decoder->state.from_ip = 0;
1870	decoder->state.to_ip = decoder->ip;
1871	intel_pt_log_to("Setting IP", decoder->ip);
1872
1873	return 0;
1874}
1875
1876static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
1877{
1878	const unsigned char *end = decoder->buf + decoder->len;
1879	size_t i;
1880
1881	for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
1882		if (i > decoder->len)
1883			continue;
1884		if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
1885			return i;
1886	}
1887	return 0;
1888}
1889
1890static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
1891{
1892	size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
1893	const char *psb = INTEL_PT_PSB_STR;
1894
1895	if (rest_psb > decoder->len ||
1896	    memcmp(decoder->buf, psb + part_psb, rest_psb))
1897		return 0;
1898
1899	return rest_psb;
1900}
1901
1902static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
1903				  int part_psb)
1904{
1905	int rest_psb, ret;
1906
1907	decoder->pos += decoder->len;
1908	decoder->len = 0;
1909
1910	ret = intel_pt_get_next_data(decoder);
1911	if (ret)
1912		return ret;
1913
1914	rest_psb = intel_pt_rest_psb(decoder, part_psb);
1915	if (!rest_psb)
1916		return 0;
1917
1918	decoder->pos -= part_psb;
1919	decoder->next_buf = decoder->buf + rest_psb;
1920	decoder->next_len = decoder->len - rest_psb;
1921	memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
1922	decoder->buf = decoder->temp_buf;
1923	decoder->len = INTEL_PT_PSB_LEN;
1924
1925	return 0;
1926}
1927
1928static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
1929{
1930	unsigned char *next;
1931	int ret;
1932
1933	intel_pt_log("Scanning for PSB\n");
1934	while (1) {
1935		if (!decoder->len) {
1936			ret = intel_pt_get_next_data(decoder);
1937			if (ret)
1938				return ret;
1939		}
1940
1941		next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
1942			      INTEL_PT_PSB_LEN);
1943		if (!next) {
1944			int part_psb;
1945
1946			part_psb = intel_pt_part_psb(decoder);
1947			if (part_psb) {
1948				ret = intel_pt_get_split_psb(decoder, part_psb);
1949				if (ret)
1950					return ret;
1951			} else {
1952				decoder->pos += decoder->len;
1953				decoder->len = 0;
1954			}
1955			continue;
1956		}
1957
1958		decoder->pkt_step = next - decoder->buf;
1959		return intel_pt_get_next_packet(decoder);
1960	}
1961}
1962
1963static int intel_pt_sync(struct intel_pt_decoder *decoder)
1964{
1965	int err;
1966
1967	decoder->pge = false;
1968	decoder->continuous_period = false;
 
1969	decoder->last_ip = 0;
1970	decoder->ip = 0;
1971	intel_pt_clear_stack(&decoder->stack);
1972
1973	err = intel_pt_scan_for_psb(decoder);
1974	if (err)
1975		return err;
1976
 
1977	decoder->pkt_state = INTEL_PT_STATE_NO_IP;
1978
1979	err = intel_pt_walk_psb(decoder);
1980	if (err)
1981		return err;
1982
1983	if (decoder->ip) {
1984		decoder->state.type = 0; /* Do not have a sample */
1985		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1986	} else {
1987		return intel_pt_sync_ip(decoder);
1988	}
1989
1990	return 0;
1991}
1992
1993static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
1994{
1995	uint64_t est = decoder->timestamp_insn_cnt << 1;
1996
1997	if (!decoder->cbr || !decoder->max_non_turbo_ratio)
1998		goto out;
1999
2000	est *= decoder->max_non_turbo_ratio;
2001	est /= decoder->cbr;
2002out:
2003	return decoder->timestamp + est;
2004}
2005
2006const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2007{
2008	int err;
2009
2010	do {
2011		decoder->state.type = INTEL_PT_BRANCH;
2012		decoder->state.flags = 0;
2013
2014		switch (decoder->pkt_state) {
2015		case INTEL_PT_STATE_NO_PSB:
2016			err = intel_pt_sync(decoder);
2017			break;
2018		case INTEL_PT_STATE_NO_IP:
 
2019			decoder->last_ip = 0;
2020			/* Fall through */
 
2021		case INTEL_PT_STATE_ERR_RESYNC:
2022			err = intel_pt_sync_ip(decoder);
2023			break;
2024		case INTEL_PT_STATE_IN_SYNC:
2025			err = intel_pt_walk_trace(decoder);
2026			break;
2027		case INTEL_PT_STATE_TNT:
 
2028			err = intel_pt_walk_tnt(decoder);
2029			if (err == -EAGAIN)
2030				err = intel_pt_walk_trace(decoder);
2031			break;
2032		case INTEL_PT_STATE_TIP:
2033		case INTEL_PT_STATE_TIP_PGD:
2034			err = intel_pt_walk_tip(decoder);
2035			break;
2036		case INTEL_PT_STATE_FUP:
2037			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2038			err = intel_pt_walk_fup(decoder);
2039			if (err == -EAGAIN)
2040				err = intel_pt_walk_fup_tip(decoder);
2041			else if (!err)
2042				decoder->pkt_state = INTEL_PT_STATE_FUP;
2043			break;
2044		case INTEL_PT_STATE_FUP_NO_TIP:
2045			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2046			err = intel_pt_walk_fup(decoder);
2047			if (err == -EAGAIN)
2048				err = intel_pt_walk_trace(decoder);
2049			break;
2050		default:
2051			err = intel_pt_bug(decoder);
2052			break;
2053		}
2054	} while (err == -ENOLINK);
2055
2056	decoder->state.err = err ? intel_pt_ext_err(err) : 0;
2057	decoder->state.timestamp = decoder->timestamp;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2058	decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2059	decoder->state.cr3 = decoder->cr3;
2060	decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2061
2062	if (err)
2063		decoder->state.from_ip = decoder->ip;
2064
2065	return &decoder->state;
2066}
2067
2068static bool intel_pt_at_psb(unsigned char *buf, size_t len)
2069{
2070	if (len < INTEL_PT_PSB_LEN)
2071		return false;
2072	return memmem(buf, INTEL_PT_PSB_LEN, INTEL_PT_PSB_STR,
2073		      INTEL_PT_PSB_LEN);
2074}
2075
2076/**
2077 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2078 * @buf: pointer to buffer pointer
2079 * @len: size of buffer
2080 *
2081 * Updates the buffer pointer to point to the start of the next PSB packet if
2082 * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
2083 * @len is adjusted accordingly.
2084 *
2085 * Return: %true if a PSB packet is found, %false otherwise.
2086 */
2087static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2088{
2089	unsigned char *next;
2090
2091	next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2092	if (next) {
2093		*len -= next - *buf;
2094		*buf = next;
2095		return true;
2096	}
2097	return false;
2098}
2099
2100/**
2101 * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2102 *                     packet.
2103 * @buf: pointer to buffer pointer
2104 * @len: size of buffer
2105 *
2106 * Updates the buffer pointer to point to the start of the following PSB packet
2107 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2108 * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
2109 *
2110 * Return: %true if a PSB packet is found, %false otherwise.
2111 */
2112static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2113{
2114	unsigned char *next;
2115
2116	if (!*len)
2117		return false;
2118
2119	next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2120	if (next) {
2121		*len -= next - *buf;
2122		*buf = next;
2123		return true;
2124	}
2125	return false;
2126}
2127
2128/**
2129 * intel_pt_last_psb - find the last PSB packet in a buffer.
2130 * @buf: buffer
2131 * @len: size of buffer
2132 *
2133 * This function finds the last PSB in a buffer.
2134 *
2135 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2136 */
2137static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2138{
2139	const char *n = INTEL_PT_PSB_STR;
2140	unsigned char *p;
2141	size_t k;
2142
2143	if (len < INTEL_PT_PSB_LEN)
2144		return NULL;
2145
2146	k = len - INTEL_PT_PSB_LEN + 1;
2147	while (1) {
2148		p = memrchr(buf, n[0], k);
2149		if (!p)
2150			return NULL;
2151		if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2152			return p;
2153		k = p - buf;
2154		if (!k)
2155			return NULL;
2156	}
2157}
2158
2159/**
2160 * intel_pt_next_tsc - find and return next TSC.
2161 * @buf: buffer
2162 * @len: size of buffer
2163 * @tsc: TSC value returned
 
2164 *
2165 * Find a TSC packet in @buf and return the TSC value.  This function assumes
2166 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2167 * PSBEND packet is found.
2168 *
2169 * Return: %true if TSC is found, false otherwise.
2170 */
2171static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc)
 
2172{
 
2173	struct intel_pt_pkt packet;
2174	int ret;
2175
2176	while (len) {
2177		ret = intel_pt_get_packet(buf, len, &packet);
2178		if (ret <= 0)
2179			return false;
2180		if (packet.type == INTEL_PT_TSC) {
2181			*tsc = packet.payload;
 
2182			return true;
2183		}
2184		if (packet.type == INTEL_PT_PSBEND)
2185			return false;
2186		buf += ret;
2187		len -= ret;
2188	}
2189	return false;
2190}
2191
2192/**
2193 * intel_pt_tsc_cmp - compare 7-byte TSCs.
2194 * @tsc1: first TSC to compare
2195 * @tsc2: second TSC to compare
2196 *
2197 * This function compares 7-byte TSC values allowing for the possibility that
2198 * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
2199 * around so for that purpose this function assumes the absolute difference is
2200 * less than half the maximum difference.
2201 *
2202 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2203 * after @tsc2.
2204 */
2205static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2206{
2207	const uint64_t halfway = (1ULL << 55);
2208
2209	if (tsc1 == tsc2)
2210		return 0;
2211
2212	if (tsc1 < tsc2) {
2213		if (tsc2 - tsc1 < halfway)
2214			return -1;
2215		else
2216			return 1;
2217	} else {
2218		if (tsc1 - tsc2 < halfway)
2219			return 1;
2220		else
2221			return -1;
2222	}
2223}
2224
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2225/**
2226 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2227 *                             using TSC.
2228 * @buf_a: first buffer
2229 * @len_a: size of first buffer
2230 * @buf_b: second buffer
2231 * @len_b: size of second buffer
 
 
2232 *
2233 * If the trace contains TSC we can look at the last TSC of @buf_a and the
2234 * first TSC of @buf_b in order to determine if the buffers overlap, and then
2235 * walk forward in @buf_b until a later TSC is found.  A precondition is that
2236 * @buf_a and @buf_b are positioned at a PSB.
2237 *
2238 * Return: A pointer into @buf_b from where non-overlapped data starts, or
2239 * @buf_b + @len_b if there is no non-overlapped data.
2240 */
2241static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2242						size_t len_a,
2243						unsigned char *buf_b,
2244						size_t len_b)
2245{
2246	uint64_t tsc_a, tsc_b;
2247	unsigned char *p;
2248	size_t len;
2249
2250	p = intel_pt_last_psb(buf_a, len_a);
2251	if (!p)
2252		return buf_b; /* No PSB in buf_a => no overlap */
2253
2254	len = len_a - (p - buf_a);
2255	if (!intel_pt_next_tsc(p, len, &tsc_a)) {
2256		/* The last PSB+ in buf_a is incomplete, so go back one more */
2257		len_a -= len;
2258		p = intel_pt_last_psb(buf_a, len_a);
2259		if (!p)
2260			return buf_b; /* No full PSB+ => assume no overlap */
2261		len = len_a - (p - buf_a);
2262		if (!intel_pt_next_tsc(p, len, &tsc_a))
2263			return buf_b; /* No TSC in buf_a => assume no overlap */
2264	}
2265
2266	while (1) {
2267		/* Ignore PSB+ with no TSC */
2268		if (intel_pt_next_tsc(buf_b, len_b, &tsc_b) &&
2269		    intel_pt_tsc_cmp(tsc_a, tsc_b) < 0)
2270			return buf_b; /* tsc_a < tsc_b => no overlap */
 
 
 
 
 
 
 
 
 
 
 
2271
2272		if (!intel_pt_step_psb(&buf_b, &len_b))
2273			return buf_b + len_b; /* No PSB in buf_b => no data */
2274	}
2275}
2276
2277/**
2278 * intel_pt_find_overlap - determine start of non-overlapped trace data.
2279 * @buf_a: first buffer
2280 * @len_a: size of first buffer
2281 * @buf_b: second buffer
2282 * @len_b: size of second buffer
2283 * @have_tsc: can use TSC packets to detect overlap
 
 
2284 *
2285 * When trace samples or snapshots are recorded there is the possibility that
2286 * the data overlaps.  Note that, for the purposes of decoding, data is only
2287 * useful if it begins with a PSB packet.
2288 *
2289 * Return: A pointer into @buf_b from where non-overlapped data starts, or
2290 * @buf_b + @len_b if there is no non-overlapped data.
2291 */
2292unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2293				     unsigned char *buf_b, size_t len_b,
2294				     bool have_tsc)
2295{
2296	unsigned char *found;
2297
2298	/* Buffer 'b' must start at PSB so throw away everything before that */
2299	if (!intel_pt_next_psb(&buf_b, &len_b))
2300		return buf_b + len_b; /* No PSB */
2301
2302	if (!intel_pt_next_psb(&buf_a, &len_a))
2303		return buf_b; /* No overlap */
2304
2305	if (have_tsc) {
2306		found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b);
 
2307		if (found)
2308			return found;
2309	}
2310
2311	/*
2312	 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2313	 * we can ignore the first part of buffer 'a'.
2314	 */
2315	while (len_b < len_a) {
2316		if (!intel_pt_step_psb(&buf_a, &len_a))
2317			return buf_b; /* No overlap */
2318	}
2319
2320	/* Now len_b >= len_a */
2321	if (len_b > len_a) {
2322		/* The leftover buffer 'b' must start at a PSB */
2323		while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
2324			if (!intel_pt_step_psb(&buf_a, &len_a))
2325				return buf_b; /* No overlap */
2326		}
2327	}
2328
2329	while (1) {
2330		/* Potential overlap so check the bytes */
2331		found = memmem(buf_a, len_a, buf_b, len_a);
2332		if (found)
2333			return buf_b + len_a;
 
 
2334
2335		/* Try again at next PSB in buffer 'a' */
2336		if (!intel_pt_step_psb(&buf_a, &len_a))
2337			return buf_b; /* No overlap */
 
 
2338
2339		/* The leftover buffer 'b' must start at a PSB */
2340		while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
2341			if (!intel_pt_step_psb(&buf_a, &len_a))
2342				return buf_b; /* No overlap */
2343		}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2344	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2345}