Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Arm Statistical Profiling Extensions (SPE) support
4 * Copyright (c) 2017-2018, Arm Ltd.
5 */
6
7#include <byteswap.h>
8#include <endian.h>
9#include <errno.h>
10#include <inttypes.h>
11#include <linux/bitops.h>
12#include <linux/kernel.h>
13#include <linux/log2.h>
14#include <linux/types.h>
15#include <linux/zalloc.h>
16#include <stdlib.h>
17#include <unistd.h>
18
19#include "auxtrace.h"
20#include "color.h"
21#include "debug.h"
22#include "evlist.h"
23#include "evsel.h"
24#include "machine.h"
25#include "session.h"
26#include "symbol.h"
27#include "thread.h"
28#include "thread-stack.h"
29#include "tsc.h"
30#include "tool.h"
31#include "util/synthetic-events.h"
32
33#include "arm-spe.h"
34#include "arm-spe-decoder/arm-spe-decoder.h"
35#include "arm-spe-decoder/arm-spe-pkt-decoder.h"
36
37#include "../../arch/arm64/include/asm/cputype.h"
38#define MAX_TIMESTAMP (~0ULL)
39
40struct arm_spe {
41 struct auxtrace auxtrace;
42 struct auxtrace_queues queues;
43 struct auxtrace_heap heap;
44 struct itrace_synth_opts synth_opts;
45 u32 auxtrace_type;
46 struct perf_session *session;
47 struct machine *machine;
48 u32 pmu_type;
49 u64 midr;
50
51 struct perf_tsc_conversion tc;
52
53 u8 timeless_decoding;
54 u8 data_queued;
55
56 u64 sample_type;
57 u8 sample_flc;
58 u8 sample_llc;
59 u8 sample_tlb;
60 u8 sample_branch;
61 u8 sample_remote_access;
62 u8 sample_memory;
63 u8 sample_instructions;
64 u64 instructions_sample_period;
65
66 u64 l1d_miss_id;
67 u64 l1d_access_id;
68 u64 llc_miss_id;
69 u64 llc_access_id;
70 u64 tlb_miss_id;
71 u64 tlb_access_id;
72 u64 branch_miss_id;
73 u64 remote_access_id;
74 u64 memory_id;
75 u64 instructions_id;
76
77 u64 kernel_start;
78
79 unsigned long num_events;
80 u8 use_ctx_pkt_for_pid;
81};
82
83struct arm_spe_queue {
84 struct arm_spe *spe;
85 unsigned int queue_nr;
86 struct auxtrace_buffer *buffer;
87 struct auxtrace_buffer *old_buffer;
88 union perf_event *event_buf;
89 bool on_heap;
90 bool done;
91 pid_t pid;
92 pid_t tid;
93 int cpu;
94 struct arm_spe_decoder *decoder;
95 u64 time;
96 u64 timestamp;
97 struct thread *thread;
98 u64 period_instructions;
99};
100
101static void arm_spe_dump(struct arm_spe *spe __maybe_unused,
102 unsigned char *buf, size_t len)
103{
104 struct arm_spe_pkt packet;
105 size_t pos = 0;
106 int ret, pkt_len, i;
107 char desc[ARM_SPE_PKT_DESC_MAX];
108 const char *color = PERF_COLOR_BLUE;
109
110 color_fprintf(stdout, color,
111 ". ... ARM SPE data: size %#zx bytes\n",
112 len);
113
114 while (len) {
115 ret = arm_spe_get_packet(buf, len, &packet);
116 if (ret > 0)
117 pkt_len = ret;
118 else
119 pkt_len = 1;
120 printf(".");
121 color_fprintf(stdout, color, " %08x: ", pos);
122 for (i = 0; i < pkt_len; i++)
123 color_fprintf(stdout, color, " %02x", buf[i]);
124 for (; i < 16; i++)
125 color_fprintf(stdout, color, " ");
126 if (ret > 0) {
127 ret = arm_spe_pkt_desc(&packet, desc,
128 ARM_SPE_PKT_DESC_MAX);
129 if (!ret)
130 color_fprintf(stdout, color, " %s\n", desc);
131 } else {
132 color_fprintf(stdout, color, " Bad packet!\n");
133 }
134 pos += pkt_len;
135 buf += pkt_len;
136 len -= pkt_len;
137 }
138}
139
140static void arm_spe_dump_event(struct arm_spe *spe, unsigned char *buf,
141 size_t len)
142{
143 printf(".\n");
144 arm_spe_dump(spe, buf, len);
145}
146
147static int arm_spe_get_trace(struct arm_spe_buffer *b, void *data)
148{
149 struct arm_spe_queue *speq = data;
150 struct auxtrace_buffer *buffer = speq->buffer;
151 struct auxtrace_buffer *old_buffer = speq->old_buffer;
152 struct auxtrace_queue *queue;
153
154 queue = &speq->spe->queues.queue_array[speq->queue_nr];
155
156 buffer = auxtrace_buffer__next(queue, buffer);
157 /* If no more data, drop the previous auxtrace_buffer and return */
158 if (!buffer) {
159 if (old_buffer)
160 auxtrace_buffer__drop_data(old_buffer);
161 b->len = 0;
162 return 0;
163 }
164
165 speq->buffer = buffer;
166
167 /* If the aux_buffer doesn't have data associated, try to load it */
168 if (!buffer->data) {
169 /* get the file desc associated with the perf data file */
170 int fd = perf_data__fd(speq->spe->session->data);
171
172 buffer->data = auxtrace_buffer__get_data(buffer, fd);
173 if (!buffer->data)
174 return -ENOMEM;
175 }
176
177 b->len = buffer->size;
178 b->buf = buffer->data;
179
180 if (b->len) {
181 if (old_buffer)
182 auxtrace_buffer__drop_data(old_buffer);
183 speq->old_buffer = buffer;
184 } else {
185 auxtrace_buffer__drop_data(buffer);
186 return arm_spe_get_trace(b, data);
187 }
188
189 return 0;
190}
191
192static struct arm_spe_queue *arm_spe__alloc_queue(struct arm_spe *spe,
193 unsigned int queue_nr)
194{
195 struct arm_spe_params params = { .get_trace = 0, };
196 struct arm_spe_queue *speq;
197
198 speq = zalloc(sizeof(*speq));
199 if (!speq)
200 return NULL;
201
202 speq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
203 if (!speq->event_buf)
204 goto out_free;
205
206 speq->spe = spe;
207 speq->queue_nr = queue_nr;
208 speq->pid = -1;
209 speq->tid = -1;
210 speq->cpu = -1;
211 speq->period_instructions = 0;
212
213 /* params set */
214 params.get_trace = arm_spe_get_trace;
215 params.data = speq;
216
217 /* create new decoder */
218 speq->decoder = arm_spe_decoder_new(¶ms);
219 if (!speq->decoder)
220 goto out_free;
221
222 return speq;
223
224out_free:
225 zfree(&speq->event_buf);
226 free(speq);
227
228 return NULL;
229}
230
231static inline u8 arm_spe_cpumode(struct arm_spe *spe, u64 ip)
232{
233 return ip >= spe->kernel_start ?
234 PERF_RECORD_MISC_KERNEL :
235 PERF_RECORD_MISC_USER;
236}
237
238static void arm_spe_set_pid_tid_cpu(struct arm_spe *spe,
239 struct auxtrace_queue *queue)
240{
241 struct arm_spe_queue *speq = queue->priv;
242 pid_t tid;
243
244 tid = machine__get_current_tid(spe->machine, speq->cpu);
245 if (tid != -1) {
246 speq->tid = tid;
247 thread__zput(speq->thread);
248 } else
249 speq->tid = queue->tid;
250
251 if ((!speq->thread) && (speq->tid != -1)) {
252 speq->thread = machine__find_thread(spe->machine, -1,
253 speq->tid);
254 }
255
256 if (speq->thread) {
257 speq->pid = speq->thread->pid_;
258 if (queue->cpu == -1)
259 speq->cpu = speq->thread->cpu;
260 }
261}
262
263static int arm_spe_set_tid(struct arm_spe_queue *speq, pid_t tid)
264{
265 struct arm_spe *spe = speq->spe;
266 int err = machine__set_current_tid(spe->machine, speq->cpu, -1, tid);
267
268 if (err)
269 return err;
270
271 arm_spe_set_pid_tid_cpu(spe, &spe->queues.queue_array[speq->queue_nr]);
272
273 return 0;
274}
275
276static void arm_spe_prep_sample(struct arm_spe *spe,
277 struct arm_spe_queue *speq,
278 union perf_event *event,
279 struct perf_sample *sample)
280{
281 struct arm_spe_record *record = &speq->decoder->record;
282
283 if (!spe->timeless_decoding)
284 sample->time = tsc_to_perf_time(record->timestamp, &spe->tc);
285
286 sample->ip = record->from_ip;
287 sample->cpumode = arm_spe_cpumode(spe, sample->ip);
288 sample->pid = speq->pid;
289 sample->tid = speq->tid;
290 sample->period = 1;
291 sample->cpu = speq->cpu;
292
293 event->sample.header.type = PERF_RECORD_SAMPLE;
294 event->sample.header.misc = sample->cpumode;
295 event->sample.header.size = sizeof(struct perf_event_header);
296}
297
298static int arm_spe__inject_event(union perf_event *event, struct perf_sample *sample, u64 type)
299{
300 event->header.size = perf_event__sample_event_size(sample, type, 0);
301 return perf_event__synthesize_sample(event, type, 0, sample);
302}
303
304static inline int
305arm_spe_deliver_synth_event(struct arm_spe *spe,
306 struct arm_spe_queue *speq __maybe_unused,
307 union perf_event *event,
308 struct perf_sample *sample)
309{
310 int ret;
311
312 if (spe->synth_opts.inject) {
313 ret = arm_spe__inject_event(event, sample, spe->sample_type);
314 if (ret)
315 return ret;
316 }
317
318 ret = perf_session__deliver_synth_event(spe->session, event, sample);
319 if (ret)
320 pr_err("ARM SPE: failed to deliver event, error %d\n", ret);
321
322 return ret;
323}
324
325static int arm_spe__synth_mem_sample(struct arm_spe_queue *speq,
326 u64 spe_events_id, u64 data_src)
327{
328 struct arm_spe *spe = speq->spe;
329 struct arm_spe_record *record = &speq->decoder->record;
330 union perf_event *event = speq->event_buf;
331 struct perf_sample sample = { .ip = 0, };
332
333 arm_spe_prep_sample(spe, speq, event, &sample);
334
335 sample.id = spe_events_id;
336 sample.stream_id = spe_events_id;
337 sample.addr = record->virt_addr;
338 sample.phys_addr = record->phys_addr;
339 sample.data_src = data_src;
340 sample.weight = record->latency;
341
342 return arm_spe_deliver_synth_event(spe, speq, event, &sample);
343}
344
345static int arm_spe__synth_branch_sample(struct arm_spe_queue *speq,
346 u64 spe_events_id)
347{
348 struct arm_spe *spe = speq->spe;
349 struct arm_spe_record *record = &speq->decoder->record;
350 union perf_event *event = speq->event_buf;
351 struct perf_sample sample = { .ip = 0, };
352
353 arm_spe_prep_sample(spe, speq, event, &sample);
354
355 sample.id = spe_events_id;
356 sample.stream_id = spe_events_id;
357 sample.addr = record->to_ip;
358 sample.weight = record->latency;
359
360 return arm_spe_deliver_synth_event(spe, speq, event, &sample);
361}
362
363static int arm_spe__synth_instruction_sample(struct arm_spe_queue *speq,
364 u64 spe_events_id, u64 data_src)
365{
366 struct arm_spe *spe = speq->spe;
367 struct arm_spe_record *record = &speq->decoder->record;
368 union perf_event *event = speq->event_buf;
369 struct perf_sample sample = { .ip = 0, };
370
371 /*
372 * Handles perf instruction sampling period.
373 */
374 speq->period_instructions++;
375 if (speq->period_instructions < spe->instructions_sample_period)
376 return 0;
377 speq->period_instructions = 0;
378
379 arm_spe_prep_sample(spe, speq, event, &sample);
380
381 sample.id = spe_events_id;
382 sample.stream_id = spe_events_id;
383 sample.addr = record->virt_addr;
384 sample.phys_addr = record->phys_addr;
385 sample.data_src = data_src;
386 sample.period = spe->instructions_sample_period;
387 sample.weight = record->latency;
388
389 return arm_spe_deliver_synth_event(spe, speq, event, &sample);
390}
391
392static const struct midr_range neoverse_spe[] = {
393 MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1),
394 MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2),
395 MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1),
396 {},
397};
398
399static void arm_spe__synth_data_source_neoverse(const struct arm_spe_record *record,
400 union perf_mem_data_src *data_src)
401{
402 /*
403 * Even though four levels of cache hierarchy are possible, no known
404 * production Neoverse systems currently include more than three levels
405 * so for the time being we assume three exist. If a production system
406 * is built with four the this function would have to be changed to
407 * detect the number of levels for reporting.
408 */
409
410 /*
411 * We have no data on the hit level or data source for stores in the
412 * Neoverse SPE records.
413 */
414 if (record->op & ARM_SPE_ST) {
415 data_src->mem_lvl = PERF_MEM_LVL_NA;
416 data_src->mem_lvl_num = PERF_MEM_LVLNUM_NA;
417 data_src->mem_snoop = PERF_MEM_SNOOP_NA;
418 return;
419 }
420
421 switch (record->source) {
422 case ARM_SPE_NV_L1D:
423 data_src->mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT;
424 data_src->mem_lvl_num = PERF_MEM_LVLNUM_L1;
425 data_src->mem_snoop = PERF_MEM_SNOOP_NONE;
426 break;
427 case ARM_SPE_NV_L2:
428 data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT;
429 data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2;
430 data_src->mem_snoop = PERF_MEM_SNOOP_NONE;
431 break;
432 case ARM_SPE_NV_PEER_CORE:
433 data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT;
434 data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2;
435 data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER;
436 break;
437 /*
438 * We don't know if this is L1, L2 but we do know it was a cache-2-cache
439 * transfer, so set SNOOPX_PEER
440 */
441 case ARM_SPE_NV_LOCAL_CLUSTER:
442 case ARM_SPE_NV_PEER_CLUSTER:
443 data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT;
444 data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3;
445 data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER;
446 break;
447 /*
448 * System cache is assumed to be L3
449 */
450 case ARM_SPE_NV_SYS_CACHE:
451 data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT;
452 data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3;
453 data_src->mem_snoop = PERF_MEM_SNOOP_HIT;
454 break;
455 /*
456 * We don't know what level it hit in, except it came from the other
457 * socket
458 */
459 case ARM_SPE_NV_REMOTE:
460 data_src->mem_lvl = PERF_MEM_LVL_REM_CCE1;
461 data_src->mem_lvl_num = PERF_MEM_LVLNUM_ANY_CACHE;
462 data_src->mem_remote = PERF_MEM_REMOTE_REMOTE;
463 data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER;
464 break;
465 case ARM_SPE_NV_DRAM:
466 data_src->mem_lvl = PERF_MEM_LVL_LOC_RAM | PERF_MEM_LVL_HIT;
467 data_src->mem_lvl_num = PERF_MEM_LVLNUM_RAM;
468 data_src->mem_snoop = PERF_MEM_SNOOP_NONE;
469 break;
470 default:
471 break;
472 }
473}
474
475static void arm_spe__synth_data_source_generic(const struct arm_spe_record *record,
476 union perf_mem_data_src *data_src)
477{
478 if (record->type & (ARM_SPE_LLC_ACCESS | ARM_SPE_LLC_MISS)) {
479 data_src->mem_lvl = PERF_MEM_LVL_L3;
480
481 if (record->type & ARM_SPE_LLC_MISS)
482 data_src->mem_lvl |= PERF_MEM_LVL_MISS;
483 else
484 data_src->mem_lvl |= PERF_MEM_LVL_HIT;
485 } else if (record->type & (ARM_SPE_L1D_ACCESS | ARM_SPE_L1D_MISS)) {
486 data_src->mem_lvl = PERF_MEM_LVL_L1;
487
488 if (record->type & ARM_SPE_L1D_MISS)
489 data_src->mem_lvl |= PERF_MEM_LVL_MISS;
490 else
491 data_src->mem_lvl |= PERF_MEM_LVL_HIT;
492 }
493
494 if (record->type & ARM_SPE_REMOTE_ACCESS)
495 data_src->mem_lvl |= PERF_MEM_LVL_REM_CCE1;
496}
497
498static u64 arm_spe__synth_data_source(const struct arm_spe_record *record, u64 midr)
499{
500 union perf_mem_data_src data_src = { 0 };
501 bool is_neoverse = is_midr_in_range_list(midr, neoverse_spe);
502
503 if (record->op == ARM_SPE_LD)
504 data_src.mem_op = PERF_MEM_OP_LOAD;
505 else if (record->op == ARM_SPE_ST)
506 data_src.mem_op = PERF_MEM_OP_STORE;
507 else
508 return 0;
509
510 if (is_neoverse)
511 arm_spe__synth_data_source_neoverse(record, &data_src);
512 else
513 arm_spe__synth_data_source_generic(record, &data_src);
514
515 if (record->type & (ARM_SPE_TLB_ACCESS | ARM_SPE_TLB_MISS)) {
516 data_src.mem_dtlb = PERF_MEM_TLB_WK;
517
518 if (record->type & ARM_SPE_TLB_MISS)
519 data_src.mem_dtlb |= PERF_MEM_TLB_MISS;
520 else
521 data_src.mem_dtlb |= PERF_MEM_TLB_HIT;
522 }
523
524 return data_src.val;
525}
526
527static int arm_spe_sample(struct arm_spe_queue *speq)
528{
529 const struct arm_spe_record *record = &speq->decoder->record;
530 struct arm_spe *spe = speq->spe;
531 u64 data_src;
532 int err;
533
534 data_src = arm_spe__synth_data_source(record, spe->midr);
535
536 if (spe->sample_flc) {
537 if (record->type & ARM_SPE_L1D_MISS) {
538 err = arm_spe__synth_mem_sample(speq, spe->l1d_miss_id,
539 data_src);
540 if (err)
541 return err;
542 }
543
544 if (record->type & ARM_SPE_L1D_ACCESS) {
545 err = arm_spe__synth_mem_sample(speq, spe->l1d_access_id,
546 data_src);
547 if (err)
548 return err;
549 }
550 }
551
552 if (spe->sample_llc) {
553 if (record->type & ARM_SPE_LLC_MISS) {
554 err = arm_spe__synth_mem_sample(speq, spe->llc_miss_id,
555 data_src);
556 if (err)
557 return err;
558 }
559
560 if (record->type & ARM_SPE_LLC_ACCESS) {
561 err = arm_spe__synth_mem_sample(speq, spe->llc_access_id,
562 data_src);
563 if (err)
564 return err;
565 }
566 }
567
568 if (spe->sample_tlb) {
569 if (record->type & ARM_SPE_TLB_MISS) {
570 err = arm_spe__synth_mem_sample(speq, spe->tlb_miss_id,
571 data_src);
572 if (err)
573 return err;
574 }
575
576 if (record->type & ARM_SPE_TLB_ACCESS) {
577 err = arm_spe__synth_mem_sample(speq, spe->tlb_access_id,
578 data_src);
579 if (err)
580 return err;
581 }
582 }
583
584 if (spe->sample_branch && (record->type & ARM_SPE_BRANCH_MISS)) {
585 err = arm_spe__synth_branch_sample(speq, spe->branch_miss_id);
586 if (err)
587 return err;
588 }
589
590 if (spe->sample_remote_access &&
591 (record->type & ARM_SPE_REMOTE_ACCESS)) {
592 err = arm_spe__synth_mem_sample(speq, spe->remote_access_id,
593 data_src);
594 if (err)
595 return err;
596 }
597
598 /*
599 * When data_src is zero it means the record is not a memory operation,
600 * skip to synthesize memory sample for this case.
601 */
602 if (spe->sample_memory && data_src) {
603 err = arm_spe__synth_mem_sample(speq, spe->memory_id, data_src);
604 if (err)
605 return err;
606 }
607
608 if (spe->sample_instructions) {
609 err = arm_spe__synth_instruction_sample(speq, spe->instructions_id, data_src);
610 if (err)
611 return err;
612 }
613
614 return 0;
615}
616
617static int arm_spe_run_decoder(struct arm_spe_queue *speq, u64 *timestamp)
618{
619 struct arm_spe *spe = speq->spe;
620 struct arm_spe_record *record;
621 int ret;
622
623 if (!spe->kernel_start)
624 spe->kernel_start = machine__kernel_start(spe->machine);
625
626 while (1) {
627 /*
628 * The usual logic is firstly to decode the packets, and then
629 * based the record to synthesize sample; but here the flow is
630 * reversed: it calls arm_spe_sample() for synthesizing samples
631 * prior to arm_spe_decode().
632 *
633 * Two reasons for this code logic:
634 * 1. Firstly, when setup queue in arm_spe__setup_queue(), it
635 * has decoded trace data and generated a record, but the record
636 * is left to generate sample until run to here, so it's correct
637 * to synthesize sample for the left record.
638 * 2. After decoding trace data, it needs to compare the record
639 * timestamp with the coming perf event, if the record timestamp
640 * is later than the perf event, it needs bail out and pushs the
641 * record into auxtrace heap, thus the record can be deferred to
642 * synthesize sample until run to here at the next time; so this
643 * can correlate samples between Arm SPE trace data and other
644 * perf events with correct time ordering.
645 */
646
647 /*
648 * Update pid/tid info.
649 */
650 record = &speq->decoder->record;
651 if (!spe->timeless_decoding && record->context_id != (u64)-1) {
652 ret = arm_spe_set_tid(speq, record->context_id);
653 if (ret)
654 return ret;
655
656 spe->use_ctx_pkt_for_pid = true;
657 }
658
659 ret = arm_spe_sample(speq);
660 if (ret)
661 return ret;
662
663 ret = arm_spe_decode(speq->decoder);
664 if (!ret) {
665 pr_debug("No data or all data has been processed.\n");
666 return 1;
667 }
668
669 /*
670 * Error is detected when decode SPE trace data, continue to
671 * the next trace data and find out more records.
672 */
673 if (ret < 0)
674 continue;
675
676 record = &speq->decoder->record;
677
678 /* Update timestamp for the last record */
679 if (record->timestamp > speq->timestamp)
680 speq->timestamp = record->timestamp;
681
682 /*
683 * If the timestamp of the queue is later than timestamp of the
684 * coming perf event, bail out so can allow the perf event to
685 * be processed ahead.
686 */
687 if (!spe->timeless_decoding && speq->timestamp >= *timestamp) {
688 *timestamp = speq->timestamp;
689 return 0;
690 }
691 }
692
693 return 0;
694}
695
696static int arm_spe__setup_queue(struct arm_spe *spe,
697 struct auxtrace_queue *queue,
698 unsigned int queue_nr)
699{
700 struct arm_spe_queue *speq = queue->priv;
701 struct arm_spe_record *record;
702
703 if (list_empty(&queue->head) || speq)
704 return 0;
705
706 speq = arm_spe__alloc_queue(spe, queue_nr);
707
708 if (!speq)
709 return -ENOMEM;
710
711 queue->priv = speq;
712
713 if (queue->cpu != -1)
714 speq->cpu = queue->cpu;
715
716 if (!speq->on_heap) {
717 int ret;
718
719 if (spe->timeless_decoding)
720 return 0;
721
722retry:
723 ret = arm_spe_decode(speq->decoder);
724
725 if (!ret)
726 return 0;
727
728 if (ret < 0)
729 goto retry;
730
731 record = &speq->decoder->record;
732
733 speq->timestamp = record->timestamp;
734 ret = auxtrace_heap__add(&spe->heap, queue_nr, speq->timestamp);
735 if (ret)
736 return ret;
737 speq->on_heap = true;
738 }
739
740 return 0;
741}
742
743static int arm_spe__setup_queues(struct arm_spe *spe)
744{
745 unsigned int i;
746 int ret;
747
748 for (i = 0; i < spe->queues.nr_queues; i++) {
749 ret = arm_spe__setup_queue(spe, &spe->queues.queue_array[i], i);
750 if (ret)
751 return ret;
752 }
753
754 return 0;
755}
756
757static int arm_spe__update_queues(struct arm_spe *spe)
758{
759 if (spe->queues.new_data) {
760 spe->queues.new_data = false;
761 return arm_spe__setup_queues(spe);
762 }
763
764 return 0;
765}
766
767static bool arm_spe__is_timeless_decoding(struct arm_spe *spe)
768{
769 struct evsel *evsel;
770 struct evlist *evlist = spe->session->evlist;
771 bool timeless_decoding = true;
772
773 /*
774 * Circle through the list of event and complain if we find one
775 * with the time bit set.
776 */
777 evlist__for_each_entry(evlist, evsel) {
778 if ((evsel->core.attr.sample_type & PERF_SAMPLE_TIME))
779 timeless_decoding = false;
780 }
781
782 return timeless_decoding;
783}
784
785static int arm_spe_process_queues(struct arm_spe *spe, u64 timestamp)
786{
787 unsigned int queue_nr;
788 u64 ts;
789 int ret;
790
791 while (1) {
792 struct auxtrace_queue *queue;
793 struct arm_spe_queue *speq;
794
795 if (!spe->heap.heap_cnt)
796 return 0;
797
798 if (spe->heap.heap_array[0].ordinal >= timestamp)
799 return 0;
800
801 queue_nr = spe->heap.heap_array[0].queue_nr;
802 queue = &spe->queues.queue_array[queue_nr];
803 speq = queue->priv;
804
805 auxtrace_heap__pop(&spe->heap);
806
807 if (spe->heap.heap_cnt) {
808 ts = spe->heap.heap_array[0].ordinal + 1;
809 if (ts > timestamp)
810 ts = timestamp;
811 } else {
812 ts = timestamp;
813 }
814
815 /*
816 * A previous context-switch event has set pid/tid in the machine's context, so
817 * here we need to update the pid/tid in the thread and SPE queue.
818 */
819 if (!spe->use_ctx_pkt_for_pid)
820 arm_spe_set_pid_tid_cpu(spe, queue);
821
822 ret = arm_spe_run_decoder(speq, &ts);
823 if (ret < 0) {
824 auxtrace_heap__add(&spe->heap, queue_nr, ts);
825 return ret;
826 }
827
828 if (!ret) {
829 ret = auxtrace_heap__add(&spe->heap, queue_nr, ts);
830 if (ret < 0)
831 return ret;
832 } else {
833 speq->on_heap = false;
834 }
835 }
836
837 return 0;
838}
839
840static int arm_spe_process_timeless_queues(struct arm_spe *spe, pid_t tid,
841 u64 time_)
842{
843 struct auxtrace_queues *queues = &spe->queues;
844 unsigned int i;
845 u64 ts = 0;
846
847 for (i = 0; i < queues->nr_queues; i++) {
848 struct auxtrace_queue *queue = &spe->queues.queue_array[i];
849 struct arm_spe_queue *speq = queue->priv;
850
851 if (speq && (tid == -1 || speq->tid == tid)) {
852 speq->time = time_;
853 arm_spe_set_pid_tid_cpu(spe, queue);
854 arm_spe_run_decoder(speq, &ts);
855 }
856 }
857 return 0;
858}
859
860static int arm_spe_context_switch(struct arm_spe *spe, union perf_event *event,
861 struct perf_sample *sample)
862{
863 pid_t pid, tid;
864 int cpu;
865
866 if (!(event->header.misc & PERF_RECORD_MISC_SWITCH_OUT))
867 return 0;
868
869 pid = event->context_switch.next_prev_pid;
870 tid = event->context_switch.next_prev_tid;
871 cpu = sample->cpu;
872
873 if (tid == -1)
874 pr_warning("context_switch event has no tid\n");
875
876 return machine__set_current_tid(spe->machine, cpu, pid, tid);
877}
878
879static int arm_spe_process_event(struct perf_session *session,
880 union perf_event *event,
881 struct perf_sample *sample,
882 struct perf_tool *tool)
883{
884 int err = 0;
885 u64 timestamp;
886 struct arm_spe *spe = container_of(session->auxtrace,
887 struct arm_spe, auxtrace);
888
889 if (dump_trace)
890 return 0;
891
892 if (!tool->ordered_events) {
893 pr_err("SPE trace requires ordered events\n");
894 return -EINVAL;
895 }
896
897 if (sample->time && (sample->time != (u64) -1))
898 timestamp = perf_time_to_tsc(sample->time, &spe->tc);
899 else
900 timestamp = 0;
901
902 if (timestamp || spe->timeless_decoding) {
903 err = arm_spe__update_queues(spe);
904 if (err)
905 return err;
906 }
907
908 if (spe->timeless_decoding) {
909 if (event->header.type == PERF_RECORD_EXIT) {
910 err = arm_spe_process_timeless_queues(spe,
911 event->fork.tid,
912 sample->time);
913 }
914 } else if (timestamp) {
915 err = arm_spe_process_queues(spe, timestamp);
916 if (err)
917 return err;
918
919 if (!spe->use_ctx_pkt_for_pid &&
920 (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE ||
921 event->header.type == PERF_RECORD_SWITCH))
922 err = arm_spe_context_switch(spe, event, sample);
923 }
924
925 return err;
926}
927
928static int arm_spe_process_auxtrace_event(struct perf_session *session,
929 union perf_event *event,
930 struct perf_tool *tool __maybe_unused)
931{
932 struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
933 auxtrace);
934
935 if (!spe->data_queued) {
936 struct auxtrace_buffer *buffer;
937 off_t data_offset;
938 int fd = perf_data__fd(session->data);
939 int err;
940
941 if (perf_data__is_pipe(session->data)) {
942 data_offset = 0;
943 } else {
944 data_offset = lseek(fd, 0, SEEK_CUR);
945 if (data_offset == -1)
946 return -errno;
947 }
948
949 err = auxtrace_queues__add_event(&spe->queues, session, event,
950 data_offset, &buffer);
951 if (err)
952 return err;
953
954 /* Dump here now we have copied a piped trace out of the pipe */
955 if (dump_trace) {
956 if (auxtrace_buffer__get_data(buffer, fd)) {
957 arm_spe_dump_event(spe, buffer->data,
958 buffer->size);
959 auxtrace_buffer__put_data(buffer);
960 }
961 }
962 }
963
964 return 0;
965}
966
967static int arm_spe_flush(struct perf_session *session __maybe_unused,
968 struct perf_tool *tool __maybe_unused)
969{
970 struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
971 auxtrace);
972 int ret;
973
974 if (dump_trace)
975 return 0;
976
977 if (!tool->ordered_events)
978 return -EINVAL;
979
980 ret = arm_spe__update_queues(spe);
981 if (ret < 0)
982 return ret;
983
984 if (spe->timeless_decoding)
985 return arm_spe_process_timeless_queues(spe, -1,
986 MAX_TIMESTAMP - 1);
987
988 ret = arm_spe_process_queues(spe, MAX_TIMESTAMP);
989 if (ret)
990 return ret;
991
992 if (!spe->use_ctx_pkt_for_pid)
993 ui__warning("Arm SPE CONTEXT packets not found in the traces.\n"
994 "Matching of TIDs to SPE events could be inaccurate.\n");
995
996 return 0;
997}
998
999static void arm_spe_free_queue(void *priv)
1000{
1001 struct arm_spe_queue *speq = priv;
1002
1003 if (!speq)
1004 return;
1005 thread__zput(speq->thread);
1006 arm_spe_decoder_free(speq->decoder);
1007 zfree(&speq->event_buf);
1008 free(speq);
1009}
1010
1011static void arm_spe_free_events(struct perf_session *session)
1012{
1013 struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
1014 auxtrace);
1015 struct auxtrace_queues *queues = &spe->queues;
1016 unsigned int i;
1017
1018 for (i = 0; i < queues->nr_queues; i++) {
1019 arm_spe_free_queue(queues->queue_array[i].priv);
1020 queues->queue_array[i].priv = NULL;
1021 }
1022 auxtrace_queues__free(queues);
1023}
1024
1025static void arm_spe_free(struct perf_session *session)
1026{
1027 struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
1028 auxtrace);
1029
1030 auxtrace_heap__free(&spe->heap);
1031 arm_spe_free_events(session);
1032 session->auxtrace = NULL;
1033 free(spe);
1034}
1035
1036static bool arm_spe_evsel_is_auxtrace(struct perf_session *session,
1037 struct evsel *evsel)
1038{
1039 struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, auxtrace);
1040
1041 return evsel->core.attr.type == spe->pmu_type;
1042}
1043
1044static const char * const arm_spe_info_fmts[] = {
1045 [ARM_SPE_PMU_TYPE] = " PMU Type %"PRId64"\n",
1046};
1047
1048static void arm_spe_print_info(__u64 *arr)
1049{
1050 if (!dump_trace)
1051 return;
1052
1053 fprintf(stdout, arm_spe_info_fmts[ARM_SPE_PMU_TYPE], arr[ARM_SPE_PMU_TYPE]);
1054}
1055
1056struct arm_spe_synth {
1057 struct perf_tool dummy_tool;
1058 struct perf_session *session;
1059};
1060
1061static int arm_spe_event_synth(struct perf_tool *tool,
1062 union perf_event *event,
1063 struct perf_sample *sample __maybe_unused,
1064 struct machine *machine __maybe_unused)
1065{
1066 struct arm_spe_synth *arm_spe_synth =
1067 container_of(tool, struct arm_spe_synth, dummy_tool);
1068
1069 return perf_session__deliver_synth_event(arm_spe_synth->session,
1070 event, NULL);
1071}
1072
1073static int arm_spe_synth_event(struct perf_session *session,
1074 struct perf_event_attr *attr, u64 id)
1075{
1076 struct arm_spe_synth arm_spe_synth;
1077
1078 memset(&arm_spe_synth, 0, sizeof(struct arm_spe_synth));
1079 arm_spe_synth.session = session;
1080
1081 return perf_event__synthesize_attr(&arm_spe_synth.dummy_tool, attr, 1,
1082 &id, arm_spe_event_synth);
1083}
1084
1085static void arm_spe_set_event_name(struct evlist *evlist, u64 id,
1086 const char *name)
1087{
1088 struct evsel *evsel;
1089
1090 evlist__for_each_entry(evlist, evsel) {
1091 if (evsel->core.id && evsel->core.id[0] == id) {
1092 if (evsel->name)
1093 zfree(&evsel->name);
1094 evsel->name = strdup(name);
1095 break;
1096 }
1097 }
1098}
1099
1100static int
1101arm_spe_synth_events(struct arm_spe *spe, struct perf_session *session)
1102{
1103 struct evlist *evlist = session->evlist;
1104 struct evsel *evsel;
1105 struct perf_event_attr attr;
1106 bool found = false;
1107 u64 id;
1108 int err;
1109
1110 evlist__for_each_entry(evlist, evsel) {
1111 if (evsel->core.attr.type == spe->pmu_type) {
1112 found = true;
1113 break;
1114 }
1115 }
1116
1117 if (!found) {
1118 pr_debug("No selected events with SPE trace data\n");
1119 return 0;
1120 }
1121
1122 memset(&attr, 0, sizeof(struct perf_event_attr));
1123 attr.size = sizeof(struct perf_event_attr);
1124 attr.type = PERF_TYPE_HARDWARE;
1125 attr.sample_type = evsel->core.attr.sample_type &
1126 (PERF_SAMPLE_MASK | PERF_SAMPLE_PHYS_ADDR);
1127 attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1128 PERF_SAMPLE_PERIOD | PERF_SAMPLE_DATA_SRC |
1129 PERF_SAMPLE_WEIGHT | PERF_SAMPLE_ADDR;
1130 if (spe->timeless_decoding)
1131 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1132 else
1133 attr.sample_type |= PERF_SAMPLE_TIME;
1134
1135 spe->sample_type = attr.sample_type;
1136
1137 attr.exclude_user = evsel->core.attr.exclude_user;
1138 attr.exclude_kernel = evsel->core.attr.exclude_kernel;
1139 attr.exclude_hv = evsel->core.attr.exclude_hv;
1140 attr.exclude_host = evsel->core.attr.exclude_host;
1141 attr.exclude_guest = evsel->core.attr.exclude_guest;
1142 attr.sample_id_all = evsel->core.attr.sample_id_all;
1143 attr.read_format = evsel->core.attr.read_format;
1144
1145 /* create new id val to be a fixed offset from evsel id */
1146 id = evsel->core.id[0] + 1000000000;
1147
1148 if (!id)
1149 id = 1;
1150
1151 if (spe->synth_opts.flc) {
1152 spe->sample_flc = true;
1153
1154 /* Level 1 data cache miss */
1155 err = arm_spe_synth_event(session, &attr, id);
1156 if (err)
1157 return err;
1158 spe->l1d_miss_id = id;
1159 arm_spe_set_event_name(evlist, id, "l1d-miss");
1160 id += 1;
1161
1162 /* Level 1 data cache access */
1163 err = arm_spe_synth_event(session, &attr, id);
1164 if (err)
1165 return err;
1166 spe->l1d_access_id = id;
1167 arm_spe_set_event_name(evlist, id, "l1d-access");
1168 id += 1;
1169 }
1170
1171 if (spe->synth_opts.llc) {
1172 spe->sample_llc = true;
1173
1174 /* Last level cache miss */
1175 err = arm_spe_synth_event(session, &attr, id);
1176 if (err)
1177 return err;
1178 spe->llc_miss_id = id;
1179 arm_spe_set_event_name(evlist, id, "llc-miss");
1180 id += 1;
1181
1182 /* Last level cache access */
1183 err = arm_spe_synth_event(session, &attr, id);
1184 if (err)
1185 return err;
1186 spe->llc_access_id = id;
1187 arm_spe_set_event_name(evlist, id, "llc-access");
1188 id += 1;
1189 }
1190
1191 if (spe->synth_opts.tlb) {
1192 spe->sample_tlb = true;
1193
1194 /* TLB miss */
1195 err = arm_spe_synth_event(session, &attr, id);
1196 if (err)
1197 return err;
1198 spe->tlb_miss_id = id;
1199 arm_spe_set_event_name(evlist, id, "tlb-miss");
1200 id += 1;
1201
1202 /* TLB access */
1203 err = arm_spe_synth_event(session, &attr, id);
1204 if (err)
1205 return err;
1206 spe->tlb_access_id = id;
1207 arm_spe_set_event_name(evlist, id, "tlb-access");
1208 id += 1;
1209 }
1210
1211 if (spe->synth_opts.branches) {
1212 spe->sample_branch = true;
1213
1214 /* Branch miss */
1215 err = arm_spe_synth_event(session, &attr, id);
1216 if (err)
1217 return err;
1218 spe->branch_miss_id = id;
1219 arm_spe_set_event_name(evlist, id, "branch-miss");
1220 id += 1;
1221 }
1222
1223 if (spe->synth_opts.remote_access) {
1224 spe->sample_remote_access = true;
1225
1226 /* Remote access */
1227 err = arm_spe_synth_event(session, &attr, id);
1228 if (err)
1229 return err;
1230 spe->remote_access_id = id;
1231 arm_spe_set_event_name(evlist, id, "remote-access");
1232 id += 1;
1233 }
1234
1235 if (spe->synth_opts.mem) {
1236 spe->sample_memory = true;
1237
1238 err = arm_spe_synth_event(session, &attr, id);
1239 if (err)
1240 return err;
1241 spe->memory_id = id;
1242 arm_spe_set_event_name(evlist, id, "memory");
1243 id += 1;
1244 }
1245
1246 if (spe->synth_opts.instructions) {
1247 if (spe->synth_opts.period_type != PERF_ITRACE_PERIOD_INSTRUCTIONS) {
1248 pr_warning("Only instruction-based sampling period is currently supported by Arm SPE.\n");
1249 goto synth_instructions_out;
1250 }
1251 if (spe->synth_opts.period > 1)
1252 pr_warning("Arm SPE has a hardware-based sample period.\n"
1253 "Additional instruction events will be discarded by --itrace\n");
1254
1255 spe->sample_instructions = true;
1256 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1257 attr.sample_period = spe->synth_opts.period;
1258 spe->instructions_sample_period = attr.sample_period;
1259 err = arm_spe_synth_event(session, &attr, id);
1260 if (err)
1261 return err;
1262 spe->instructions_id = id;
1263 arm_spe_set_event_name(evlist, id, "instructions");
1264 }
1265synth_instructions_out:
1266
1267 return 0;
1268}
1269
1270int arm_spe_process_auxtrace_info(union perf_event *event,
1271 struct perf_session *session)
1272{
1273 struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;
1274 size_t min_sz = sizeof(u64) * ARM_SPE_AUXTRACE_PRIV_MAX;
1275 struct perf_record_time_conv *tc = &session->time_conv;
1276 const char *cpuid = perf_env__cpuid(session->evlist->env);
1277 u64 midr = strtol(cpuid, NULL, 16);
1278 struct arm_spe *spe;
1279 int err;
1280
1281 if (auxtrace_info->header.size < sizeof(struct perf_record_auxtrace_info) +
1282 min_sz)
1283 return -EINVAL;
1284
1285 spe = zalloc(sizeof(struct arm_spe));
1286 if (!spe)
1287 return -ENOMEM;
1288
1289 err = auxtrace_queues__init(&spe->queues);
1290 if (err)
1291 goto err_free;
1292
1293 spe->session = session;
1294 spe->machine = &session->machines.host; /* No kvm support */
1295 spe->auxtrace_type = auxtrace_info->type;
1296 spe->pmu_type = auxtrace_info->priv[ARM_SPE_PMU_TYPE];
1297 spe->midr = midr;
1298
1299 spe->timeless_decoding = arm_spe__is_timeless_decoding(spe);
1300
1301 /*
1302 * The synthesized event PERF_RECORD_TIME_CONV has been handled ahead
1303 * and the parameters for hardware clock are stored in the session
1304 * context. Passes these parameters to the struct perf_tsc_conversion
1305 * in "spe->tc", which is used for later conversion between clock
1306 * counter and timestamp.
1307 *
1308 * For backward compatibility, copies the fields starting from
1309 * "time_cycles" only if they are contained in the event.
1310 */
1311 spe->tc.time_shift = tc->time_shift;
1312 spe->tc.time_mult = tc->time_mult;
1313 spe->tc.time_zero = tc->time_zero;
1314
1315 if (event_contains(*tc, time_cycles)) {
1316 spe->tc.time_cycles = tc->time_cycles;
1317 spe->tc.time_mask = tc->time_mask;
1318 spe->tc.cap_user_time_zero = tc->cap_user_time_zero;
1319 spe->tc.cap_user_time_short = tc->cap_user_time_short;
1320 }
1321
1322 spe->auxtrace.process_event = arm_spe_process_event;
1323 spe->auxtrace.process_auxtrace_event = arm_spe_process_auxtrace_event;
1324 spe->auxtrace.flush_events = arm_spe_flush;
1325 spe->auxtrace.free_events = arm_spe_free_events;
1326 spe->auxtrace.free = arm_spe_free;
1327 spe->auxtrace.evsel_is_auxtrace = arm_spe_evsel_is_auxtrace;
1328 session->auxtrace = &spe->auxtrace;
1329
1330 arm_spe_print_info(&auxtrace_info->priv[0]);
1331
1332 if (dump_trace)
1333 return 0;
1334
1335 if (session->itrace_synth_opts && session->itrace_synth_opts->set)
1336 spe->synth_opts = *session->itrace_synth_opts;
1337 else
1338 itrace_synth_opts__set_default(&spe->synth_opts, false);
1339
1340 err = arm_spe_synth_events(spe, session);
1341 if (err)
1342 goto err_free_queues;
1343
1344 err = auxtrace_queues__process_index(&spe->queues, session);
1345 if (err)
1346 goto err_free_queues;
1347
1348 if (spe->queues.populated)
1349 spe->data_queued = true;
1350
1351 return 0;
1352
1353err_free_queues:
1354 auxtrace_queues__free(&spe->queues);
1355 session->auxtrace = NULL;
1356err_free:
1357 free(spe);
1358 return err;
1359}