Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright(C) 2015-2018 Linaro Limited.
4 *
5 * Author: Tor Jeremiassen <tor@ti.com>
6 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
7 */
8
9#include <linux/bitops.h>
10#include <linux/err.h>
11#include <linux/kernel.h>
12#include <linux/log2.h>
13#include <linux/types.h>
14#include <linux/zalloc.h>
15
16#include <opencsd/ocsd_if_types.h>
17#include <stdlib.h>
18
19#include "auxtrace.h"
20#include "color.h"
21#include "cs-etm.h"
22#include "cs-etm-decoder/cs-etm-decoder.h"
23#include "debug.h"
24#include "dso.h"
25#include "evlist.h"
26#include "intlist.h"
27#include "machine.h"
28#include "map.h"
29#include "perf.h"
30#include "session.h"
31#include "map_symbol.h"
32#include "branch.h"
33#include "symbol.h"
34#include "tool.h"
35#include "thread.h"
36#include "thread-stack.h"
37#include <tools/libc_compat.h>
38#include "util/synthetic-events.h"
39
40#define MAX_TIMESTAMP (~0ULL)
41
42struct cs_etm_auxtrace {
43 struct auxtrace auxtrace;
44 struct auxtrace_queues queues;
45 struct auxtrace_heap heap;
46 struct itrace_synth_opts synth_opts;
47 struct perf_session *session;
48 struct machine *machine;
49 struct thread *unknown_thread;
50
51 u8 timeless_decoding;
52 u8 snapshot_mode;
53 u8 data_queued;
54 u8 sample_branches;
55 u8 sample_instructions;
56
57 int num_cpu;
58 u32 auxtrace_type;
59 u64 branches_sample_type;
60 u64 branches_id;
61 u64 instructions_sample_type;
62 u64 instructions_sample_period;
63 u64 instructions_id;
64 u64 **metadata;
65 u64 kernel_start;
66 unsigned int pmu_type;
67};
68
69struct cs_etm_traceid_queue {
70 u8 trace_chan_id;
71 pid_t pid, tid;
72 u64 period_instructions;
73 size_t last_branch_pos;
74 union perf_event *event_buf;
75 struct thread *thread;
76 struct branch_stack *last_branch;
77 struct branch_stack *last_branch_rb;
78 struct cs_etm_packet *prev_packet;
79 struct cs_etm_packet *packet;
80 struct cs_etm_packet_queue packet_queue;
81};
82
83struct cs_etm_queue {
84 struct cs_etm_auxtrace *etm;
85 struct cs_etm_decoder *decoder;
86 struct auxtrace_buffer *buffer;
87 unsigned int queue_nr;
88 u8 pending_timestamp;
89 u64 offset;
90 const unsigned char *buf;
91 size_t buf_len, buf_used;
92 /* Conversion between traceID and index in traceid_queues array */
93 struct intlist *traceid_queues_list;
94 struct cs_etm_traceid_queue **traceid_queues;
95};
96
97static int cs_etm__update_queues(struct cs_etm_auxtrace *etm);
98static int cs_etm__process_queues(struct cs_etm_auxtrace *etm);
99static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
100 pid_t tid);
101static int cs_etm__get_data_block(struct cs_etm_queue *etmq);
102static int cs_etm__decode_data_block(struct cs_etm_queue *etmq);
103
104/* PTMs ETMIDR [11:8] set to b0011 */
105#define ETMIDR_PTM_VERSION 0x00000300
106
107/*
108 * A struct auxtrace_heap_item only has a queue_nr and a timestamp to
109 * work with. One option is to modify to auxtrace_heap_XYZ() API or simply
110 * encode the etm queue number as the upper 16 bit and the channel as
111 * the lower 16 bit.
112 */
113#define TO_CS_QUEUE_NR(queue_nr, trace_id_chan) \
114 (queue_nr << 16 | trace_chan_id)
115#define TO_QUEUE_NR(cs_queue_nr) (cs_queue_nr >> 16)
116#define TO_TRACE_CHAN_ID(cs_queue_nr) (cs_queue_nr & 0x0000ffff)
117
118static u32 cs_etm__get_v7_protocol_version(u32 etmidr)
119{
120 etmidr &= ETMIDR_PTM_VERSION;
121
122 if (etmidr == ETMIDR_PTM_VERSION)
123 return CS_ETM_PROTO_PTM;
124
125 return CS_ETM_PROTO_ETMV3;
126}
127
128static int cs_etm__get_magic(u8 trace_chan_id, u64 *magic)
129{
130 struct int_node *inode;
131 u64 *metadata;
132
133 inode = intlist__find(traceid_list, trace_chan_id);
134 if (!inode)
135 return -EINVAL;
136
137 metadata = inode->priv;
138 *magic = metadata[CS_ETM_MAGIC];
139 return 0;
140}
141
142int cs_etm__get_cpu(u8 trace_chan_id, int *cpu)
143{
144 struct int_node *inode;
145 u64 *metadata;
146
147 inode = intlist__find(traceid_list, trace_chan_id);
148 if (!inode)
149 return -EINVAL;
150
151 metadata = inode->priv;
152 *cpu = (int)metadata[CS_ETM_CPU];
153 return 0;
154}
155
156void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq,
157 u8 trace_chan_id)
158{
159 /*
160 * Wnen a timestamp packet is encountered the backend code
161 * is stopped so that the front end has time to process packets
162 * that were accumulated in the traceID queue. Since there can
163 * be more than one channel per cs_etm_queue, we need to specify
164 * what traceID queue needs servicing.
165 */
166 etmq->pending_timestamp = trace_chan_id;
167}
168
169static u64 cs_etm__etmq_get_timestamp(struct cs_etm_queue *etmq,
170 u8 *trace_chan_id)
171{
172 struct cs_etm_packet_queue *packet_queue;
173
174 if (!etmq->pending_timestamp)
175 return 0;
176
177 if (trace_chan_id)
178 *trace_chan_id = etmq->pending_timestamp;
179
180 packet_queue = cs_etm__etmq_get_packet_queue(etmq,
181 etmq->pending_timestamp);
182 if (!packet_queue)
183 return 0;
184
185 /* Acknowledge pending status */
186 etmq->pending_timestamp = 0;
187
188 /* See function cs_etm_decoder__do_{hard|soft}_timestamp() */
189 return packet_queue->timestamp;
190}
191
192static void cs_etm__clear_packet_queue(struct cs_etm_packet_queue *queue)
193{
194 int i;
195
196 queue->head = 0;
197 queue->tail = 0;
198 queue->packet_count = 0;
199 for (i = 0; i < CS_ETM_PACKET_MAX_BUFFER; i++) {
200 queue->packet_buffer[i].isa = CS_ETM_ISA_UNKNOWN;
201 queue->packet_buffer[i].start_addr = CS_ETM_INVAL_ADDR;
202 queue->packet_buffer[i].end_addr = CS_ETM_INVAL_ADDR;
203 queue->packet_buffer[i].instr_count = 0;
204 queue->packet_buffer[i].last_instr_taken_branch = false;
205 queue->packet_buffer[i].last_instr_size = 0;
206 queue->packet_buffer[i].last_instr_type = 0;
207 queue->packet_buffer[i].last_instr_subtype = 0;
208 queue->packet_buffer[i].last_instr_cond = 0;
209 queue->packet_buffer[i].flags = 0;
210 queue->packet_buffer[i].exception_number = UINT32_MAX;
211 queue->packet_buffer[i].trace_chan_id = UINT8_MAX;
212 queue->packet_buffer[i].cpu = INT_MIN;
213 }
214}
215
216static void cs_etm__clear_all_packet_queues(struct cs_etm_queue *etmq)
217{
218 int idx;
219 struct int_node *inode;
220 struct cs_etm_traceid_queue *tidq;
221 struct intlist *traceid_queues_list = etmq->traceid_queues_list;
222
223 intlist__for_each_entry(inode, traceid_queues_list) {
224 idx = (int)(intptr_t)inode->priv;
225 tidq = etmq->traceid_queues[idx];
226 cs_etm__clear_packet_queue(&tidq->packet_queue);
227 }
228}
229
230static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq,
231 struct cs_etm_traceid_queue *tidq,
232 u8 trace_chan_id)
233{
234 int rc = -ENOMEM;
235 struct auxtrace_queue *queue;
236 struct cs_etm_auxtrace *etm = etmq->etm;
237
238 cs_etm__clear_packet_queue(&tidq->packet_queue);
239
240 queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
241 tidq->tid = queue->tid;
242 tidq->pid = -1;
243 tidq->trace_chan_id = trace_chan_id;
244
245 tidq->packet = zalloc(sizeof(struct cs_etm_packet));
246 if (!tidq->packet)
247 goto out;
248
249 tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet));
250 if (!tidq->prev_packet)
251 goto out_free;
252
253 if (etm->synth_opts.last_branch) {
254 size_t sz = sizeof(struct branch_stack);
255
256 sz += etm->synth_opts.last_branch_sz *
257 sizeof(struct branch_entry);
258 tidq->last_branch = zalloc(sz);
259 if (!tidq->last_branch)
260 goto out_free;
261 tidq->last_branch_rb = zalloc(sz);
262 if (!tidq->last_branch_rb)
263 goto out_free;
264 }
265
266 tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
267 if (!tidq->event_buf)
268 goto out_free;
269
270 return 0;
271
272out_free:
273 zfree(&tidq->last_branch_rb);
274 zfree(&tidq->last_branch);
275 zfree(&tidq->prev_packet);
276 zfree(&tidq->packet);
277out:
278 return rc;
279}
280
281static struct cs_etm_traceid_queue
282*cs_etm__etmq_get_traceid_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
283{
284 int idx;
285 struct int_node *inode;
286 struct intlist *traceid_queues_list;
287 struct cs_etm_traceid_queue *tidq, **traceid_queues;
288 struct cs_etm_auxtrace *etm = etmq->etm;
289
290 if (etm->timeless_decoding)
291 trace_chan_id = CS_ETM_PER_THREAD_TRACEID;
292
293 traceid_queues_list = etmq->traceid_queues_list;
294
295 /*
296 * Check if the traceid_queue exist for this traceID by looking
297 * in the queue list.
298 */
299 inode = intlist__find(traceid_queues_list, trace_chan_id);
300 if (inode) {
301 idx = (int)(intptr_t)inode->priv;
302 return etmq->traceid_queues[idx];
303 }
304
305 /* We couldn't find a traceid_queue for this traceID, allocate one */
306 tidq = malloc(sizeof(*tidq));
307 if (!tidq)
308 return NULL;
309
310 memset(tidq, 0, sizeof(*tidq));
311
312 /* Get a valid index for the new traceid_queue */
313 idx = intlist__nr_entries(traceid_queues_list);
314 /* Memory for the inode is free'ed in cs_etm_free_traceid_queues () */
315 inode = intlist__findnew(traceid_queues_list, trace_chan_id);
316 if (!inode)
317 goto out_free;
318
319 /* Associate this traceID with this index */
320 inode->priv = (void *)(intptr_t)idx;
321
322 if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id))
323 goto out_free;
324
325 /* Grow the traceid_queues array by one unit */
326 traceid_queues = etmq->traceid_queues;
327 traceid_queues = reallocarray(traceid_queues,
328 idx + 1,
329 sizeof(*traceid_queues));
330
331 /*
332 * On failure reallocarray() returns NULL and the original block of
333 * memory is left untouched.
334 */
335 if (!traceid_queues)
336 goto out_free;
337
338 traceid_queues[idx] = tidq;
339 etmq->traceid_queues = traceid_queues;
340
341 return etmq->traceid_queues[idx];
342
343out_free:
344 /*
345 * Function intlist__remove() removes the inode from the list
346 * and delete the memory associated to it.
347 */
348 intlist__remove(traceid_queues_list, inode);
349 free(tidq);
350
351 return NULL;
352}
353
354struct cs_etm_packet_queue
355*cs_etm__etmq_get_packet_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
356{
357 struct cs_etm_traceid_queue *tidq;
358
359 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
360 if (tidq)
361 return &tidq->packet_queue;
362
363 return NULL;
364}
365
366static void cs_etm__packet_dump(const char *pkt_string)
367{
368 const char *color = PERF_COLOR_BLUE;
369 int len = strlen(pkt_string);
370
371 if (len && (pkt_string[len-1] == '\n'))
372 color_fprintf(stdout, color, " %s", pkt_string);
373 else
374 color_fprintf(stdout, color, " %s\n", pkt_string);
375
376 fflush(stdout);
377}
378
379static void cs_etm__set_trace_param_etmv3(struct cs_etm_trace_params *t_params,
380 struct cs_etm_auxtrace *etm, int idx,
381 u32 etmidr)
382{
383 u64 **metadata = etm->metadata;
384
385 t_params[idx].protocol = cs_etm__get_v7_protocol_version(etmidr);
386 t_params[idx].etmv3.reg_ctrl = metadata[idx][CS_ETM_ETMCR];
387 t_params[idx].etmv3.reg_trc_id = metadata[idx][CS_ETM_ETMTRACEIDR];
388}
389
390static void cs_etm__set_trace_param_etmv4(struct cs_etm_trace_params *t_params,
391 struct cs_etm_auxtrace *etm, int idx)
392{
393 u64 **metadata = etm->metadata;
394
395 t_params[idx].protocol = CS_ETM_PROTO_ETMV4i;
396 t_params[idx].etmv4.reg_idr0 = metadata[idx][CS_ETMV4_TRCIDR0];
397 t_params[idx].etmv4.reg_idr1 = metadata[idx][CS_ETMV4_TRCIDR1];
398 t_params[idx].etmv4.reg_idr2 = metadata[idx][CS_ETMV4_TRCIDR2];
399 t_params[idx].etmv4.reg_idr8 = metadata[idx][CS_ETMV4_TRCIDR8];
400 t_params[idx].etmv4.reg_configr = metadata[idx][CS_ETMV4_TRCCONFIGR];
401 t_params[idx].etmv4.reg_traceidr = metadata[idx][CS_ETMV4_TRCTRACEIDR];
402}
403
404static int cs_etm__init_trace_params(struct cs_etm_trace_params *t_params,
405 struct cs_etm_auxtrace *etm)
406{
407 int i;
408 u32 etmidr;
409 u64 architecture;
410
411 for (i = 0; i < etm->num_cpu; i++) {
412 architecture = etm->metadata[i][CS_ETM_MAGIC];
413
414 switch (architecture) {
415 case __perf_cs_etmv3_magic:
416 etmidr = etm->metadata[i][CS_ETM_ETMIDR];
417 cs_etm__set_trace_param_etmv3(t_params, etm, i, etmidr);
418 break;
419 case __perf_cs_etmv4_magic:
420 cs_etm__set_trace_param_etmv4(t_params, etm, i);
421 break;
422 default:
423 return -EINVAL;
424 }
425 }
426
427 return 0;
428}
429
430static int cs_etm__init_decoder_params(struct cs_etm_decoder_params *d_params,
431 struct cs_etm_queue *etmq,
432 enum cs_etm_decoder_operation mode)
433{
434 int ret = -EINVAL;
435
436 if (!(mode < CS_ETM_OPERATION_MAX))
437 goto out;
438
439 d_params->packet_printer = cs_etm__packet_dump;
440 d_params->operation = mode;
441 d_params->data = etmq;
442 d_params->formatted = true;
443 d_params->fsyncs = false;
444 d_params->hsyncs = false;
445 d_params->frame_aligned = true;
446
447 ret = 0;
448out:
449 return ret;
450}
451
452static void cs_etm__dump_event(struct cs_etm_auxtrace *etm,
453 struct auxtrace_buffer *buffer)
454{
455 int ret;
456 const char *color = PERF_COLOR_BLUE;
457 struct cs_etm_decoder_params d_params;
458 struct cs_etm_trace_params *t_params;
459 struct cs_etm_decoder *decoder;
460 size_t buffer_used = 0;
461
462 fprintf(stdout, "\n");
463 color_fprintf(stdout, color,
464 ". ... CoreSight ETM Trace data: size %zu bytes\n",
465 buffer->size);
466
467 /* Use metadata to fill in trace parameters for trace decoder */
468 t_params = zalloc(sizeof(*t_params) * etm->num_cpu);
469
470 if (!t_params)
471 return;
472
473 if (cs_etm__init_trace_params(t_params, etm))
474 goto out_free;
475
476 /* Set decoder parameters to simply print the trace packets */
477 if (cs_etm__init_decoder_params(&d_params, NULL,
478 CS_ETM_OPERATION_PRINT))
479 goto out_free;
480
481 decoder = cs_etm_decoder__new(etm->num_cpu, &d_params, t_params);
482
483 if (!decoder)
484 goto out_free;
485 do {
486 size_t consumed;
487
488 ret = cs_etm_decoder__process_data_block(
489 decoder, buffer->offset,
490 &((u8 *)buffer->data)[buffer_used],
491 buffer->size - buffer_used, &consumed);
492 if (ret)
493 break;
494
495 buffer_used += consumed;
496 } while (buffer_used < buffer->size);
497
498 cs_etm_decoder__free(decoder);
499
500out_free:
501 zfree(&t_params);
502}
503
504static int cs_etm__flush_events(struct perf_session *session,
505 struct perf_tool *tool)
506{
507 int ret;
508 struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
509 struct cs_etm_auxtrace,
510 auxtrace);
511 if (dump_trace)
512 return 0;
513
514 if (!tool->ordered_events)
515 return -EINVAL;
516
517 ret = cs_etm__update_queues(etm);
518
519 if (ret < 0)
520 return ret;
521
522 if (etm->timeless_decoding)
523 return cs_etm__process_timeless_queues(etm, -1);
524
525 return cs_etm__process_queues(etm);
526}
527
528static void cs_etm__free_traceid_queues(struct cs_etm_queue *etmq)
529{
530 int idx;
531 uintptr_t priv;
532 struct int_node *inode, *tmp;
533 struct cs_etm_traceid_queue *tidq;
534 struct intlist *traceid_queues_list = etmq->traceid_queues_list;
535
536 intlist__for_each_entry_safe(inode, tmp, traceid_queues_list) {
537 priv = (uintptr_t)inode->priv;
538 idx = priv;
539
540 /* Free this traceid_queue from the array */
541 tidq = etmq->traceid_queues[idx];
542 thread__zput(tidq->thread);
543 zfree(&tidq->event_buf);
544 zfree(&tidq->last_branch);
545 zfree(&tidq->last_branch_rb);
546 zfree(&tidq->prev_packet);
547 zfree(&tidq->packet);
548 zfree(&tidq);
549
550 /*
551 * Function intlist__remove() removes the inode from the list
552 * and delete the memory associated to it.
553 */
554 intlist__remove(traceid_queues_list, inode);
555 }
556
557 /* Then the RB tree itself */
558 intlist__delete(traceid_queues_list);
559 etmq->traceid_queues_list = NULL;
560
561 /* finally free the traceid_queues array */
562 zfree(&etmq->traceid_queues);
563}
564
565static void cs_etm__free_queue(void *priv)
566{
567 struct cs_etm_queue *etmq = priv;
568
569 if (!etmq)
570 return;
571
572 cs_etm_decoder__free(etmq->decoder);
573 cs_etm__free_traceid_queues(etmq);
574 free(etmq);
575}
576
577static void cs_etm__free_events(struct perf_session *session)
578{
579 unsigned int i;
580 struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
581 struct cs_etm_auxtrace,
582 auxtrace);
583 struct auxtrace_queues *queues = &aux->queues;
584
585 for (i = 0; i < queues->nr_queues; i++) {
586 cs_etm__free_queue(queues->queue_array[i].priv);
587 queues->queue_array[i].priv = NULL;
588 }
589
590 auxtrace_queues__free(queues);
591}
592
593static void cs_etm__free(struct perf_session *session)
594{
595 int i;
596 struct int_node *inode, *tmp;
597 struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
598 struct cs_etm_auxtrace,
599 auxtrace);
600 cs_etm__free_events(session);
601 session->auxtrace = NULL;
602
603 /* First remove all traceID/metadata nodes for the RB tree */
604 intlist__for_each_entry_safe(inode, tmp, traceid_list)
605 intlist__remove(traceid_list, inode);
606 /* Then the RB tree itself */
607 intlist__delete(traceid_list);
608
609 for (i = 0; i < aux->num_cpu; i++)
610 zfree(&aux->metadata[i]);
611
612 thread__zput(aux->unknown_thread);
613 zfree(&aux->metadata);
614 zfree(&aux);
615}
616
617static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address)
618{
619 struct machine *machine;
620
621 machine = etmq->etm->machine;
622
623 if (address >= etmq->etm->kernel_start) {
624 if (machine__is_host(machine))
625 return PERF_RECORD_MISC_KERNEL;
626 else
627 return PERF_RECORD_MISC_GUEST_KERNEL;
628 } else {
629 if (machine__is_host(machine))
630 return PERF_RECORD_MISC_USER;
631 else if (perf_guest)
632 return PERF_RECORD_MISC_GUEST_USER;
633 else
634 return PERF_RECORD_MISC_HYPERVISOR;
635 }
636}
637
638static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id,
639 u64 address, size_t size, u8 *buffer)
640{
641 u8 cpumode;
642 u64 offset;
643 int len;
644 struct thread *thread;
645 struct machine *machine;
646 struct addr_location al;
647 struct cs_etm_traceid_queue *tidq;
648
649 if (!etmq)
650 return 0;
651
652 machine = etmq->etm->machine;
653 cpumode = cs_etm__cpu_mode(etmq, address);
654 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
655 if (!tidq)
656 return 0;
657
658 thread = tidq->thread;
659 if (!thread) {
660 if (cpumode != PERF_RECORD_MISC_KERNEL)
661 return 0;
662 thread = etmq->etm->unknown_thread;
663 }
664
665 if (!thread__find_map(thread, cpumode, address, &al) || !al.map->dso)
666 return 0;
667
668 if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
669 dso__data_status_seen(al.map->dso, DSO_DATA_STATUS_SEEN_ITRACE))
670 return 0;
671
672 offset = al.map->map_ip(al.map, address);
673
674 map__load(al.map);
675
676 len = dso__data_read_offset(al.map->dso, machine, offset, buffer, size);
677
678 if (len <= 0)
679 return 0;
680
681 return len;
682}
683
684static struct cs_etm_queue *cs_etm__alloc_queue(struct cs_etm_auxtrace *etm)
685{
686 struct cs_etm_decoder_params d_params;
687 struct cs_etm_trace_params *t_params = NULL;
688 struct cs_etm_queue *etmq;
689
690 etmq = zalloc(sizeof(*etmq));
691 if (!etmq)
692 return NULL;
693
694 etmq->traceid_queues_list = intlist__new(NULL);
695 if (!etmq->traceid_queues_list)
696 goto out_free;
697
698 /* Use metadata to fill in trace parameters for trace decoder */
699 t_params = zalloc(sizeof(*t_params) * etm->num_cpu);
700
701 if (!t_params)
702 goto out_free;
703
704 if (cs_etm__init_trace_params(t_params, etm))
705 goto out_free;
706
707 /* Set decoder parameters to decode trace packets */
708 if (cs_etm__init_decoder_params(&d_params, etmq,
709 CS_ETM_OPERATION_DECODE))
710 goto out_free;
711
712 etmq->decoder = cs_etm_decoder__new(etm->num_cpu, &d_params, t_params);
713
714 if (!etmq->decoder)
715 goto out_free;
716
717 /*
718 * Register a function to handle all memory accesses required by
719 * the trace decoder library.
720 */
721 if (cs_etm_decoder__add_mem_access_cb(etmq->decoder,
722 0x0L, ((u64) -1L),
723 cs_etm__mem_access))
724 goto out_free_decoder;
725
726 zfree(&t_params);
727 return etmq;
728
729out_free_decoder:
730 cs_etm_decoder__free(etmq->decoder);
731out_free:
732 intlist__delete(etmq->traceid_queues_list);
733 free(etmq);
734
735 return NULL;
736}
737
738static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm,
739 struct auxtrace_queue *queue,
740 unsigned int queue_nr)
741{
742 int ret = 0;
743 unsigned int cs_queue_nr;
744 u8 trace_chan_id;
745 u64 timestamp;
746 struct cs_etm_queue *etmq = queue->priv;
747
748 if (list_empty(&queue->head) || etmq)
749 goto out;
750
751 etmq = cs_etm__alloc_queue(etm);
752
753 if (!etmq) {
754 ret = -ENOMEM;
755 goto out;
756 }
757
758 queue->priv = etmq;
759 etmq->etm = etm;
760 etmq->queue_nr = queue_nr;
761 etmq->offset = 0;
762
763 if (etm->timeless_decoding)
764 goto out;
765
766 /*
767 * We are under a CPU-wide trace scenario. As such we need to know
768 * when the code that generated the traces started to execute so that
769 * it can be correlated with execution on other CPUs. So we get a
770 * handle on the beginning of traces and decode until we find a
771 * timestamp. The timestamp is then added to the auxtrace min heap
772 * in order to know what nibble (of all the etmqs) to decode first.
773 */
774 while (1) {
775 /*
776 * Fetch an aux_buffer from this etmq. Bail if no more
777 * blocks or an error has been encountered.
778 */
779 ret = cs_etm__get_data_block(etmq);
780 if (ret <= 0)
781 goto out;
782
783 /*
784 * Run decoder on the trace block. The decoder will stop when
785 * encountering a timestamp, a full packet queue or the end of
786 * trace for that block.
787 */
788 ret = cs_etm__decode_data_block(etmq);
789 if (ret)
790 goto out;
791
792 /*
793 * Function cs_etm_decoder__do_{hard|soft}_timestamp() does all
794 * the timestamp calculation for us.
795 */
796 timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
797
798 /* We found a timestamp, no need to continue. */
799 if (timestamp)
800 break;
801
802 /*
803 * We didn't find a timestamp so empty all the traceid packet
804 * queues before looking for another timestamp packet, either
805 * in the current data block or a new one. Packets that were
806 * just decoded are useless since no timestamp has been
807 * associated with them. As such simply discard them.
808 */
809 cs_etm__clear_all_packet_queues(etmq);
810 }
811
812 /*
813 * We have a timestamp. Add it to the min heap to reflect when
814 * instructions conveyed by the range packets of this traceID queue
815 * started to execute. Once the same has been done for all the traceID
816 * queues of each etmq, redenring and decoding can start in
817 * chronological order.
818 *
819 * Note that packets decoded above are still in the traceID's packet
820 * queue and will be processed in cs_etm__process_queues().
821 */
822 cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_id_chan);
823 ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, timestamp);
824out:
825 return ret;
826}
827
828static int cs_etm__setup_queues(struct cs_etm_auxtrace *etm)
829{
830 unsigned int i;
831 int ret;
832
833 if (!etm->kernel_start)
834 etm->kernel_start = machine__kernel_start(etm->machine);
835
836 for (i = 0; i < etm->queues.nr_queues; i++) {
837 ret = cs_etm__setup_queue(etm, &etm->queues.queue_array[i], i);
838 if (ret)
839 return ret;
840 }
841
842 return 0;
843}
844
845static int cs_etm__update_queues(struct cs_etm_auxtrace *etm)
846{
847 if (etm->queues.new_data) {
848 etm->queues.new_data = false;
849 return cs_etm__setup_queues(etm);
850 }
851
852 return 0;
853}
854
855static inline
856void cs_etm__copy_last_branch_rb(struct cs_etm_queue *etmq,
857 struct cs_etm_traceid_queue *tidq)
858{
859 struct branch_stack *bs_src = tidq->last_branch_rb;
860 struct branch_stack *bs_dst = tidq->last_branch;
861 size_t nr = 0;
862
863 /*
864 * Set the number of records before early exit: ->nr is used to
865 * determine how many branches to copy from ->entries.
866 */
867 bs_dst->nr = bs_src->nr;
868
869 /*
870 * Early exit when there is nothing to copy.
871 */
872 if (!bs_src->nr)
873 return;
874
875 /*
876 * As bs_src->entries is a circular buffer, we need to copy from it in
877 * two steps. First, copy the branches from the most recently inserted
878 * branch ->last_branch_pos until the end of bs_src->entries buffer.
879 */
880 nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos;
881 memcpy(&bs_dst->entries[0],
882 &bs_src->entries[tidq->last_branch_pos],
883 sizeof(struct branch_entry) * nr);
884
885 /*
886 * If we wrapped around at least once, the branches from the beginning
887 * of the bs_src->entries buffer and until the ->last_branch_pos element
888 * are older valid branches: copy them over. The total number of
889 * branches copied over will be equal to the number of branches asked by
890 * the user in last_branch_sz.
891 */
892 if (bs_src->nr >= etmq->etm->synth_opts.last_branch_sz) {
893 memcpy(&bs_dst->entries[nr],
894 &bs_src->entries[0],
895 sizeof(struct branch_entry) * tidq->last_branch_pos);
896 }
897}
898
899static inline
900void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq)
901{
902 tidq->last_branch_pos = 0;
903 tidq->last_branch_rb->nr = 0;
904}
905
906static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq,
907 u8 trace_chan_id, u64 addr)
908{
909 u8 instrBytes[2];
910
911 cs_etm__mem_access(etmq, trace_chan_id, addr,
912 ARRAY_SIZE(instrBytes), instrBytes);
913 /*
914 * T32 instruction size is indicated by bits[15:11] of the first
915 * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111
916 * denote a 32-bit instruction.
917 */
918 return ((instrBytes[1] & 0xF8) >= 0xE8) ? 4 : 2;
919}
920
921static inline u64 cs_etm__first_executed_instr(struct cs_etm_packet *packet)
922{
923 /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
924 if (packet->sample_type == CS_ETM_DISCONTINUITY)
925 return 0;
926
927 return packet->start_addr;
928}
929
930static inline
931u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet)
932{
933 /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
934 if (packet->sample_type == CS_ETM_DISCONTINUITY)
935 return 0;
936
937 return packet->end_addr - packet->last_instr_size;
938}
939
940static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq,
941 u64 trace_chan_id,
942 const struct cs_etm_packet *packet,
943 u64 offset)
944{
945 if (packet->isa == CS_ETM_ISA_T32) {
946 u64 addr = packet->start_addr;
947
948 while (offset > 0) {
949 addr += cs_etm__t32_instr_size(etmq,
950 trace_chan_id, addr);
951 offset--;
952 }
953 return addr;
954 }
955
956 /* Assume a 4 byte instruction size (A32/A64) */
957 return packet->start_addr + offset * 4;
958}
959
960static void cs_etm__update_last_branch_rb(struct cs_etm_queue *etmq,
961 struct cs_etm_traceid_queue *tidq)
962{
963 struct branch_stack *bs = tidq->last_branch_rb;
964 struct branch_entry *be;
965
966 /*
967 * The branches are recorded in a circular buffer in reverse
968 * chronological order: we start recording from the last element of the
969 * buffer down. After writing the first element of the stack, move the
970 * insert position back to the end of the buffer.
971 */
972 if (!tidq->last_branch_pos)
973 tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz;
974
975 tidq->last_branch_pos -= 1;
976
977 be = &bs->entries[tidq->last_branch_pos];
978 be->from = cs_etm__last_executed_instr(tidq->prev_packet);
979 be->to = cs_etm__first_executed_instr(tidq->packet);
980 /* No support for mispredict */
981 be->flags.mispred = 0;
982 be->flags.predicted = 1;
983
984 /*
985 * Increment bs->nr until reaching the number of last branches asked by
986 * the user on the command line.
987 */
988 if (bs->nr < etmq->etm->synth_opts.last_branch_sz)
989 bs->nr += 1;
990}
991
992static int cs_etm__inject_event(union perf_event *event,
993 struct perf_sample *sample, u64 type)
994{
995 event->header.size = perf_event__sample_event_size(sample, type, 0);
996 return perf_event__synthesize_sample(event, type, 0, sample);
997}
998
999
1000static int
1001cs_etm__get_trace(struct cs_etm_queue *etmq)
1002{
1003 struct auxtrace_buffer *aux_buffer = etmq->buffer;
1004 struct auxtrace_buffer *old_buffer = aux_buffer;
1005 struct auxtrace_queue *queue;
1006
1007 queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
1008
1009 aux_buffer = auxtrace_buffer__next(queue, aux_buffer);
1010
1011 /* If no more data, drop the previous auxtrace_buffer and return */
1012 if (!aux_buffer) {
1013 if (old_buffer)
1014 auxtrace_buffer__drop_data(old_buffer);
1015 etmq->buf_len = 0;
1016 return 0;
1017 }
1018
1019 etmq->buffer = aux_buffer;
1020
1021 /* If the aux_buffer doesn't have data associated, try to load it */
1022 if (!aux_buffer->data) {
1023 /* get the file desc associated with the perf data file */
1024 int fd = perf_data__fd(etmq->etm->session->data);
1025
1026 aux_buffer->data = auxtrace_buffer__get_data(aux_buffer, fd);
1027 if (!aux_buffer->data)
1028 return -ENOMEM;
1029 }
1030
1031 /* If valid, drop the previous buffer */
1032 if (old_buffer)
1033 auxtrace_buffer__drop_data(old_buffer);
1034
1035 etmq->buf_used = 0;
1036 etmq->buf_len = aux_buffer->size;
1037 etmq->buf = aux_buffer->data;
1038
1039 return etmq->buf_len;
1040}
1041
1042static void cs_etm__set_pid_tid_cpu(struct cs_etm_auxtrace *etm,
1043 struct cs_etm_traceid_queue *tidq)
1044{
1045 if ((!tidq->thread) && (tidq->tid != -1))
1046 tidq->thread = machine__find_thread(etm->machine, -1,
1047 tidq->tid);
1048
1049 if (tidq->thread)
1050 tidq->pid = tidq->thread->pid_;
1051}
1052
1053int cs_etm__etmq_set_tid(struct cs_etm_queue *etmq,
1054 pid_t tid, u8 trace_chan_id)
1055{
1056 int cpu, err = -EINVAL;
1057 struct cs_etm_auxtrace *etm = etmq->etm;
1058 struct cs_etm_traceid_queue *tidq;
1059
1060 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
1061 if (!tidq)
1062 return err;
1063
1064 if (cs_etm__get_cpu(trace_chan_id, &cpu) < 0)
1065 return err;
1066
1067 err = machine__set_current_tid(etm->machine, cpu, tid, tid);
1068 if (err)
1069 return err;
1070
1071 tidq->tid = tid;
1072 thread__zput(tidq->thread);
1073
1074 cs_etm__set_pid_tid_cpu(etm, tidq);
1075 return 0;
1076}
1077
1078bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq)
1079{
1080 return !!etmq->etm->timeless_decoding;
1081}
1082
1083static void cs_etm__copy_insn(struct cs_etm_queue *etmq,
1084 u64 trace_chan_id,
1085 const struct cs_etm_packet *packet,
1086 struct perf_sample *sample)
1087{
1088 /*
1089 * It's pointless to read instructions for the CS_ETM_DISCONTINUITY
1090 * packet, so directly bail out with 'insn_len' = 0.
1091 */
1092 if (packet->sample_type == CS_ETM_DISCONTINUITY) {
1093 sample->insn_len = 0;
1094 return;
1095 }
1096
1097 /*
1098 * T32 instruction size might be 32-bit or 16-bit, decide by calling
1099 * cs_etm__t32_instr_size().
1100 */
1101 if (packet->isa == CS_ETM_ISA_T32)
1102 sample->insn_len = cs_etm__t32_instr_size(etmq, trace_chan_id,
1103 sample->ip);
1104 /* Otherwise, A64 and A32 instruction size are always 32-bit. */
1105 else
1106 sample->insn_len = 4;
1107
1108 cs_etm__mem_access(etmq, trace_chan_id, sample->ip,
1109 sample->insn_len, (void *)sample->insn);
1110}
1111
1112static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq,
1113 struct cs_etm_traceid_queue *tidq,
1114 u64 addr, u64 period)
1115{
1116 int ret = 0;
1117 struct cs_etm_auxtrace *etm = etmq->etm;
1118 union perf_event *event = tidq->event_buf;
1119 struct perf_sample sample = {.ip = 0,};
1120
1121 event->sample.header.type = PERF_RECORD_SAMPLE;
1122 event->sample.header.misc = cs_etm__cpu_mode(etmq, addr);
1123 event->sample.header.size = sizeof(struct perf_event_header);
1124
1125 sample.ip = addr;
1126 sample.pid = tidq->pid;
1127 sample.tid = tidq->tid;
1128 sample.id = etmq->etm->instructions_id;
1129 sample.stream_id = etmq->etm->instructions_id;
1130 sample.period = period;
1131 sample.cpu = tidq->packet->cpu;
1132 sample.flags = tidq->prev_packet->flags;
1133 sample.cpumode = event->sample.header.misc;
1134
1135 cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->packet, &sample);
1136
1137 if (etm->synth_opts.last_branch) {
1138 cs_etm__copy_last_branch_rb(etmq, tidq);
1139 sample.branch_stack = tidq->last_branch;
1140 }
1141
1142 if (etm->synth_opts.inject) {
1143 ret = cs_etm__inject_event(event, &sample,
1144 etm->instructions_sample_type);
1145 if (ret)
1146 return ret;
1147 }
1148
1149 ret = perf_session__deliver_synth_event(etm->session, event, &sample);
1150
1151 if (ret)
1152 pr_err(
1153 "CS ETM Trace: failed to deliver instruction event, error %d\n",
1154 ret);
1155
1156 if (etm->synth_opts.last_branch)
1157 cs_etm__reset_last_branch_rb(tidq);
1158
1159 return ret;
1160}
1161
1162/*
1163 * The cs etm packet encodes an instruction range between a branch target
1164 * and the next taken branch. Generate sample accordingly.
1165 */
1166static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq,
1167 struct cs_etm_traceid_queue *tidq)
1168{
1169 int ret = 0;
1170 struct cs_etm_auxtrace *etm = etmq->etm;
1171 struct perf_sample sample = {.ip = 0,};
1172 union perf_event *event = tidq->event_buf;
1173 struct dummy_branch_stack {
1174 u64 nr;
1175 struct branch_entry entries;
1176 } dummy_bs;
1177 u64 ip;
1178
1179 ip = cs_etm__last_executed_instr(tidq->prev_packet);
1180
1181 event->sample.header.type = PERF_RECORD_SAMPLE;
1182 event->sample.header.misc = cs_etm__cpu_mode(etmq, ip);
1183 event->sample.header.size = sizeof(struct perf_event_header);
1184
1185 sample.ip = ip;
1186 sample.pid = tidq->pid;
1187 sample.tid = tidq->tid;
1188 sample.addr = cs_etm__first_executed_instr(tidq->packet);
1189 sample.id = etmq->etm->branches_id;
1190 sample.stream_id = etmq->etm->branches_id;
1191 sample.period = 1;
1192 sample.cpu = tidq->packet->cpu;
1193 sample.flags = tidq->prev_packet->flags;
1194 sample.cpumode = event->sample.header.misc;
1195
1196 cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->prev_packet,
1197 &sample);
1198
1199 /*
1200 * perf report cannot handle events without a branch stack
1201 */
1202 if (etm->synth_opts.last_branch) {
1203 dummy_bs = (struct dummy_branch_stack){
1204 .nr = 1,
1205 .entries = {
1206 .from = sample.ip,
1207 .to = sample.addr,
1208 },
1209 };
1210 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1211 }
1212
1213 if (etm->synth_opts.inject) {
1214 ret = cs_etm__inject_event(event, &sample,
1215 etm->branches_sample_type);
1216 if (ret)
1217 return ret;
1218 }
1219
1220 ret = perf_session__deliver_synth_event(etm->session, event, &sample);
1221
1222 if (ret)
1223 pr_err(
1224 "CS ETM Trace: failed to deliver instruction event, error %d\n",
1225 ret);
1226
1227 return ret;
1228}
1229
1230struct cs_etm_synth {
1231 struct perf_tool dummy_tool;
1232 struct perf_session *session;
1233};
1234
1235static int cs_etm__event_synth(struct perf_tool *tool,
1236 union perf_event *event,
1237 struct perf_sample *sample __maybe_unused,
1238 struct machine *machine __maybe_unused)
1239{
1240 struct cs_etm_synth *cs_etm_synth =
1241 container_of(tool, struct cs_etm_synth, dummy_tool);
1242
1243 return perf_session__deliver_synth_event(cs_etm_synth->session,
1244 event, NULL);
1245}
1246
1247static int cs_etm__synth_event(struct perf_session *session,
1248 struct perf_event_attr *attr, u64 id)
1249{
1250 struct cs_etm_synth cs_etm_synth;
1251
1252 memset(&cs_etm_synth, 0, sizeof(struct cs_etm_synth));
1253 cs_etm_synth.session = session;
1254
1255 return perf_event__synthesize_attr(&cs_etm_synth.dummy_tool, attr, 1,
1256 &id, cs_etm__event_synth);
1257}
1258
1259static int cs_etm__synth_events(struct cs_etm_auxtrace *etm,
1260 struct perf_session *session)
1261{
1262 struct evlist *evlist = session->evlist;
1263 struct evsel *evsel;
1264 struct perf_event_attr attr;
1265 bool found = false;
1266 u64 id;
1267 int err;
1268
1269 evlist__for_each_entry(evlist, evsel) {
1270 if (evsel->core.attr.type == etm->pmu_type) {
1271 found = true;
1272 break;
1273 }
1274 }
1275
1276 if (!found) {
1277 pr_debug("No selected events with CoreSight Trace data\n");
1278 return 0;
1279 }
1280
1281 memset(&attr, 0, sizeof(struct perf_event_attr));
1282 attr.size = sizeof(struct perf_event_attr);
1283 attr.type = PERF_TYPE_HARDWARE;
1284 attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK;
1285 attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1286 PERF_SAMPLE_PERIOD;
1287 if (etm->timeless_decoding)
1288 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1289 else
1290 attr.sample_type |= PERF_SAMPLE_TIME;
1291
1292 attr.exclude_user = evsel->core.attr.exclude_user;
1293 attr.exclude_kernel = evsel->core.attr.exclude_kernel;
1294 attr.exclude_hv = evsel->core.attr.exclude_hv;
1295 attr.exclude_host = evsel->core.attr.exclude_host;
1296 attr.exclude_guest = evsel->core.attr.exclude_guest;
1297 attr.sample_id_all = evsel->core.attr.sample_id_all;
1298 attr.read_format = evsel->core.attr.read_format;
1299
1300 /* create new id val to be a fixed offset from evsel id */
1301 id = evsel->core.id[0] + 1000000000;
1302
1303 if (!id)
1304 id = 1;
1305
1306 if (etm->synth_opts.branches) {
1307 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
1308 attr.sample_period = 1;
1309 attr.sample_type |= PERF_SAMPLE_ADDR;
1310 err = cs_etm__synth_event(session, &attr, id);
1311 if (err)
1312 return err;
1313 etm->sample_branches = true;
1314 etm->branches_sample_type = attr.sample_type;
1315 etm->branches_id = id;
1316 id += 1;
1317 attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR;
1318 }
1319
1320 if (etm->synth_opts.last_branch)
1321 attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1322
1323 if (etm->synth_opts.instructions) {
1324 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1325 attr.sample_period = etm->synth_opts.period;
1326 etm->instructions_sample_period = attr.sample_period;
1327 err = cs_etm__synth_event(session, &attr, id);
1328 if (err)
1329 return err;
1330 etm->sample_instructions = true;
1331 etm->instructions_sample_type = attr.sample_type;
1332 etm->instructions_id = id;
1333 id += 1;
1334 }
1335
1336 return 0;
1337}
1338
1339static int cs_etm__sample(struct cs_etm_queue *etmq,
1340 struct cs_etm_traceid_queue *tidq)
1341{
1342 struct cs_etm_auxtrace *etm = etmq->etm;
1343 struct cs_etm_packet *tmp;
1344 int ret;
1345 u8 trace_chan_id = tidq->trace_chan_id;
1346 u64 instrs_executed = tidq->packet->instr_count;
1347
1348 tidq->period_instructions += instrs_executed;
1349
1350 /*
1351 * Record a branch when the last instruction in
1352 * PREV_PACKET is a branch.
1353 */
1354 if (etm->synth_opts.last_branch &&
1355 tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1356 tidq->prev_packet->last_instr_taken_branch)
1357 cs_etm__update_last_branch_rb(etmq, tidq);
1358
1359 if (etm->sample_instructions &&
1360 tidq->period_instructions >= etm->instructions_sample_period) {
1361 /*
1362 * Emit instruction sample periodically
1363 * TODO: allow period to be defined in cycles and clock time
1364 */
1365
1366 /* Get number of instructions executed after the sample point */
1367 u64 instrs_over = tidq->period_instructions -
1368 etm->instructions_sample_period;
1369
1370 /*
1371 * Calculate the address of the sampled instruction (-1 as
1372 * sample is reported as though instruction has just been
1373 * executed, but PC has not advanced to next instruction)
1374 */
1375 u64 offset = (instrs_executed - instrs_over - 1);
1376 u64 addr = cs_etm__instr_addr(etmq, trace_chan_id,
1377 tidq->packet, offset);
1378
1379 ret = cs_etm__synth_instruction_sample(
1380 etmq, tidq, addr, etm->instructions_sample_period);
1381 if (ret)
1382 return ret;
1383
1384 /* Carry remaining instructions into next sample period */
1385 tidq->period_instructions = instrs_over;
1386 }
1387
1388 if (etm->sample_branches) {
1389 bool generate_sample = false;
1390
1391 /* Generate sample for tracing on packet */
1392 if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY)
1393 generate_sample = true;
1394
1395 /* Generate sample for branch taken packet */
1396 if (tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1397 tidq->prev_packet->last_instr_taken_branch)
1398 generate_sample = true;
1399
1400 if (generate_sample) {
1401 ret = cs_etm__synth_branch_sample(etmq, tidq);
1402 if (ret)
1403 return ret;
1404 }
1405 }
1406
1407 if (etm->sample_branches || etm->synth_opts.last_branch) {
1408 /*
1409 * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
1410 * the next incoming packet.
1411 */
1412 tmp = tidq->packet;
1413 tidq->packet = tidq->prev_packet;
1414 tidq->prev_packet = tmp;
1415 }
1416
1417 return 0;
1418}
1419
1420static int cs_etm__exception(struct cs_etm_traceid_queue *tidq)
1421{
1422 /*
1423 * When the exception packet is inserted, whether the last instruction
1424 * in previous range packet is taken branch or not, we need to force
1425 * to set 'prev_packet->last_instr_taken_branch' to true. This ensures
1426 * to generate branch sample for the instruction range before the
1427 * exception is trapped to kernel or before the exception returning.
1428 *
1429 * The exception packet includes the dummy address values, so don't
1430 * swap PACKET with PREV_PACKET. This keeps PREV_PACKET to be useful
1431 * for generating instruction and branch samples.
1432 */
1433 if (tidq->prev_packet->sample_type == CS_ETM_RANGE)
1434 tidq->prev_packet->last_instr_taken_branch = true;
1435
1436 return 0;
1437}
1438
1439static int cs_etm__flush(struct cs_etm_queue *etmq,
1440 struct cs_etm_traceid_queue *tidq)
1441{
1442 int err = 0;
1443 struct cs_etm_auxtrace *etm = etmq->etm;
1444 struct cs_etm_packet *tmp;
1445
1446 /* Handle start tracing packet */
1447 if (tidq->prev_packet->sample_type == CS_ETM_EMPTY)
1448 goto swap_packet;
1449
1450 if (etmq->etm->synth_opts.last_branch &&
1451 tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1452 /*
1453 * Generate a last branch event for the branches left in the
1454 * circular buffer at the end of the trace.
1455 *
1456 * Use the address of the end of the last reported execution
1457 * range
1458 */
1459 u64 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1460
1461 err = cs_etm__synth_instruction_sample(
1462 etmq, tidq, addr,
1463 tidq->period_instructions);
1464 if (err)
1465 return err;
1466
1467 tidq->period_instructions = 0;
1468
1469 }
1470
1471 if (etm->sample_branches &&
1472 tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1473 err = cs_etm__synth_branch_sample(etmq, tidq);
1474 if (err)
1475 return err;
1476 }
1477
1478swap_packet:
1479 if (etm->sample_branches || etm->synth_opts.last_branch) {
1480 /*
1481 * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
1482 * the next incoming packet.
1483 */
1484 tmp = tidq->packet;
1485 tidq->packet = tidq->prev_packet;
1486 tidq->prev_packet = tmp;
1487 }
1488
1489 return err;
1490}
1491
1492static int cs_etm__end_block(struct cs_etm_queue *etmq,
1493 struct cs_etm_traceid_queue *tidq)
1494{
1495 int err;
1496
1497 /*
1498 * It has no new packet coming and 'etmq->packet' contains the stale
1499 * packet which was set at the previous time with packets swapping;
1500 * so skip to generate branch sample to avoid stale packet.
1501 *
1502 * For this case only flush branch stack and generate a last branch
1503 * event for the branches left in the circular buffer at the end of
1504 * the trace.
1505 */
1506 if (etmq->etm->synth_opts.last_branch &&
1507 tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1508 /*
1509 * Use the address of the end of the last reported execution
1510 * range.
1511 */
1512 u64 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1513
1514 err = cs_etm__synth_instruction_sample(
1515 etmq, tidq, addr,
1516 tidq->period_instructions);
1517 if (err)
1518 return err;
1519
1520 tidq->period_instructions = 0;
1521 }
1522
1523 return 0;
1524}
1525/*
1526 * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue
1527 * if need be.
1528 * Returns: < 0 if error
1529 * = 0 if no more auxtrace_buffer to read
1530 * > 0 if the current buffer isn't empty yet
1531 */
1532static int cs_etm__get_data_block(struct cs_etm_queue *etmq)
1533{
1534 int ret;
1535
1536 if (!etmq->buf_len) {
1537 ret = cs_etm__get_trace(etmq);
1538 if (ret <= 0)
1539 return ret;
1540 /*
1541 * We cannot assume consecutive blocks in the data file
1542 * are contiguous, reset the decoder to force re-sync.
1543 */
1544 ret = cs_etm_decoder__reset(etmq->decoder);
1545 if (ret)
1546 return ret;
1547 }
1548
1549 return etmq->buf_len;
1550}
1551
1552static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id,
1553 struct cs_etm_packet *packet,
1554 u64 end_addr)
1555{
1556 /* Initialise to keep compiler happy */
1557 u16 instr16 = 0;
1558 u32 instr32 = 0;
1559 u64 addr;
1560
1561 switch (packet->isa) {
1562 case CS_ETM_ISA_T32:
1563 /*
1564 * The SVC of T32 is defined in ARM DDI 0487D.a, F5.1.247:
1565 *
1566 * b'15 b'8
1567 * +-----------------+--------+
1568 * | 1 1 0 1 1 1 1 1 | imm8 |
1569 * +-----------------+--------+
1570 *
1571 * According to the specifiction, it only defines SVC for T32
1572 * with 16 bits instruction and has no definition for 32bits;
1573 * so below only read 2 bytes as instruction size for T32.
1574 */
1575 addr = end_addr - 2;
1576 cs_etm__mem_access(etmq, trace_chan_id, addr,
1577 sizeof(instr16), (u8 *)&instr16);
1578 if ((instr16 & 0xFF00) == 0xDF00)
1579 return true;
1580
1581 break;
1582 case CS_ETM_ISA_A32:
1583 /*
1584 * The SVC of A32 is defined in ARM DDI 0487D.a, F5.1.247:
1585 *
1586 * b'31 b'28 b'27 b'24
1587 * +---------+---------+-------------------------+
1588 * | !1111 | 1 1 1 1 | imm24 |
1589 * +---------+---------+-------------------------+
1590 */
1591 addr = end_addr - 4;
1592 cs_etm__mem_access(etmq, trace_chan_id, addr,
1593 sizeof(instr32), (u8 *)&instr32);
1594 if ((instr32 & 0x0F000000) == 0x0F000000 &&
1595 (instr32 & 0xF0000000) != 0xF0000000)
1596 return true;
1597
1598 break;
1599 case CS_ETM_ISA_A64:
1600 /*
1601 * The SVC of A64 is defined in ARM DDI 0487D.a, C6.2.294:
1602 *
1603 * b'31 b'21 b'4 b'0
1604 * +-----------------------+---------+-----------+
1605 * | 1 1 0 1 0 1 0 0 0 0 0 | imm16 | 0 0 0 0 1 |
1606 * +-----------------------+---------+-----------+
1607 */
1608 addr = end_addr - 4;
1609 cs_etm__mem_access(etmq, trace_chan_id, addr,
1610 sizeof(instr32), (u8 *)&instr32);
1611 if ((instr32 & 0xFFE0001F) == 0xd4000001)
1612 return true;
1613
1614 break;
1615 case CS_ETM_ISA_UNKNOWN:
1616 default:
1617 break;
1618 }
1619
1620 return false;
1621}
1622
1623static bool cs_etm__is_syscall(struct cs_etm_queue *etmq,
1624 struct cs_etm_traceid_queue *tidq, u64 magic)
1625{
1626 u8 trace_chan_id = tidq->trace_chan_id;
1627 struct cs_etm_packet *packet = tidq->packet;
1628 struct cs_etm_packet *prev_packet = tidq->prev_packet;
1629
1630 if (magic == __perf_cs_etmv3_magic)
1631 if (packet->exception_number == CS_ETMV3_EXC_SVC)
1632 return true;
1633
1634 /*
1635 * ETMv4 exception type CS_ETMV4_EXC_CALL covers SVC, SMC and
1636 * HVC cases; need to check if it's SVC instruction based on
1637 * packet address.
1638 */
1639 if (magic == __perf_cs_etmv4_magic) {
1640 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
1641 cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
1642 prev_packet->end_addr))
1643 return true;
1644 }
1645
1646 return false;
1647}
1648
1649static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq,
1650 u64 magic)
1651{
1652 struct cs_etm_packet *packet = tidq->packet;
1653
1654 if (magic == __perf_cs_etmv3_magic)
1655 if (packet->exception_number == CS_ETMV3_EXC_DEBUG_HALT ||
1656 packet->exception_number == CS_ETMV3_EXC_ASYNC_DATA_ABORT ||
1657 packet->exception_number == CS_ETMV3_EXC_PE_RESET ||
1658 packet->exception_number == CS_ETMV3_EXC_IRQ ||
1659 packet->exception_number == CS_ETMV3_EXC_FIQ)
1660 return true;
1661
1662 if (magic == __perf_cs_etmv4_magic)
1663 if (packet->exception_number == CS_ETMV4_EXC_RESET ||
1664 packet->exception_number == CS_ETMV4_EXC_DEBUG_HALT ||
1665 packet->exception_number == CS_ETMV4_EXC_SYSTEM_ERROR ||
1666 packet->exception_number == CS_ETMV4_EXC_INST_DEBUG ||
1667 packet->exception_number == CS_ETMV4_EXC_DATA_DEBUG ||
1668 packet->exception_number == CS_ETMV4_EXC_IRQ ||
1669 packet->exception_number == CS_ETMV4_EXC_FIQ)
1670 return true;
1671
1672 return false;
1673}
1674
1675static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq,
1676 struct cs_etm_traceid_queue *tidq,
1677 u64 magic)
1678{
1679 u8 trace_chan_id = tidq->trace_chan_id;
1680 struct cs_etm_packet *packet = tidq->packet;
1681 struct cs_etm_packet *prev_packet = tidq->prev_packet;
1682
1683 if (magic == __perf_cs_etmv3_magic)
1684 if (packet->exception_number == CS_ETMV3_EXC_SMC ||
1685 packet->exception_number == CS_ETMV3_EXC_HYP ||
1686 packet->exception_number == CS_ETMV3_EXC_JAZELLE_THUMBEE ||
1687 packet->exception_number == CS_ETMV3_EXC_UNDEFINED_INSTR ||
1688 packet->exception_number == CS_ETMV3_EXC_PREFETCH_ABORT ||
1689 packet->exception_number == CS_ETMV3_EXC_DATA_FAULT ||
1690 packet->exception_number == CS_ETMV3_EXC_GENERIC)
1691 return true;
1692
1693 if (magic == __perf_cs_etmv4_magic) {
1694 if (packet->exception_number == CS_ETMV4_EXC_TRAP ||
1695 packet->exception_number == CS_ETMV4_EXC_ALIGNMENT ||
1696 packet->exception_number == CS_ETMV4_EXC_INST_FAULT ||
1697 packet->exception_number == CS_ETMV4_EXC_DATA_FAULT)
1698 return true;
1699
1700 /*
1701 * For CS_ETMV4_EXC_CALL, except SVC other instructions
1702 * (SMC, HVC) are taken as sync exceptions.
1703 */
1704 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
1705 !cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
1706 prev_packet->end_addr))
1707 return true;
1708
1709 /*
1710 * ETMv4 has 5 bits for exception number; if the numbers
1711 * are in the range ( CS_ETMV4_EXC_FIQ, CS_ETMV4_EXC_END ]
1712 * they are implementation defined exceptions.
1713 *
1714 * For this case, simply take it as sync exception.
1715 */
1716 if (packet->exception_number > CS_ETMV4_EXC_FIQ &&
1717 packet->exception_number <= CS_ETMV4_EXC_END)
1718 return true;
1719 }
1720
1721 return false;
1722}
1723
1724static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq,
1725 struct cs_etm_traceid_queue *tidq)
1726{
1727 struct cs_etm_packet *packet = tidq->packet;
1728 struct cs_etm_packet *prev_packet = tidq->prev_packet;
1729 u8 trace_chan_id = tidq->trace_chan_id;
1730 u64 magic;
1731 int ret;
1732
1733 switch (packet->sample_type) {
1734 case CS_ETM_RANGE:
1735 /*
1736 * Immediate branch instruction without neither link nor
1737 * return flag, it's normal branch instruction within
1738 * the function.
1739 */
1740 if (packet->last_instr_type == OCSD_INSTR_BR &&
1741 packet->last_instr_subtype == OCSD_S_INSTR_NONE) {
1742 packet->flags = PERF_IP_FLAG_BRANCH;
1743
1744 if (packet->last_instr_cond)
1745 packet->flags |= PERF_IP_FLAG_CONDITIONAL;
1746 }
1747
1748 /*
1749 * Immediate branch instruction with link (e.g. BL), this is
1750 * branch instruction for function call.
1751 */
1752 if (packet->last_instr_type == OCSD_INSTR_BR &&
1753 packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
1754 packet->flags = PERF_IP_FLAG_BRANCH |
1755 PERF_IP_FLAG_CALL;
1756
1757 /*
1758 * Indirect branch instruction with link (e.g. BLR), this is
1759 * branch instruction for function call.
1760 */
1761 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1762 packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
1763 packet->flags = PERF_IP_FLAG_BRANCH |
1764 PERF_IP_FLAG_CALL;
1765
1766 /*
1767 * Indirect branch instruction with subtype of
1768 * OCSD_S_INSTR_V7_IMPLIED_RET, this is explicit hint for
1769 * function return for A32/T32.
1770 */
1771 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1772 packet->last_instr_subtype == OCSD_S_INSTR_V7_IMPLIED_RET)
1773 packet->flags = PERF_IP_FLAG_BRANCH |
1774 PERF_IP_FLAG_RETURN;
1775
1776 /*
1777 * Indirect branch instruction without link (e.g. BR), usually
1778 * this is used for function return, especially for functions
1779 * within dynamic link lib.
1780 */
1781 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1782 packet->last_instr_subtype == OCSD_S_INSTR_NONE)
1783 packet->flags = PERF_IP_FLAG_BRANCH |
1784 PERF_IP_FLAG_RETURN;
1785
1786 /* Return instruction for function return. */
1787 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1788 packet->last_instr_subtype == OCSD_S_INSTR_V8_RET)
1789 packet->flags = PERF_IP_FLAG_BRANCH |
1790 PERF_IP_FLAG_RETURN;
1791
1792 /*
1793 * Decoder might insert a discontinuity in the middle of
1794 * instruction packets, fixup prev_packet with flag
1795 * PERF_IP_FLAG_TRACE_BEGIN to indicate restarting trace.
1796 */
1797 if (prev_packet->sample_type == CS_ETM_DISCONTINUITY)
1798 prev_packet->flags |= PERF_IP_FLAG_BRANCH |
1799 PERF_IP_FLAG_TRACE_BEGIN;
1800
1801 /*
1802 * If the previous packet is an exception return packet
1803 * and the return address just follows SVC instuction,
1804 * it needs to calibrate the previous packet sample flags
1805 * as PERF_IP_FLAG_SYSCALLRET.
1806 */
1807 if (prev_packet->flags == (PERF_IP_FLAG_BRANCH |
1808 PERF_IP_FLAG_RETURN |
1809 PERF_IP_FLAG_INTERRUPT) &&
1810 cs_etm__is_svc_instr(etmq, trace_chan_id,
1811 packet, packet->start_addr))
1812 prev_packet->flags = PERF_IP_FLAG_BRANCH |
1813 PERF_IP_FLAG_RETURN |
1814 PERF_IP_FLAG_SYSCALLRET;
1815 break;
1816 case CS_ETM_DISCONTINUITY:
1817 /*
1818 * The trace is discontinuous, if the previous packet is
1819 * instruction packet, set flag PERF_IP_FLAG_TRACE_END
1820 * for previous packet.
1821 */
1822 if (prev_packet->sample_type == CS_ETM_RANGE)
1823 prev_packet->flags |= PERF_IP_FLAG_BRANCH |
1824 PERF_IP_FLAG_TRACE_END;
1825 break;
1826 case CS_ETM_EXCEPTION:
1827 ret = cs_etm__get_magic(packet->trace_chan_id, &magic);
1828 if (ret)
1829 return ret;
1830
1831 /* The exception is for system call. */
1832 if (cs_etm__is_syscall(etmq, tidq, magic))
1833 packet->flags = PERF_IP_FLAG_BRANCH |
1834 PERF_IP_FLAG_CALL |
1835 PERF_IP_FLAG_SYSCALLRET;
1836 /*
1837 * The exceptions are triggered by external signals from bus,
1838 * interrupt controller, debug module, PE reset or halt.
1839 */
1840 else if (cs_etm__is_async_exception(tidq, magic))
1841 packet->flags = PERF_IP_FLAG_BRANCH |
1842 PERF_IP_FLAG_CALL |
1843 PERF_IP_FLAG_ASYNC |
1844 PERF_IP_FLAG_INTERRUPT;
1845 /*
1846 * Otherwise, exception is caused by trap, instruction &
1847 * data fault, or alignment errors.
1848 */
1849 else if (cs_etm__is_sync_exception(etmq, tidq, magic))
1850 packet->flags = PERF_IP_FLAG_BRANCH |
1851 PERF_IP_FLAG_CALL |
1852 PERF_IP_FLAG_INTERRUPT;
1853
1854 /*
1855 * When the exception packet is inserted, since exception
1856 * packet is not used standalone for generating samples
1857 * and it's affiliation to the previous instruction range
1858 * packet; so set previous range packet flags to tell perf
1859 * it is an exception taken branch.
1860 */
1861 if (prev_packet->sample_type == CS_ETM_RANGE)
1862 prev_packet->flags = packet->flags;
1863 break;
1864 case CS_ETM_EXCEPTION_RET:
1865 /*
1866 * When the exception return packet is inserted, since
1867 * exception return packet is not used standalone for
1868 * generating samples and it's affiliation to the previous
1869 * instruction range packet; so set previous range packet
1870 * flags to tell perf it is an exception return branch.
1871 *
1872 * The exception return can be for either system call or
1873 * other exception types; unfortunately the packet doesn't
1874 * contain exception type related info so we cannot decide
1875 * the exception type purely based on exception return packet.
1876 * If we record the exception number from exception packet and
1877 * reuse it for excpetion return packet, this is not reliable
1878 * due the trace can be discontinuity or the interrupt can
1879 * be nested, thus the recorded exception number cannot be
1880 * used for exception return packet for these two cases.
1881 *
1882 * For exception return packet, we only need to distinguish the
1883 * packet is for system call or for other types. Thus the
1884 * decision can be deferred when receive the next packet which
1885 * contains the return address, based on the return address we
1886 * can read out the previous instruction and check if it's a
1887 * system call instruction and then calibrate the sample flag
1888 * as needed.
1889 */
1890 if (prev_packet->sample_type == CS_ETM_RANGE)
1891 prev_packet->flags = PERF_IP_FLAG_BRANCH |
1892 PERF_IP_FLAG_RETURN |
1893 PERF_IP_FLAG_INTERRUPT;
1894 break;
1895 case CS_ETM_EMPTY:
1896 default:
1897 break;
1898 }
1899
1900 return 0;
1901}
1902
1903static int cs_etm__decode_data_block(struct cs_etm_queue *etmq)
1904{
1905 int ret = 0;
1906 size_t processed = 0;
1907
1908 /*
1909 * Packets are decoded and added to the decoder's packet queue
1910 * until the decoder packet processing callback has requested that
1911 * processing stops or there is nothing left in the buffer. Normal
1912 * operations that stop processing are a timestamp packet or a full
1913 * decoder buffer queue.
1914 */
1915 ret = cs_etm_decoder__process_data_block(etmq->decoder,
1916 etmq->offset,
1917 &etmq->buf[etmq->buf_used],
1918 etmq->buf_len,
1919 &processed);
1920 if (ret)
1921 goto out;
1922
1923 etmq->offset += processed;
1924 etmq->buf_used += processed;
1925 etmq->buf_len -= processed;
1926
1927out:
1928 return ret;
1929}
1930
1931static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq,
1932 struct cs_etm_traceid_queue *tidq)
1933{
1934 int ret;
1935 struct cs_etm_packet_queue *packet_queue;
1936
1937 packet_queue = &tidq->packet_queue;
1938
1939 /* Process each packet in this chunk */
1940 while (1) {
1941 ret = cs_etm_decoder__get_packet(packet_queue,
1942 tidq->packet);
1943 if (ret <= 0)
1944 /*
1945 * Stop processing this chunk on
1946 * end of data or error
1947 */
1948 break;
1949
1950 /*
1951 * Since packet addresses are swapped in packet
1952 * handling within below switch() statements,
1953 * thus setting sample flags must be called
1954 * prior to switch() statement to use address
1955 * information before packets swapping.
1956 */
1957 ret = cs_etm__set_sample_flags(etmq, tidq);
1958 if (ret < 0)
1959 break;
1960
1961 switch (tidq->packet->sample_type) {
1962 case CS_ETM_RANGE:
1963 /*
1964 * If the packet contains an instruction
1965 * range, generate instruction sequence
1966 * events.
1967 */
1968 cs_etm__sample(etmq, tidq);
1969 break;
1970 case CS_ETM_EXCEPTION:
1971 case CS_ETM_EXCEPTION_RET:
1972 /*
1973 * If the exception packet is coming,
1974 * make sure the previous instruction
1975 * range packet to be handled properly.
1976 */
1977 cs_etm__exception(tidq);
1978 break;
1979 case CS_ETM_DISCONTINUITY:
1980 /*
1981 * Discontinuity in trace, flush
1982 * previous branch stack
1983 */
1984 cs_etm__flush(etmq, tidq);
1985 break;
1986 case CS_ETM_EMPTY:
1987 /*
1988 * Should not receive empty packet,
1989 * report error.
1990 */
1991 pr_err("CS ETM Trace: empty packet\n");
1992 return -EINVAL;
1993 default:
1994 break;
1995 }
1996 }
1997
1998 return ret;
1999}
2000
2001static void cs_etm__clear_all_traceid_queues(struct cs_etm_queue *etmq)
2002{
2003 int idx;
2004 struct int_node *inode;
2005 struct cs_etm_traceid_queue *tidq;
2006 struct intlist *traceid_queues_list = etmq->traceid_queues_list;
2007
2008 intlist__for_each_entry(inode, traceid_queues_list) {
2009 idx = (int)(intptr_t)inode->priv;
2010 tidq = etmq->traceid_queues[idx];
2011
2012 /* Ignore return value */
2013 cs_etm__process_traceid_queue(etmq, tidq);
2014
2015 /*
2016 * Generate an instruction sample with the remaining
2017 * branchstack entries.
2018 */
2019 cs_etm__flush(etmq, tidq);
2020 }
2021}
2022
2023static int cs_etm__run_decoder(struct cs_etm_queue *etmq)
2024{
2025 int err = 0;
2026 struct cs_etm_traceid_queue *tidq;
2027
2028 tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID);
2029 if (!tidq)
2030 return -EINVAL;
2031
2032 /* Go through each buffer in the queue and decode them one by one */
2033 while (1) {
2034 err = cs_etm__get_data_block(etmq);
2035 if (err <= 0)
2036 return err;
2037
2038 /* Run trace decoder until buffer consumed or end of trace */
2039 do {
2040 err = cs_etm__decode_data_block(etmq);
2041 if (err)
2042 return err;
2043
2044 /*
2045 * Process each packet in this chunk, nothing to do if
2046 * an error occurs other than hoping the next one will
2047 * be better.
2048 */
2049 err = cs_etm__process_traceid_queue(etmq, tidq);
2050
2051 } while (etmq->buf_len);
2052
2053 if (err == 0)
2054 /* Flush any remaining branch stack entries */
2055 err = cs_etm__end_block(etmq, tidq);
2056 }
2057
2058 return err;
2059}
2060
2061static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
2062 pid_t tid)
2063{
2064 unsigned int i;
2065 struct auxtrace_queues *queues = &etm->queues;
2066
2067 for (i = 0; i < queues->nr_queues; i++) {
2068 struct auxtrace_queue *queue = &etm->queues.queue_array[i];
2069 struct cs_etm_queue *etmq = queue->priv;
2070 struct cs_etm_traceid_queue *tidq;
2071
2072 if (!etmq)
2073 continue;
2074
2075 tidq = cs_etm__etmq_get_traceid_queue(etmq,
2076 CS_ETM_PER_THREAD_TRACEID);
2077
2078 if (!tidq)
2079 continue;
2080
2081 if ((tid == -1) || (tidq->tid == tid)) {
2082 cs_etm__set_pid_tid_cpu(etm, tidq);
2083 cs_etm__run_decoder(etmq);
2084 }
2085 }
2086
2087 return 0;
2088}
2089
2090static int cs_etm__process_queues(struct cs_etm_auxtrace *etm)
2091{
2092 int ret = 0;
2093 unsigned int cs_queue_nr, queue_nr;
2094 u8 trace_chan_id;
2095 u64 timestamp;
2096 struct auxtrace_queue *queue;
2097 struct cs_etm_queue *etmq;
2098 struct cs_etm_traceid_queue *tidq;
2099
2100 while (1) {
2101 if (!etm->heap.heap_cnt)
2102 goto out;
2103
2104 /* Take the entry at the top of the min heap */
2105 cs_queue_nr = etm->heap.heap_array[0].queue_nr;
2106 queue_nr = TO_QUEUE_NR(cs_queue_nr);
2107 trace_chan_id = TO_TRACE_CHAN_ID(cs_queue_nr);
2108 queue = &etm->queues.queue_array[queue_nr];
2109 etmq = queue->priv;
2110
2111 /*
2112 * Remove the top entry from the heap since we are about
2113 * to process it.
2114 */
2115 auxtrace_heap__pop(&etm->heap);
2116
2117 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
2118 if (!tidq) {
2119 /*
2120 * No traceID queue has been allocated for this traceID,
2121 * which means something somewhere went very wrong. No
2122 * other choice than simply exit.
2123 */
2124 ret = -EINVAL;
2125 goto out;
2126 }
2127
2128 /*
2129 * Packets associated with this timestamp are already in
2130 * the etmq's traceID queue, so process them.
2131 */
2132 ret = cs_etm__process_traceid_queue(etmq, tidq);
2133 if (ret < 0)
2134 goto out;
2135
2136 /*
2137 * Packets for this timestamp have been processed, time to
2138 * move on to the next timestamp, fetching a new auxtrace_buffer
2139 * if need be.
2140 */
2141refetch:
2142 ret = cs_etm__get_data_block(etmq);
2143 if (ret < 0)
2144 goto out;
2145
2146 /*
2147 * No more auxtrace_buffers to process in this etmq, simply
2148 * move on to another entry in the auxtrace_heap.
2149 */
2150 if (!ret)
2151 continue;
2152
2153 ret = cs_etm__decode_data_block(etmq);
2154 if (ret)
2155 goto out;
2156
2157 timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
2158
2159 if (!timestamp) {
2160 /*
2161 * Function cs_etm__decode_data_block() returns when
2162 * there is no more traces to decode in the current
2163 * auxtrace_buffer OR when a timestamp has been
2164 * encountered on any of the traceID queues. Since we
2165 * did not get a timestamp, there is no more traces to
2166 * process in this auxtrace_buffer. As such empty and
2167 * flush all traceID queues.
2168 */
2169 cs_etm__clear_all_traceid_queues(etmq);
2170
2171 /* Fetch another auxtrace_buffer for this etmq */
2172 goto refetch;
2173 }
2174
2175 /*
2176 * Add to the min heap the timestamp for packets that have
2177 * just been decoded. They will be processed and synthesized
2178 * during the next call to cs_etm__process_traceid_queue() for
2179 * this queue/traceID.
2180 */
2181 cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id);
2182 ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, timestamp);
2183 }
2184
2185out:
2186 return ret;
2187}
2188
2189static int cs_etm__process_itrace_start(struct cs_etm_auxtrace *etm,
2190 union perf_event *event)
2191{
2192 struct thread *th;
2193
2194 if (etm->timeless_decoding)
2195 return 0;
2196
2197 /*
2198 * Add the tid/pid to the log so that we can get a match when
2199 * we get a contextID from the decoder.
2200 */
2201 th = machine__findnew_thread(etm->machine,
2202 event->itrace_start.pid,
2203 event->itrace_start.tid);
2204 if (!th)
2205 return -ENOMEM;
2206
2207 thread__put(th);
2208
2209 return 0;
2210}
2211
2212static int cs_etm__process_switch_cpu_wide(struct cs_etm_auxtrace *etm,
2213 union perf_event *event)
2214{
2215 struct thread *th;
2216 bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
2217
2218 /*
2219 * Context switch in per-thread mode are irrelevant since perf
2220 * will start/stop tracing as the process is scheduled.
2221 */
2222 if (etm->timeless_decoding)
2223 return 0;
2224
2225 /*
2226 * SWITCH_IN events carry the next process to be switched out while
2227 * SWITCH_OUT events carry the process to be switched in. As such
2228 * we don't care about IN events.
2229 */
2230 if (!out)
2231 return 0;
2232
2233 /*
2234 * Add the tid/pid to the log so that we can get a match when
2235 * we get a contextID from the decoder.
2236 */
2237 th = machine__findnew_thread(etm->machine,
2238 event->context_switch.next_prev_pid,
2239 event->context_switch.next_prev_tid);
2240 if (!th)
2241 return -ENOMEM;
2242
2243 thread__put(th);
2244
2245 return 0;
2246}
2247
2248static int cs_etm__process_event(struct perf_session *session,
2249 union perf_event *event,
2250 struct perf_sample *sample,
2251 struct perf_tool *tool)
2252{
2253 int err = 0;
2254 u64 timestamp;
2255 struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2256 struct cs_etm_auxtrace,
2257 auxtrace);
2258
2259 if (dump_trace)
2260 return 0;
2261
2262 if (!tool->ordered_events) {
2263 pr_err("CoreSight ETM Trace requires ordered events\n");
2264 return -EINVAL;
2265 }
2266
2267 if (sample->time && (sample->time != (u64) -1))
2268 timestamp = sample->time;
2269 else
2270 timestamp = 0;
2271
2272 if (timestamp || etm->timeless_decoding) {
2273 err = cs_etm__update_queues(etm);
2274 if (err)
2275 return err;
2276 }
2277
2278 if (etm->timeless_decoding &&
2279 event->header.type == PERF_RECORD_EXIT)
2280 return cs_etm__process_timeless_queues(etm,
2281 event->fork.tid);
2282
2283 if (event->header.type == PERF_RECORD_ITRACE_START)
2284 return cs_etm__process_itrace_start(etm, event);
2285 else if (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
2286 return cs_etm__process_switch_cpu_wide(etm, event);
2287
2288 if (!etm->timeless_decoding &&
2289 event->header.type == PERF_RECORD_AUX)
2290 return cs_etm__process_queues(etm);
2291
2292 return 0;
2293}
2294
2295static int cs_etm__process_auxtrace_event(struct perf_session *session,
2296 union perf_event *event,
2297 struct perf_tool *tool __maybe_unused)
2298{
2299 struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2300 struct cs_etm_auxtrace,
2301 auxtrace);
2302 if (!etm->data_queued) {
2303 struct auxtrace_buffer *buffer;
2304 off_t data_offset;
2305 int fd = perf_data__fd(session->data);
2306 bool is_pipe = perf_data__is_pipe(session->data);
2307 int err;
2308
2309 if (is_pipe)
2310 data_offset = 0;
2311 else {
2312 data_offset = lseek(fd, 0, SEEK_CUR);
2313 if (data_offset == -1)
2314 return -errno;
2315 }
2316
2317 err = auxtrace_queues__add_event(&etm->queues, session,
2318 event, data_offset, &buffer);
2319 if (err)
2320 return err;
2321
2322 if (dump_trace)
2323 if (auxtrace_buffer__get_data(buffer, fd)) {
2324 cs_etm__dump_event(etm, buffer);
2325 auxtrace_buffer__put_data(buffer);
2326 }
2327 }
2328
2329 return 0;
2330}
2331
2332static bool cs_etm__is_timeless_decoding(struct cs_etm_auxtrace *etm)
2333{
2334 struct evsel *evsel;
2335 struct evlist *evlist = etm->session->evlist;
2336 bool timeless_decoding = true;
2337
2338 /*
2339 * Circle through the list of event and complain if we find one
2340 * with the time bit set.
2341 */
2342 evlist__for_each_entry(evlist, evsel) {
2343 if ((evsel->core.attr.sample_type & PERF_SAMPLE_TIME))
2344 timeless_decoding = false;
2345 }
2346
2347 return timeless_decoding;
2348}
2349
2350static const char * const cs_etm_global_header_fmts[] = {
2351 [CS_HEADER_VERSION_0] = " Header version %llx\n",
2352 [CS_PMU_TYPE_CPUS] = " PMU type/num cpus %llx\n",
2353 [CS_ETM_SNAPSHOT] = " Snapshot %llx\n",
2354};
2355
2356static const char * const cs_etm_priv_fmts[] = {
2357 [CS_ETM_MAGIC] = " Magic number %llx\n",
2358 [CS_ETM_CPU] = " CPU %lld\n",
2359 [CS_ETM_ETMCR] = " ETMCR %llx\n",
2360 [CS_ETM_ETMTRACEIDR] = " ETMTRACEIDR %llx\n",
2361 [CS_ETM_ETMCCER] = " ETMCCER %llx\n",
2362 [CS_ETM_ETMIDR] = " ETMIDR %llx\n",
2363};
2364
2365static const char * const cs_etmv4_priv_fmts[] = {
2366 [CS_ETM_MAGIC] = " Magic number %llx\n",
2367 [CS_ETM_CPU] = " CPU %lld\n",
2368 [CS_ETMV4_TRCCONFIGR] = " TRCCONFIGR %llx\n",
2369 [CS_ETMV4_TRCTRACEIDR] = " TRCTRACEIDR %llx\n",
2370 [CS_ETMV4_TRCIDR0] = " TRCIDR0 %llx\n",
2371 [CS_ETMV4_TRCIDR1] = " TRCIDR1 %llx\n",
2372 [CS_ETMV4_TRCIDR2] = " TRCIDR2 %llx\n",
2373 [CS_ETMV4_TRCIDR8] = " TRCIDR8 %llx\n",
2374 [CS_ETMV4_TRCAUTHSTATUS] = " TRCAUTHSTATUS %llx\n",
2375};
2376
2377static void cs_etm__print_auxtrace_info(__u64 *val, int num)
2378{
2379 int i, j, cpu = 0;
2380
2381 for (i = 0; i < CS_HEADER_VERSION_0_MAX; i++)
2382 fprintf(stdout, cs_etm_global_header_fmts[i], val[i]);
2383
2384 for (i = CS_HEADER_VERSION_0_MAX; cpu < num; cpu++) {
2385 if (val[i] == __perf_cs_etmv3_magic)
2386 for (j = 0; j < CS_ETM_PRIV_MAX; j++, i++)
2387 fprintf(stdout, cs_etm_priv_fmts[j], val[i]);
2388 else if (val[i] == __perf_cs_etmv4_magic)
2389 for (j = 0; j < CS_ETMV4_PRIV_MAX; j++, i++)
2390 fprintf(stdout, cs_etmv4_priv_fmts[j], val[i]);
2391 else
2392 /* failure.. return */
2393 return;
2394 }
2395}
2396
2397int cs_etm__process_auxtrace_info(union perf_event *event,
2398 struct perf_session *session)
2399{
2400 struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;
2401 struct cs_etm_auxtrace *etm = NULL;
2402 struct int_node *inode;
2403 unsigned int pmu_type;
2404 int event_header_size = sizeof(struct perf_event_header);
2405 int info_header_size;
2406 int total_size = auxtrace_info->header.size;
2407 int priv_size = 0;
2408 int num_cpu;
2409 int err = 0, idx = -1;
2410 int i, j, k;
2411 u64 *ptr, *hdr = NULL;
2412 u64 **metadata = NULL;
2413
2414 /*
2415 * sizeof(auxtrace_info_event::type) +
2416 * sizeof(auxtrace_info_event::reserved) == 8
2417 */
2418 info_header_size = 8;
2419
2420 if (total_size < (event_header_size + info_header_size))
2421 return -EINVAL;
2422
2423 priv_size = total_size - event_header_size - info_header_size;
2424
2425 /* First the global part */
2426 ptr = (u64 *) auxtrace_info->priv;
2427
2428 /* Look for version '0' of the header */
2429 if (ptr[0] != 0)
2430 return -EINVAL;
2431
2432 hdr = zalloc(sizeof(*hdr) * CS_HEADER_VERSION_0_MAX);
2433 if (!hdr)
2434 return -ENOMEM;
2435
2436 /* Extract header information - see cs-etm.h for format */
2437 for (i = 0; i < CS_HEADER_VERSION_0_MAX; i++)
2438 hdr[i] = ptr[i];
2439 num_cpu = hdr[CS_PMU_TYPE_CPUS] & 0xffffffff;
2440 pmu_type = (unsigned int) ((hdr[CS_PMU_TYPE_CPUS] >> 32) &
2441 0xffffffff);
2442
2443 /*
2444 * Create an RB tree for traceID-metadata tuple. Since the conversion
2445 * has to be made for each packet that gets decoded, optimizing access
2446 * in anything other than a sequential array is worth doing.
2447 */
2448 traceid_list = intlist__new(NULL);
2449 if (!traceid_list) {
2450 err = -ENOMEM;
2451 goto err_free_hdr;
2452 }
2453
2454 metadata = zalloc(sizeof(*metadata) * num_cpu);
2455 if (!metadata) {
2456 err = -ENOMEM;
2457 goto err_free_traceid_list;
2458 }
2459
2460 /*
2461 * The metadata is stored in the auxtrace_info section and encodes
2462 * the configuration of the ARM embedded trace macrocell which is
2463 * required by the trace decoder to properly decode the trace due
2464 * to its highly compressed nature.
2465 */
2466 for (j = 0; j < num_cpu; j++) {
2467 if (ptr[i] == __perf_cs_etmv3_magic) {
2468 metadata[j] = zalloc(sizeof(*metadata[j]) *
2469 CS_ETM_PRIV_MAX);
2470 if (!metadata[j]) {
2471 err = -ENOMEM;
2472 goto err_free_metadata;
2473 }
2474 for (k = 0; k < CS_ETM_PRIV_MAX; k++)
2475 metadata[j][k] = ptr[i + k];
2476
2477 /* The traceID is our handle */
2478 idx = metadata[j][CS_ETM_ETMTRACEIDR];
2479 i += CS_ETM_PRIV_MAX;
2480 } else if (ptr[i] == __perf_cs_etmv4_magic) {
2481 metadata[j] = zalloc(sizeof(*metadata[j]) *
2482 CS_ETMV4_PRIV_MAX);
2483 if (!metadata[j]) {
2484 err = -ENOMEM;
2485 goto err_free_metadata;
2486 }
2487 for (k = 0; k < CS_ETMV4_PRIV_MAX; k++)
2488 metadata[j][k] = ptr[i + k];
2489
2490 /* The traceID is our handle */
2491 idx = metadata[j][CS_ETMV4_TRCTRACEIDR];
2492 i += CS_ETMV4_PRIV_MAX;
2493 }
2494
2495 /* Get an RB node for this CPU */
2496 inode = intlist__findnew(traceid_list, idx);
2497
2498 /* Something went wrong, no need to continue */
2499 if (!inode) {
2500 err = -ENOMEM;
2501 goto err_free_metadata;
2502 }
2503
2504 /*
2505 * The node for that CPU should not be taken.
2506 * Back out if that's the case.
2507 */
2508 if (inode->priv) {
2509 err = -EINVAL;
2510 goto err_free_metadata;
2511 }
2512 /* All good, associate the traceID with the metadata pointer */
2513 inode->priv = metadata[j];
2514 }
2515
2516 /*
2517 * Each of CS_HEADER_VERSION_0_MAX, CS_ETM_PRIV_MAX and
2518 * CS_ETMV4_PRIV_MAX mark how many double words are in the
2519 * global metadata, and each cpu's metadata respectively.
2520 * The following tests if the correct number of double words was
2521 * present in the auxtrace info section.
2522 */
2523 if (i * 8 != priv_size) {
2524 err = -EINVAL;
2525 goto err_free_metadata;
2526 }
2527
2528 etm = zalloc(sizeof(*etm));
2529
2530 if (!etm) {
2531 err = -ENOMEM;
2532 goto err_free_metadata;
2533 }
2534
2535 err = auxtrace_queues__init(&etm->queues);
2536 if (err)
2537 goto err_free_etm;
2538
2539 etm->session = session;
2540 etm->machine = &session->machines.host;
2541
2542 etm->num_cpu = num_cpu;
2543 etm->pmu_type = pmu_type;
2544 etm->snapshot_mode = (hdr[CS_ETM_SNAPSHOT] != 0);
2545 etm->metadata = metadata;
2546 etm->auxtrace_type = auxtrace_info->type;
2547 etm->timeless_decoding = cs_etm__is_timeless_decoding(etm);
2548
2549 etm->auxtrace.process_event = cs_etm__process_event;
2550 etm->auxtrace.process_auxtrace_event = cs_etm__process_auxtrace_event;
2551 etm->auxtrace.flush_events = cs_etm__flush_events;
2552 etm->auxtrace.free_events = cs_etm__free_events;
2553 etm->auxtrace.free = cs_etm__free;
2554 session->auxtrace = &etm->auxtrace;
2555
2556 etm->unknown_thread = thread__new(999999999, 999999999);
2557 if (!etm->unknown_thread) {
2558 err = -ENOMEM;
2559 goto err_free_queues;
2560 }
2561
2562 /*
2563 * Initialize list node so that at thread__zput() we can avoid
2564 * segmentation fault at list_del_init().
2565 */
2566 INIT_LIST_HEAD(&etm->unknown_thread->node);
2567
2568 err = thread__set_comm(etm->unknown_thread, "unknown", 0);
2569 if (err)
2570 goto err_delete_thread;
2571
2572 if (thread__init_map_groups(etm->unknown_thread, etm->machine)) {
2573 err = -ENOMEM;
2574 goto err_delete_thread;
2575 }
2576
2577 if (dump_trace) {
2578 cs_etm__print_auxtrace_info(auxtrace_info->priv, num_cpu);
2579 return 0;
2580 }
2581
2582 if (session->itrace_synth_opts->set) {
2583 etm->synth_opts = *session->itrace_synth_opts;
2584 } else {
2585 itrace_synth_opts__set_default(&etm->synth_opts,
2586 session->itrace_synth_opts->default_no_sample);
2587 etm->synth_opts.callchain = false;
2588 }
2589
2590 err = cs_etm__synth_events(etm, session);
2591 if (err)
2592 goto err_delete_thread;
2593
2594 err = auxtrace_queues__process_index(&etm->queues, session);
2595 if (err)
2596 goto err_delete_thread;
2597
2598 etm->data_queued = etm->queues.populated;
2599
2600 return 0;
2601
2602err_delete_thread:
2603 thread__zput(etm->unknown_thread);
2604err_free_queues:
2605 auxtrace_queues__free(&etm->queues);
2606 session->auxtrace = NULL;
2607err_free_etm:
2608 zfree(&etm);
2609err_free_metadata:
2610 /* No need to check @metadata[j], free(NULL) is supported */
2611 for (j = 0; j < num_cpu; j++)
2612 zfree(&metadata[j]);
2613 zfree(&metadata);
2614err_free_traceid_list:
2615 intlist__delete(traceid_list);
2616err_free_hdr:
2617 zfree(&hdr);
2618
2619 return err;
2620}
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright(C) 2015-2018 Linaro Limited.
4 *
5 * Author: Tor Jeremiassen <tor@ti.com>
6 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
7 */
8
9#include <linux/kernel.h>
10#include <linux/bitfield.h>
11#include <linux/bitops.h>
12#include <linux/coresight-pmu.h>
13#include <linux/err.h>
14#include <linux/log2.h>
15#include <linux/types.h>
16#include <linux/zalloc.h>
17
18#include <stdlib.h>
19
20#include "auxtrace.h"
21#include "color.h"
22#include "cs-etm.h"
23#include "cs-etm-decoder/cs-etm-decoder.h"
24#include "debug.h"
25#include "dso.h"
26#include "evlist.h"
27#include "intlist.h"
28#include "machine.h"
29#include "map.h"
30#include "perf.h"
31#include "session.h"
32#include "map_symbol.h"
33#include "branch.h"
34#include "symbol.h"
35#include "tool.h"
36#include "thread.h"
37#include "thread-stack.h"
38#include "tsc.h"
39#include <tools/libc_compat.h>
40#include "util/synthetic-events.h"
41#include "util/util.h"
42
43struct cs_etm_auxtrace {
44 struct auxtrace auxtrace;
45 struct auxtrace_queues queues;
46 struct auxtrace_heap heap;
47 struct itrace_synth_opts synth_opts;
48 struct perf_session *session;
49 struct perf_tsc_conversion tc;
50
51 /*
52 * Timeless has no timestamps in the trace so overlapping mmap lookups
53 * are less accurate but produces smaller trace data. We use context IDs
54 * in the trace instead of matching timestamps with fork records so
55 * they're not really needed in the general case. Overlapping mmaps
56 * happen in cases like between a fork and an exec.
57 */
58 bool timeless_decoding;
59
60 /*
61 * Per-thread ignores the trace channel ID and instead assumes that
62 * everything in a buffer comes from the same process regardless of
63 * which CPU it ran on. It also implies no context IDs so the TID is
64 * taken from the auxtrace buffer.
65 */
66 bool per_thread_decoding;
67 bool snapshot_mode;
68 bool data_queued;
69 bool has_virtual_ts; /* Virtual/Kernel timestamps in the trace. */
70
71 int num_cpu;
72 u64 latest_kernel_timestamp;
73 u32 auxtrace_type;
74 u64 branches_sample_type;
75 u64 branches_id;
76 u64 instructions_sample_type;
77 u64 instructions_sample_period;
78 u64 instructions_id;
79 u64 **metadata;
80 unsigned int pmu_type;
81 enum cs_etm_pid_fmt pid_fmt;
82};
83
84struct cs_etm_traceid_queue {
85 u8 trace_chan_id;
86 u64 period_instructions;
87 size_t last_branch_pos;
88 union perf_event *event_buf;
89 struct thread *thread;
90 struct thread *prev_packet_thread;
91 ocsd_ex_level prev_packet_el;
92 ocsd_ex_level el;
93 struct branch_stack *last_branch;
94 struct branch_stack *last_branch_rb;
95 struct cs_etm_packet *prev_packet;
96 struct cs_etm_packet *packet;
97 struct cs_etm_packet_queue packet_queue;
98};
99
100struct cs_etm_queue {
101 struct cs_etm_auxtrace *etm;
102 struct cs_etm_decoder *decoder;
103 struct auxtrace_buffer *buffer;
104 unsigned int queue_nr;
105 u8 pending_timestamp_chan_id;
106 u64 offset;
107 const unsigned char *buf;
108 size_t buf_len, buf_used;
109 /* Conversion between traceID and index in traceid_queues array */
110 struct intlist *traceid_queues_list;
111 struct cs_etm_traceid_queue **traceid_queues;
112};
113
114/* RB tree for quick conversion between traceID and metadata pointers */
115static struct intlist *traceid_list;
116
117static int cs_etm__process_timestamped_queues(struct cs_etm_auxtrace *etm);
118static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
119 pid_t tid);
120static int cs_etm__get_data_block(struct cs_etm_queue *etmq);
121static int cs_etm__decode_data_block(struct cs_etm_queue *etmq);
122
123/* PTMs ETMIDR [11:8] set to b0011 */
124#define ETMIDR_PTM_VERSION 0x00000300
125
126/*
127 * A struct auxtrace_heap_item only has a queue_nr and a timestamp to
128 * work with. One option is to modify to auxtrace_heap_XYZ() API or simply
129 * encode the etm queue number as the upper 16 bit and the channel as
130 * the lower 16 bit.
131 */
132#define TO_CS_QUEUE_NR(queue_nr, trace_chan_id) \
133 (queue_nr << 16 | trace_chan_id)
134#define TO_QUEUE_NR(cs_queue_nr) (cs_queue_nr >> 16)
135#define TO_TRACE_CHAN_ID(cs_queue_nr) (cs_queue_nr & 0x0000ffff)
136
137static u32 cs_etm__get_v7_protocol_version(u32 etmidr)
138{
139 etmidr &= ETMIDR_PTM_VERSION;
140
141 if (etmidr == ETMIDR_PTM_VERSION)
142 return CS_ETM_PROTO_PTM;
143
144 return CS_ETM_PROTO_ETMV3;
145}
146
147static int cs_etm__get_magic(u8 trace_chan_id, u64 *magic)
148{
149 struct int_node *inode;
150 u64 *metadata;
151
152 inode = intlist__find(traceid_list, trace_chan_id);
153 if (!inode)
154 return -EINVAL;
155
156 metadata = inode->priv;
157 *magic = metadata[CS_ETM_MAGIC];
158 return 0;
159}
160
161int cs_etm__get_cpu(u8 trace_chan_id, int *cpu)
162{
163 struct int_node *inode;
164 u64 *metadata;
165
166 inode = intlist__find(traceid_list, trace_chan_id);
167 if (!inode)
168 return -EINVAL;
169
170 metadata = inode->priv;
171 *cpu = (int)metadata[CS_ETM_CPU];
172 return 0;
173}
174
175/*
176 * The returned PID format is presented as an enum:
177 *
178 * CS_ETM_PIDFMT_CTXTID: CONTEXTIDR or CONTEXTIDR_EL1 is traced.
179 * CS_ETM_PIDFMT_CTXTID2: CONTEXTIDR_EL2 is traced.
180 * CS_ETM_PIDFMT_NONE: No context IDs
181 *
182 * It's possible that the two bits ETM_OPT_CTXTID and ETM_OPT_CTXTID2
183 * are enabled at the same time when the session runs on an EL2 kernel.
184 * This means the CONTEXTIDR_EL1 and CONTEXTIDR_EL2 both will be
185 * recorded in the trace data, the tool will selectively use
186 * CONTEXTIDR_EL2 as PID.
187 *
188 * The result is cached in etm->pid_fmt so this function only needs to be called
189 * when processing the aux info.
190 */
191static enum cs_etm_pid_fmt cs_etm__init_pid_fmt(u64 *metadata)
192{
193 u64 val;
194
195 if (metadata[CS_ETM_MAGIC] == __perf_cs_etmv3_magic) {
196 val = metadata[CS_ETM_ETMCR];
197 /* CONTEXTIDR is traced */
198 if (val & BIT(ETM_OPT_CTXTID))
199 return CS_ETM_PIDFMT_CTXTID;
200 } else {
201 val = metadata[CS_ETMV4_TRCCONFIGR];
202 /* CONTEXTIDR_EL2 is traced */
203 if (val & (BIT(ETM4_CFG_BIT_VMID) | BIT(ETM4_CFG_BIT_VMID_OPT)))
204 return CS_ETM_PIDFMT_CTXTID2;
205 /* CONTEXTIDR_EL1 is traced */
206 else if (val & BIT(ETM4_CFG_BIT_CTXTID))
207 return CS_ETM_PIDFMT_CTXTID;
208 }
209
210 return CS_ETM_PIDFMT_NONE;
211}
212
213enum cs_etm_pid_fmt cs_etm__get_pid_fmt(struct cs_etm_queue *etmq)
214{
215 return etmq->etm->pid_fmt;
216}
217
218static int cs_etm__map_trace_id(u8 trace_chan_id, u64 *cpu_metadata)
219{
220 struct int_node *inode;
221
222 /* Get an RB node for this CPU */
223 inode = intlist__findnew(traceid_list, trace_chan_id);
224
225 /* Something went wrong, no need to continue */
226 if (!inode)
227 return -ENOMEM;
228
229 /*
230 * The node for that CPU should not be taken.
231 * Back out if that's the case.
232 */
233 if (inode->priv)
234 return -EINVAL;
235
236 /* All good, associate the traceID with the metadata pointer */
237 inode->priv = cpu_metadata;
238
239 return 0;
240}
241
242static int cs_etm__metadata_get_trace_id(u8 *trace_chan_id, u64 *cpu_metadata)
243{
244 u64 cs_etm_magic = cpu_metadata[CS_ETM_MAGIC];
245
246 switch (cs_etm_magic) {
247 case __perf_cs_etmv3_magic:
248 *trace_chan_id = (u8)(cpu_metadata[CS_ETM_ETMTRACEIDR] &
249 CORESIGHT_TRACE_ID_VAL_MASK);
250 break;
251 case __perf_cs_etmv4_magic:
252 case __perf_cs_ete_magic:
253 *trace_chan_id = (u8)(cpu_metadata[CS_ETMV4_TRCTRACEIDR] &
254 CORESIGHT_TRACE_ID_VAL_MASK);
255 break;
256 default:
257 return -EINVAL;
258 }
259 return 0;
260}
261
262/*
263 * update metadata trace ID from the value found in the AUX_HW_INFO packet.
264 * This will also clear the CORESIGHT_TRACE_ID_UNUSED_FLAG flag if present.
265 */
266static int cs_etm__metadata_set_trace_id(u8 trace_chan_id, u64 *cpu_metadata)
267{
268 u64 cs_etm_magic = cpu_metadata[CS_ETM_MAGIC];
269
270 switch (cs_etm_magic) {
271 case __perf_cs_etmv3_magic:
272 cpu_metadata[CS_ETM_ETMTRACEIDR] = trace_chan_id;
273 break;
274 case __perf_cs_etmv4_magic:
275 case __perf_cs_ete_magic:
276 cpu_metadata[CS_ETMV4_TRCTRACEIDR] = trace_chan_id;
277 break;
278
279 default:
280 return -EINVAL;
281 }
282 return 0;
283}
284
285/*
286 * Get a metadata index for a specific cpu from an array.
287 *
288 */
289static int get_cpu_data_idx(struct cs_etm_auxtrace *etm, int cpu)
290{
291 int i;
292
293 for (i = 0; i < etm->num_cpu; i++) {
294 if (etm->metadata[i][CS_ETM_CPU] == (u64)cpu) {
295 return i;
296 }
297 }
298
299 return -1;
300}
301
302/*
303 * Get a metadata for a specific cpu from an array.
304 *
305 */
306static u64 *get_cpu_data(struct cs_etm_auxtrace *etm, int cpu)
307{
308 int idx = get_cpu_data_idx(etm, cpu);
309
310 return (idx != -1) ? etm->metadata[idx] : NULL;
311}
312
313/*
314 * Handle the PERF_RECORD_AUX_OUTPUT_HW_ID event.
315 *
316 * The payload associates the Trace ID and the CPU.
317 * The routine is tolerant of seeing multiple packets with the same association,
318 * but a CPU / Trace ID association changing during a session is an error.
319 */
320static int cs_etm__process_aux_output_hw_id(struct perf_session *session,
321 union perf_event *event)
322{
323 struct cs_etm_auxtrace *etm;
324 struct perf_sample sample;
325 struct int_node *inode;
326 struct evsel *evsel;
327 u64 *cpu_data;
328 u64 hw_id;
329 int cpu, version, err;
330 u8 trace_chan_id, curr_chan_id;
331
332 /* extract and parse the HW ID */
333 hw_id = event->aux_output_hw_id.hw_id;
334 version = FIELD_GET(CS_AUX_HW_ID_VERSION_MASK, hw_id);
335 trace_chan_id = FIELD_GET(CS_AUX_HW_ID_TRACE_ID_MASK, hw_id);
336
337 /* check that we can handle this version */
338 if (version > CS_AUX_HW_ID_CURR_VERSION)
339 return -EINVAL;
340
341 /* get access to the etm metadata */
342 etm = container_of(session->auxtrace, struct cs_etm_auxtrace, auxtrace);
343 if (!etm || !etm->metadata)
344 return -EINVAL;
345
346 /* parse the sample to get the CPU */
347 evsel = evlist__event2evsel(session->evlist, event);
348 if (!evsel)
349 return -EINVAL;
350 err = evsel__parse_sample(evsel, event, &sample);
351 if (err)
352 return err;
353 cpu = sample.cpu;
354 if (cpu == -1) {
355 /* no CPU in the sample - possibly recorded with an old version of perf */
356 pr_err("CS_ETM: no CPU AUX_OUTPUT_HW_ID sample. Use compatible perf to record.");
357 return -EINVAL;
358 }
359
360 /* See if the ID is mapped to a CPU, and it matches the current CPU */
361 inode = intlist__find(traceid_list, trace_chan_id);
362 if (inode) {
363 cpu_data = inode->priv;
364 if ((int)cpu_data[CS_ETM_CPU] != cpu) {
365 pr_err("CS_ETM: map mismatch between HW_ID packet CPU and Trace ID\n");
366 return -EINVAL;
367 }
368
369 /* check that the mapped ID matches */
370 err = cs_etm__metadata_get_trace_id(&curr_chan_id, cpu_data);
371 if (err)
372 return err;
373 if (curr_chan_id != trace_chan_id) {
374 pr_err("CS_ETM: mismatch between CPU trace ID and HW_ID packet ID\n");
375 return -EINVAL;
376 }
377
378 /* mapped and matched - return OK */
379 return 0;
380 }
381
382 cpu_data = get_cpu_data(etm, cpu);
383 if (cpu_data == NULL)
384 return err;
385
386 /* not one we've seen before - lets map it */
387 err = cs_etm__map_trace_id(trace_chan_id, cpu_data);
388 if (err)
389 return err;
390
391 /*
392 * if we are picking up the association from the packet, need to plug
393 * the correct trace ID into the metadata for setting up decoders later.
394 */
395 err = cs_etm__metadata_set_trace_id(trace_chan_id, cpu_data);
396 return err;
397}
398
399void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq,
400 u8 trace_chan_id)
401{
402 /*
403 * When a timestamp packet is encountered the backend code
404 * is stopped so that the front end has time to process packets
405 * that were accumulated in the traceID queue. Since there can
406 * be more than one channel per cs_etm_queue, we need to specify
407 * what traceID queue needs servicing.
408 */
409 etmq->pending_timestamp_chan_id = trace_chan_id;
410}
411
412static u64 cs_etm__etmq_get_timestamp(struct cs_etm_queue *etmq,
413 u8 *trace_chan_id)
414{
415 struct cs_etm_packet_queue *packet_queue;
416
417 if (!etmq->pending_timestamp_chan_id)
418 return 0;
419
420 if (trace_chan_id)
421 *trace_chan_id = etmq->pending_timestamp_chan_id;
422
423 packet_queue = cs_etm__etmq_get_packet_queue(etmq,
424 etmq->pending_timestamp_chan_id);
425 if (!packet_queue)
426 return 0;
427
428 /* Acknowledge pending status */
429 etmq->pending_timestamp_chan_id = 0;
430
431 /* See function cs_etm_decoder__do_{hard|soft}_timestamp() */
432 return packet_queue->cs_timestamp;
433}
434
435static void cs_etm__clear_packet_queue(struct cs_etm_packet_queue *queue)
436{
437 int i;
438
439 queue->head = 0;
440 queue->tail = 0;
441 queue->packet_count = 0;
442 for (i = 0; i < CS_ETM_PACKET_MAX_BUFFER; i++) {
443 queue->packet_buffer[i].isa = CS_ETM_ISA_UNKNOWN;
444 queue->packet_buffer[i].start_addr = CS_ETM_INVAL_ADDR;
445 queue->packet_buffer[i].end_addr = CS_ETM_INVAL_ADDR;
446 queue->packet_buffer[i].instr_count = 0;
447 queue->packet_buffer[i].last_instr_taken_branch = false;
448 queue->packet_buffer[i].last_instr_size = 0;
449 queue->packet_buffer[i].last_instr_type = 0;
450 queue->packet_buffer[i].last_instr_subtype = 0;
451 queue->packet_buffer[i].last_instr_cond = 0;
452 queue->packet_buffer[i].flags = 0;
453 queue->packet_buffer[i].exception_number = UINT32_MAX;
454 queue->packet_buffer[i].trace_chan_id = UINT8_MAX;
455 queue->packet_buffer[i].cpu = INT_MIN;
456 }
457}
458
459static void cs_etm__clear_all_packet_queues(struct cs_etm_queue *etmq)
460{
461 int idx;
462 struct int_node *inode;
463 struct cs_etm_traceid_queue *tidq;
464 struct intlist *traceid_queues_list = etmq->traceid_queues_list;
465
466 intlist__for_each_entry(inode, traceid_queues_list) {
467 idx = (int)(intptr_t)inode->priv;
468 tidq = etmq->traceid_queues[idx];
469 cs_etm__clear_packet_queue(&tidq->packet_queue);
470 }
471}
472
473static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq,
474 struct cs_etm_traceid_queue *tidq,
475 u8 trace_chan_id)
476{
477 int rc = -ENOMEM;
478 struct auxtrace_queue *queue;
479 struct cs_etm_auxtrace *etm = etmq->etm;
480
481 cs_etm__clear_packet_queue(&tidq->packet_queue);
482
483 queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
484 tidq->trace_chan_id = trace_chan_id;
485 tidq->el = tidq->prev_packet_el = ocsd_EL_unknown;
486 tidq->thread = machine__findnew_thread(&etm->session->machines.host, -1,
487 queue->tid);
488 tidq->prev_packet_thread = machine__idle_thread(&etm->session->machines.host);
489
490 tidq->packet = zalloc(sizeof(struct cs_etm_packet));
491 if (!tidq->packet)
492 goto out;
493
494 tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet));
495 if (!tidq->prev_packet)
496 goto out_free;
497
498 if (etm->synth_opts.last_branch) {
499 size_t sz = sizeof(struct branch_stack);
500
501 sz += etm->synth_opts.last_branch_sz *
502 sizeof(struct branch_entry);
503 tidq->last_branch = zalloc(sz);
504 if (!tidq->last_branch)
505 goto out_free;
506 tidq->last_branch_rb = zalloc(sz);
507 if (!tidq->last_branch_rb)
508 goto out_free;
509 }
510
511 tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
512 if (!tidq->event_buf)
513 goto out_free;
514
515 return 0;
516
517out_free:
518 zfree(&tidq->last_branch_rb);
519 zfree(&tidq->last_branch);
520 zfree(&tidq->prev_packet);
521 zfree(&tidq->packet);
522out:
523 return rc;
524}
525
526static struct cs_etm_traceid_queue
527*cs_etm__etmq_get_traceid_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
528{
529 int idx;
530 struct int_node *inode;
531 struct intlist *traceid_queues_list;
532 struct cs_etm_traceid_queue *tidq, **traceid_queues;
533 struct cs_etm_auxtrace *etm = etmq->etm;
534
535 if (etm->per_thread_decoding)
536 trace_chan_id = CS_ETM_PER_THREAD_TRACEID;
537
538 traceid_queues_list = etmq->traceid_queues_list;
539
540 /*
541 * Check if the traceid_queue exist for this traceID by looking
542 * in the queue list.
543 */
544 inode = intlist__find(traceid_queues_list, trace_chan_id);
545 if (inode) {
546 idx = (int)(intptr_t)inode->priv;
547 return etmq->traceid_queues[idx];
548 }
549
550 /* We couldn't find a traceid_queue for this traceID, allocate one */
551 tidq = malloc(sizeof(*tidq));
552 if (!tidq)
553 return NULL;
554
555 memset(tidq, 0, sizeof(*tidq));
556
557 /* Get a valid index for the new traceid_queue */
558 idx = intlist__nr_entries(traceid_queues_list);
559 /* Memory for the inode is free'ed in cs_etm_free_traceid_queues () */
560 inode = intlist__findnew(traceid_queues_list, trace_chan_id);
561 if (!inode)
562 goto out_free;
563
564 /* Associate this traceID with this index */
565 inode->priv = (void *)(intptr_t)idx;
566
567 if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id))
568 goto out_free;
569
570 /* Grow the traceid_queues array by one unit */
571 traceid_queues = etmq->traceid_queues;
572 traceid_queues = reallocarray(traceid_queues,
573 idx + 1,
574 sizeof(*traceid_queues));
575
576 /*
577 * On failure reallocarray() returns NULL and the original block of
578 * memory is left untouched.
579 */
580 if (!traceid_queues)
581 goto out_free;
582
583 traceid_queues[idx] = tidq;
584 etmq->traceid_queues = traceid_queues;
585
586 return etmq->traceid_queues[idx];
587
588out_free:
589 /*
590 * Function intlist__remove() removes the inode from the list
591 * and delete the memory associated to it.
592 */
593 intlist__remove(traceid_queues_list, inode);
594 free(tidq);
595
596 return NULL;
597}
598
599struct cs_etm_packet_queue
600*cs_etm__etmq_get_packet_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
601{
602 struct cs_etm_traceid_queue *tidq;
603
604 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
605 if (tidq)
606 return &tidq->packet_queue;
607
608 return NULL;
609}
610
611static void cs_etm__packet_swap(struct cs_etm_auxtrace *etm,
612 struct cs_etm_traceid_queue *tidq)
613{
614 struct cs_etm_packet *tmp;
615
616 if (etm->synth_opts.branches || etm->synth_opts.last_branch ||
617 etm->synth_opts.instructions) {
618 /*
619 * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
620 * the next incoming packet.
621 *
622 * Threads and exception levels are also tracked for both the
623 * previous and current packets. This is because the previous
624 * packet is used for the 'from' IP for branch samples, so the
625 * thread at that time must also be assigned to that sample.
626 * Across discontinuity packets the thread can change, so by
627 * tracking the thread for the previous packet the branch sample
628 * will have the correct info.
629 */
630 tmp = tidq->packet;
631 tidq->packet = tidq->prev_packet;
632 tidq->prev_packet = tmp;
633 tidq->prev_packet_el = tidq->el;
634 thread__put(tidq->prev_packet_thread);
635 tidq->prev_packet_thread = thread__get(tidq->thread);
636 }
637}
638
639static void cs_etm__packet_dump(const char *pkt_string)
640{
641 const char *color = PERF_COLOR_BLUE;
642 int len = strlen(pkt_string);
643
644 if (len && (pkt_string[len-1] == '\n'))
645 color_fprintf(stdout, color, " %s", pkt_string);
646 else
647 color_fprintf(stdout, color, " %s\n", pkt_string);
648
649 fflush(stdout);
650}
651
652static void cs_etm__set_trace_param_etmv3(struct cs_etm_trace_params *t_params,
653 struct cs_etm_auxtrace *etm, int t_idx,
654 int m_idx, u32 etmidr)
655{
656 u64 **metadata = etm->metadata;
657
658 t_params[t_idx].protocol = cs_etm__get_v7_protocol_version(etmidr);
659 t_params[t_idx].etmv3.reg_ctrl = metadata[m_idx][CS_ETM_ETMCR];
660 t_params[t_idx].etmv3.reg_trc_id = metadata[m_idx][CS_ETM_ETMTRACEIDR];
661}
662
663static void cs_etm__set_trace_param_etmv4(struct cs_etm_trace_params *t_params,
664 struct cs_etm_auxtrace *etm, int t_idx,
665 int m_idx)
666{
667 u64 **metadata = etm->metadata;
668
669 t_params[t_idx].protocol = CS_ETM_PROTO_ETMV4i;
670 t_params[t_idx].etmv4.reg_idr0 = metadata[m_idx][CS_ETMV4_TRCIDR0];
671 t_params[t_idx].etmv4.reg_idr1 = metadata[m_idx][CS_ETMV4_TRCIDR1];
672 t_params[t_idx].etmv4.reg_idr2 = metadata[m_idx][CS_ETMV4_TRCIDR2];
673 t_params[t_idx].etmv4.reg_idr8 = metadata[m_idx][CS_ETMV4_TRCIDR8];
674 t_params[t_idx].etmv4.reg_configr = metadata[m_idx][CS_ETMV4_TRCCONFIGR];
675 t_params[t_idx].etmv4.reg_traceidr = metadata[m_idx][CS_ETMV4_TRCTRACEIDR];
676}
677
678static void cs_etm__set_trace_param_ete(struct cs_etm_trace_params *t_params,
679 struct cs_etm_auxtrace *etm, int t_idx,
680 int m_idx)
681{
682 u64 **metadata = etm->metadata;
683
684 t_params[t_idx].protocol = CS_ETM_PROTO_ETE;
685 t_params[t_idx].ete.reg_idr0 = metadata[m_idx][CS_ETE_TRCIDR0];
686 t_params[t_idx].ete.reg_idr1 = metadata[m_idx][CS_ETE_TRCIDR1];
687 t_params[t_idx].ete.reg_idr2 = metadata[m_idx][CS_ETE_TRCIDR2];
688 t_params[t_idx].ete.reg_idr8 = metadata[m_idx][CS_ETE_TRCIDR8];
689 t_params[t_idx].ete.reg_configr = metadata[m_idx][CS_ETE_TRCCONFIGR];
690 t_params[t_idx].ete.reg_traceidr = metadata[m_idx][CS_ETE_TRCTRACEIDR];
691 t_params[t_idx].ete.reg_devarch = metadata[m_idx][CS_ETE_TRCDEVARCH];
692}
693
694static int cs_etm__init_trace_params(struct cs_etm_trace_params *t_params,
695 struct cs_etm_auxtrace *etm,
696 bool formatted,
697 int sample_cpu,
698 int decoders)
699{
700 int t_idx, m_idx;
701 u32 etmidr;
702 u64 architecture;
703
704 for (t_idx = 0; t_idx < decoders; t_idx++) {
705 if (formatted)
706 m_idx = t_idx;
707 else {
708 m_idx = get_cpu_data_idx(etm, sample_cpu);
709 if (m_idx == -1) {
710 pr_warning("CS_ETM: unknown CPU, falling back to first metadata\n");
711 m_idx = 0;
712 }
713 }
714
715 architecture = etm->metadata[m_idx][CS_ETM_MAGIC];
716
717 switch (architecture) {
718 case __perf_cs_etmv3_magic:
719 etmidr = etm->metadata[m_idx][CS_ETM_ETMIDR];
720 cs_etm__set_trace_param_etmv3(t_params, etm, t_idx, m_idx, etmidr);
721 break;
722 case __perf_cs_etmv4_magic:
723 cs_etm__set_trace_param_etmv4(t_params, etm, t_idx, m_idx);
724 break;
725 case __perf_cs_ete_magic:
726 cs_etm__set_trace_param_ete(t_params, etm, t_idx, m_idx);
727 break;
728 default:
729 return -EINVAL;
730 }
731 }
732
733 return 0;
734}
735
736static int cs_etm__init_decoder_params(struct cs_etm_decoder_params *d_params,
737 struct cs_etm_queue *etmq,
738 enum cs_etm_decoder_operation mode,
739 bool formatted)
740{
741 int ret = -EINVAL;
742
743 if (!(mode < CS_ETM_OPERATION_MAX))
744 goto out;
745
746 d_params->packet_printer = cs_etm__packet_dump;
747 d_params->operation = mode;
748 d_params->data = etmq;
749 d_params->formatted = formatted;
750 d_params->fsyncs = false;
751 d_params->hsyncs = false;
752 d_params->frame_aligned = true;
753
754 ret = 0;
755out:
756 return ret;
757}
758
759static void cs_etm__dump_event(struct cs_etm_queue *etmq,
760 struct auxtrace_buffer *buffer)
761{
762 int ret;
763 const char *color = PERF_COLOR_BLUE;
764 size_t buffer_used = 0;
765
766 fprintf(stdout, "\n");
767 color_fprintf(stdout, color,
768 ". ... CoreSight %s Trace data: size %#zx bytes\n",
769 cs_etm_decoder__get_name(etmq->decoder), buffer->size);
770
771 do {
772 size_t consumed;
773
774 ret = cs_etm_decoder__process_data_block(
775 etmq->decoder, buffer->offset,
776 &((u8 *)buffer->data)[buffer_used],
777 buffer->size - buffer_used, &consumed);
778 if (ret)
779 break;
780
781 buffer_used += consumed;
782 } while (buffer_used < buffer->size);
783
784 cs_etm_decoder__reset(etmq->decoder);
785}
786
787static int cs_etm__flush_events(struct perf_session *session,
788 struct perf_tool *tool)
789{
790 struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
791 struct cs_etm_auxtrace,
792 auxtrace);
793 if (dump_trace)
794 return 0;
795
796 if (!tool->ordered_events)
797 return -EINVAL;
798
799 if (etm->timeless_decoding) {
800 /*
801 * Pass tid = -1 to process all queues. But likely they will have
802 * already been processed on PERF_RECORD_EXIT anyway.
803 */
804 return cs_etm__process_timeless_queues(etm, -1);
805 }
806
807 return cs_etm__process_timestamped_queues(etm);
808}
809
810static void cs_etm__free_traceid_queues(struct cs_etm_queue *etmq)
811{
812 int idx;
813 uintptr_t priv;
814 struct int_node *inode, *tmp;
815 struct cs_etm_traceid_queue *tidq;
816 struct intlist *traceid_queues_list = etmq->traceid_queues_list;
817
818 intlist__for_each_entry_safe(inode, tmp, traceid_queues_list) {
819 priv = (uintptr_t)inode->priv;
820 idx = priv;
821
822 /* Free this traceid_queue from the array */
823 tidq = etmq->traceid_queues[idx];
824 thread__zput(tidq->thread);
825 thread__zput(tidq->prev_packet_thread);
826 zfree(&tidq->event_buf);
827 zfree(&tidq->last_branch);
828 zfree(&tidq->last_branch_rb);
829 zfree(&tidq->prev_packet);
830 zfree(&tidq->packet);
831 zfree(&tidq);
832
833 /*
834 * Function intlist__remove() removes the inode from the list
835 * and delete the memory associated to it.
836 */
837 intlist__remove(traceid_queues_list, inode);
838 }
839
840 /* Then the RB tree itself */
841 intlist__delete(traceid_queues_list);
842 etmq->traceid_queues_list = NULL;
843
844 /* finally free the traceid_queues array */
845 zfree(&etmq->traceid_queues);
846}
847
848static void cs_etm__free_queue(void *priv)
849{
850 struct cs_etm_queue *etmq = priv;
851
852 if (!etmq)
853 return;
854
855 cs_etm_decoder__free(etmq->decoder);
856 cs_etm__free_traceid_queues(etmq);
857 free(etmq);
858}
859
860static void cs_etm__free_events(struct perf_session *session)
861{
862 unsigned int i;
863 struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
864 struct cs_etm_auxtrace,
865 auxtrace);
866 struct auxtrace_queues *queues = &aux->queues;
867
868 for (i = 0; i < queues->nr_queues; i++) {
869 cs_etm__free_queue(queues->queue_array[i].priv);
870 queues->queue_array[i].priv = NULL;
871 }
872
873 auxtrace_queues__free(queues);
874}
875
876static void cs_etm__free(struct perf_session *session)
877{
878 int i;
879 struct int_node *inode, *tmp;
880 struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
881 struct cs_etm_auxtrace,
882 auxtrace);
883 cs_etm__free_events(session);
884 session->auxtrace = NULL;
885
886 /* First remove all traceID/metadata nodes for the RB tree */
887 intlist__for_each_entry_safe(inode, tmp, traceid_list)
888 intlist__remove(traceid_list, inode);
889 /* Then the RB tree itself */
890 intlist__delete(traceid_list);
891
892 for (i = 0; i < aux->num_cpu; i++)
893 zfree(&aux->metadata[i]);
894
895 zfree(&aux->metadata);
896 zfree(&aux);
897}
898
899static bool cs_etm__evsel_is_auxtrace(struct perf_session *session,
900 struct evsel *evsel)
901{
902 struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
903 struct cs_etm_auxtrace,
904 auxtrace);
905
906 return evsel->core.attr.type == aux->pmu_type;
907}
908
909static struct machine *cs_etm__get_machine(struct cs_etm_queue *etmq,
910 ocsd_ex_level el)
911{
912 enum cs_etm_pid_fmt pid_fmt = cs_etm__get_pid_fmt(etmq);
913
914 /*
915 * For any virtualisation based on nVHE (e.g. pKVM), or host kernels
916 * running at EL1 assume everything is the host.
917 */
918 if (pid_fmt == CS_ETM_PIDFMT_CTXTID)
919 return &etmq->etm->session->machines.host;
920
921 /*
922 * Not perfect, but otherwise assume anything in EL1 is the default
923 * guest, and everything else is the host. Distinguishing between guest
924 * and host userspaces isn't currently supported either. Neither is
925 * multiple guest support. All this does is reduce the likeliness of
926 * decode errors where we look into the host kernel maps when it should
927 * have been the guest maps.
928 */
929 switch (el) {
930 case ocsd_EL1:
931 return machines__find_guest(&etmq->etm->session->machines,
932 DEFAULT_GUEST_KERNEL_ID);
933 case ocsd_EL3:
934 case ocsd_EL2:
935 case ocsd_EL0:
936 case ocsd_EL_unknown:
937 default:
938 return &etmq->etm->session->machines.host;
939 }
940}
941
942static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address,
943 ocsd_ex_level el)
944{
945 struct machine *machine = cs_etm__get_machine(etmq, el);
946
947 if (address >= machine__kernel_start(machine)) {
948 if (machine__is_host(machine))
949 return PERF_RECORD_MISC_KERNEL;
950 else
951 return PERF_RECORD_MISC_GUEST_KERNEL;
952 } else {
953 if (machine__is_host(machine))
954 return PERF_RECORD_MISC_USER;
955 else {
956 /*
957 * Can't really happen at the moment because
958 * cs_etm__get_machine() will always return
959 * machines.host for any non EL1 trace.
960 */
961 return PERF_RECORD_MISC_GUEST_USER;
962 }
963 }
964}
965
966static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id,
967 u64 address, size_t size, u8 *buffer,
968 const ocsd_mem_space_acc_t mem_space)
969{
970 u8 cpumode;
971 u64 offset;
972 int len;
973 struct addr_location al;
974 struct dso *dso;
975 struct cs_etm_traceid_queue *tidq;
976 int ret = 0;
977
978 if (!etmq)
979 return 0;
980
981 addr_location__init(&al);
982 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
983 if (!tidq)
984 goto out;
985
986 /*
987 * We've already tracked EL along side the PID in cs_etm__set_thread()
988 * so double check that it matches what OpenCSD thinks as well. It
989 * doesn't distinguish between EL0 and EL1 for this mem access callback
990 * so we had to do the extra tracking. Skip validation if it's any of
991 * the 'any' values.
992 */
993 if (!(mem_space == OCSD_MEM_SPACE_ANY ||
994 mem_space == OCSD_MEM_SPACE_N || mem_space == OCSD_MEM_SPACE_S)) {
995 if (mem_space & OCSD_MEM_SPACE_EL1N) {
996 /* Includes both non secure EL1 and EL0 */
997 assert(tidq->el == ocsd_EL1 || tidq->el == ocsd_EL0);
998 } else if (mem_space & OCSD_MEM_SPACE_EL2)
999 assert(tidq->el == ocsd_EL2);
1000 else if (mem_space & OCSD_MEM_SPACE_EL3)
1001 assert(tidq->el == ocsd_EL3);
1002 }
1003
1004 cpumode = cs_etm__cpu_mode(etmq, address, tidq->el);
1005
1006 if (!thread__find_map(tidq->thread, cpumode, address, &al))
1007 goto out;
1008
1009 dso = map__dso(al.map);
1010 if (!dso)
1011 goto out;
1012
1013 if (dso->data.status == DSO_DATA_STATUS_ERROR &&
1014 dso__data_status_seen(dso, DSO_DATA_STATUS_SEEN_ITRACE))
1015 goto out;
1016
1017 offset = map__map_ip(al.map, address);
1018
1019 map__load(al.map);
1020
1021 len = dso__data_read_offset(dso, maps__machine(thread__maps(tidq->thread)),
1022 offset, buffer, size);
1023
1024 if (len <= 0) {
1025 ui__warning_once("CS ETM Trace: Missing DSO. Use 'perf archive' or debuginfod to export data from the traced system.\n"
1026 " Enable CONFIG_PROC_KCORE or use option '-k /path/to/vmlinux' for kernel symbols.\n");
1027 if (!dso->auxtrace_warned) {
1028 pr_err("CS ETM Trace: Debug data not found for address %#"PRIx64" in %s\n",
1029 address,
1030 dso->long_name ? dso->long_name : "Unknown");
1031 dso->auxtrace_warned = true;
1032 }
1033 goto out;
1034 }
1035 ret = len;
1036out:
1037 addr_location__exit(&al);
1038 return ret;
1039}
1040
1041static struct cs_etm_queue *cs_etm__alloc_queue(struct cs_etm_auxtrace *etm,
1042 bool formatted, int sample_cpu)
1043{
1044 struct cs_etm_decoder_params d_params;
1045 struct cs_etm_trace_params *t_params = NULL;
1046 struct cs_etm_queue *etmq;
1047 /*
1048 * Each queue can only contain data from one CPU when unformatted, so only one decoder is
1049 * needed.
1050 */
1051 int decoders = formatted ? etm->num_cpu : 1;
1052
1053 etmq = zalloc(sizeof(*etmq));
1054 if (!etmq)
1055 return NULL;
1056
1057 etmq->traceid_queues_list = intlist__new(NULL);
1058 if (!etmq->traceid_queues_list)
1059 goto out_free;
1060
1061 /* Use metadata to fill in trace parameters for trace decoder */
1062 t_params = zalloc(sizeof(*t_params) * decoders);
1063
1064 if (!t_params)
1065 goto out_free;
1066
1067 if (cs_etm__init_trace_params(t_params, etm, formatted, sample_cpu, decoders))
1068 goto out_free;
1069
1070 /* Set decoder parameters to decode trace packets */
1071 if (cs_etm__init_decoder_params(&d_params, etmq,
1072 dump_trace ? CS_ETM_OPERATION_PRINT :
1073 CS_ETM_OPERATION_DECODE,
1074 formatted))
1075 goto out_free;
1076
1077 etmq->decoder = cs_etm_decoder__new(decoders, &d_params,
1078 t_params);
1079
1080 if (!etmq->decoder)
1081 goto out_free;
1082
1083 /*
1084 * Register a function to handle all memory accesses required by
1085 * the trace decoder library.
1086 */
1087 if (cs_etm_decoder__add_mem_access_cb(etmq->decoder,
1088 0x0L, ((u64) -1L),
1089 cs_etm__mem_access))
1090 goto out_free_decoder;
1091
1092 zfree(&t_params);
1093 return etmq;
1094
1095out_free_decoder:
1096 cs_etm_decoder__free(etmq->decoder);
1097out_free:
1098 intlist__delete(etmq->traceid_queues_list);
1099 free(etmq);
1100
1101 return NULL;
1102}
1103
1104static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm,
1105 struct auxtrace_queue *queue,
1106 unsigned int queue_nr,
1107 bool formatted,
1108 int sample_cpu)
1109{
1110 struct cs_etm_queue *etmq = queue->priv;
1111
1112 if (list_empty(&queue->head) || etmq)
1113 return 0;
1114
1115 etmq = cs_etm__alloc_queue(etm, formatted, sample_cpu);
1116
1117 if (!etmq)
1118 return -ENOMEM;
1119
1120 queue->priv = etmq;
1121 etmq->etm = etm;
1122 etmq->queue_nr = queue_nr;
1123 etmq->offset = 0;
1124
1125 return 0;
1126}
1127
1128static int cs_etm__queue_first_cs_timestamp(struct cs_etm_auxtrace *etm,
1129 struct cs_etm_queue *etmq,
1130 unsigned int queue_nr)
1131{
1132 int ret = 0;
1133 unsigned int cs_queue_nr;
1134 u8 trace_chan_id;
1135 u64 cs_timestamp;
1136
1137 /*
1138 * We are under a CPU-wide trace scenario. As such we need to know
1139 * when the code that generated the traces started to execute so that
1140 * it can be correlated with execution on other CPUs. So we get a
1141 * handle on the beginning of traces and decode until we find a
1142 * timestamp. The timestamp is then added to the auxtrace min heap
1143 * in order to know what nibble (of all the etmqs) to decode first.
1144 */
1145 while (1) {
1146 /*
1147 * Fetch an aux_buffer from this etmq. Bail if no more
1148 * blocks or an error has been encountered.
1149 */
1150 ret = cs_etm__get_data_block(etmq);
1151 if (ret <= 0)
1152 goto out;
1153
1154 /*
1155 * Run decoder on the trace block. The decoder will stop when
1156 * encountering a CS timestamp, a full packet queue or the end of
1157 * trace for that block.
1158 */
1159 ret = cs_etm__decode_data_block(etmq);
1160 if (ret)
1161 goto out;
1162
1163 /*
1164 * Function cs_etm_decoder__do_{hard|soft}_timestamp() does all
1165 * the timestamp calculation for us.
1166 */
1167 cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
1168
1169 /* We found a timestamp, no need to continue. */
1170 if (cs_timestamp)
1171 break;
1172
1173 /*
1174 * We didn't find a timestamp so empty all the traceid packet
1175 * queues before looking for another timestamp packet, either
1176 * in the current data block or a new one. Packets that were
1177 * just decoded are useless since no timestamp has been
1178 * associated with them. As such simply discard them.
1179 */
1180 cs_etm__clear_all_packet_queues(etmq);
1181 }
1182
1183 /*
1184 * We have a timestamp. Add it to the min heap to reflect when
1185 * instructions conveyed by the range packets of this traceID queue
1186 * started to execute. Once the same has been done for all the traceID
1187 * queues of each etmq, redenring and decoding can start in
1188 * chronological order.
1189 *
1190 * Note that packets decoded above are still in the traceID's packet
1191 * queue and will be processed in cs_etm__process_timestamped_queues().
1192 */
1193 cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id);
1194 ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp);
1195out:
1196 return ret;
1197}
1198
1199static inline
1200void cs_etm__copy_last_branch_rb(struct cs_etm_queue *etmq,
1201 struct cs_etm_traceid_queue *tidq)
1202{
1203 struct branch_stack *bs_src = tidq->last_branch_rb;
1204 struct branch_stack *bs_dst = tidq->last_branch;
1205 size_t nr = 0;
1206
1207 /*
1208 * Set the number of records before early exit: ->nr is used to
1209 * determine how many branches to copy from ->entries.
1210 */
1211 bs_dst->nr = bs_src->nr;
1212
1213 /*
1214 * Early exit when there is nothing to copy.
1215 */
1216 if (!bs_src->nr)
1217 return;
1218
1219 /*
1220 * As bs_src->entries is a circular buffer, we need to copy from it in
1221 * two steps. First, copy the branches from the most recently inserted
1222 * branch ->last_branch_pos until the end of bs_src->entries buffer.
1223 */
1224 nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos;
1225 memcpy(&bs_dst->entries[0],
1226 &bs_src->entries[tidq->last_branch_pos],
1227 sizeof(struct branch_entry) * nr);
1228
1229 /*
1230 * If we wrapped around at least once, the branches from the beginning
1231 * of the bs_src->entries buffer and until the ->last_branch_pos element
1232 * are older valid branches: copy them over. The total number of
1233 * branches copied over will be equal to the number of branches asked by
1234 * the user in last_branch_sz.
1235 */
1236 if (bs_src->nr >= etmq->etm->synth_opts.last_branch_sz) {
1237 memcpy(&bs_dst->entries[nr],
1238 &bs_src->entries[0],
1239 sizeof(struct branch_entry) * tidq->last_branch_pos);
1240 }
1241}
1242
1243static inline
1244void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq)
1245{
1246 tidq->last_branch_pos = 0;
1247 tidq->last_branch_rb->nr = 0;
1248}
1249
1250static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq,
1251 u8 trace_chan_id, u64 addr)
1252{
1253 u8 instrBytes[2];
1254
1255 cs_etm__mem_access(etmq, trace_chan_id, addr, ARRAY_SIZE(instrBytes),
1256 instrBytes, 0);
1257 /*
1258 * T32 instruction size is indicated by bits[15:11] of the first
1259 * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111
1260 * denote a 32-bit instruction.
1261 */
1262 return ((instrBytes[1] & 0xF8) >= 0xE8) ? 4 : 2;
1263}
1264
1265static inline u64 cs_etm__first_executed_instr(struct cs_etm_packet *packet)
1266{
1267 /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
1268 if (packet->sample_type == CS_ETM_DISCONTINUITY)
1269 return 0;
1270
1271 return packet->start_addr;
1272}
1273
1274static inline
1275u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet)
1276{
1277 /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
1278 if (packet->sample_type == CS_ETM_DISCONTINUITY)
1279 return 0;
1280
1281 return packet->end_addr - packet->last_instr_size;
1282}
1283
1284static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq,
1285 u64 trace_chan_id,
1286 const struct cs_etm_packet *packet,
1287 u64 offset)
1288{
1289 if (packet->isa == CS_ETM_ISA_T32) {
1290 u64 addr = packet->start_addr;
1291
1292 while (offset) {
1293 addr += cs_etm__t32_instr_size(etmq,
1294 trace_chan_id, addr);
1295 offset--;
1296 }
1297 return addr;
1298 }
1299
1300 /* Assume a 4 byte instruction size (A32/A64) */
1301 return packet->start_addr + offset * 4;
1302}
1303
1304static void cs_etm__update_last_branch_rb(struct cs_etm_queue *etmq,
1305 struct cs_etm_traceid_queue *tidq)
1306{
1307 struct branch_stack *bs = tidq->last_branch_rb;
1308 struct branch_entry *be;
1309
1310 /*
1311 * The branches are recorded in a circular buffer in reverse
1312 * chronological order: we start recording from the last element of the
1313 * buffer down. After writing the first element of the stack, move the
1314 * insert position back to the end of the buffer.
1315 */
1316 if (!tidq->last_branch_pos)
1317 tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz;
1318
1319 tidq->last_branch_pos -= 1;
1320
1321 be = &bs->entries[tidq->last_branch_pos];
1322 be->from = cs_etm__last_executed_instr(tidq->prev_packet);
1323 be->to = cs_etm__first_executed_instr(tidq->packet);
1324 /* No support for mispredict */
1325 be->flags.mispred = 0;
1326 be->flags.predicted = 1;
1327
1328 /*
1329 * Increment bs->nr until reaching the number of last branches asked by
1330 * the user on the command line.
1331 */
1332 if (bs->nr < etmq->etm->synth_opts.last_branch_sz)
1333 bs->nr += 1;
1334}
1335
1336static int cs_etm__inject_event(union perf_event *event,
1337 struct perf_sample *sample, u64 type)
1338{
1339 event->header.size = perf_event__sample_event_size(sample, type, 0);
1340 return perf_event__synthesize_sample(event, type, 0, sample);
1341}
1342
1343
1344static int
1345cs_etm__get_trace(struct cs_etm_queue *etmq)
1346{
1347 struct auxtrace_buffer *aux_buffer = etmq->buffer;
1348 struct auxtrace_buffer *old_buffer = aux_buffer;
1349 struct auxtrace_queue *queue;
1350
1351 queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
1352
1353 aux_buffer = auxtrace_buffer__next(queue, aux_buffer);
1354
1355 /* If no more data, drop the previous auxtrace_buffer and return */
1356 if (!aux_buffer) {
1357 if (old_buffer)
1358 auxtrace_buffer__drop_data(old_buffer);
1359 etmq->buf_len = 0;
1360 return 0;
1361 }
1362
1363 etmq->buffer = aux_buffer;
1364
1365 /* If the aux_buffer doesn't have data associated, try to load it */
1366 if (!aux_buffer->data) {
1367 /* get the file desc associated with the perf data file */
1368 int fd = perf_data__fd(etmq->etm->session->data);
1369
1370 aux_buffer->data = auxtrace_buffer__get_data(aux_buffer, fd);
1371 if (!aux_buffer->data)
1372 return -ENOMEM;
1373 }
1374
1375 /* If valid, drop the previous buffer */
1376 if (old_buffer)
1377 auxtrace_buffer__drop_data(old_buffer);
1378
1379 etmq->buf_used = 0;
1380 etmq->buf_len = aux_buffer->size;
1381 etmq->buf = aux_buffer->data;
1382
1383 return etmq->buf_len;
1384}
1385
1386static void cs_etm__set_thread(struct cs_etm_queue *etmq,
1387 struct cs_etm_traceid_queue *tidq, pid_t tid,
1388 ocsd_ex_level el)
1389{
1390 struct machine *machine = cs_etm__get_machine(etmq, el);
1391
1392 if (tid != -1) {
1393 thread__zput(tidq->thread);
1394 tidq->thread = machine__find_thread(machine, -1, tid);
1395 }
1396
1397 /* Couldn't find a known thread */
1398 if (!tidq->thread)
1399 tidq->thread = machine__idle_thread(machine);
1400
1401 tidq->el = el;
1402}
1403
1404int cs_etm__etmq_set_tid_el(struct cs_etm_queue *etmq, pid_t tid,
1405 u8 trace_chan_id, ocsd_ex_level el)
1406{
1407 struct cs_etm_traceid_queue *tidq;
1408
1409 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
1410 if (!tidq)
1411 return -EINVAL;
1412
1413 cs_etm__set_thread(etmq, tidq, tid, el);
1414 return 0;
1415}
1416
1417bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq)
1418{
1419 return !!etmq->etm->timeless_decoding;
1420}
1421
1422static void cs_etm__copy_insn(struct cs_etm_queue *etmq,
1423 u64 trace_chan_id,
1424 const struct cs_etm_packet *packet,
1425 struct perf_sample *sample)
1426{
1427 /*
1428 * It's pointless to read instructions for the CS_ETM_DISCONTINUITY
1429 * packet, so directly bail out with 'insn_len' = 0.
1430 */
1431 if (packet->sample_type == CS_ETM_DISCONTINUITY) {
1432 sample->insn_len = 0;
1433 return;
1434 }
1435
1436 /*
1437 * T32 instruction size might be 32-bit or 16-bit, decide by calling
1438 * cs_etm__t32_instr_size().
1439 */
1440 if (packet->isa == CS_ETM_ISA_T32)
1441 sample->insn_len = cs_etm__t32_instr_size(etmq, trace_chan_id,
1442 sample->ip);
1443 /* Otherwise, A64 and A32 instruction size are always 32-bit. */
1444 else
1445 sample->insn_len = 4;
1446
1447 cs_etm__mem_access(etmq, trace_chan_id, sample->ip, sample->insn_len,
1448 (void *)sample->insn, 0);
1449}
1450
1451u64 cs_etm__convert_sample_time(struct cs_etm_queue *etmq, u64 cs_timestamp)
1452{
1453 struct cs_etm_auxtrace *etm = etmq->etm;
1454
1455 if (etm->has_virtual_ts)
1456 return tsc_to_perf_time(cs_timestamp, &etm->tc);
1457 else
1458 return cs_timestamp;
1459}
1460
1461static inline u64 cs_etm__resolve_sample_time(struct cs_etm_queue *etmq,
1462 struct cs_etm_traceid_queue *tidq)
1463{
1464 struct cs_etm_auxtrace *etm = etmq->etm;
1465 struct cs_etm_packet_queue *packet_queue = &tidq->packet_queue;
1466
1467 if (!etm->timeless_decoding && etm->has_virtual_ts)
1468 return packet_queue->cs_timestamp;
1469 else
1470 return etm->latest_kernel_timestamp;
1471}
1472
1473static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq,
1474 struct cs_etm_traceid_queue *tidq,
1475 u64 addr, u64 period)
1476{
1477 int ret = 0;
1478 struct cs_etm_auxtrace *etm = etmq->etm;
1479 union perf_event *event = tidq->event_buf;
1480 struct perf_sample sample = {.ip = 0,};
1481
1482 event->sample.header.type = PERF_RECORD_SAMPLE;
1483 event->sample.header.misc = cs_etm__cpu_mode(etmq, addr, tidq->el);
1484 event->sample.header.size = sizeof(struct perf_event_header);
1485
1486 /* Set time field based on etm auxtrace config. */
1487 sample.time = cs_etm__resolve_sample_time(etmq, tidq);
1488
1489 sample.ip = addr;
1490 sample.pid = thread__pid(tidq->thread);
1491 sample.tid = thread__tid(tidq->thread);
1492 sample.id = etmq->etm->instructions_id;
1493 sample.stream_id = etmq->etm->instructions_id;
1494 sample.period = period;
1495 sample.cpu = tidq->packet->cpu;
1496 sample.flags = tidq->prev_packet->flags;
1497 sample.cpumode = event->sample.header.misc;
1498
1499 cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->packet, &sample);
1500
1501 if (etm->synth_opts.last_branch)
1502 sample.branch_stack = tidq->last_branch;
1503
1504 if (etm->synth_opts.inject) {
1505 ret = cs_etm__inject_event(event, &sample,
1506 etm->instructions_sample_type);
1507 if (ret)
1508 return ret;
1509 }
1510
1511 ret = perf_session__deliver_synth_event(etm->session, event, &sample);
1512
1513 if (ret)
1514 pr_err(
1515 "CS ETM Trace: failed to deliver instruction event, error %d\n",
1516 ret);
1517
1518 return ret;
1519}
1520
1521/*
1522 * The cs etm packet encodes an instruction range between a branch target
1523 * and the next taken branch. Generate sample accordingly.
1524 */
1525static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq,
1526 struct cs_etm_traceid_queue *tidq)
1527{
1528 int ret = 0;
1529 struct cs_etm_auxtrace *etm = etmq->etm;
1530 struct perf_sample sample = {.ip = 0,};
1531 union perf_event *event = tidq->event_buf;
1532 struct dummy_branch_stack {
1533 u64 nr;
1534 u64 hw_idx;
1535 struct branch_entry entries;
1536 } dummy_bs;
1537 u64 ip;
1538
1539 ip = cs_etm__last_executed_instr(tidq->prev_packet);
1540
1541 event->sample.header.type = PERF_RECORD_SAMPLE;
1542 event->sample.header.misc = cs_etm__cpu_mode(etmq, ip,
1543 tidq->prev_packet_el);
1544 event->sample.header.size = sizeof(struct perf_event_header);
1545
1546 /* Set time field based on etm auxtrace config. */
1547 sample.time = cs_etm__resolve_sample_time(etmq, tidq);
1548
1549 sample.ip = ip;
1550 sample.pid = thread__pid(tidq->prev_packet_thread);
1551 sample.tid = thread__tid(tidq->prev_packet_thread);
1552 sample.addr = cs_etm__first_executed_instr(tidq->packet);
1553 sample.id = etmq->etm->branches_id;
1554 sample.stream_id = etmq->etm->branches_id;
1555 sample.period = 1;
1556 sample.cpu = tidq->packet->cpu;
1557 sample.flags = tidq->prev_packet->flags;
1558 sample.cpumode = event->sample.header.misc;
1559
1560 cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->prev_packet,
1561 &sample);
1562
1563 /*
1564 * perf report cannot handle events without a branch stack
1565 */
1566 if (etm->synth_opts.last_branch) {
1567 dummy_bs = (struct dummy_branch_stack){
1568 .nr = 1,
1569 .hw_idx = -1ULL,
1570 .entries = {
1571 .from = sample.ip,
1572 .to = sample.addr,
1573 },
1574 };
1575 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1576 }
1577
1578 if (etm->synth_opts.inject) {
1579 ret = cs_etm__inject_event(event, &sample,
1580 etm->branches_sample_type);
1581 if (ret)
1582 return ret;
1583 }
1584
1585 ret = perf_session__deliver_synth_event(etm->session, event, &sample);
1586
1587 if (ret)
1588 pr_err(
1589 "CS ETM Trace: failed to deliver instruction event, error %d\n",
1590 ret);
1591
1592 return ret;
1593}
1594
1595struct cs_etm_synth {
1596 struct perf_tool dummy_tool;
1597 struct perf_session *session;
1598};
1599
1600static int cs_etm__event_synth(struct perf_tool *tool,
1601 union perf_event *event,
1602 struct perf_sample *sample __maybe_unused,
1603 struct machine *machine __maybe_unused)
1604{
1605 struct cs_etm_synth *cs_etm_synth =
1606 container_of(tool, struct cs_etm_synth, dummy_tool);
1607
1608 return perf_session__deliver_synth_event(cs_etm_synth->session,
1609 event, NULL);
1610}
1611
1612static int cs_etm__synth_event(struct perf_session *session,
1613 struct perf_event_attr *attr, u64 id)
1614{
1615 struct cs_etm_synth cs_etm_synth;
1616
1617 memset(&cs_etm_synth, 0, sizeof(struct cs_etm_synth));
1618 cs_etm_synth.session = session;
1619
1620 return perf_event__synthesize_attr(&cs_etm_synth.dummy_tool, attr, 1,
1621 &id, cs_etm__event_synth);
1622}
1623
1624static int cs_etm__synth_events(struct cs_etm_auxtrace *etm,
1625 struct perf_session *session)
1626{
1627 struct evlist *evlist = session->evlist;
1628 struct evsel *evsel;
1629 struct perf_event_attr attr;
1630 bool found = false;
1631 u64 id;
1632 int err;
1633
1634 evlist__for_each_entry(evlist, evsel) {
1635 if (evsel->core.attr.type == etm->pmu_type) {
1636 found = true;
1637 break;
1638 }
1639 }
1640
1641 if (!found) {
1642 pr_debug("No selected events with CoreSight Trace data\n");
1643 return 0;
1644 }
1645
1646 memset(&attr, 0, sizeof(struct perf_event_attr));
1647 attr.size = sizeof(struct perf_event_attr);
1648 attr.type = PERF_TYPE_HARDWARE;
1649 attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK;
1650 attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1651 PERF_SAMPLE_PERIOD;
1652 if (etm->timeless_decoding)
1653 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1654 else
1655 attr.sample_type |= PERF_SAMPLE_TIME;
1656
1657 attr.exclude_user = evsel->core.attr.exclude_user;
1658 attr.exclude_kernel = evsel->core.attr.exclude_kernel;
1659 attr.exclude_hv = evsel->core.attr.exclude_hv;
1660 attr.exclude_host = evsel->core.attr.exclude_host;
1661 attr.exclude_guest = evsel->core.attr.exclude_guest;
1662 attr.sample_id_all = evsel->core.attr.sample_id_all;
1663 attr.read_format = evsel->core.attr.read_format;
1664
1665 /* create new id val to be a fixed offset from evsel id */
1666 id = evsel->core.id[0] + 1000000000;
1667
1668 if (!id)
1669 id = 1;
1670
1671 if (etm->synth_opts.branches) {
1672 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
1673 attr.sample_period = 1;
1674 attr.sample_type |= PERF_SAMPLE_ADDR;
1675 err = cs_etm__synth_event(session, &attr, id);
1676 if (err)
1677 return err;
1678 etm->branches_sample_type = attr.sample_type;
1679 etm->branches_id = id;
1680 id += 1;
1681 attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR;
1682 }
1683
1684 if (etm->synth_opts.last_branch) {
1685 attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1686 /*
1687 * We don't use the hardware index, but the sample generation
1688 * code uses the new format branch_stack with this field,
1689 * so the event attributes must indicate that it's present.
1690 */
1691 attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX;
1692 }
1693
1694 if (etm->synth_opts.instructions) {
1695 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1696 attr.sample_period = etm->synth_opts.period;
1697 etm->instructions_sample_period = attr.sample_period;
1698 err = cs_etm__synth_event(session, &attr, id);
1699 if (err)
1700 return err;
1701 etm->instructions_sample_type = attr.sample_type;
1702 etm->instructions_id = id;
1703 id += 1;
1704 }
1705
1706 return 0;
1707}
1708
1709static int cs_etm__sample(struct cs_etm_queue *etmq,
1710 struct cs_etm_traceid_queue *tidq)
1711{
1712 struct cs_etm_auxtrace *etm = etmq->etm;
1713 int ret;
1714 u8 trace_chan_id = tidq->trace_chan_id;
1715 u64 instrs_prev;
1716
1717 /* Get instructions remainder from previous packet */
1718 instrs_prev = tidq->period_instructions;
1719
1720 tidq->period_instructions += tidq->packet->instr_count;
1721
1722 /*
1723 * Record a branch when the last instruction in
1724 * PREV_PACKET is a branch.
1725 */
1726 if (etm->synth_opts.last_branch &&
1727 tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1728 tidq->prev_packet->last_instr_taken_branch)
1729 cs_etm__update_last_branch_rb(etmq, tidq);
1730
1731 if (etm->synth_opts.instructions &&
1732 tidq->period_instructions >= etm->instructions_sample_period) {
1733 /*
1734 * Emit instruction sample periodically
1735 * TODO: allow period to be defined in cycles and clock time
1736 */
1737
1738 /*
1739 * Below diagram demonstrates the instruction samples
1740 * generation flows:
1741 *
1742 * Instrs Instrs Instrs Instrs
1743 * Sample(n) Sample(n+1) Sample(n+2) Sample(n+3)
1744 * | | | |
1745 * V V V V
1746 * --------------------------------------------------
1747 * ^ ^
1748 * | |
1749 * Period Period
1750 * instructions(Pi) instructions(Pi')
1751 *
1752 * | |
1753 * \---------------- -----------------/
1754 * V
1755 * tidq->packet->instr_count
1756 *
1757 * Instrs Sample(n...) are the synthesised samples occurring
1758 * every etm->instructions_sample_period instructions - as
1759 * defined on the perf command line. Sample(n) is being the
1760 * last sample before the current etm packet, n+1 to n+3
1761 * samples are generated from the current etm packet.
1762 *
1763 * tidq->packet->instr_count represents the number of
1764 * instructions in the current etm packet.
1765 *
1766 * Period instructions (Pi) contains the number of
1767 * instructions executed after the sample point(n) from the
1768 * previous etm packet. This will always be less than
1769 * etm->instructions_sample_period.
1770 *
1771 * When generate new samples, it combines with two parts
1772 * instructions, one is the tail of the old packet and another
1773 * is the head of the new coming packet, to generate
1774 * sample(n+1); sample(n+2) and sample(n+3) consume the
1775 * instructions with sample period. After sample(n+3), the rest
1776 * instructions will be used by later packet and it is assigned
1777 * to tidq->period_instructions for next round calculation.
1778 */
1779
1780 /*
1781 * Get the initial offset into the current packet instructions;
1782 * entry conditions ensure that instrs_prev is less than
1783 * etm->instructions_sample_period.
1784 */
1785 u64 offset = etm->instructions_sample_period - instrs_prev;
1786 u64 addr;
1787
1788 /* Prepare last branches for instruction sample */
1789 if (etm->synth_opts.last_branch)
1790 cs_etm__copy_last_branch_rb(etmq, tidq);
1791
1792 while (tidq->period_instructions >=
1793 etm->instructions_sample_period) {
1794 /*
1795 * Calculate the address of the sampled instruction (-1
1796 * as sample is reported as though instruction has just
1797 * been executed, but PC has not advanced to next
1798 * instruction)
1799 */
1800 addr = cs_etm__instr_addr(etmq, trace_chan_id,
1801 tidq->packet, offset - 1);
1802 ret = cs_etm__synth_instruction_sample(
1803 etmq, tidq, addr,
1804 etm->instructions_sample_period);
1805 if (ret)
1806 return ret;
1807
1808 offset += etm->instructions_sample_period;
1809 tidq->period_instructions -=
1810 etm->instructions_sample_period;
1811 }
1812 }
1813
1814 if (etm->synth_opts.branches) {
1815 bool generate_sample = false;
1816
1817 /* Generate sample for tracing on packet */
1818 if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY)
1819 generate_sample = true;
1820
1821 /* Generate sample for branch taken packet */
1822 if (tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1823 tidq->prev_packet->last_instr_taken_branch)
1824 generate_sample = true;
1825
1826 if (generate_sample) {
1827 ret = cs_etm__synth_branch_sample(etmq, tidq);
1828 if (ret)
1829 return ret;
1830 }
1831 }
1832
1833 cs_etm__packet_swap(etm, tidq);
1834
1835 return 0;
1836}
1837
1838static int cs_etm__exception(struct cs_etm_traceid_queue *tidq)
1839{
1840 /*
1841 * When the exception packet is inserted, whether the last instruction
1842 * in previous range packet is taken branch or not, we need to force
1843 * to set 'prev_packet->last_instr_taken_branch' to true. This ensures
1844 * to generate branch sample for the instruction range before the
1845 * exception is trapped to kernel or before the exception returning.
1846 *
1847 * The exception packet includes the dummy address values, so don't
1848 * swap PACKET with PREV_PACKET. This keeps PREV_PACKET to be useful
1849 * for generating instruction and branch samples.
1850 */
1851 if (tidq->prev_packet->sample_type == CS_ETM_RANGE)
1852 tidq->prev_packet->last_instr_taken_branch = true;
1853
1854 return 0;
1855}
1856
1857static int cs_etm__flush(struct cs_etm_queue *etmq,
1858 struct cs_etm_traceid_queue *tidq)
1859{
1860 int err = 0;
1861 struct cs_etm_auxtrace *etm = etmq->etm;
1862
1863 /* Handle start tracing packet */
1864 if (tidq->prev_packet->sample_type == CS_ETM_EMPTY)
1865 goto swap_packet;
1866
1867 if (etmq->etm->synth_opts.last_branch &&
1868 etmq->etm->synth_opts.instructions &&
1869 tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1870 u64 addr;
1871
1872 /* Prepare last branches for instruction sample */
1873 cs_etm__copy_last_branch_rb(etmq, tidq);
1874
1875 /*
1876 * Generate a last branch event for the branches left in the
1877 * circular buffer at the end of the trace.
1878 *
1879 * Use the address of the end of the last reported execution
1880 * range
1881 */
1882 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1883
1884 err = cs_etm__synth_instruction_sample(
1885 etmq, tidq, addr,
1886 tidq->period_instructions);
1887 if (err)
1888 return err;
1889
1890 tidq->period_instructions = 0;
1891
1892 }
1893
1894 if (etm->synth_opts.branches &&
1895 tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1896 err = cs_etm__synth_branch_sample(etmq, tidq);
1897 if (err)
1898 return err;
1899 }
1900
1901swap_packet:
1902 cs_etm__packet_swap(etm, tidq);
1903
1904 /* Reset last branches after flush the trace */
1905 if (etm->synth_opts.last_branch)
1906 cs_etm__reset_last_branch_rb(tidq);
1907
1908 return err;
1909}
1910
1911static int cs_etm__end_block(struct cs_etm_queue *etmq,
1912 struct cs_etm_traceid_queue *tidq)
1913{
1914 int err;
1915
1916 /*
1917 * It has no new packet coming and 'etmq->packet' contains the stale
1918 * packet which was set at the previous time with packets swapping;
1919 * so skip to generate branch sample to avoid stale packet.
1920 *
1921 * For this case only flush branch stack and generate a last branch
1922 * event for the branches left in the circular buffer at the end of
1923 * the trace.
1924 */
1925 if (etmq->etm->synth_opts.last_branch &&
1926 etmq->etm->synth_opts.instructions &&
1927 tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1928 u64 addr;
1929
1930 /* Prepare last branches for instruction sample */
1931 cs_etm__copy_last_branch_rb(etmq, tidq);
1932
1933 /*
1934 * Use the address of the end of the last reported execution
1935 * range.
1936 */
1937 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1938
1939 err = cs_etm__synth_instruction_sample(
1940 etmq, tidq, addr,
1941 tidq->period_instructions);
1942 if (err)
1943 return err;
1944
1945 tidq->period_instructions = 0;
1946 }
1947
1948 return 0;
1949}
1950/*
1951 * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue
1952 * if need be.
1953 * Returns: < 0 if error
1954 * = 0 if no more auxtrace_buffer to read
1955 * > 0 if the current buffer isn't empty yet
1956 */
1957static int cs_etm__get_data_block(struct cs_etm_queue *etmq)
1958{
1959 int ret;
1960
1961 if (!etmq->buf_len) {
1962 ret = cs_etm__get_trace(etmq);
1963 if (ret <= 0)
1964 return ret;
1965 /*
1966 * We cannot assume consecutive blocks in the data file
1967 * are contiguous, reset the decoder to force re-sync.
1968 */
1969 ret = cs_etm_decoder__reset(etmq->decoder);
1970 if (ret)
1971 return ret;
1972 }
1973
1974 return etmq->buf_len;
1975}
1976
1977static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id,
1978 struct cs_etm_packet *packet,
1979 u64 end_addr)
1980{
1981 /* Initialise to keep compiler happy */
1982 u16 instr16 = 0;
1983 u32 instr32 = 0;
1984 u64 addr;
1985
1986 switch (packet->isa) {
1987 case CS_ETM_ISA_T32:
1988 /*
1989 * The SVC of T32 is defined in ARM DDI 0487D.a, F5.1.247:
1990 *
1991 * b'15 b'8
1992 * +-----------------+--------+
1993 * | 1 1 0 1 1 1 1 1 | imm8 |
1994 * +-----------------+--------+
1995 *
1996 * According to the specification, it only defines SVC for T32
1997 * with 16 bits instruction and has no definition for 32bits;
1998 * so below only read 2 bytes as instruction size for T32.
1999 */
2000 addr = end_addr - 2;
2001 cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr16),
2002 (u8 *)&instr16, 0);
2003 if ((instr16 & 0xFF00) == 0xDF00)
2004 return true;
2005
2006 break;
2007 case CS_ETM_ISA_A32:
2008 /*
2009 * The SVC of A32 is defined in ARM DDI 0487D.a, F5.1.247:
2010 *
2011 * b'31 b'28 b'27 b'24
2012 * +---------+---------+-------------------------+
2013 * | !1111 | 1 1 1 1 | imm24 |
2014 * +---------+---------+-------------------------+
2015 */
2016 addr = end_addr - 4;
2017 cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr32),
2018 (u8 *)&instr32, 0);
2019 if ((instr32 & 0x0F000000) == 0x0F000000 &&
2020 (instr32 & 0xF0000000) != 0xF0000000)
2021 return true;
2022
2023 break;
2024 case CS_ETM_ISA_A64:
2025 /*
2026 * The SVC of A64 is defined in ARM DDI 0487D.a, C6.2.294:
2027 *
2028 * b'31 b'21 b'4 b'0
2029 * +-----------------------+---------+-----------+
2030 * | 1 1 0 1 0 1 0 0 0 0 0 | imm16 | 0 0 0 0 1 |
2031 * +-----------------------+---------+-----------+
2032 */
2033 addr = end_addr - 4;
2034 cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr32),
2035 (u8 *)&instr32, 0);
2036 if ((instr32 & 0xFFE0001F) == 0xd4000001)
2037 return true;
2038
2039 break;
2040 case CS_ETM_ISA_UNKNOWN:
2041 default:
2042 break;
2043 }
2044
2045 return false;
2046}
2047
2048static bool cs_etm__is_syscall(struct cs_etm_queue *etmq,
2049 struct cs_etm_traceid_queue *tidq, u64 magic)
2050{
2051 u8 trace_chan_id = tidq->trace_chan_id;
2052 struct cs_etm_packet *packet = tidq->packet;
2053 struct cs_etm_packet *prev_packet = tidq->prev_packet;
2054
2055 if (magic == __perf_cs_etmv3_magic)
2056 if (packet->exception_number == CS_ETMV3_EXC_SVC)
2057 return true;
2058
2059 /*
2060 * ETMv4 exception type CS_ETMV4_EXC_CALL covers SVC, SMC and
2061 * HVC cases; need to check if it's SVC instruction based on
2062 * packet address.
2063 */
2064 if (magic == __perf_cs_etmv4_magic) {
2065 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
2066 cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
2067 prev_packet->end_addr))
2068 return true;
2069 }
2070
2071 return false;
2072}
2073
2074static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq,
2075 u64 magic)
2076{
2077 struct cs_etm_packet *packet = tidq->packet;
2078
2079 if (magic == __perf_cs_etmv3_magic)
2080 if (packet->exception_number == CS_ETMV3_EXC_DEBUG_HALT ||
2081 packet->exception_number == CS_ETMV3_EXC_ASYNC_DATA_ABORT ||
2082 packet->exception_number == CS_ETMV3_EXC_PE_RESET ||
2083 packet->exception_number == CS_ETMV3_EXC_IRQ ||
2084 packet->exception_number == CS_ETMV3_EXC_FIQ)
2085 return true;
2086
2087 if (magic == __perf_cs_etmv4_magic)
2088 if (packet->exception_number == CS_ETMV4_EXC_RESET ||
2089 packet->exception_number == CS_ETMV4_EXC_DEBUG_HALT ||
2090 packet->exception_number == CS_ETMV4_EXC_SYSTEM_ERROR ||
2091 packet->exception_number == CS_ETMV4_EXC_INST_DEBUG ||
2092 packet->exception_number == CS_ETMV4_EXC_DATA_DEBUG ||
2093 packet->exception_number == CS_ETMV4_EXC_IRQ ||
2094 packet->exception_number == CS_ETMV4_EXC_FIQ)
2095 return true;
2096
2097 return false;
2098}
2099
2100static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq,
2101 struct cs_etm_traceid_queue *tidq,
2102 u64 magic)
2103{
2104 u8 trace_chan_id = tidq->trace_chan_id;
2105 struct cs_etm_packet *packet = tidq->packet;
2106 struct cs_etm_packet *prev_packet = tidq->prev_packet;
2107
2108 if (magic == __perf_cs_etmv3_magic)
2109 if (packet->exception_number == CS_ETMV3_EXC_SMC ||
2110 packet->exception_number == CS_ETMV3_EXC_HYP ||
2111 packet->exception_number == CS_ETMV3_EXC_JAZELLE_THUMBEE ||
2112 packet->exception_number == CS_ETMV3_EXC_UNDEFINED_INSTR ||
2113 packet->exception_number == CS_ETMV3_EXC_PREFETCH_ABORT ||
2114 packet->exception_number == CS_ETMV3_EXC_DATA_FAULT ||
2115 packet->exception_number == CS_ETMV3_EXC_GENERIC)
2116 return true;
2117
2118 if (magic == __perf_cs_etmv4_magic) {
2119 if (packet->exception_number == CS_ETMV4_EXC_TRAP ||
2120 packet->exception_number == CS_ETMV4_EXC_ALIGNMENT ||
2121 packet->exception_number == CS_ETMV4_EXC_INST_FAULT ||
2122 packet->exception_number == CS_ETMV4_EXC_DATA_FAULT)
2123 return true;
2124
2125 /*
2126 * For CS_ETMV4_EXC_CALL, except SVC other instructions
2127 * (SMC, HVC) are taken as sync exceptions.
2128 */
2129 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
2130 !cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
2131 prev_packet->end_addr))
2132 return true;
2133
2134 /*
2135 * ETMv4 has 5 bits for exception number; if the numbers
2136 * are in the range ( CS_ETMV4_EXC_FIQ, CS_ETMV4_EXC_END ]
2137 * they are implementation defined exceptions.
2138 *
2139 * For this case, simply take it as sync exception.
2140 */
2141 if (packet->exception_number > CS_ETMV4_EXC_FIQ &&
2142 packet->exception_number <= CS_ETMV4_EXC_END)
2143 return true;
2144 }
2145
2146 return false;
2147}
2148
2149static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq,
2150 struct cs_etm_traceid_queue *tidq)
2151{
2152 struct cs_etm_packet *packet = tidq->packet;
2153 struct cs_etm_packet *prev_packet = tidq->prev_packet;
2154 u8 trace_chan_id = tidq->trace_chan_id;
2155 u64 magic;
2156 int ret;
2157
2158 switch (packet->sample_type) {
2159 case CS_ETM_RANGE:
2160 /*
2161 * Immediate branch instruction without neither link nor
2162 * return flag, it's normal branch instruction within
2163 * the function.
2164 */
2165 if (packet->last_instr_type == OCSD_INSTR_BR &&
2166 packet->last_instr_subtype == OCSD_S_INSTR_NONE) {
2167 packet->flags = PERF_IP_FLAG_BRANCH;
2168
2169 if (packet->last_instr_cond)
2170 packet->flags |= PERF_IP_FLAG_CONDITIONAL;
2171 }
2172
2173 /*
2174 * Immediate branch instruction with link (e.g. BL), this is
2175 * branch instruction for function call.
2176 */
2177 if (packet->last_instr_type == OCSD_INSTR_BR &&
2178 packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
2179 packet->flags = PERF_IP_FLAG_BRANCH |
2180 PERF_IP_FLAG_CALL;
2181
2182 /*
2183 * Indirect branch instruction with link (e.g. BLR), this is
2184 * branch instruction for function call.
2185 */
2186 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2187 packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
2188 packet->flags = PERF_IP_FLAG_BRANCH |
2189 PERF_IP_FLAG_CALL;
2190
2191 /*
2192 * Indirect branch instruction with subtype of
2193 * OCSD_S_INSTR_V7_IMPLIED_RET, this is explicit hint for
2194 * function return for A32/T32.
2195 */
2196 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2197 packet->last_instr_subtype == OCSD_S_INSTR_V7_IMPLIED_RET)
2198 packet->flags = PERF_IP_FLAG_BRANCH |
2199 PERF_IP_FLAG_RETURN;
2200
2201 /*
2202 * Indirect branch instruction without link (e.g. BR), usually
2203 * this is used for function return, especially for functions
2204 * within dynamic link lib.
2205 */
2206 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2207 packet->last_instr_subtype == OCSD_S_INSTR_NONE)
2208 packet->flags = PERF_IP_FLAG_BRANCH |
2209 PERF_IP_FLAG_RETURN;
2210
2211 /* Return instruction for function return. */
2212 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2213 packet->last_instr_subtype == OCSD_S_INSTR_V8_RET)
2214 packet->flags = PERF_IP_FLAG_BRANCH |
2215 PERF_IP_FLAG_RETURN;
2216
2217 /*
2218 * Decoder might insert a discontinuity in the middle of
2219 * instruction packets, fixup prev_packet with flag
2220 * PERF_IP_FLAG_TRACE_BEGIN to indicate restarting trace.
2221 */
2222 if (prev_packet->sample_type == CS_ETM_DISCONTINUITY)
2223 prev_packet->flags |= PERF_IP_FLAG_BRANCH |
2224 PERF_IP_FLAG_TRACE_BEGIN;
2225
2226 /*
2227 * If the previous packet is an exception return packet
2228 * and the return address just follows SVC instruction,
2229 * it needs to calibrate the previous packet sample flags
2230 * as PERF_IP_FLAG_SYSCALLRET.
2231 */
2232 if (prev_packet->flags == (PERF_IP_FLAG_BRANCH |
2233 PERF_IP_FLAG_RETURN |
2234 PERF_IP_FLAG_INTERRUPT) &&
2235 cs_etm__is_svc_instr(etmq, trace_chan_id,
2236 packet, packet->start_addr))
2237 prev_packet->flags = PERF_IP_FLAG_BRANCH |
2238 PERF_IP_FLAG_RETURN |
2239 PERF_IP_FLAG_SYSCALLRET;
2240 break;
2241 case CS_ETM_DISCONTINUITY:
2242 /*
2243 * The trace is discontinuous, if the previous packet is
2244 * instruction packet, set flag PERF_IP_FLAG_TRACE_END
2245 * for previous packet.
2246 */
2247 if (prev_packet->sample_type == CS_ETM_RANGE)
2248 prev_packet->flags |= PERF_IP_FLAG_BRANCH |
2249 PERF_IP_FLAG_TRACE_END;
2250 break;
2251 case CS_ETM_EXCEPTION:
2252 ret = cs_etm__get_magic(packet->trace_chan_id, &magic);
2253 if (ret)
2254 return ret;
2255
2256 /* The exception is for system call. */
2257 if (cs_etm__is_syscall(etmq, tidq, magic))
2258 packet->flags = PERF_IP_FLAG_BRANCH |
2259 PERF_IP_FLAG_CALL |
2260 PERF_IP_FLAG_SYSCALLRET;
2261 /*
2262 * The exceptions are triggered by external signals from bus,
2263 * interrupt controller, debug module, PE reset or halt.
2264 */
2265 else if (cs_etm__is_async_exception(tidq, magic))
2266 packet->flags = PERF_IP_FLAG_BRANCH |
2267 PERF_IP_FLAG_CALL |
2268 PERF_IP_FLAG_ASYNC |
2269 PERF_IP_FLAG_INTERRUPT;
2270 /*
2271 * Otherwise, exception is caused by trap, instruction &
2272 * data fault, or alignment errors.
2273 */
2274 else if (cs_etm__is_sync_exception(etmq, tidq, magic))
2275 packet->flags = PERF_IP_FLAG_BRANCH |
2276 PERF_IP_FLAG_CALL |
2277 PERF_IP_FLAG_INTERRUPT;
2278
2279 /*
2280 * When the exception packet is inserted, since exception
2281 * packet is not used standalone for generating samples
2282 * and it's affiliation to the previous instruction range
2283 * packet; so set previous range packet flags to tell perf
2284 * it is an exception taken branch.
2285 */
2286 if (prev_packet->sample_type == CS_ETM_RANGE)
2287 prev_packet->flags = packet->flags;
2288 break;
2289 case CS_ETM_EXCEPTION_RET:
2290 /*
2291 * When the exception return packet is inserted, since
2292 * exception return packet is not used standalone for
2293 * generating samples and it's affiliation to the previous
2294 * instruction range packet; so set previous range packet
2295 * flags to tell perf it is an exception return branch.
2296 *
2297 * The exception return can be for either system call or
2298 * other exception types; unfortunately the packet doesn't
2299 * contain exception type related info so we cannot decide
2300 * the exception type purely based on exception return packet.
2301 * If we record the exception number from exception packet and
2302 * reuse it for exception return packet, this is not reliable
2303 * due the trace can be discontinuity or the interrupt can
2304 * be nested, thus the recorded exception number cannot be
2305 * used for exception return packet for these two cases.
2306 *
2307 * For exception return packet, we only need to distinguish the
2308 * packet is for system call or for other types. Thus the
2309 * decision can be deferred when receive the next packet which
2310 * contains the return address, based on the return address we
2311 * can read out the previous instruction and check if it's a
2312 * system call instruction and then calibrate the sample flag
2313 * as needed.
2314 */
2315 if (prev_packet->sample_type == CS_ETM_RANGE)
2316 prev_packet->flags = PERF_IP_FLAG_BRANCH |
2317 PERF_IP_FLAG_RETURN |
2318 PERF_IP_FLAG_INTERRUPT;
2319 break;
2320 case CS_ETM_EMPTY:
2321 default:
2322 break;
2323 }
2324
2325 return 0;
2326}
2327
2328static int cs_etm__decode_data_block(struct cs_etm_queue *etmq)
2329{
2330 int ret = 0;
2331 size_t processed = 0;
2332
2333 /*
2334 * Packets are decoded and added to the decoder's packet queue
2335 * until the decoder packet processing callback has requested that
2336 * processing stops or there is nothing left in the buffer. Normal
2337 * operations that stop processing are a timestamp packet or a full
2338 * decoder buffer queue.
2339 */
2340 ret = cs_etm_decoder__process_data_block(etmq->decoder,
2341 etmq->offset,
2342 &etmq->buf[etmq->buf_used],
2343 etmq->buf_len,
2344 &processed);
2345 if (ret)
2346 goto out;
2347
2348 etmq->offset += processed;
2349 etmq->buf_used += processed;
2350 etmq->buf_len -= processed;
2351
2352out:
2353 return ret;
2354}
2355
2356static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq,
2357 struct cs_etm_traceid_queue *tidq)
2358{
2359 int ret;
2360 struct cs_etm_packet_queue *packet_queue;
2361
2362 packet_queue = &tidq->packet_queue;
2363
2364 /* Process each packet in this chunk */
2365 while (1) {
2366 ret = cs_etm_decoder__get_packet(packet_queue,
2367 tidq->packet);
2368 if (ret <= 0)
2369 /*
2370 * Stop processing this chunk on
2371 * end of data or error
2372 */
2373 break;
2374
2375 /*
2376 * Since packet addresses are swapped in packet
2377 * handling within below switch() statements,
2378 * thus setting sample flags must be called
2379 * prior to switch() statement to use address
2380 * information before packets swapping.
2381 */
2382 ret = cs_etm__set_sample_flags(etmq, tidq);
2383 if (ret < 0)
2384 break;
2385
2386 switch (tidq->packet->sample_type) {
2387 case CS_ETM_RANGE:
2388 /*
2389 * If the packet contains an instruction
2390 * range, generate instruction sequence
2391 * events.
2392 */
2393 cs_etm__sample(etmq, tidq);
2394 break;
2395 case CS_ETM_EXCEPTION:
2396 case CS_ETM_EXCEPTION_RET:
2397 /*
2398 * If the exception packet is coming,
2399 * make sure the previous instruction
2400 * range packet to be handled properly.
2401 */
2402 cs_etm__exception(tidq);
2403 break;
2404 case CS_ETM_DISCONTINUITY:
2405 /*
2406 * Discontinuity in trace, flush
2407 * previous branch stack
2408 */
2409 cs_etm__flush(etmq, tidq);
2410 break;
2411 case CS_ETM_EMPTY:
2412 /*
2413 * Should not receive empty packet,
2414 * report error.
2415 */
2416 pr_err("CS ETM Trace: empty packet\n");
2417 return -EINVAL;
2418 default:
2419 break;
2420 }
2421 }
2422
2423 return ret;
2424}
2425
2426static void cs_etm__clear_all_traceid_queues(struct cs_etm_queue *etmq)
2427{
2428 int idx;
2429 struct int_node *inode;
2430 struct cs_etm_traceid_queue *tidq;
2431 struct intlist *traceid_queues_list = etmq->traceid_queues_list;
2432
2433 intlist__for_each_entry(inode, traceid_queues_list) {
2434 idx = (int)(intptr_t)inode->priv;
2435 tidq = etmq->traceid_queues[idx];
2436
2437 /* Ignore return value */
2438 cs_etm__process_traceid_queue(etmq, tidq);
2439
2440 /*
2441 * Generate an instruction sample with the remaining
2442 * branchstack entries.
2443 */
2444 cs_etm__flush(etmq, tidq);
2445 }
2446}
2447
2448static int cs_etm__run_per_thread_timeless_decoder(struct cs_etm_queue *etmq)
2449{
2450 int err = 0;
2451 struct cs_etm_traceid_queue *tidq;
2452
2453 tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID);
2454 if (!tidq)
2455 return -EINVAL;
2456
2457 /* Go through each buffer in the queue and decode them one by one */
2458 while (1) {
2459 err = cs_etm__get_data_block(etmq);
2460 if (err <= 0)
2461 return err;
2462
2463 /* Run trace decoder until buffer consumed or end of trace */
2464 do {
2465 err = cs_etm__decode_data_block(etmq);
2466 if (err)
2467 return err;
2468
2469 /*
2470 * Process each packet in this chunk, nothing to do if
2471 * an error occurs other than hoping the next one will
2472 * be better.
2473 */
2474 err = cs_etm__process_traceid_queue(etmq, tidq);
2475
2476 } while (etmq->buf_len);
2477
2478 if (err == 0)
2479 /* Flush any remaining branch stack entries */
2480 err = cs_etm__end_block(etmq, tidq);
2481 }
2482
2483 return err;
2484}
2485
2486static int cs_etm__run_per_cpu_timeless_decoder(struct cs_etm_queue *etmq)
2487{
2488 int idx, err = 0;
2489 struct cs_etm_traceid_queue *tidq;
2490 struct int_node *inode;
2491
2492 /* Go through each buffer in the queue and decode them one by one */
2493 while (1) {
2494 err = cs_etm__get_data_block(etmq);
2495 if (err <= 0)
2496 return err;
2497
2498 /* Run trace decoder until buffer consumed or end of trace */
2499 do {
2500 err = cs_etm__decode_data_block(etmq);
2501 if (err)
2502 return err;
2503
2504 /*
2505 * cs_etm__run_per_thread_timeless_decoder() runs on a
2506 * single traceID queue because each TID has a separate
2507 * buffer. But here in per-cpu mode we need to iterate
2508 * over each channel instead.
2509 */
2510 intlist__for_each_entry(inode,
2511 etmq->traceid_queues_list) {
2512 idx = (int)(intptr_t)inode->priv;
2513 tidq = etmq->traceid_queues[idx];
2514 cs_etm__process_traceid_queue(etmq, tidq);
2515 }
2516 } while (etmq->buf_len);
2517
2518 intlist__for_each_entry(inode, etmq->traceid_queues_list) {
2519 idx = (int)(intptr_t)inode->priv;
2520 tidq = etmq->traceid_queues[idx];
2521 /* Flush any remaining branch stack entries */
2522 err = cs_etm__end_block(etmq, tidq);
2523 if (err)
2524 return err;
2525 }
2526 }
2527
2528 return err;
2529}
2530
2531static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
2532 pid_t tid)
2533{
2534 unsigned int i;
2535 struct auxtrace_queues *queues = &etm->queues;
2536
2537 for (i = 0; i < queues->nr_queues; i++) {
2538 struct auxtrace_queue *queue = &etm->queues.queue_array[i];
2539 struct cs_etm_queue *etmq = queue->priv;
2540 struct cs_etm_traceid_queue *tidq;
2541
2542 if (!etmq)
2543 continue;
2544
2545 if (etm->per_thread_decoding) {
2546 tidq = cs_etm__etmq_get_traceid_queue(
2547 etmq, CS_ETM_PER_THREAD_TRACEID);
2548
2549 if (!tidq)
2550 continue;
2551
2552 if (tid == -1 || thread__tid(tidq->thread) == tid)
2553 cs_etm__run_per_thread_timeless_decoder(etmq);
2554 } else
2555 cs_etm__run_per_cpu_timeless_decoder(etmq);
2556 }
2557
2558 return 0;
2559}
2560
2561static int cs_etm__process_timestamped_queues(struct cs_etm_auxtrace *etm)
2562{
2563 int ret = 0;
2564 unsigned int cs_queue_nr, queue_nr, i;
2565 u8 trace_chan_id;
2566 u64 cs_timestamp;
2567 struct auxtrace_queue *queue;
2568 struct cs_etm_queue *etmq;
2569 struct cs_etm_traceid_queue *tidq;
2570
2571 /*
2572 * Pre-populate the heap with one entry from each queue so that we can
2573 * start processing in time order across all queues.
2574 */
2575 for (i = 0; i < etm->queues.nr_queues; i++) {
2576 etmq = etm->queues.queue_array[i].priv;
2577 if (!etmq)
2578 continue;
2579
2580 ret = cs_etm__queue_first_cs_timestamp(etm, etmq, i);
2581 if (ret)
2582 return ret;
2583 }
2584
2585 while (1) {
2586 if (!etm->heap.heap_cnt)
2587 goto out;
2588
2589 /* Take the entry at the top of the min heap */
2590 cs_queue_nr = etm->heap.heap_array[0].queue_nr;
2591 queue_nr = TO_QUEUE_NR(cs_queue_nr);
2592 trace_chan_id = TO_TRACE_CHAN_ID(cs_queue_nr);
2593 queue = &etm->queues.queue_array[queue_nr];
2594 etmq = queue->priv;
2595
2596 /*
2597 * Remove the top entry from the heap since we are about
2598 * to process it.
2599 */
2600 auxtrace_heap__pop(&etm->heap);
2601
2602 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
2603 if (!tidq) {
2604 /*
2605 * No traceID queue has been allocated for this traceID,
2606 * which means something somewhere went very wrong. No
2607 * other choice than simply exit.
2608 */
2609 ret = -EINVAL;
2610 goto out;
2611 }
2612
2613 /*
2614 * Packets associated with this timestamp are already in
2615 * the etmq's traceID queue, so process them.
2616 */
2617 ret = cs_etm__process_traceid_queue(etmq, tidq);
2618 if (ret < 0)
2619 goto out;
2620
2621 /*
2622 * Packets for this timestamp have been processed, time to
2623 * move on to the next timestamp, fetching a new auxtrace_buffer
2624 * if need be.
2625 */
2626refetch:
2627 ret = cs_etm__get_data_block(etmq);
2628 if (ret < 0)
2629 goto out;
2630
2631 /*
2632 * No more auxtrace_buffers to process in this etmq, simply
2633 * move on to another entry in the auxtrace_heap.
2634 */
2635 if (!ret)
2636 continue;
2637
2638 ret = cs_etm__decode_data_block(etmq);
2639 if (ret)
2640 goto out;
2641
2642 cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
2643
2644 if (!cs_timestamp) {
2645 /*
2646 * Function cs_etm__decode_data_block() returns when
2647 * there is no more traces to decode in the current
2648 * auxtrace_buffer OR when a timestamp has been
2649 * encountered on any of the traceID queues. Since we
2650 * did not get a timestamp, there is no more traces to
2651 * process in this auxtrace_buffer. As such empty and
2652 * flush all traceID queues.
2653 */
2654 cs_etm__clear_all_traceid_queues(etmq);
2655
2656 /* Fetch another auxtrace_buffer for this etmq */
2657 goto refetch;
2658 }
2659
2660 /*
2661 * Add to the min heap the timestamp for packets that have
2662 * just been decoded. They will be processed and synthesized
2663 * during the next call to cs_etm__process_traceid_queue() for
2664 * this queue/traceID.
2665 */
2666 cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id);
2667 ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp);
2668 }
2669
2670out:
2671 return ret;
2672}
2673
2674static int cs_etm__process_itrace_start(struct cs_etm_auxtrace *etm,
2675 union perf_event *event)
2676{
2677 struct thread *th;
2678
2679 if (etm->timeless_decoding)
2680 return 0;
2681
2682 /*
2683 * Add the tid/pid to the log so that we can get a match when we get a
2684 * contextID from the decoder. Only track for the host: only kernel
2685 * trace is supported for guests which wouldn't need pids so this should
2686 * be fine.
2687 */
2688 th = machine__findnew_thread(&etm->session->machines.host,
2689 event->itrace_start.pid,
2690 event->itrace_start.tid);
2691 if (!th)
2692 return -ENOMEM;
2693
2694 thread__put(th);
2695
2696 return 0;
2697}
2698
2699static int cs_etm__process_switch_cpu_wide(struct cs_etm_auxtrace *etm,
2700 union perf_event *event)
2701{
2702 struct thread *th;
2703 bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
2704
2705 /*
2706 * Context switch in per-thread mode are irrelevant since perf
2707 * will start/stop tracing as the process is scheduled.
2708 */
2709 if (etm->timeless_decoding)
2710 return 0;
2711
2712 /*
2713 * SWITCH_IN events carry the next process to be switched out while
2714 * SWITCH_OUT events carry the process to be switched in. As such
2715 * we don't care about IN events.
2716 */
2717 if (!out)
2718 return 0;
2719
2720 /*
2721 * Add the tid/pid to the log so that we can get a match when we get a
2722 * contextID from the decoder. Only track for the host: only kernel
2723 * trace is supported for guests which wouldn't need pids so this should
2724 * be fine.
2725 */
2726 th = machine__findnew_thread(&etm->session->machines.host,
2727 event->context_switch.next_prev_pid,
2728 event->context_switch.next_prev_tid);
2729 if (!th)
2730 return -ENOMEM;
2731
2732 thread__put(th);
2733
2734 return 0;
2735}
2736
2737static int cs_etm__process_event(struct perf_session *session,
2738 union perf_event *event,
2739 struct perf_sample *sample,
2740 struct perf_tool *tool)
2741{
2742 struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2743 struct cs_etm_auxtrace,
2744 auxtrace);
2745
2746 if (dump_trace)
2747 return 0;
2748
2749 if (!tool->ordered_events) {
2750 pr_err("CoreSight ETM Trace requires ordered events\n");
2751 return -EINVAL;
2752 }
2753
2754 switch (event->header.type) {
2755 case PERF_RECORD_EXIT:
2756 /*
2757 * Don't need to wait for cs_etm__flush_events() in per-thread mode to
2758 * start the decode because we know there will be no more trace from
2759 * this thread. All this does is emit samples earlier than waiting for
2760 * the flush in other modes, but with timestamps it makes sense to wait
2761 * for flush so that events from different threads are interleaved
2762 * properly.
2763 */
2764 if (etm->per_thread_decoding && etm->timeless_decoding)
2765 return cs_etm__process_timeless_queues(etm,
2766 event->fork.tid);
2767 break;
2768
2769 case PERF_RECORD_ITRACE_START:
2770 return cs_etm__process_itrace_start(etm, event);
2771
2772 case PERF_RECORD_SWITCH_CPU_WIDE:
2773 return cs_etm__process_switch_cpu_wide(etm, event);
2774
2775 case PERF_RECORD_AUX:
2776 /*
2777 * Record the latest kernel timestamp available in the header
2778 * for samples so that synthesised samples occur from this point
2779 * onwards.
2780 */
2781 if (sample->time && (sample->time != (u64)-1))
2782 etm->latest_kernel_timestamp = sample->time;
2783 break;
2784
2785 default:
2786 break;
2787 }
2788
2789 return 0;
2790}
2791
2792static void dump_queued_data(struct cs_etm_auxtrace *etm,
2793 struct perf_record_auxtrace *event)
2794{
2795 struct auxtrace_buffer *buf;
2796 unsigned int i;
2797 /*
2798 * Find all buffers with same reference in the queues and dump them.
2799 * This is because the queues can contain multiple entries of the same
2800 * buffer that were split on aux records.
2801 */
2802 for (i = 0; i < etm->queues.nr_queues; ++i)
2803 list_for_each_entry(buf, &etm->queues.queue_array[i].head, list)
2804 if (buf->reference == event->reference)
2805 cs_etm__dump_event(etm->queues.queue_array[i].priv, buf);
2806}
2807
2808static int cs_etm__process_auxtrace_event(struct perf_session *session,
2809 union perf_event *event,
2810 struct perf_tool *tool __maybe_unused)
2811{
2812 struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2813 struct cs_etm_auxtrace,
2814 auxtrace);
2815 if (!etm->data_queued) {
2816 struct auxtrace_buffer *buffer;
2817 off_t data_offset;
2818 int fd = perf_data__fd(session->data);
2819 bool is_pipe = perf_data__is_pipe(session->data);
2820 int err;
2821 int idx = event->auxtrace.idx;
2822
2823 if (is_pipe)
2824 data_offset = 0;
2825 else {
2826 data_offset = lseek(fd, 0, SEEK_CUR);
2827 if (data_offset == -1)
2828 return -errno;
2829 }
2830
2831 err = auxtrace_queues__add_event(&etm->queues, session,
2832 event, data_offset, &buffer);
2833 if (err)
2834 return err;
2835
2836 /*
2837 * Knowing if the trace is formatted or not requires a lookup of
2838 * the aux record so only works in non-piped mode where data is
2839 * queued in cs_etm__queue_aux_records(). Always assume
2840 * formatted in piped mode (true).
2841 */
2842 err = cs_etm__setup_queue(etm, &etm->queues.queue_array[idx],
2843 idx, true, -1);
2844 if (err)
2845 return err;
2846
2847 if (dump_trace)
2848 if (auxtrace_buffer__get_data(buffer, fd)) {
2849 cs_etm__dump_event(etm->queues.queue_array[idx].priv, buffer);
2850 auxtrace_buffer__put_data(buffer);
2851 }
2852 } else if (dump_trace)
2853 dump_queued_data(etm, &event->auxtrace);
2854
2855 return 0;
2856}
2857
2858static int cs_etm__setup_timeless_decoding(struct cs_etm_auxtrace *etm)
2859{
2860 struct evsel *evsel;
2861 struct evlist *evlist = etm->session->evlist;
2862
2863 /* Override timeless mode with user input from --itrace=Z */
2864 if (etm->synth_opts.timeless_decoding) {
2865 etm->timeless_decoding = true;
2866 return 0;
2867 }
2868
2869 /*
2870 * Find the cs_etm evsel and look at what its timestamp setting was
2871 */
2872 evlist__for_each_entry(evlist, evsel)
2873 if (cs_etm__evsel_is_auxtrace(etm->session, evsel)) {
2874 etm->timeless_decoding =
2875 !(evsel->core.attr.config & BIT(ETM_OPT_TS));
2876 return 0;
2877 }
2878
2879 pr_err("CS ETM: Couldn't find ETM evsel\n");
2880 return -EINVAL;
2881}
2882
2883/*
2884 * Read a single cpu parameter block from the auxtrace_info priv block.
2885 *
2886 * For version 1 there is a per cpu nr_params entry. If we are handling
2887 * version 1 file, then there may be less, the same, or more params
2888 * indicated by this value than the compile time number we understand.
2889 *
2890 * For a version 0 info block, there are a fixed number, and we need to
2891 * fill out the nr_param value in the metadata we create.
2892 */
2893static u64 *cs_etm__create_meta_blk(u64 *buff_in, int *buff_in_offset,
2894 int out_blk_size, int nr_params_v0)
2895{
2896 u64 *metadata = NULL;
2897 int hdr_version;
2898 int nr_in_params, nr_out_params, nr_cmn_params;
2899 int i, k;
2900
2901 metadata = zalloc(sizeof(*metadata) * out_blk_size);
2902 if (!metadata)
2903 return NULL;
2904
2905 /* read block current index & version */
2906 i = *buff_in_offset;
2907 hdr_version = buff_in[CS_HEADER_VERSION];
2908
2909 if (!hdr_version) {
2910 /* read version 0 info block into a version 1 metadata block */
2911 nr_in_params = nr_params_v0;
2912 metadata[CS_ETM_MAGIC] = buff_in[i + CS_ETM_MAGIC];
2913 metadata[CS_ETM_CPU] = buff_in[i + CS_ETM_CPU];
2914 metadata[CS_ETM_NR_TRC_PARAMS] = nr_in_params;
2915 /* remaining block params at offset +1 from source */
2916 for (k = CS_ETM_COMMON_BLK_MAX_V1 - 1; k < nr_in_params; k++)
2917 metadata[k + 1] = buff_in[i + k];
2918 /* version 0 has 2 common params */
2919 nr_cmn_params = 2;
2920 } else {
2921 /* read version 1 info block - input and output nr_params may differ */
2922 /* version 1 has 3 common params */
2923 nr_cmn_params = 3;
2924 nr_in_params = buff_in[i + CS_ETM_NR_TRC_PARAMS];
2925
2926 /* if input has more params than output - skip excess */
2927 nr_out_params = nr_in_params + nr_cmn_params;
2928 if (nr_out_params > out_blk_size)
2929 nr_out_params = out_blk_size;
2930
2931 for (k = CS_ETM_MAGIC; k < nr_out_params; k++)
2932 metadata[k] = buff_in[i + k];
2933
2934 /* record the actual nr params we copied */
2935 metadata[CS_ETM_NR_TRC_PARAMS] = nr_out_params - nr_cmn_params;
2936 }
2937
2938 /* adjust in offset by number of in params used */
2939 i += nr_in_params + nr_cmn_params;
2940 *buff_in_offset = i;
2941 return metadata;
2942}
2943
2944/**
2945 * Puts a fragment of an auxtrace buffer into the auxtrace queues based
2946 * on the bounds of aux_event, if it matches with the buffer that's at
2947 * file_offset.
2948 *
2949 * Normally, whole auxtrace buffers would be added to the queue. But we
2950 * want to reset the decoder for every PERF_RECORD_AUX event, and the decoder
2951 * is reset across each buffer, so splitting the buffers up in advance has
2952 * the same effect.
2953 */
2954static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_offset, size_t sz,
2955 struct perf_record_aux *aux_event, struct perf_sample *sample)
2956{
2957 int err;
2958 char buf[PERF_SAMPLE_MAX_SIZE];
2959 union perf_event *auxtrace_event_union;
2960 struct perf_record_auxtrace *auxtrace_event;
2961 union perf_event auxtrace_fragment;
2962 __u64 aux_offset, aux_size;
2963 __u32 idx;
2964 bool formatted;
2965
2966 struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2967 struct cs_etm_auxtrace,
2968 auxtrace);
2969
2970 /*
2971 * There should be a PERF_RECORD_AUXTRACE event at the file_offset that we got
2972 * from looping through the auxtrace index.
2973 */
2974 err = perf_session__peek_event(session, file_offset, buf,
2975 PERF_SAMPLE_MAX_SIZE, &auxtrace_event_union, NULL);
2976 if (err)
2977 return err;
2978 auxtrace_event = &auxtrace_event_union->auxtrace;
2979 if (auxtrace_event->header.type != PERF_RECORD_AUXTRACE)
2980 return -EINVAL;
2981
2982 if (auxtrace_event->header.size < sizeof(struct perf_record_auxtrace) ||
2983 auxtrace_event->header.size != sz) {
2984 return -EINVAL;
2985 }
2986
2987 /*
2988 * In per-thread mode, auxtrace CPU is set to -1, but TID will be set instead. See
2989 * auxtrace_mmap_params__set_idx(). However, the sample AUX event will contain a
2990 * CPU as we set this always for the AUX_OUTPUT_HW_ID event.
2991 * So now compare only TIDs if auxtrace CPU is -1, and CPUs if auxtrace CPU is not -1.
2992 * Return 'not found' if mismatch.
2993 */
2994 if (auxtrace_event->cpu == (__u32) -1) {
2995 etm->per_thread_decoding = true;
2996 if (auxtrace_event->tid != sample->tid)
2997 return 1;
2998 } else if (auxtrace_event->cpu != sample->cpu) {
2999 if (etm->per_thread_decoding) {
3000 /*
3001 * Found a per-cpu buffer after a per-thread one was
3002 * already found
3003 */
3004 pr_err("CS ETM: Inconsistent per-thread/per-cpu mode.\n");
3005 return -EINVAL;
3006 }
3007 return 1;
3008 }
3009
3010 if (aux_event->flags & PERF_AUX_FLAG_OVERWRITE) {
3011 /*
3012 * Clamp size in snapshot mode. The buffer size is clamped in
3013 * __auxtrace_mmap__read() for snapshots, so the aux record size doesn't reflect
3014 * the buffer size.
3015 */
3016 aux_size = min(aux_event->aux_size, auxtrace_event->size);
3017
3018 /*
3019 * In this mode, the head also points to the end of the buffer so aux_offset
3020 * needs to have the size subtracted so it points to the beginning as in normal mode
3021 */
3022 aux_offset = aux_event->aux_offset - aux_size;
3023 } else {
3024 aux_size = aux_event->aux_size;
3025 aux_offset = aux_event->aux_offset;
3026 }
3027
3028 if (aux_offset >= auxtrace_event->offset &&
3029 aux_offset + aux_size <= auxtrace_event->offset + auxtrace_event->size) {
3030 /*
3031 * If this AUX event was inside this buffer somewhere, create a new auxtrace event
3032 * based on the sizes of the aux event, and queue that fragment.
3033 */
3034 auxtrace_fragment.auxtrace = *auxtrace_event;
3035 auxtrace_fragment.auxtrace.size = aux_size;
3036 auxtrace_fragment.auxtrace.offset = aux_offset;
3037 file_offset += aux_offset - auxtrace_event->offset + auxtrace_event->header.size;
3038
3039 pr_debug3("CS ETM: Queue buffer size: %#"PRI_lx64" offset: %#"PRI_lx64
3040 " tid: %d cpu: %d\n", aux_size, aux_offset, sample->tid, sample->cpu);
3041 err = auxtrace_queues__add_event(&etm->queues, session, &auxtrace_fragment,
3042 file_offset, NULL);
3043 if (err)
3044 return err;
3045
3046 idx = auxtrace_event->idx;
3047 formatted = !(aux_event->flags & PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW);
3048 return cs_etm__setup_queue(etm, &etm->queues.queue_array[idx],
3049 idx, formatted, sample->cpu);
3050 }
3051
3052 /* Wasn't inside this buffer, but there were no parse errors. 1 == 'not found' */
3053 return 1;
3054}
3055
3056static int cs_etm__process_aux_hw_id_cb(struct perf_session *session, union perf_event *event,
3057 u64 offset __maybe_unused, void *data __maybe_unused)
3058{
3059 /* look to handle PERF_RECORD_AUX_OUTPUT_HW_ID early to ensure decoders can be set up */
3060 if (event->header.type == PERF_RECORD_AUX_OUTPUT_HW_ID) {
3061 (*(int *)data)++; /* increment found count */
3062 return cs_etm__process_aux_output_hw_id(session, event);
3063 }
3064 return 0;
3065}
3066
3067static int cs_etm__queue_aux_records_cb(struct perf_session *session, union perf_event *event,
3068 u64 offset __maybe_unused, void *data __maybe_unused)
3069{
3070 struct perf_sample sample;
3071 int ret;
3072 struct auxtrace_index_entry *ent;
3073 struct auxtrace_index *auxtrace_index;
3074 struct evsel *evsel;
3075 size_t i;
3076
3077 /* Don't care about any other events, we're only queuing buffers for AUX events */
3078 if (event->header.type != PERF_RECORD_AUX)
3079 return 0;
3080
3081 if (event->header.size < sizeof(struct perf_record_aux))
3082 return -EINVAL;
3083
3084 /* Truncated Aux records can have 0 size and shouldn't result in anything being queued. */
3085 if (!event->aux.aux_size)
3086 return 0;
3087
3088 /*
3089 * Parse the sample, we need the sample_id_all data that comes after the event so that the
3090 * CPU or PID can be matched to an AUXTRACE buffer's CPU or PID.
3091 */
3092 evsel = evlist__event2evsel(session->evlist, event);
3093 if (!evsel)
3094 return -EINVAL;
3095 ret = evsel__parse_sample(evsel, event, &sample);
3096 if (ret)
3097 return ret;
3098
3099 /*
3100 * Loop through the auxtrace index to find the buffer that matches up with this aux event.
3101 */
3102 list_for_each_entry(auxtrace_index, &session->auxtrace_index, list) {
3103 for (i = 0; i < auxtrace_index->nr; i++) {
3104 ent = &auxtrace_index->entries[i];
3105 ret = cs_etm__queue_aux_fragment(session, ent->file_offset,
3106 ent->sz, &event->aux, &sample);
3107 /*
3108 * Stop search on error or successful values. Continue search on
3109 * 1 ('not found')
3110 */
3111 if (ret != 1)
3112 return ret;
3113 }
3114 }
3115
3116 /*
3117 * Couldn't find the buffer corresponding to this aux record, something went wrong. Warn but
3118 * don't exit with an error because it will still be possible to decode other aux records.
3119 */
3120 pr_err("CS ETM: Couldn't find auxtrace buffer for aux_offset: %#"PRI_lx64
3121 " tid: %d cpu: %d\n", event->aux.aux_offset, sample.tid, sample.cpu);
3122 return 0;
3123}
3124
3125static int cs_etm__queue_aux_records(struct perf_session *session)
3126{
3127 struct auxtrace_index *index = list_first_entry_or_null(&session->auxtrace_index,
3128 struct auxtrace_index, list);
3129 if (index && index->nr > 0)
3130 return perf_session__peek_events(session, session->header.data_offset,
3131 session->header.data_size,
3132 cs_etm__queue_aux_records_cb, NULL);
3133
3134 /*
3135 * We would get here if there are no entries in the index (either no auxtrace
3136 * buffers or no index at all). Fail silently as there is the possibility of
3137 * queueing them in cs_etm__process_auxtrace_event() if etm->data_queued is still
3138 * false.
3139 *
3140 * In that scenario, buffers will not be split by AUX records.
3141 */
3142 return 0;
3143}
3144
3145#define HAS_PARAM(j, type, param) (metadata[(j)][CS_ETM_NR_TRC_PARAMS] <= \
3146 (CS_##type##_##param - CS_ETM_COMMON_BLK_MAX_V1))
3147
3148/*
3149 * Loop through the ETMs and complain if we find at least one where ts_source != 1 (virtual
3150 * timestamps).
3151 */
3152static bool cs_etm__has_virtual_ts(u64 **metadata, int num_cpu)
3153{
3154 int j;
3155
3156 for (j = 0; j < num_cpu; j++) {
3157 switch (metadata[j][CS_ETM_MAGIC]) {
3158 case __perf_cs_etmv4_magic:
3159 if (HAS_PARAM(j, ETMV4, TS_SOURCE) || metadata[j][CS_ETMV4_TS_SOURCE] != 1)
3160 return false;
3161 break;
3162 case __perf_cs_ete_magic:
3163 if (HAS_PARAM(j, ETE, TS_SOURCE) || metadata[j][CS_ETE_TS_SOURCE] != 1)
3164 return false;
3165 break;
3166 default:
3167 /* Unknown / unsupported magic number. */
3168 return false;
3169 }
3170 }
3171 return true;
3172}
3173
3174/* map trace ids to correct metadata block, from information in metadata */
3175static int cs_etm__map_trace_ids_metadata(int num_cpu, u64 **metadata)
3176{
3177 u64 cs_etm_magic;
3178 u8 trace_chan_id;
3179 int i, err;
3180
3181 for (i = 0; i < num_cpu; i++) {
3182 cs_etm_magic = metadata[i][CS_ETM_MAGIC];
3183 switch (cs_etm_magic) {
3184 case __perf_cs_etmv3_magic:
3185 metadata[i][CS_ETM_ETMTRACEIDR] &= CORESIGHT_TRACE_ID_VAL_MASK;
3186 trace_chan_id = (u8)(metadata[i][CS_ETM_ETMTRACEIDR]);
3187 break;
3188 case __perf_cs_etmv4_magic:
3189 case __perf_cs_ete_magic:
3190 metadata[i][CS_ETMV4_TRCTRACEIDR] &= CORESIGHT_TRACE_ID_VAL_MASK;
3191 trace_chan_id = (u8)(metadata[i][CS_ETMV4_TRCTRACEIDR]);
3192 break;
3193 default:
3194 /* unknown magic number */
3195 return -EINVAL;
3196 }
3197 err = cs_etm__map_trace_id(trace_chan_id, metadata[i]);
3198 if (err)
3199 return err;
3200 }
3201 return 0;
3202}
3203
3204/*
3205 * If we found AUX_HW_ID packets, then set any metadata marked as unused to the
3206 * unused value to reduce the number of unneeded decoders created.
3207 */
3208static int cs_etm__clear_unused_trace_ids_metadata(int num_cpu, u64 **metadata)
3209{
3210 u64 cs_etm_magic;
3211 int i;
3212
3213 for (i = 0; i < num_cpu; i++) {
3214 cs_etm_magic = metadata[i][CS_ETM_MAGIC];
3215 switch (cs_etm_magic) {
3216 case __perf_cs_etmv3_magic:
3217 if (metadata[i][CS_ETM_ETMTRACEIDR] & CORESIGHT_TRACE_ID_UNUSED_FLAG)
3218 metadata[i][CS_ETM_ETMTRACEIDR] = CORESIGHT_TRACE_ID_UNUSED_VAL;
3219 break;
3220 case __perf_cs_etmv4_magic:
3221 case __perf_cs_ete_magic:
3222 if (metadata[i][CS_ETMV4_TRCTRACEIDR] & CORESIGHT_TRACE_ID_UNUSED_FLAG)
3223 metadata[i][CS_ETMV4_TRCTRACEIDR] = CORESIGHT_TRACE_ID_UNUSED_VAL;
3224 break;
3225 default:
3226 /* unknown magic number */
3227 return -EINVAL;
3228 }
3229 }
3230 return 0;
3231}
3232
3233int cs_etm__process_auxtrace_info_full(union perf_event *event,
3234 struct perf_session *session)
3235{
3236 struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;
3237 struct cs_etm_auxtrace *etm = NULL;
3238 struct perf_record_time_conv *tc = &session->time_conv;
3239 int event_header_size = sizeof(struct perf_event_header);
3240 int total_size = auxtrace_info->header.size;
3241 int priv_size = 0;
3242 int num_cpu;
3243 int err = 0;
3244 int aux_hw_id_found;
3245 int i, j;
3246 u64 *ptr = NULL;
3247 u64 **metadata = NULL;
3248
3249 /*
3250 * Create an RB tree for traceID-metadata tuple. Since the conversion
3251 * has to be made for each packet that gets decoded, optimizing access
3252 * in anything other than a sequential array is worth doing.
3253 */
3254 traceid_list = intlist__new(NULL);
3255 if (!traceid_list)
3256 return -ENOMEM;
3257
3258 /* First the global part */
3259 ptr = (u64 *) auxtrace_info->priv;
3260 num_cpu = ptr[CS_PMU_TYPE_CPUS] & 0xffffffff;
3261 metadata = zalloc(sizeof(*metadata) * num_cpu);
3262 if (!metadata) {
3263 err = -ENOMEM;
3264 goto err_free_traceid_list;
3265 }
3266
3267 /* Start parsing after the common part of the header */
3268 i = CS_HEADER_VERSION_MAX;
3269
3270 /*
3271 * The metadata is stored in the auxtrace_info section and encodes
3272 * the configuration of the ARM embedded trace macrocell which is
3273 * required by the trace decoder to properly decode the trace due
3274 * to its highly compressed nature.
3275 */
3276 for (j = 0; j < num_cpu; j++) {
3277 if (ptr[i] == __perf_cs_etmv3_magic) {
3278 metadata[j] =
3279 cs_etm__create_meta_blk(ptr, &i,
3280 CS_ETM_PRIV_MAX,
3281 CS_ETM_NR_TRC_PARAMS_V0);
3282 } else if (ptr[i] == __perf_cs_etmv4_magic) {
3283 metadata[j] =
3284 cs_etm__create_meta_blk(ptr, &i,
3285 CS_ETMV4_PRIV_MAX,
3286 CS_ETMV4_NR_TRC_PARAMS_V0);
3287 } else if (ptr[i] == __perf_cs_ete_magic) {
3288 metadata[j] = cs_etm__create_meta_blk(ptr, &i, CS_ETE_PRIV_MAX, -1);
3289 } else {
3290 ui__error("CS ETM Trace: Unrecognised magic number %#"PRIx64". File could be from a newer version of perf.\n",
3291 ptr[i]);
3292 err = -EINVAL;
3293 goto err_free_metadata;
3294 }
3295
3296 if (!metadata[j]) {
3297 err = -ENOMEM;
3298 goto err_free_metadata;
3299 }
3300 }
3301
3302 /*
3303 * Each of CS_HEADER_VERSION_MAX, CS_ETM_PRIV_MAX and
3304 * CS_ETMV4_PRIV_MAX mark how many double words are in the
3305 * global metadata, and each cpu's metadata respectively.
3306 * The following tests if the correct number of double words was
3307 * present in the auxtrace info section.
3308 */
3309 priv_size = total_size - event_header_size - INFO_HEADER_SIZE;
3310 if (i * 8 != priv_size) {
3311 err = -EINVAL;
3312 goto err_free_metadata;
3313 }
3314
3315 etm = zalloc(sizeof(*etm));
3316
3317 if (!etm) {
3318 err = -ENOMEM;
3319 goto err_free_metadata;
3320 }
3321
3322 /*
3323 * As all the ETMs run at the same exception level, the system should
3324 * have the same PID format crossing CPUs. So cache the PID format
3325 * and reuse it for sequential decoding.
3326 */
3327 etm->pid_fmt = cs_etm__init_pid_fmt(metadata[0]);
3328
3329 err = auxtrace_queues__init(&etm->queues);
3330 if (err)
3331 goto err_free_etm;
3332
3333 if (session->itrace_synth_opts->set) {
3334 etm->synth_opts = *session->itrace_synth_opts;
3335 } else {
3336 itrace_synth_opts__set_default(&etm->synth_opts,
3337 session->itrace_synth_opts->default_no_sample);
3338 etm->synth_opts.callchain = false;
3339 }
3340
3341 etm->session = session;
3342
3343 etm->num_cpu = num_cpu;
3344 etm->pmu_type = (unsigned int) ((ptr[CS_PMU_TYPE_CPUS] >> 32) & 0xffffffff);
3345 etm->snapshot_mode = (ptr[CS_ETM_SNAPSHOT] != 0);
3346 etm->metadata = metadata;
3347 etm->auxtrace_type = auxtrace_info->type;
3348
3349 if (etm->synth_opts.use_timestamp)
3350 /*
3351 * Prior to Armv8.4, Arm CPUs don't support FEAT_TRF feature,
3352 * therefore the decoder cannot know if the timestamp trace is
3353 * same with the kernel time.
3354 *
3355 * If a user has knowledge for the working platform and can
3356 * specify itrace option 'T' to tell decoder to forcely use the
3357 * traced timestamp as the kernel time.
3358 */
3359 etm->has_virtual_ts = true;
3360 else
3361 /* Use virtual timestamps if all ETMs report ts_source = 1 */
3362 etm->has_virtual_ts = cs_etm__has_virtual_ts(metadata, num_cpu);
3363
3364 if (!etm->has_virtual_ts)
3365 ui__warning("Virtual timestamps are not enabled, or not supported by the traced system.\n"
3366 "The time field of the samples will not be set accurately.\n"
3367 "For Arm CPUs prior to Armv8.4 or without support FEAT_TRF,\n"
3368 "you can specify the itrace option 'T' for timestamp decoding\n"
3369 "if the Coresight timestamp on the platform is same with the kernel time.\n\n");
3370
3371 etm->auxtrace.process_event = cs_etm__process_event;
3372 etm->auxtrace.process_auxtrace_event = cs_etm__process_auxtrace_event;
3373 etm->auxtrace.flush_events = cs_etm__flush_events;
3374 etm->auxtrace.free_events = cs_etm__free_events;
3375 etm->auxtrace.free = cs_etm__free;
3376 etm->auxtrace.evsel_is_auxtrace = cs_etm__evsel_is_auxtrace;
3377 session->auxtrace = &etm->auxtrace;
3378
3379 err = cs_etm__setup_timeless_decoding(etm);
3380 if (err)
3381 return err;
3382
3383 etm->tc.time_shift = tc->time_shift;
3384 etm->tc.time_mult = tc->time_mult;
3385 etm->tc.time_zero = tc->time_zero;
3386 if (event_contains(*tc, time_cycles)) {
3387 etm->tc.time_cycles = tc->time_cycles;
3388 etm->tc.time_mask = tc->time_mask;
3389 etm->tc.cap_user_time_zero = tc->cap_user_time_zero;
3390 etm->tc.cap_user_time_short = tc->cap_user_time_short;
3391 }
3392 err = cs_etm__synth_events(etm, session);
3393 if (err)
3394 goto err_free_queues;
3395
3396 /*
3397 * Map Trace ID values to CPU metadata.
3398 *
3399 * Trace metadata will always contain Trace ID values from the legacy algorithm. If the
3400 * files has been recorded by a "new" perf updated to handle AUX_HW_ID then the metadata
3401 * ID value will also have the CORESIGHT_TRACE_ID_UNUSED_FLAG set.
3402 *
3403 * The updated kernel drivers that use AUX_HW_ID to sent Trace IDs will attempt to use
3404 * the same IDs as the old algorithm as far as is possible, unless there are clashes
3405 * in which case a different value will be used. This means an older perf may still
3406 * be able to record and read files generate on a newer system.
3407 *
3408 * For a perf able to interpret AUX_HW_ID packets we first check for the presence of
3409 * those packets. If they are there then the values will be mapped and plugged into
3410 * the metadata. We then set any remaining metadata values with the used flag to a
3411 * value CORESIGHT_TRACE_ID_UNUSED_VAL - which indicates no decoder is required.
3412 *
3413 * If no AUX_HW_ID packets are present - which means a file recorded on an old kernel
3414 * then we map Trace ID values to CPU directly from the metadata - clearing any unused
3415 * flags if present.
3416 */
3417
3418 /* first scan for AUX_OUTPUT_HW_ID records to map trace ID values to CPU metadata */
3419 aux_hw_id_found = 0;
3420 err = perf_session__peek_events(session, session->header.data_offset,
3421 session->header.data_size,
3422 cs_etm__process_aux_hw_id_cb, &aux_hw_id_found);
3423 if (err)
3424 goto err_free_queues;
3425
3426 /* if HW ID found then clear any unused metadata ID values */
3427 if (aux_hw_id_found)
3428 err = cs_etm__clear_unused_trace_ids_metadata(num_cpu, metadata);
3429 /* otherwise, this is a file with metadata values only, map from metadata */
3430 else
3431 err = cs_etm__map_trace_ids_metadata(num_cpu, metadata);
3432
3433 if (err)
3434 goto err_free_queues;
3435
3436 err = cs_etm__queue_aux_records(session);
3437 if (err)
3438 goto err_free_queues;
3439
3440 etm->data_queued = etm->queues.populated;
3441 return 0;
3442
3443err_free_queues:
3444 auxtrace_queues__free(&etm->queues);
3445 session->auxtrace = NULL;
3446err_free_etm:
3447 zfree(&etm);
3448err_free_metadata:
3449 /* No need to check @metadata[j], free(NULL) is supported */
3450 for (j = 0; j < num_cpu; j++)
3451 zfree(&metadata[j]);
3452 zfree(&metadata);
3453err_free_traceid_list:
3454 intlist__delete(traceid_list);
3455 return err;
3456}